From ec6995ddaa3db2c5f028184ed9b74c0c448c4947 Mon Sep 17 00:00:00 2001 From: Steve Wise Date: Wed, 29 Apr 2009 15:15:59 -0700 Subject: [PATCH 1/2] RDMA/cxgb3: Don't complete flushed send work requests twice When the SQ is flushed, mark the flushed entries as not signaled so the poll logic doesn't re-insert the CQ entry thinking its an out of order completion. The bug can cause the NFS/RDMA server to crash due to processing the same completed work request twice. Signed-off-by: Steve Wise Signed-off-by: Roland Dreier --- drivers/infiniband/hw/cxgb3/cxio_hal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c index 8d71086f5a1c..62f9cf2f94ec 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.c +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c @@ -410,6 +410,7 @@ int cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count) ptr = wq->sq_rptr + count; sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2); while (ptr != wq->sq_wptr) { + sqp->signaled = 0; insert_sq_cqe(wq, cq, sqp); ptr++; sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2); From 2b6b7d4be487bada8c727df829c25068c7b5e5a3 Mon Sep 17 00:00:00 2001 From: Jack Morgenstein Date: Thu, 7 May 2009 21:35:13 -0700 Subject: [PATCH 2/2] IB/mlx4: Don't overwrite fast registration page list when posting work request The low-level mlx4 driver modified the page-list addresses for fast register work requests post send to big-endian, and set a "present" bit. This caused problems later when the consumer attempted to unmap the pages using the page-list (using the list addresses which were assumed to be still in CPU-endian order). Fix the mlx4 driver to allocate two buffers and use a private buffer for the hardware-format bus addresses. This patch fixes , an NFS/RDMA server crash. The cause of the crash was found by Vu Pham of Mellanox. The fix is along the lines suggested by Steve Wise in comment #21 in bug 1571. Signed-off-by: Jack Morgenstein Signed-off-by: Roland Dreier --- drivers/infiniband/hw/mlx4/mlx4_ib.h | 1 + drivers/infiniband/hw/mlx4/mr.c | 10 ++++++++-- drivers/infiniband/hw/mlx4/qp.c | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 9974e886b8de..8a7dd6795fa0 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -86,6 +86,7 @@ struct mlx4_ib_mr { struct mlx4_ib_fast_reg_page_list { struct ib_fast_reg_page_list ibfrpl; + __be64 *mapped_page_list; dma_addr_t map; }; diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 8e4d26d56a95..8f3666b20ea4 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -231,7 +231,11 @@ struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device if (!mfrpl) return ERR_PTR(-ENOMEM); - mfrpl->ibfrpl.page_list = dma_alloc_coherent(&dev->dev->pdev->dev, + mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL); + if (!mfrpl->ibfrpl.page_list) + goto err_free; + + mfrpl->mapped_page_list = dma_alloc_coherent(&dev->dev->pdev->dev, size, &mfrpl->map, GFP_KERNEL); if (!mfrpl->ibfrpl.page_list) @@ -242,6 +246,7 @@ struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device return &mfrpl->ibfrpl; err_free: + kfree(mfrpl->ibfrpl.page_list); kfree(mfrpl); return ERR_PTR(-ENOMEM); } @@ -252,8 +257,9 @@ void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list) struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list); int size = page_list->max_page_list_len * sizeof (u64); - dma_free_coherent(&dev->dev->pdev->dev, size, page_list->page_list, + dma_free_coherent(&dev->dev->pdev->dev, size, mfrpl->mapped_page_list, mfrpl->map); + kfree(mfrpl->ibfrpl.page_list); kfree(mfrpl); } diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index f385a24d31d2..20724aee76f4 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -1365,7 +1365,7 @@ static void set_fmr_seg(struct mlx4_wqe_fmr_seg *fseg, struct ib_send_wr *wr) int i; for (i = 0; i < wr->wr.fast_reg.page_list_len; ++i) - wr->wr.fast_reg.page_list->page_list[i] = + mfrpl->mapped_page_list[i] = cpu_to_be64(wr->wr.fast_reg.page_list->page_list[i] | MLX4_MTT_FLAG_PRESENT);