UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

IB/isert: Initialize T10-PI resources 

Introduce pi_context to hold relevant RDMA protection resources.
We eliminate data_key_valid boolean and replace it with indicators
container to indicate:
- Is the descriptor protected (registered via signature MR)
- Is the data_mr key valid (can spare LOCAL_INV WR)
- Is the prot_mr key valid (can spare LOCAL_INV WR)
- Is the sig_mr key valid (can spare LOCAL_INV WR)

Upon connection establishment check if network portal is T10-PI
enabled and allocate T10-PI resources if necessary, allocate
signature enabled memory regions and mark connection queue-pair
as signature enabled.

(Fix context change for v3.14-rc6 code - nab)

Signed-off-by: Sagi Grimberg <sagig@mellanox.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
This commit is contained in:
Sagi Grimberg 2014-02-19 17:50:23 +02:00 committed by Nicholas Bellinger
parent e3d7e4c30c
commit d3e125dac1
2 changed files with 110 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
drivers/infiniband/ulp/isert/ib_isert.c
View File

@ -87,7 +87,8 @@ isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr)
}
static int
isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id,
u8 protection)
{
struct isert_device *device = isert_conn->conn_device;
struct ib_qp_init_attr attr;
@ -119,6 +120,8 @@ isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
attr.cap.max_recv_sge = 1;
attr.sq_sig_type = IB_SIGNAL_REQ_WR;
attr.qp_type = IB_QPT_RC;
if (protection)
attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN;
pr_debug("isert_conn_setup_qp cma_id->device: %p\n",
cma_id->device);
@ -236,13 +239,18 @@ isert_create_device_ib_res(struct isert_device *device)
device->unreg_rdma_mem = isert_unmap_cmd;
}
/* Check signature cap */
device->pi_capable = dev_attr->device_cap_flags &
IB_DEVICE_SIGNATURE_HANDOVER ? true : false;
device->cqs_used = min_t(int, num_online_cpus(),
device->ib_device->num_comp_vectors);
device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used);
pr_debug("Using %d CQs, device %s supports %d vectors support "
"Fast registration %d\n",
"Fast registration %d pi_capable %d\n",
device->cqs_used, device->ib_device->name,
device->ib_device->num_comp_vectors, device->use_fastreg);
device->ib_device->num_comp_vectors, device->use_fastreg,
device->pi_capable);
device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) *
device->cqs_used, GFP_KERNEL);
if (!device->cq_desc) {
@ -395,6 +403,12 @@ isert_conn_free_fastreg_pool(struct isert_conn *isert_conn)
list_del(&fr_desc->list);
ib_free_fast_reg_page_list(fr_desc->data_frpl);
ib_dereg_mr(fr_desc->data_mr);
if (fr_desc->pi_ctx) {
ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl);
ib_dereg_mr(fr_desc->pi_ctx->prot_mr);
ib_destroy_mr(fr_desc->pi_ctx->sig_mr);
kfree(fr_desc->pi_ctx);
}
kfree(fr_desc);
++i;
}
@ -406,8 +420,10 @@ isert_conn_free_fastreg_pool(struct isert_conn *isert_conn)
static int
isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd,
struct fast_reg_descriptor *fr_desc)
struct fast_reg_descriptor *fr_desc, u8 protection)
{
int ret;
fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device,
ISCSI_ISER_SG_TABLESIZE);
if (IS_ERR(fr_desc->data_frpl)) {
@ -420,19 +436,73 @@ isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd,
if (IS_ERR(fr_desc->data_mr)) {
pr_err("Failed to allocate data frmr err=%ld\n",
PTR_ERR(fr_desc->data_mr));
ib_free_fast_reg_page_list(fr_desc->data_frpl);
return PTR_ERR(fr_desc->data_mr);
ret = PTR_ERR(fr_desc->data_mr);
goto err_data_frpl;
}
pr_debug("Create fr_desc %p page_list %p\n",
fr_desc, fr_desc->data_frpl->page_list);
fr_desc->ind |= ISERT_DATA_KEY_VALID;
fr_desc->valid = true;
if (protection) {
struct ib_mr_init_attr mr_init_attr = {0};
struct pi_context *pi_ctx;
fr_desc->pi_ctx = kzalloc(sizeof(*fr_desc->pi_ctx), GFP_KERNEL);
if (!fr_desc->pi_ctx) {
pr_err("Failed to allocate pi context\n");
ret = -ENOMEM;
goto err_data_mr;
}
pi_ctx = fr_desc->pi_ctx;
pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(ib_device,
ISCSI_ISER_SG_TABLESIZE);
if (IS_ERR(pi_ctx->prot_frpl)) {
pr_err("Failed to allocate prot frpl err=%ld\n",
PTR_ERR(pi_ctx->prot_frpl));
ret = PTR_ERR(pi_ctx->prot_frpl);
goto err_pi_ctx;
}
pi_ctx->prot_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE);
if (IS_ERR(pi_ctx->prot_mr)) {
pr_err("Failed to allocate prot frmr err=%ld\n",
PTR_ERR(pi_ctx->prot_mr));
ret = PTR_ERR(pi_ctx->prot_mr);
goto err_prot_frpl;
}
fr_desc->ind |= ISERT_PROT_KEY_VALID;
mr_init_attr.max_reg_descriptors = 2;
mr_init_attr.flags |= IB_MR_SIGNATURE_EN;
pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr);
if (IS_ERR(pi_ctx->sig_mr)) {
pr_err("Failed to allocate signature enabled mr err=%ld\n",
PTR_ERR(pi_ctx->sig_mr));
ret = PTR_ERR(pi_ctx->sig_mr);
goto err_prot_mr;
}
fr_desc->ind |= ISERT_SIG_KEY_VALID;
}
fr_desc->ind &= ~ISERT_PROTECTED;
return 0;
err_prot_mr:
ib_dereg_mr(fr_desc->pi_ctx->prot_mr);
err_prot_frpl:
ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl);
err_pi_ctx:
kfree(fr_desc->pi_ctx);
err_data_mr:
ib_dereg_mr(fr_desc->data_mr);
err_data_frpl:
ib_free_fast_reg_page_list(fr_desc->data_frpl);
return ret;
}
static int
isert_conn_create_fastreg_pool(struct isert_conn *isert_conn)
isert_conn_create_fastreg_pool(struct isert_conn *isert_conn, u8 pi_support)
{
struct fast_reg_descriptor *fr_desc;
struct isert_device *device = isert_conn->conn_device;
@ -449,7 +519,8 @@ isert_conn_create_fastreg_pool(struct isert_conn *isert_conn)
}
ret = isert_create_fr_desc(device->ib_device,
isert_conn->conn_pd, fr_desc);
isert_conn->conn_pd, fr_desc,
pi_support);
if (ret) {
pr_err("Failed to create fastreg descriptor err=%d\n",
ret);
@ -480,6 +551,7 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
struct isert_device *device;
struct ib_device *ib_dev = cma_id->device;
int ret = 0;
u8 pi_support = np->tpg_np->tpg->tpg_attrib.t10_pi;
pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n",
cma_id, cma_id->context);
@ -569,8 +641,14 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
goto out_mr;
}
if (pi_support && !device->pi_capable) {
pr_err("Protection information requested but not supported\n");
ret = -EINVAL;
goto out_mr;
}
if (device->use_fastreg) {
ret = isert_conn_create_fastreg_pool(isert_conn);
ret = isert_conn_create_fastreg_pool(isert_conn, pi_support);
if (ret) {
pr_err("Conn: %p failed to create fastreg pool\n",
isert_conn);
@ -578,7 +656,7 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
}
}
ret = isert_conn_setup_qp(isert_conn, cma_id);
ret = isert_conn_setup_qp(isert_conn, cma_id, pi_support);
if (ret)
goto out_conn_dev;
@ -2280,7 +2358,7 @@ isert_fast_reg_mr(struct isert_conn *isert_conn,
pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents,
&fr_desc->data_frpl->page_list[0]);
if (!fr_desc->valid) {
if (!(fr_desc->ind & ISERT_DATA_KEY_VALID)) {
memset(&inv_wr, 0, sizeof(inv_wr));
inv_wr.wr_id = ISER_FASTREG_LI_WRID;
inv_wr.opcode = IB_WR_LOCAL_INV;
@ -2314,7 +2392,7 @@ isert_fast_reg_mr(struct isert_conn *isert_conn,
pr_err("fast registration failed, ret:%d\n", ret);
return ret;
}
fr_desc->valid = false;
fr_desc->ind &= ~ISERT_DATA_KEY_VALID;
sge->lkey = fr_desc->data_mr->lkey;
sge->addr = fr_desc->data_frpl->page_list[0] + page_off;

17
drivers/infiniband/ulp/isert/ib_isert.h
View File

@ -50,11 +50,25 @@ struct iser_tx_desc {
struct ib_send_wr send_wr;
} __packed;
enum isert_indicator {
ISERT_PROTECTED = 1 << 0,
ISERT_DATA_KEY_VALID = 1 << 1,
ISERT_PROT_KEY_VALID = 1 << 2,
ISERT_SIG_KEY_VALID = 1 << 3,
};
struct pi_context {
struct ib_mr *prot_mr;
struct ib_fast_reg_page_list *prot_frpl;
struct ib_mr *sig_mr;
};
struct fast_reg_descriptor {
struct list_head list;
struct ib_mr *data_mr;
struct ib_fast_reg_page_list *data_frpl;
bool valid;
u8 ind;
struct pi_context *pi_ctx;
};
struct isert_data_buf {
@ -149,6 +163,7 @@ struct isert_cq_desc {
struct isert_device {
int use_fastreg;
bool pi_capable;
int cqs_used;
int refcount;
int cq_active_qps[ISERT_MAX_CQ];