UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

xprtrdma: Invoke rpcrdma_ia_open in the connect worker 

Move rdma_cm_id creation into rpcrdma_ep_create() so that it is now
responsible for allocating all per-connection hardware resources.

With this clean-up, all three arms of the switch statement in
rpcrdma_ep_connect are exactly the same now, thus the switch can be
removed.

Because device removal behaves a little differently than
disconnection, there is a little more work to be done before
rpcrdma_ep_destroy() can release the connection's rdma_cm_id. So
it is not quite symmetrical with rpcrdma_ep_create() yet.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
This commit is contained in:
Chuck Lever 2020-02-21 17:00:38 -05:00 committed by Anna Schumaker
parent 9ba373ee24
commit 81fe0c57f4
4 changed files with 20 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/trace/events/rpcrdma.h
View File

@ -415,7 +415,6 @@ DEFINE_CONN_EVENT(disconnect);
DEFINE_RXPRT_EVENT(xprtrdma_create);
DEFINE_RXPRT_EVENT(xprtrdma_op_destroy);
DEFINE_RXPRT_EVENT(xprtrdma_remove);
DEFINE_RXPRT_EVENT(xprtrdma_reinsert);
DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc);
DEFINE_RXPRT_EVENT(xprtrdma_op_close);
DEFINE_RXPRT_EVENT(xprtrdma_op_setport);

7
net/sunrpc/xprtrdma/transport.c
View File

@ -286,7 +286,6 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
rpcrdma_xprt_disconnect(r_xprt);
rpcrdma_buffer_destroy(&r_xprt->rx_buf);
rpcrdma_ia_close(&r_xprt->rx_ia);
xprt_rdma_free_addresses(xprt);
xprt_free(xprt);
@ -347,10 +346,6 @@ xprt_setup_rdma(struct xprt_create *args)
xprt_rdma_format_addresses(xprt, sap);
new_xprt = rpcx_to_rdmax(xprt);
rc = rpcrdma_ia_open(new_xprt);
if (rc)
goto out1;
rc = rpcrdma_buffer_create(new_xprt);
if (rc)
goto out2;
@ -372,8 +367,6 @@ out4:
rpcrdma_buffer_destroy(&new_xprt->rx_buf);
rc = -ENODEV;
out2:
rpcrdma_ia_close(&new_xprt->rx_ia);
out1:
trace_xprtrdma_op_destroy(new_xprt);
xprt_rdma_free_addresses(xprt);
xprt_free(xprt);

153
net/sunrpc/xprtrdma/verbs.c
View File

@ -345,31 +345,6 @@ out:
* Exported functions.
*/
/**
* rpcrdma_ia_open - Open and initialize an Interface Adapter.
* @xprt: transport with IA to (re)initialize
*
* Returns 0 on success, negative errno if an appropriate
* Interface Adapter could not be found and opened.
*/
int
rpcrdma_ia_open(struct rpcrdma_xprt *xprt)
{
struct rpcrdma_ia *ia = &xprt->rx_ia;
int rc;
ia->ri_id = rpcrdma_create_id(xprt, ia);
if (IS_ERR(ia->ri_id)) {
rc = PTR_ERR(ia->ri_id);
goto out_err;
}
return 0;
out_err:
rpcrdma_ia_close(ia);
return rc;
}
/**
* rpcrdma_ia_remove - Handle device driver unload
* @ia: interface adapter being removed
@ -401,34 +376,26 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia)
trace_xprtrdma_remove(r_xprt);
}
/**
* rpcrdma_ia_close - Clean up/close an IA.
* @ia: interface adapter to close
*
*/
void
rpcrdma_ia_close(struct rpcrdma_ia *ia)
{
if (ia->ri_id && !IS_ERR(ia->ri_id))
rdma_destroy_id(ia->ri_id);
ia->ri_id = NULL;
}
static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt,
struct rdma_cm_id *id)
static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
{
struct rpcrdma_ep *ep = &r_xprt->rx_ep;
struct rpcrdma_ia *ia = &r_xprt->rx_ia;
struct rpcrdma_connect_private *pmsg = &ep->rep_cm_private;
struct rdma_cm_id *id;
int rc;
id = rpcrdma_create_id(r_xprt, ia);
if (IS_ERR(id))
return PTR_ERR(id);
ep->rep_max_requests = r_xprt->rx_xprt.max_reqs;
ep->rep_inline_send = xprt_rdma_max_inline_write;
ep->rep_inline_recv = xprt_rdma_max_inline_read;
rc = frwr_query_device(r_xprt, id->device);
if (rc)
return rc;
goto out_destroy;
r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->rep_max_requests);
ep->rep_attr.event_handler = rpcrdma_qp_event_handler;
@ -507,10 +474,12 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt,
rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr);
if (rc)
goto out_destroy;
ia->ri_id = id;
return 0;
out_destroy:
rpcrdma_ep_destroy(r_xprt);
rdma_destroy_id(id);
return rc;
}
@ -536,79 +505,8 @@ static void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt)
ia->ri_pd = NULL;
}
/* Re-establish a connection after a device removal event.
* Unlike a normal reconnection, a fresh PD and a new set
* of MRs and buffers is needed.
*/
static int rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt)
{
struct rpcrdma_ia *ia = &r_xprt->rx_ia;
int rc, err;
trace_xprtrdma_reinsert(r_xprt);
rc = -EHOSTUNREACH;
if (rpcrdma_ia_open(r_xprt))
goto out1;
rc = -ENETUNREACH;
err = rpcrdma_ep_create(r_xprt, ia->ri_id);
if (err)
goto out2;
return 0;
out2:
rpcrdma_ia_close(ia);
out1:
return rc;
}
static int rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt)
{
struct rpcrdma_ia *ia = &r_xprt->rx_ia;
struct rdma_cm_id *id, *old;
int err, rc;
rc = -EHOSTUNREACH;
id = rpcrdma_create_id(r_xprt, ia);
if (IS_ERR(id))
goto out;
/* As long as the new ID points to the same device as the
* old ID, we can reuse the transport's existing PD and all
* previously allocated MRs. Also, the same device means
* the transport's previous DMA mappings are still valid.
*
* This is a sanity check only. There should be no way these
* point to two different devices here.
*/
old = id;
rc = -ENETUNREACH;
if (ia->ri_id->device != id->device) {
pr_err("rpcrdma: can't reconnect on different device!\n");
goto out_destroy;
}
err = rpcrdma_ep_create(r_xprt, id);
if (err)
goto out_destroy;
/* Atomically replace the transport's ID. */
rc = 0;
old = ia->ri_id;
ia->ri_id = id;
out_destroy:
rdma_destroy_id(old);
out:
return rc;
}
/**
* rpcrdma_xprt_connect - Connect an unconnected transport
* @r_xprt: controlling transport instance
*
* Returns 0 on success or a negative errno.
/*
* Connect unconnected endpoint.
*/
int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)
{
@ -618,25 +516,10 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)
int rc;
retry:
switch (ep->rep_connected) {
case 0:
rc = -ENETUNREACH;
if (rpcrdma_ep_create(r_xprt, ia->ri_id))
goto out_noupdate;
break;
case -ENODEV:
rc = rpcrdma_ep_recreate_xprt(r_xprt);
if (rc)
goto out_noupdate;
break;
case 1:
rpcrdma_xprt_disconnect(r_xprt);
/* fall through */
default:
rc = rpcrdma_ep_reconnect(r_xprt);
if (rc)
goto out;
}
rpcrdma_xprt_disconnect(r_xprt);
rc = rpcrdma_ep_create(r_xprt);
if (rc)
goto out_noupdate;
ep->rep_connected = 0;
xprt_clear_connected(xprt);
@ -712,6 +595,10 @@ out:
rpcrdma_sendctxs_destroy(r_xprt);
rpcrdma_ep_destroy(r_xprt);
if (ia->ri_id)
rdma_destroy_id(ia->ri_id);
ia->ri_id = NULL;
}
/* Fixed-size circular FIFO queue. This implementation is wait-free and

2
net/sunrpc/xprtrdma/xprt_rdma.h
View File

@ -457,9 +457,7 @@ extern unsigned int xprt_rdma_memreg_strategy;
/*
* Interface Adapter calls - xprtrdma/verbs.c
*/
int rpcrdma_ia_open(struct rpcrdma_xprt *xprt);
void rpcrdma_ia_remove(struct rpcrdma_ia *ia);
void rpcrdma_ia_close(struct rpcrdma_ia *);
/*
* Endpoint calls - xprtrdma/verbs.c