From 6ff8da088766d70f0441feb982b82978a6cbf7ef Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Fri, 4 Jun 2010 20:04:45 -0400 Subject: [PATCH] nfsd4: Move callback setup to callback queue Instead of creating the new rpc client from a regular server thread, set a flag, kick off a null call, and allow the null call to do the work of setting up the client on the callback workqueue. Use a spinlock to ensure the callback work gets a consistent view of the callback parameters. This allows, for example, changing the callback from contexts where sleeping is not allowed. I hope it will also keep the locking simple as we add more session and trunking features, by serializing most of the callback-specific work. This also closes a small race where the the new cb_ident could be used with an old connection (or vice-versa). Signed-off-by: J. Bruce Fields --- fs/nfsd/nfs4callback.c | 73 ++++++++++++++++++++++++++++++------------ fs/nfsd/nfs4state.c | 7 ++-- fs/nfsd/state.h | 10 ++++-- 3 files changed, 63 insertions(+), 27 deletions(-) diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 07c3be6eea64..a269dbeff150 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -284,7 +284,7 @@ nfs4_xdr_enc_cb_recall(struct rpc_rqst *req, __be32 *p, struct xdr_stream xdr; struct nfs4_delegation *args = cb->cb_op; struct nfs4_cb_compound_hdr hdr = { - .ident = args->dl_ident, + .ident = cb->cb_clp->cl_cb_ident, .minorversion = cb->cb_minorversion, }; @@ -506,7 +506,8 @@ int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *conn) PTR_ERR(client)); return PTR_ERR(client); } - nfsd4_set_callback_client(clp, client); + clp->cl_cb_ident = conn->cb_ident; + clp->cl_cb_client = client; return 0; } @@ -569,15 +570,12 @@ void do_probe_callback(struct nfs4_client *clp) */ void nfsd4_probe_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn) { - int status; - BUG_ON(atomic_read(&clp->cl_cb_set)); - status = setup_callback_client(clp, conn); - if (status) { - warn_no_callback_path(clp, status); - return; - } + spin_lock(&clp->cl_lock); + memcpy(&clp->cl_cb_conn, conn, sizeof(struct nfs4_cb_conn)); + set_bit(NFSD4_CLIENT_CB_UPDATE, &clp->cl_cb_flags); + spin_unlock(&clp->cl_lock); do_probe_callback(clp); } @@ -730,19 +728,16 @@ void nfsd4_destroy_callback_queue(void) } /* must be called under the state lock */ -void nfsd4_set_callback_client(struct nfs4_client *clp, struct rpc_clnt *new) +void nfsd4_shutdown_callback(struct nfs4_client *clp) { - struct rpc_clnt *old = clp->cl_cb_client; - - clp->cl_cb_client = new; + set_bit(NFSD4_CLIENT_KILL, &clp->cl_cb_flags); /* - * After this, any work that saw the old value of cl_cb_client will - * be gone: + * Note this won't actually result in a null callback; + * instead, nfsd4_do_callback_rpc() will detect the killed + * client, destroy the rpc client, and stop: */ + do_probe_callback(clp); flush_workqueue(callback_wq); - /* So we can safely shut it down: */ - if (old) - rpc_shutdown_client(old); } void nfsd4_release_cb(struct nfsd4_callback *cb) @@ -751,15 +746,51 @@ void nfsd4_release_cb(struct nfsd4_callback *cb) cb->cb_ops->rpc_release(cb); } +void nfsd4_process_cb_update(struct nfsd4_callback *cb) +{ + struct nfs4_cb_conn conn; + struct nfs4_client *clp = cb->cb_clp; + int err; + + /* + * This is either an update, or the client dying; in either case, + * kill the old client: + */ + if (clp->cl_cb_client) { + rpc_shutdown_client(clp->cl_cb_client); + clp->cl_cb_client = NULL; + } + if (test_bit(NFSD4_CLIENT_KILL, &clp->cl_cb_flags)) + return; + spin_lock(&clp->cl_lock); + /* + * Only serialized callback code is allowed to clear these + * flags; main nfsd code can only set them: + */ + BUG_ON(!clp->cl_cb_flags); + clear_bit(NFSD4_CLIENT_CB_UPDATE, &clp->cl_cb_flags); + memcpy(&conn, &cb->cb_clp->cl_cb_conn, sizeof(struct nfs4_cb_conn)); + spin_unlock(&clp->cl_lock); + + err = setup_callback_client(clp, &conn); + if (err) + warn_no_callback_path(clp, err); +} + void nfsd4_do_callback_rpc(struct work_struct *w) { struct nfsd4_callback *cb = container_of(w, struct nfsd4_callback, cb_work); struct nfs4_client *clp = cb->cb_clp; - struct rpc_clnt *clnt = clp->cl_cb_client; + struct rpc_clnt *clnt; - if (clnt == NULL) { + if (clp->cl_cb_flags) + nfsd4_process_cb_update(cb); + + clnt = clp->cl_cb_client; + if (!clnt) { + /* Callback channel broken, or client killed; give up: */ nfsd4_release_cb(cb); - return; /* Client is shutting down; give up. */ + return; } rpc_call_async(clnt, &cb->cb_msg, RPC_TASK_SOFT | RPC_TASK_SOFTCONN, cb->cb_ops, cb); diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 2f464fb26afc..d3f12dcc1696 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -207,7 +207,6 @@ alloc_init_deleg(struct nfs4_client *clp, struct nfs4_stateid *stp, struct svc_f { struct nfs4_delegation *dp; struct nfs4_file *fp = stp->st_file; - struct nfs4_cb_conn *conn = &stp->st_stateowner->so_client->cl_cb_conn; dprintk("NFSD alloc_init_deleg\n"); /* @@ -234,7 +233,6 @@ alloc_init_deleg(struct nfs4_client *clp, struct nfs4_stateid *stp, struct svc_f nfs4_file_get_access(fp, O_RDONLY); dp->dl_flock = NULL; dp->dl_type = type; - dp->dl_ident = conn->cb_ident; dp->dl_stateid.si_boot = boot_time; dp->dl_stateid.si_stateownerid = current_delegid++; dp->dl_stateid.si_fileid = 0; @@ -875,7 +873,7 @@ expire_client(struct nfs4_client *clp) sop = list_entry(clp->cl_openowners.next, struct nfs4_stateowner, so_perclient); release_openowner(sop); } - nfsd4_set_callback_client(clp, NULL); + nfsd4_shutdown_callback(clp); if (clp->cl_cb_conn.cb_xprt) svc_xprt_put(clp->cl_cb_conn.cb_xprt); list_del(&clp->cl_idhash); @@ -978,6 +976,7 @@ static struct nfs4_client *create_client(struct xdr_netobj name, char *recdir, INIT_LIST_HEAD(&clp->cl_delegations); INIT_LIST_HEAD(&clp->cl_sessions); INIT_LIST_HEAD(&clp->cl_lru); + spin_lock_init(&clp->cl_lock); INIT_WORK(&clp->cl_cb_null.cb_work, nfsd4_do_callback_rpc); clp->cl_time = get_seconds(); clear_bit(0, &clp->cl_cb_slot_busy); @@ -1547,7 +1546,7 @@ nfsd4_destroy_session(struct svc_rqst *r, nfs4_lock_state(); /* wait for callbacks */ - nfsd4_set_callback_client(ses->se_client, NULL); + nfsd4_shutdown_callback(ses->se_client); nfs4_unlock_state(); nfsd4_put_session(ses); status = nfs_ok; diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index 2ece6bee65f7..58bc2a63ca14 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -84,7 +84,6 @@ struct nfs4_delegation { u32 dl_type; time_t dl_time; /* For recall: */ - u32 dl_ident; stateid_t dl_stateid; struct knfsd_fh dl_fh; int dl_retries; @@ -217,10 +216,17 @@ struct nfs4_client { /* for v4.0 and v4.1 callbacks: */ struct nfs4_cb_conn cl_cb_conn; +#define NFSD4_CLIENT_CB_UPDATE 1 +#define NFSD4_CLIENT_KILL 2 + unsigned long cl_cb_flags; struct rpc_clnt *cl_cb_client; + u32 cl_cb_ident; atomic_t cl_cb_set; struct nfsd4_callback cl_cb_null; + /* for all client information that callback code might need: */ + spinlock_t cl_lock; + /* for nfs41 */ struct list_head cl_sessions; struct nfsd4_clid_slot cl_cs_slot; /* create_session slot */ @@ -439,7 +445,7 @@ extern void nfsd4_do_callback_rpc(struct work_struct *); extern void nfsd4_cb_recall(struct nfs4_delegation *dp); extern int nfsd4_create_callback_queue(void); extern void nfsd4_destroy_callback_queue(void); -extern void nfsd4_set_callback_client(struct nfs4_client *, struct rpc_clnt *); +extern void nfsd4_shutdown_callback(struct nfs4_client *); extern void nfs4_put_delegation(struct nfs4_delegation *dp); extern __be32 nfs4_make_rec_clidname(char *clidname, struct xdr_netobj *clname); extern void nfsd4_init_recdir(char *recdir_name);