hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

io_uring: remove @nxt from handlers 

There will be no use for @nxt in the handlers, and it's doesn't work
anyway, so purge it

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2020-03-03 21:33:12 +03:00 committed by Jens Axboe
parent 594506fec5
commit 014db0073c
1 changed files with 86 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

204
fs/io_uring.c
View File

@ -1804,17 +1804,6 @@ static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
io_put_req(req); io_put_req(req);
} }
static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
{
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
struct io_kiocb *nxt = NULL;
io_complete_rw_common(kiocb, res);
io_put_req_find_next(req, &nxt);
return nxt;
}
static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
{ {
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
@ -2009,14 +1998,14 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
} }
} }
static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_kiocb **nxt) static void kiocb_done(struct kiocb *kiocb, ssize_t ret)
{ {
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
if (req->flags & REQ_F_CUR_POS) if (req->flags & REQ_F_CUR_POS)
req->file->f_pos = kiocb->ki_pos; req->file->f_pos = kiocb->ki_pos;
if (ret >= 0 && kiocb->ki_complete == io_complete_rw) if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
*nxt = __io_complete_rw(kiocb, ret); io_complete_rw(kiocb, ret, 0);
else else
io_rw_done(kiocb, ret); io_rw_done(kiocb, ret);
} }
@ -2265,8 +2254,7 @@ static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
return 0; return 0;
} }
static int io_read(struct io_kiocb *req, struct io_kiocb **nxt, static int io_read(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
struct kiocb *kiocb = &req->rw.kiocb; struct kiocb *kiocb = &req->rw.kiocb;
@ -2306,7 +2294,7 @@ static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
/* Catch -EAGAIN return for forced non-blocking submission */ /* Catch -EAGAIN return for forced non-blocking submission */
if (!force_nonblock || ret2 != -EAGAIN) { if (!force_nonblock || ret2 != -EAGAIN) {
kiocb_done(kiocb, ret2, nxt); kiocb_done(kiocb, ret2);
} else { } else {
copy_iov: copy_iov:
ret = io_setup_async_rw(req, io_size, iovec, ret = io_setup_async_rw(req, io_size, iovec,
@ -2355,8 +2343,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
return 0; return 0;
} }
static int io_write(struct io_kiocb *req, struct io_kiocb **nxt, static int io_write(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
struct kiocb *kiocb = &req->rw.kiocb; struct kiocb *kiocb = &req->rw.kiocb;
@ -2420,7 +2407,7 @@ static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
ret2 = -EAGAIN; ret2 = -EAGAIN;
if (!force_nonblock || ret2 != -EAGAIN) { if (!force_nonblock || ret2 != -EAGAIN) {
kiocb_done(kiocb, ret2, nxt); kiocb_done(kiocb, ret2);
} else { } else {
copy_iov: copy_iov:
ret = io_setup_async_rw(req, io_size, iovec, ret = io_setup_async_rw(req, io_size, iovec,
@ -2477,8 +2464,7 @@ static bool io_splice_punt(struct file *file)
return !(file->f_mode & O_NONBLOCK); return !(file->f_mode & O_NONBLOCK);
} }
static int io_splice(struct io_kiocb *req, struct io_kiocb **nxt, static int io_splice(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
struct io_splice *sp = &req->splice; struct io_splice *sp = &req->splice;
struct file *in = sp->file_in; struct file *in = sp->file_in;
@ -2505,7 +2491,7 @@ static int io_splice(struct io_kiocb *req, struct io_kiocb **nxt,
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
if (ret != sp->len) if (ret != sp->len)
req_set_fail_links(req); req_set_fail_links(req);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
} }
@ -2578,7 +2564,7 @@ static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
} }
} }
static void __io_fsync(struct io_kiocb *req, struct io_kiocb **nxt) static void __io_fsync(struct io_kiocb *req)
{ {
loff_t end = req->sync.off + req->sync.len; loff_t end = req->sync.off + req->sync.len;
int ret; int ret;
@ -2589,7 +2575,7 @@ static void __io_fsync(struct io_kiocb *req, struct io_kiocb **nxt)
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
io_put_req_find_next(req, nxt); io_put_req(req);
} }
static void io_fsync_finish(struct io_wq_work **workptr) static void io_fsync_finish(struct io_wq_work **workptr)
@ -2599,25 +2585,24 @@ static void io_fsync_finish(struct io_wq_work **workptr)
if (io_req_cancelled(req)) if (io_req_cancelled(req))
return; return;
__io_fsync(req, &nxt); __io_fsync(req);
io_put_req(req); /* drop submission reference */ io_put_req(req); /* drop submission reference */
if (nxt) if (nxt)
io_wq_assign_next(workptr, nxt); io_wq_assign_next(workptr, nxt);
} }
static int io_fsync(struct io_kiocb *req, struct io_kiocb **nxt, static int io_fsync(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
/* fsync always requires a blocking context */ /* fsync always requires a blocking context */
if (force_nonblock) { if (force_nonblock) {
req->work.func = io_fsync_finish; req->work.func = io_fsync_finish;
return -EAGAIN; return -EAGAIN;
} }
__io_fsync(req, nxt); __io_fsync(req);
return 0; return 0;
} }
static void __io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt) static void __io_fallocate(struct io_kiocb *req)
{ {
int ret; int ret;
@ -2626,7 +2611,7 @@ static void __io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt)
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
io_put_req_find_next(req, nxt); io_put_req(req);
} }
static void io_fallocate_finish(struct io_wq_work **workptr) static void io_fallocate_finish(struct io_wq_work **workptr)
@ -2636,7 +2621,7 @@ static void io_fallocate_finish(struct io_wq_work **workptr)
if (io_req_cancelled(req)) if (io_req_cancelled(req))
return; return;
__io_fallocate(req, &nxt); __io_fallocate(req);
io_put_req(req); /* drop submission reference */ io_put_req(req); /* drop submission reference */
if (nxt) if (nxt)
io_wq_assign_next(workptr, nxt); io_wq_assign_next(workptr, nxt);
@ -2654,8 +2639,7 @@ static int io_fallocate_prep(struct io_kiocb *req,
return 0; return 0;
} }
static int io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt, static int io_fallocate(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
/* fallocate always requiring blocking context */ /* fallocate always requiring blocking context */
if (force_nonblock) { if (force_nonblock) {
@ -2663,7 +2647,7 @@ static int io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt,
return -EAGAIN; return -EAGAIN;
} }
__io_fallocate(req, nxt); __io_fallocate(req);
return 0; return 0;
} }
@ -2736,8 +2720,7 @@ static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0; return 0;
} }
static int io_openat2(struct io_kiocb *req, struct io_kiocb **nxt, static int io_openat2(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
struct open_flags op; struct open_flags op;
struct file *file; struct file *file;
@ -2768,15 +2751,14 @@ err:
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
} }
static int io_openat(struct io_kiocb *req, struct io_kiocb **nxt, static int io_openat(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
req->open.how = build_open_how(req->open.how.flags, req->open.how.mode); req->open.how = build_open_how(req->open.how.flags, req->open.how.mode);
return io_openat2(req, nxt, force_nonblock); return io_openat2(req, force_nonblock);
} }
static int io_epoll_ctl_prep(struct io_kiocb *req, static int io_epoll_ctl_prep(struct io_kiocb *req,
@ -2804,8 +2786,7 @@ static int io_epoll_ctl_prep(struct io_kiocb *req,
#endif #endif
} }
static int io_epoll_ctl(struct io_kiocb *req, struct io_kiocb **nxt, static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
#if defined(CONFIG_EPOLL) #if defined(CONFIG_EPOLL)
struct io_epoll *ie = &req->epoll; struct io_epoll *ie = &req->epoll;
@ -2818,7 +2799,7 @@ static int io_epoll_ctl(struct io_kiocb *req, struct io_kiocb **nxt,
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
#else #else
return -EOPNOTSUPP; return -EOPNOTSUPP;
@ -2840,8 +2821,7 @@ static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
#endif #endif
} }
static int io_madvise(struct io_kiocb *req, struct io_kiocb **nxt, static int io_madvise(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
struct io_madvise *ma = &req->madvise; struct io_madvise *ma = &req->madvise;
@ -2854,7 +2834,7 @@ static int io_madvise(struct io_kiocb *req, struct io_kiocb **nxt,
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
#else #else
return -EOPNOTSUPP; return -EOPNOTSUPP;
@ -2872,8 +2852,7 @@ static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0; return 0;
} }
static int io_fadvise(struct io_kiocb *req, struct io_kiocb **nxt, static int io_fadvise(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
struct io_fadvise *fa = &req->fadvise; struct io_fadvise *fa = &req->fadvise;
int ret; int ret;
@ -2893,7 +2872,7 @@ static int io_fadvise(struct io_kiocb *req, struct io_kiocb **nxt,
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
} }
@ -2930,8 +2909,7 @@ static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0; return 0;
} }
static int io_statx(struct io_kiocb *req, struct io_kiocb **nxt, static int io_statx(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
struct io_open *ctx = &req->open; struct io_open *ctx = &req->open;
unsigned lookup_flags; unsigned lookup_flags;
@ -2968,7 +2946,7 @@ err:
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
} }
@ -2995,7 +2973,7 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
} }
/* only called when __close_fd_get_file() is done */ /* only called when __close_fd_get_file() is done */
static void __io_close_finish(struct io_kiocb *req, struct io_kiocb **nxt) static void __io_close_finish(struct io_kiocb *req)
{ {
int ret; int ret;
@ -3004,7 +2982,7 @@ static void __io_close_finish(struct io_kiocb *req, struct io_kiocb **nxt)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
fput(req->close.put_file); fput(req->close.put_file);
io_put_req_find_next(req, nxt); io_put_req(req);
} }
static void io_close_finish(struct io_wq_work **workptr) static void io_close_finish(struct io_wq_work **workptr)
@ -3013,14 +2991,13 @@ static void io_close_finish(struct io_wq_work **workptr)
struct io_kiocb *nxt = NULL; struct io_kiocb *nxt = NULL;
/* not cancellable, don't do io_req_cancelled() */ /* not cancellable, don't do io_req_cancelled() */
__io_close_finish(req, &nxt); __io_close_finish(req);
io_put_req(req); /* drop submission reference */ io_put_req(req); /* drop submission reference */
if (nxt) if (nxt)
io_wq_assign_next(workptr, nxt); io_wq_assign_next(workptr, nxt);
} }
static int io_close(struct io_kiocb *req, struct io_kiocb **nxt, static int io_close(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
int ret; int ret;
@ -3048,7 +3025,7 @@ static int io_close(struct io_kiocb *req, struct io_kiocb **nxt,
* No ->flush(), safely close from here and just punt the * No ->flush(), safely close from here and just punt the
* fput() to async context. * fput() to async context.
*/ */
__io_close_finish(req, nxt); __io_close_finish(req);
return 0; return 0;
} }
@ -3070,7 +3047,7 @@ static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0; return 0;
} }
static void __io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt) static void __io_sync_file_range(struct io_kiocb *req)
{ {
int ret; int ret;
@ -3079,7 +3056,7 @@ static void __io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt)
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
io_put_req_find_next(req, nxt); io_put_req(req);
} }
@ -3090,14 +3067,13 @@ static void io_sync_file_range_finish(struct io_wq_work **workptr)
if (io_req_cancelled(req)) if (io_req_cancelled(req))
return; return;
__io_sync_file_range(req, &nxt); __io_sync_file_range(req);
io_put_req(req); /* put submission ref */ io_put_req(req); /* put submission ref */
if (nxt) if (nxt)
io_wq_assign_next(workptr, nxt); io_wq_assign_next(workptr, nxt);
} }
static int io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt, static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
/* sync_file_range always requires a blocking context */ /* sync_file_range always requires a blocking context */
if (force_nonblock) { if (force_nonblock) {
@ -3105,7 +3081,7 @@ static int io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt,
return -EAGAIN; return -EAGAIN;
} }
__io_sync_file_range(req, nxt); __io_sync_file_range(req);
return 0; return 0;
} }
@ -3157,8 +3133,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
#endif #endif
} }
static int io_sendmsg(struct io_kiocb *req, struct io_kiocb **nxt, static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
struct io_async_msghdr *kmsg = NULL; struct io_async_msghdr *kmsg = NULL;
@ -3212,15 +3187,14 @@ static int io_sendmsg(struct io_kiocb *req, struct io_kiocb **nxt,
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
#else #else
return -EOPNOTSUPP; return -EOPNOTSUPP;
#endif #endif
} }
static int io_send(struct io_kiocb *req, struct io_kiocb **nxt, static int io_send(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
struct socket *sock; struct socket *sock;
@ -3263,7 +3237,7 @@ static int io_send(struct io_kiocb *req, struct io_kiocb **nxt,
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
#else #else
return -EOPNOTSUPP; return -EOPNOTSUPP;
@ -3304,8 +3278,7 @@ static int io_recvmsg_prep(struct io_kiocb *req,
#endif #endif
} }
static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt, static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
struct io_async_msghdr *kmsg = NULL; struct io_async_msghdr *kmsg = NULL;
@ -3361,15 +3334,14 @@ static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt,
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
#else #else
return -EOPNOTSUPP; return -EOPNOTSUPP;
#endif #endif
} }
static int io_recv(struct io_kiocb *req, struct io_kiocb **nxt, static int io_recv(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
struct socket *sock; struct socket *sock;
@ -3413,7 +3385,7 @@ static int io_recv(struct io_kiocb *req, struct io_kiocb **nxt,
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
#else #else
return -EOPNOTSUPP; return -EOPNOTSUPP;
@ -3441,8 +3413,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
} }
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
static int __io_accept(struct io_kiocb *req, struct io_kiocb **nxt, static int __io_accept(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
struct io_accept *accept = &req->accept; struct io_accept *accept = &req->accept;
unsigned file_flags; unsigned file_flags;
@ -3458,7 +3429,7 @@ static int __io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
} }
@ -3469,20 +3440,19 @@ static void io_accept_finish(struct io_wq_work **workptr)
if (io_req_cancelled(req)) if (io_req_cancelled(req))
return; return;
__io_accept(req, &nxt, false); __io_accept(req, false);
io_put_req(req); /* drop submission reference */ io_put_req(req); /* drop submission reference */
if (nxt) if (nxt)
io_wq_assign_next(workptr, nxt); io_wq_assign_next(workptr, nxt);
} }
#endif #endif
static int io_accept(struct io_kiocb *req, struct io_kiocb **nxt, static int io_accept(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
int ret; int ret;
ret = __io_accept(req, nxt, force_nonblock); ret = __io_accept(req, force_nonblock);
if (ret == -EAGAIN && force_nonblock) { if (ret == -EAGAIN && force_nonblock) {
req->work.func = io_accept_finish; req->work.func = io_accept_finish;
return -EAGAIN; return -EAGAIN;
@ -3517,8 +3487,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
#endif #endif
} }
static int io_connect(struct io_kiocb *req, struct io_kiocb **nxt, static int io_connect(struct io_kiocb *req, bool force_nonblock)
bool force_nonblock)
{ {
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
struct io_async_ctx __io, *io; struct io_async_ctx __io, *io;
@ -3556,7 +3525,7 @@ out:
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
io_put_req_find_next(req, nxt); io_put_req(req);
return 0; return 0;
#else #else
return -EOPNOTSUPP; return -EOPNOTSUPP;
@ -3953,7 +3922,7 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe
return 0; return 0;
} }
static int io_poll_add(struct io_kiocb *req, struct io_kiocb **nxt) static int io_poll_add(struct io_kiocb *req)
{ {
struct io_poll_iocb *poll = &req->poll; struct io_poll_iocb *poll = &req->poll;
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
@ -3975,7 +3944,7 @@ static int io_poll_add(struct io_kiocb *req, struct io_kiocb **nxt)
if (mask) { if (mask) {
io_cqring_ev_posted(ctx); io_cqring_ev_posted(ctx);
io_put_req_find_next(req, nxt); io_put_req(req);
} }
return ipt.error; return ipt.error;
} }
@ -4224,7 +4193,7 @@ static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
static void io_async_find_and_cancel(struct io_ring_ctx *ctx, static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
struct io_kiocb *req, __u64 sqe_addr, struct io_kiocb *req, __u64 sqe_addr,
struct io_kiocb **nxt, int success_ret) int success_ret)
{ {
unsigned long flags; unsigned long flags;
int ret; int ret;
@ -4250,7 +4219,7 @@ done:
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
io_put_req_find_next(req, nxt); io_put_req(req);
} }
static int io_async_cancel_prep(struct io_kiocb *req, static int io_async_cancel_prep(struct io_kiocb *req,
@ -4266,11 +4235,11 @@ static int io_async_cancel_prep(struct io_kiocb *req,
return 0; return 0;
} }
static int io_async_cancel(struct io_kiocb *req, struct io_kiocb **nxt) static int io_async_cancel(struct io_kiocb *req)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
io_async_find_and_cancel(ctx, req, req->cancel.addr, nxt, 0); io_async_find_and_cancel(ctx, req, req->cancel.addr, 0);
return 0; return 0;
} }
@ -4477,7 +4446,7 @@ static void io_cleanup_req(struct io_kiocb *req)
} }
static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
struct io_kiocb **nxt, bool force_nonblock) bool force_nonblock)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
int ret; int ret;
@ -4494,7 +4463,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret < 0) if (ret < 0)
break; break;
} }
ret = io_read(req, nxt, force_nonblock); ret = io_read(req, force_nonblock);
break; break;
case IORING_OP_WRITEV: case IORING_OP_WRITEV:
case IORING_OP_WRITE_FIXED: case IORING_OP_WRITE_FIXED:
@ -4504,7 +4473,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret < 0) if (ret < 0)
break; break;
} }
ret = io_write(req, nxt, force_nonblock); ret = io_write(req, force_nonblock);
break; break;
case IORING_OP_FSYNC: case IORING_OP_FSYNC:
if (sqe) { if (sqe) {
@ -4512,7 +4481,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret < 0) if (ret < 0)
break; break;
} }
ret = io_fsync(req, nxt, force_nonblock); ret = io_fsync(req, force_nonblock);
break; break;
case IORING_OP_POLL_ADD: case IORING_OP_POLL_ADD:
if (sqe) { if (sqe) {
@ -4520,7 +4489,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_poll_add(req, nxt); ret = io_poll_add(req);
break; break;
case IORING_OP_POLL_REMOVE: case IORING_OP_POLL_REMOVE:
if (sqe) { if (sqe) {
@ -4536,7 +4505,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret < 0) if (ret < 0)
break; break;
} }
ret = io_sync_file_range(req, nxt, force_nonblock); ret = io_sync_file_range(req, force_nonblock);
break; break;
case IORING_OP_SENDMSG: case IORING_OP_SENDMSG:
case IORING_OP_SEND: case IORING_OP_SEND:
@ -4546,9 +4515,9 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
break; break;
} }
if (req->opcode == IORING_OP_SENDMSG) if (req->opcode == IORING_OP_SENDMSG)
ret = io_sendmsg(req, nxt, force_nonblock); ret = io_sendmsg(req, force_nonblock);
else else
ret = io_send(req, nxt, force_nonblock); ret = io_send(req, force_nonblock);
break; break;
case IORING_OP_RECVMSG: case IORING_OP_RECVMSG:
case IORING_OP_RECV: case IORING_OP_RECV:
@ -4558,9 +4527,9 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
break; break;
} }
if (req->opcode == IORING_OP_RECVMSG) if (req->opcode == IORING_OP_RECVMSG)
ret = io_recvmsg(req, nxt, force_nonblock); ret = io_recvmsg(req, force_nonblock);
else else
ret = io_recv(req, nxt, force_nonblock); ret = io_recv(req, force_nonblock);
break; break;
case IORING_OP_TIMEOUT: case IORING_OP_TIMEOUT:
if (sqe) { if (sqe) {
@ -4584,7 +4553,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_accept(req, nxt, force_nonblock); ret = io_accept(req, force_nonblock);
break; break;
case IORING_OP_CONNECT: case IORING_OP_CONNECT:
if (sqe) { if (sqe) {
@ -4592,7 +4561,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_connect(req, nxt, force_nonblock); ret = io_connect(req, force_nonblock);
break; break;
case IORING_OP_ASYNC_CANCEL: case IORING_OP_ASYNC_CANCEL:
if (sqe) { if (sqe) {
@ -4600,7 +4569,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_async_cancel(req, nxt); ret = io_async_cancel(req);
break; break;
case IORING_OP_FALLOCATE: case IORING_OP_FALLOCATE:
if (sqe) { if (sqe) {
@ -4608,7 +4577,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_fallocate(req, nxt, force_nonblock); ret = io_fallocate(req, force_nonblock);
break; break;
case IORING_OP_OPENAT: case IORING_OP_OPENAT:
if (sqe) { if (sqe) {
@ -4616,7 +4585,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_openat(req, nxt, force_nonblock); ret = io_openat(req, force_nonblock);
break; break;
case IORING_OP_CLOSE: case IORING_OP_CLOSE:
if (sqe) { if (sqe) {
@ -4624,7 +4593,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_close(req, nxt, force_nonblock); ret = io_close(req, force_nonblock);
break; break;
case IORING_OP_FILES_UPDATE: case IORING_OP_FILES_UPDATE:
if (sqe) { if (sqe) {
@ -4640,7 +4609,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_statx(req, nxt, force_nonblock); ret = io_statx(req, force_nonblock);
break; break;
case IORING_OP_FADVISE: case IORING_OP_FADVISE:
if (sqe) { if (sqe) {
@ -4648,7 +4617,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_fadvise(req, nxt, force_nonblock); ret = io_fadvise(req, force_nonblock);
break; break;
case IORING_OP_MADVISE: case IORING_OP_MADVISE:
if (sqe) { if (sqe) {
@ -4656,7 +4625,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_madvise(req, nxt, force_nonblock); ret = io_madvise(req, force_nonblock);
break; break;
case IORING_OP_OPENAT2: case IORING_OP_OPENAT2:
if (sqe) { if (sqe) {
@ -4664,7 +4633,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_openat2(req, nxt, force_nonblock); ret = io_openat2(req, force_nonblock);
break; break;
case IORING_OP_EPOLL_CTL: case IORING_OP_EPOLL_CTL:
if (sqe) { if (sqe) {
@ -4672,7 +4641,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
break; break;
} }
ret = io_epoll_ctl(req, nxt, force_nonblock); ret = io_epoll_ctl(req, force_nonblock);
break; break;
case IORING_OP_SPLICE: case IORING_OP_SPLICE:
if (sqe) { if (sqe) {
@ -4680,7 +4649,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret < 0) if (ret < 0)
break; break;
} }
ret = io_splice(req, nxt, force_nonblock); ret = io_splice(req, force_nonblock);
break; break;
default: default:
ret = -EINVAL; ret = -EINVAL;
@ -4724,7 +4693,7 @@ static void io_wq_submit_work(struct io_wq_work **workptr)
if (!ret) { if (!ret) {
do { do {
ret = io_issue_sqe(req, NULL, &nxt, false); ret = io_issue_sqe(req, NULL, false);
/* /*
* We can get EAGAIN for polled IO even though we're * We can get EAGAIN for polled IO even though we're
* forcing a sync submission from here, since we can't * forcing a sync submission from here, since we can't
@ -4870,8 +4839,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
if (prev) { if (prev) {
req_set_fail_links(prev); req_set_fail_links(prev);
io_async_find_and_cancel(ctx, req, prev->user_data, NULL, io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
-ETIME);
io_put_req(prev); io_put_req(prev);
} else { } else {
io_cqring_add_event(req, -ETIME); io_cqring_add_event(req, -ETIME);
@ -4940,7 +4908,7 @@ again:
old_creds = override_creds(req->work.creds); old_creds = override_creds(req->work.creds);
} }
ret = io_issue_sqe(req, sqe, &nxt, true); ret = io_issue_sqe(req, sqe, true);
/* /*
* We async punt it if the file wasn't marked NOWAIT, or if the file * We async punt it if the file wasn't marked NOWAIT, or if the file