io_uring: inline struct sqe_submit
There is no point left in keeping struct sqe_submit. Inline it into struct io_kiocb, so any req->submit.field is now just req->field - moves initialisation of ring_file into io_get_req() - removes duplicated req->sequence. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov
Jens Axboe
parent
cc42e0ac17
commit
cf6fd4bd55
169
fs/io_uring.c
169
fs/io_uring.c
|
|
@ -281,16 +281,6 @@ struct io_ring_ctx {
|
|||
} ____cacheline_aligned_in_smp;
|
||||
};
|
||||
|
||||
struct sqe_submit {
|
||||
const struct io_uring_sqe *sqe;
|
||||
struct file *ring_file;
|
||||
int ring_fd;
|
||||
u32 sequence;
|
||||
bool has_user;
|
||||
bool in_async;
|
||||
bool needs_fixed_file;
|
||||
};
|
||||
|
||||
/*
|
||||
* First field must be the file pointer in all the
|
||||
* iocb unions! See also 'struct kiocb' in <linux/fs.h>
|
||||
|
|
@ -331,7 +321,12 @@ struct io_kiocb {
|
|||
struct io_timeout timeout;
|
||||
};
|
||||
|
||||
struct sqe_submit submit;
|
||||
const struct io_uring_sqe *sqe;
|
||||
struct file *ring_file;
|
||||
int ring_fd;
|
||||
bool has_user;
|
||||
bool in_async;
|
||||
bool needs_fixed_file;
|
||||
|
||||
struct io_ring_ctx *ctx;
|
||||
union {
|
||||
|
|
@ -541,8 +536,8 @@ static inline bool io_prep_async_work(struct io_kiocb *req,
|
|||
{
|
||||
bool do_hashed = false;
|
||||
|
||||
if (req->submit.sqe) {
|
||||
switch (req->submit.sqe->opcode) {
|
||||
if (req->sqe) {
|
||||
switch (req->sqe->opcode) {
|
||||
case IORING_OP_WRITEV:
|
||||
case IORING_OP_WRITE_FIXED:
|
||||
do_hashed = true;
|
||||
|
|
@ -563,7 +558,7 @@ static inline bool io_prep_async_work(struct io_kiocb *req,
|
|||
req->work.flags |= IO_WQ_WORK_UNBOUND;
|
||||
break;
|
||||
}
|
||||
if (io_sqe_needs_user(req->submit.sqe))
|
||||
if (io_sqe_needs_user(req->sqe))
|
||||
req->work.flags |= IO_WQ_WORK_NEEDS_USER;
|
||||
}
|
||||
|
||||
|
|
@ -810,6 +805,7 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
|
|||
}
|
||||
|
||||
got_it:
|
||||
req->ring_file = NULL;
|
||||
req->file = NULL;
|
||||
req->ctx = ctx;
|
||||
req->flags = 0;
|
||||
|
|
@ -840,7 +836,7 @@ static void __io_free_req(struct io_kiocb *req)
|
|||
struct io_ring_ctx *ctx = req->ctx;
|
||||
|
||||
if (req->flags & REQ_F_FREE_SQE)
|
||||
kfree(req->submit.sqe);
|
||||
kfree(req->sqe);
|
||||
if (req->file && !(req->flags & REQ_F_FIXED_FILE))
|
||||
fput(req->file);
|
||||
if (req->flags & REQ_F_INFLIGHT) {
|
||||
|
|
@ -938,7 +934,7 @@ static void io_fail_links(struct io_kiocb *req)
|
|||
trace_io_uring_fail_link(req, link);
|
||||
|
||||
if ((req->flags & REQ_F_LINK_TIMEOUT) &&
|
||||
link->submit.sqe->opcode == IORING_OP_LINK_TIMEOUT) {
|
||||
link->sqe->opcode == IORING_OP_LINK_TIMEOUT) {
|
||||
io_link_cancel_timeout(link);
|
||||
} else {
|
||||
io_cqring_fill_event(link, -ECANCELED);
|
||||
|
|
@ -1401,7 +1397,7 @@ static bool io_file_supports_async(struct file *file)
|
|||
|
||||
static int io_prep_rw(struct io_kiocb *req, bool force_nonblock)
|
||||
{
|
||||
const struct io_uring_sqe *sqe = req->submit.sqe;
|
||||
const struct io_uring_sqe *sqe = req->sqe;
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
struct kiocb *kiocb = &req->rw;
|
||||
unsigned ioprio;
|
||||
|
|
@ -1568,11 +1564,10 @@ static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
|
|||
return len;
|
||||
}
|
||||
|
||||
static ssize_t io_import_iovec(struct io_ring_ctx *ctx, int rw,
|
||||
const struct sqe_submit *s, struct iovec **iovec,
|
||||
struct iov_iter *iter)
|
||||
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
|
||||
struct iovec **iovec, struct iov_iter *iter)
|
||||
{
|
||||
const struct io_uring_sqe *sqe = s->sqe;
|
||||
const struct io_uring_sqe *sqe = req->sqe;
|
||||
void __user *buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
|
||||
size_t sqe_len = READ_ONCE(sqe->len);
|
||||
u8 opcode;
|
||||
|
|
@ -1588,16 +1583,16 @@ static ssize_t io_import_iovec(struct io_ring_ctx *ctx, int rw,
|
|||
opcode = READ_ONCE(sqe->opcode);
|
||||
if (opcode == IORING_OP_READ_FIXED ||
|
||||
opcode == IORING_OP_WRITE_FIXED) {
|
||||
ssize_t ret = io_import_fixed(ctx, rw, sqe, iter);
|
||||
ssize_t ret = io_import_fixed(req->ctx, rw, sqe, iter);
|
||||
*iovec = NULL;
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (!s->has_user)
|
||||
if (!req->has_user)
|
||||
return -EFAULT;
|
||||
|
||||
#ifdef CONFIG_COMPAT
|
||||
if (ctx->compat)
|
||||
if (req->ctx->compat)
|
||||
return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
|
||||
iovec, iter);
|
||||
#endif
|
||||
|
|
@ -1681,7 +1676,7 @@ static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
|
|||
if (unlikely(!(file->f_mode & FMODE_READ)))
|
||||
return -EBADF;
|
||||
|
||||
ret = io_import_iovec(req->ctx, READ, &req->submit, &iovec, &iter);
|
||||
ret = io_import_iovec(READ, req, &iovec, &iter);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
|
|
@ -1713,7 +1708,7 @@ static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
|
|||
ret2 = -EAGAIN;
|
||||
/* Catch -EAGAIN return for forced non-blocking submission */
|
||||
if (!force_nonblock || ret2 != -EAGAIN)
|
||||
kiocb_done(kiocb, ret2, nxt, req->submit.in_async);
|
||||
kiocb_done(kiocb, ret2, nxt, req->in_async);
|
||||
else
|
||||
ret = -EAGAIN;
|
||||
}
|
||||
|
|
@ -1739,7 +1734,7 @@ static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
|
|||
if (unlikely(!(file->f_mode & FMODE_WRITE)))
|
||||
return -EBADF;
|
||||
|
||||
ret = io_import_iovec(req->ctx, WRITE, &req->submit, &iovec, &iter);
|
||||
ret = io_import_iovec(WRITE, req, &iovec, &iter);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
|
|
@ -1776,7 +1771,7 @@ static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
|
|||
else
|
||||
ret2 = loop_rw_iter(WRITE, file, kiocb, &iter);
|
||||
if (!force_nonblock || ret2 != -EAGAIN)
|
||||
kiocb_done(kiocb, ret2, nxt, req->submit.in_async);
|
||||
kiocb_done(kiocb, ret2, nxt, req->in_async);
|
||||
else
|
||||
ret = -EAGAIN;
|
||||
}
|
||||
|
|
@ -2259,7 +2254,7 @@ static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe,
|
|||
if (!poll->file)
|
||||
return -EBADF;
|
||||
|
||||
req->submit.sqe = NULL;
|
||||
req->sqe = NULL;
|
||||
INIT_IO_WORK(&req->work, io_poll_complete_work);
|
||||
events = READ_ONCE(sqe->poll_events);
|
||||
poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
|
||||
|
|
@ -2413,7 +2408,7 @@ static int io_timeout_remove(struct io_kiocb *req,
|
|||
|
||||
static int io_timeout_setup(struct io_kiocb *req)
|
||||
{
|
||||
const struct io_uring_sqe *sqe = req->submit.sqe;
|
||||
const struct io_uring_sqe *sqe = req->sqe;
|
||||
struct io_timeout_data *data;
|
||||
unsigned flags;
|
||||
|
||||
|
|
@ -2601,7 +2596,6 @@ static int io_async_cancel(struct io_kiocb *req, const struct io_uring_sqe *sqe,
|
|||
|
||||
static int io_req_defer(struct io_kiocb *req)
|
||||
{
|
||||
const struct io_uring_sqe *sqe = req->submit.sqe;
|
||||
struct io_uring_sqe *sqe_copy;
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
|
||||
|
|
@ -2620,9 +2614,9 @@ static int io_req_defer(struct io_kiocb *req)
|
|||
return 0;
|
||||
}
|
||||
|
||||
memcpy(sqe_copy, sqe, sizeof(*sqe_copy));
|
||||
memcpy(sqe_copy, req->sqe, sizeof(*sqe_copy));
|
||||
req->flags |= REQ_F_FREE_SQE;
|
||||
req->submit.sqe = sqe_copy;
|
||||
req->sqe = sqe_copy;
|
||||
|
||||
trace_io_uring_defer(ctx, req, req->user_data);
|
||||
list_add_tail(&req->list, &ctx->defer_list);
|
||||
|
|
@ -2635,21 +2629,20 @@ static int io_issue_sqe(struct io_kiocb *req, struct io_kiocb **nxt,
|
|||
bool force_nonblock)
|
||||
{
|
||||
int ret, opcode;
|
||||
struct sqe_submit *s = &req->submit;
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
|
||||
opcode = READ_ONCE(s->sqe->opcode);
|
||||
opcode = READ_ONCE(req->sqe->opcode);
|
||||
switch (opcode) {
|
||||
case IORING_OP_NOP:
|
||||
ret = io_nop(req);
|
||||
break;
|
||||
case IORING_OP_READV:
|
||||
if (unlikely(s->sqe->buf_index))
|
||||
if (unlikely(req->sqe->buf_index))
|
||||
return -EINVAL;
|
||||
ret = io_read(req, nxt, force_nonblock);
|
||||
break;
|
||||
case IORING_OP_WRITEV:
|
||||
if (unlikely(s->sqe->buf_index))
|
||||
if (unlikely(req->sqe->buf_index))
|
||||
return -EINVAL;
|
||||
ret = io_write(req, nxt, force_nonblock);
|
||||
break;
|
||||
|
|
@ -2660,37 +2653,37 @@ static int io_issue_sqe(struct io_kiocb *req, struct io_kiocb **nxt,
|
|||
ret = io_write(req, nxt, force_nonblock);
|
||||
break;
|
||||
case IORING_OP_FSYNC:
|
||||
ret = io_fsync(req, s->sqe, nxt, force_nonblock);
|
||||
ret = io_fsync(req, req->sqe, nxt, force_nonblock);
|
||||
break;
|
||||
case IORING_OP_POLL_ADD:
|
||||
ret = io_poll_add(req, s->sqe, nxt);
|
||||
ret = io_poll_add(req, req->sqe, nxt);
|
||||
break;
|
||||
case IORING_OP_POLL_REMOVE:
|
||||
ret = io_poll_remove(req, s->sqe);
|
||||
ret = io_poll_remove(req, req->sqe);
|
||||
break;
|
||||
case IORING_OP_SYNC_FILE_RANGE:
|
||||
ret = io_sync_file_range(req, s->sqe, nxt, force_nonblock);
|
||||
ret = io_sync_file_range(req, req->sqe, nxt, force_nonblock);
|
||||
break;
|
||||
case IORING_OP_SENDMSG:
|
||||
ret = io_sendmsg(req, s->sqe, nxt, force_nonblock);
|
||||
ret = io_sendmsg(req, req->sqe, nxt, force_nonblock);
|
||||
break;
|
||||
case IORING_OP_RECVMSG:
|
||||
ret = io_recvmsg(req, s->sqe, nxt, force_nonblock);
|
||||
ret = io_recvmsg(req, req->sqe, nxt, force_nonblock);
|
||||
break;
|
||||
case IORING_OP_TIMEOUT:
|
||||
ret = io_timeout(req, s->sqe);
|
||||
ret = io_timeout(req, req->sqe);
|
||||
break;
|
||||
case IORING_OP_TIMEOUT_REMOVE:
|
||||
ret = io_timeout_remove(req, s->sqe);
|
||||
ret = io_timeout_remove(req, req->sqe);
|
||||
break;
|
||||
case IORING_OP_ACCEPT:
|
||||
ret = io_accept(req, s->sqe, nxt, force_nonblock);
|
||||
ret = io_accept(req, req->sqe, nxt, force_nonblock);
|
||||
break;
|
||||
case IORING_OP_CONNECT:
|
||||
ret = io_connect(req, s->sqe, nxt, force_nonblock);
|
||||
ret = io_connect(req, req->sqe, nxt, force_nonblock);
|
||||
break;
|
||||
case IORING_OP_ASYNC_CANCEL:
|
||||
ret = io_async_cancel(req, s->sqe, nxt);
|
||||
ret = io_async_cancel(req, req->sqe, nxt);
|
||||
break;
|
||||
default:
|
||||
ret = -EINVAL;
|
||||
|
|
@ -2705,10 +2698,10 @@ static int io_issue_sqe(struct io_kiocb *req, struct io_kiocb **nxt,
|
|||
return -EAGAIN;
|
||||
|
||||
/* workqueue context doesn't hold uring_lock, grab it now */
|
||||
if (s->in_async)
|
||||
if (req->in_async)
|
||||
mutex_lock(&ctx->uring_lock);
|
||||
io_iopoll_req_issued(req);
|
||||
if (s->in_async)
|
||||
if (req->in_async)
|
||||
mutex_unlock(&ctx->uring_lock);
|
||||
}
|
||||
|
||||
|
|
@ -2728,7 +2721,6 @@ static void io_wq_submit_work(struct io_wq_work **workptr)
|
|||
{
|
||||
struct io_wq_work *work = *workptr;
|
||||
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
|
||||
struct sqe_submit *s = &req->submit;
|
||||
struct io_kiocb *nxt = NULL;
|
||||
int ret = 0;
|
||||
|
||||
|
|
@ -2739,8 +2731,8 @@ static void io_wq_submit_work(struct io_wq_work **workptr)
|
|||
ret = -ECANCELED;
|
||||
|
||||
if (!ret) {
|
||||
s->has_user = (work->flags & IO_WQ_WORK_HAS_MM) != 0;
|
||||
s->in_async = true;
|
||||
req->has_user = (work->flags & IO_WQ_WORK_HAS_MM) != 0;
|
||||
req->in_async = true;
|
||||
do {
|
||||
ret = io_issue_sqe(req, &nxt, false);
|
||||
/*
|
||||
|
|
@ -2806,24 +2798,17 @@ static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
|
|||
|
||||
static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req)
|
||||
{
|
||||
struct sqe_submit *s = &req->submit;
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
unsigned flags;
|
||||
int fd;
|
||||
|
||||
flags = READ_ONCE(s->sqe->flags);
|
||||
fd = READ_ONCE(s->sqe->fd);
|
||||
flags = READ_ONCE(req->sqe->flags);
|
||||
fd = READ_ONCE(req->sqe->fd);
|
||||
|
||||
if (flags & IOSQE_IO_DRAIN)
|
||||
req->flags |= REQ_F_IO_DRAIN;
|
||||
/*
|
||||
* All io need record the previous position, if LINK vs DARIN,
|
||||
* it can be used to mark the position of the first IO in the
|
||||
* link list.
|
||||
*/
|
||||
req->sequence = s->sequence;
|
||||
|
||||
if (!io_op_needs_file(s->sqe))
|
||||
if (!io_op_needs_file(req->sqe))
|
||||
return 0;
|
||||
|
||||
if (flags & IOSQE_FIXED_FILE) {
|
||||
|
|
@ -2836,7 +2821,7 @@ static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req)
|
|||
return -EBADF;
|
||||
req->flags |= REQ_F_FIXED_FILE;
|
||||
} else {
|
||||
if (s->needs_fixed_file)
|
||||
if (req->needs_fixed_file)
|
||||
return -EBADF;
|
||||
trace_io_uring_file_get(ctx, fd);
|
||||
req->file = io_file_get(state, fd);
|
||||
|
|
@ -2860,7 +2845,7 @@ static int io_grab_files(struct io_kiocb *req)
|
|||
* the fd has changed since we started down this path, and disallow
|
||||
* this operation if it has.
|
||||
*/
|
||||
if (fcheck(req->submit.ring_fd) == req->submit.ring_file) {
|
||||
if (fcheck(req->ring_fd) == req->ring_file) {
|
||||
list_add(&req->inflight_entry, &ctx->inflight_list);
|
||||
req->flags |= REQ_F_INFLIGHT;
|
||||
req->work.files = current->files;
|
||||
|
|
@ -2941,7 +2926,7 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
|
|||
return NULL;
|
||||
|
||||
nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
|
||||
if (!nxt || nxt->submit.sqe->opcode != IORING_OP_LINK_TIMEOUT)
|
||||
if (!nxt || nxt->sqe->opcode != IORING_OP_LINK_TIMEOUT)
|
||||
return NULL;
|
||||
|
||||
req->flags |= REQ_F_LINK_TIMEOUT;
|
||||
|
|
@ -2964,14 +2949,13 @@ static void __io_queue_sqe(struct io_kiocb *req)
|
|||
*/
|
||||
if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
|
||||
(req->flags & REQ_F_MUST_PUNT))) {
|
||||
struct sqe_submit *s = &req->submit;
|
||||
struct io_uring_sqe *sqe_copy;
|
||||
|
||||
sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
|
||||
sqe_copy = kmemdup(req->sqe, sizeof(*sqe_copy), GFP_KERNEL);
|
||||
if (!sqe_copy)
|
||||
goto err;
|
||||
|
||||
s->sqe = sqe_copy;
|
||||
req->sqe = sqe_copy;
|
||||
req->flags |= REQ_F_FREE_SQE;
|
||||
|
||||
if (req->work.flags & IO_WQ_WORK_NEEDS_FILES) {
|
||||
|
|
@ -3045,14 +3029,13 @@ static inline void io_queue_link_head(struct io_kiocb *req)
|
|||
static void io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
|
||||
struct io_kiocb **link)
|
||||
{
|
||||
struct sqe_submit *s = &req->submit;
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
int ret;
|
||||
|
||||
req->user_data = s->sqe->user_data;
|
||||
req->user_data = req->sqe->user_data;
|
||||
|
||||
/* enforce forwards compatibility on users */
|
||||
if (unlikely(s->sqe->flags & ~SQE_VALID_FLAGS)) {
|
||||
if (unlikely(req->sqe->flags & ~SQE_VALID_FLAGS)) {
|
||||
ret = -EINVAL;
|
||||
goto err_req;
|
||||
}
|
||||
|
|
@ -3076,13 +3059,13 @@ err_req:
|
|||
struct io_kiocb *prev = *link;
|
||||
struct io_uring_sqe *sqe_copy;
|
||||
|
||||
if (s->sqe->flags & IOSQE_IO_DRAIN)
|
||||
if (req->sqe->flags & IOSQE_IO_DRAIN)
|
||||
(*link)->flags |= REQ_F_DRAIN_LINK | REQ_F_IO_DRAIN;
|
||||
|
||||
if (READ_ONCE(s->sqe->opcode) == IORING_OP_LINK_TIMEOUT) {
|
||||
if (READ_ONCE(req->sqe->opcode) == IORING_OP_LINK_TIMEOUT) {
|
||||
ret = io_timeout_setup(req);
|
||||
/* common setup allows offset being set, we don't */
|
||||
if (!ret && s->sqe->off)
|
||||
if (!ret && req->sqe->off)
|
||||
ret = -EINVAL;
|
||||
if (ret) {
|
||||
prev->flags |= REQ_F_FAIL_LINK;
|
||||
|
|
@ -3090,17 +3073,17 @@ err_req:
|
|||
}
|
||||
}
|
||||
|
||||
sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
|
||||
sqe_copy = kmemdup(req->sqe, sizeof(*sqe_copy), GFP_KERNEL);
|
||||
if (!sqe_copy) {
|
||||
ret = -EAGAIN;
|
||||
goto err_req;
|
||||
}
|
||||
|
||||
s->sqe = sqe_copy;
|
||||
req->sqe = sqe_copy;
|
||||
req->flags |= REQ_F_FREE_SQE;
|
||||
trace_io_uring_link(ctx, req, prev);
|
||||
list_add_tail(&req->list, &prev->link_list);
|
||||
} else if (s->sqe->flags & IOSQE_IO_LINK) {
|
||||
} else if (req->sqe->flags & IOSQE_IO_LINK) {
|
||||
req->flags |= REQ_F_LINK;
|
||||
|
||||
INIT_LIST_HEAD(&req->link_list);
|
||||
|
|
@ -3156,7 +3139,7 @@ static void io_commit_sqring(struct io_ring_ctx *ctx)
|
|||
* used, it's important that those reads are done through READ_ONCE() to
|
||||
* prevent a re-load down the line.
|
||||
*/
|
||||
static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
|
||||
static bool io_get_sqring(struct io_ring_ctx *ctx, struct io_kiocb *req)
|
||||
{
|
||||
struct io_rings *rings = ctx->rings;
|
||||
u32 *sq_array = ctx->sq_array;
|
||||
|
|
@ -3177,9 +3160,13 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
|
|||
|
||||
head = READ_ONCE(sq_array[head & ctx->sq_mask]);
|
||||
if (likely(head < ctx->sq_entries)) {
|
||||
s->ring_file = NULL;
|
||||
s->sqe = &ctx->sq_sqes[head];
|
||||
s->sequence = ctx->cached_sq_head;
|
||||
/*
|
||||
* All io need record the previous position, if LINK vs DARIN,
|
||||
* it can be used to mark the position of the first IO in the
|
||||
* link list.
|
||||
*/
|
||||
req->sequence = ctx->cached_sq_head;
|
||||
req->sqe = &ctx->sq_sqes[head];
|
||||
ctx->cached_sq_head++;
|
||||
return true;
|
||||
}
|
||||
|
|
@ -3220,12 +3207,12 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
|
|||
submitted = -EAGAIN;
|
||||
break;
|
||||
}
|
||||
if (!io_get_sqring(ctx, &req->submit)) {
|
||||
if (!io_get_sqring(ctx, req)) {
|
||||
__io_free_req(req);
|
||||
break;
|
||||
}
|
||||
|
||||
if (io_sqe_needs_user(req->submit.sqe) && !*mm) {
|
||||
if (io_sqe_needs_user(req->sqe) && !*mm) {
|
||||
mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
|
||||
if (!mm_fault) {
|
||||
use_mm(ctx->sqo_mm);
|
||||
|
|
@ -3233,14 +3220,14 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
|
|||
}
|
||||
}
|
||||
|
||||
sqe_flags = req->submit.sqe->flags;
|
||||
sqe_flags = req->sqe->flags;
|
||||
|
||||
req->submit.ring_file = ring_file;
|
||||
req->submit.ring_fd = ring_fd;
|
||||
req->submit.has_user = *mm != NULL;
|
||||
req->submit.in_async = async;
|
||||
req->submit.needs_fixed_file = async;
|
||||
trace_io_uring_submit_sqe(ctx, req->submit.sqe->user_data,
|
||||
req->ring_file = ring_file;
|
||||
req->ring_fd = ring_fd;
|
||||
req->has_user = *mm != NULL;
|
||||
req->in_async = async;
|
||||
req->needs_fixed_file = async;
|
||||
trace_io_uring_submit_sqe(ctx, req->sqe->user_data,
|
||||
true, async);
|
||||
io_submit_sqe(req, statep, &link);
|
||||
submitted++;
|
||||
|
|
|
|||
Loading…
Reference in New Issue