Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

io_uring/kbuf: use struct io_br_sel for multiple buffers picking 

The networking side uses bundles, which is picking multiple buffers at
the same time. Pass in struct io_br_sel to those helpers.

Link: https://lore.kernel.org/r/20250821020750.598432-9-axboe@kernel.dk
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2025-08-20 20:03:36 -06:00
parent d8e1dec2f8
commit 429884ff35
3 changed files with 25 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
io_uring/kbuf.c
View File

@ -299,7 +299,7 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg,
} }
int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,
unsigned int issue_flags) struct io_br_sel *sel, unsigned int issue_flags)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
struct io_buffer_list *bl; struct io_buffer_list *bl;
@ -331,7 +331,8 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,
return ret; return ret;
} }
int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg) int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg,
struct io_br_sel *sel)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
struct io_buffer_list *bl; struct io_buffer_list *bl;

5
io_uring/kbuf.h
View File

@ -80,8 +80,9 @@ struct io_br_sel {
struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len, struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len,
unsigned buf_group, unsigned int issue_flags); unsigned buf_group, unsigned int issue_flags);
int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,
unsigned int issue_flags); struct io_br_sel *sel, unsigned int issue_flags);
int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg); int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg,
struct io_br_sel *sel);
void io_destroy_buffers(struct io_ring_ctx *ctx); void io_destroy_buffers(struct io_ring_ctx *ctx);
int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);

38
io_uring/net.c
View File

@ -586,17 +586,16 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
} }
static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags, static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,
struct io_async_msghdr *kmsg) struct io_br_sel *sel, struct io_async_msghdr *kmsg)
{ {
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
int ret;
struct buf_sel_arg arg = { struct buf_sel_arg arg = {
.iovs = &kmsg->fast_iov, .iovs = &kmsg->fast_iov,
.max_len = min_not_zero(sr->len, INT_MAX), .max_len = min_not_zero(sr->len, INT_MAX),
.nr_iovs = 1, .nr_iovs = 1,
.buf_group = sr->buf_group, .buf_group = sr->buf_group,
}; };
int ret;
if (kmsg->vec.iovec) { if (kmsg->vec.iovec) {
arg.nr_iovs = kmsg->vec.nr; arg.nr_iovs = kmsg->vec.nr;
@ -609,7 +608,7 @@ static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,
else else
arg.mode |= KBUF_MODE_EXPAND; arg.mode |= KBUF_MODE_EXPAND;
ret = io_buffers_select(req, &arg, issue_flags); ret = io_buffers_select(req, &arg, sel, issue_flags);
if (unlikely(ret < 0)) if (unlikely(ret < 0))
return ret; return ret;
@ -638,6 +637,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
{ {
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct io_async_msghdr *kmsg = req->async_data; struct io_async_msghdr *kmsg = req->async_data;
struct io_br_sel sel = { };
struct socket *sock; struct socket *sock;
unsigned flags; unsigned flags;
int min_ret = 0; int min_ret = 0;
@ -657,7 +657,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
retry_bundle: retry_bundle:
if (io_do_buffer_select(req)) { if (io_do_buffer_select(req)) {
ret = io_send_select_buffer(req, issue_flags, kmsg); ret = io_send_select_buffer(req, issue_flags, &sel, kmsg);
if (ret) if (ret)
return ret; return ret;
} }
@ -1015,6 +1015,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
{ {
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct io_async_msghdr *kmsg = req->async_data; struct io_async_msghdr *kmsg = req->async_data;
struct io_br_sel sel = { };
struct socket *sock; struct socket *sock;
unsigned flags; unsigned flags;
int ret, min_ret = 0; int ret, min_ret = 0;
@ -1035,7 +1036,6 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
retry_multishot: retry_multishot:
if (io_do_buffer_select(req)) { if (io_do_buffer_select(req)) {
struct io_br_sel sel;
size_t len = sr->len; size_t len = sr->len;
sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
@ -1099,7 +1099,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
} }
static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg, static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg,
size_t *len, unsigned int issue_flags) struct io_br_sel *sel, unsigned int issue_flags)
{ {
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
int ret; int ret;
@ -1124,15 +1124,15 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg
arg.mode |= KBUF_MODE_FREE; arg.mode |= KBUF_MODE_FREE;
} }
if (*len) if (sel->val)
arg.max_len = *len; arg.max_len = sel->val;
else if (kmsg->msg.msg_inq > 1) else if (kmsg->msg.msg_inq > 1)
arg.max_len = min_not_zero(*len, (size_t) kmsg->msg.msg_inq); arg.max_len = min_not_zero(sel->val, (size_t) kmsg->msg.msg_inq);
/* if mshot limited, ensure we don't go over */ /* if mshot limited, ensure we don't go over */
if (sr->flags & IORING_RECV_MSHOT_LIM) if (sr->flags & IORING_RECV_MSHOT_LIM)
arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len); arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len);
ret = io_buffers_peek(req, &arg); ret = io_buffers_peek(req, &arg, sel);
if (unlikely(ret < 0)) if (unlikely(ret < 0))
return ret; return ret;
@ -1153,14 +1153,13 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg
iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret, iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
arg.out_len); arg.out_len);
} else { } else {
struct io_br_sel sel; size_t len = sel->val;
*len = sr->len; *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
sel = io_buffer_select(req, len, sr->buf_group, issue_flags); if (!sel->addr)
if (!sel.addr)
return -ENOBUFS; return -ENOBUFS;
sr->buf = sel.addr; sr->buf = sel->addr;
sr->len = *len; sr->len = len;
map_ubuf: map_ubuf:
ret = import_ubuf(ITER_DEST, sr->buf, sr->len, ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
&kmsg->msg.msg_iter); &kmsg->msg.msg_iter);
@ -1175,11 +1174,11 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
{ {
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct io_async_msghdr *kmsg = req->async_data; struct io_async_msghdr *kmsg = req->async_data;
struct io_br_sel sel = { };
struct socket *sock; struct socket *sock;
unsigned flags; unsigned flags;
int ret, min_ret = 0; int ret, min_ret = 0;
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
size_t len = sr->len;
bool mshot_finished; bool mshot_finished;
if (!(req->flags & REQ_F_POLLED) && if (!(req->flags & REQ_F_POLLED) &&
@ -1196,7 +1195,8 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
retry_multishot: retry_multishot:
if (io_do_buffer_select(req)) { if (io_do_buffer_select(req)) {
ret = io_recv_buf_select(req, kmsg, &len, issue_flags); sel.val = sr->len;
ret = io_recv_buf_select(req, kmsg, &sel, issue_flags);
if (unlikely(ret < 0)) { if (unlikely(ret < 0)) {
kmsg->msg.msg_inq = -1; kmsg->msg.msg_inq = -1;
goto out_free; goto out_free;