Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

io_uring: pre-calculate scq layout 

Move ring layouts calculations into io_prepare_config(), so that more
misconfiguration checking can be done earlier before creating a ctx.
It also deduplicates some code with ring resizing. And as a bonus, now
it initialises params->sq_off.array, which is closer to all other user
offset init, and also applies it to ring resizing, which was previously
missing it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2025-11-12 12:45:58 +00:00 committed by Jens Axboe
parent 001b76b7e7
commit eb76ff6a68
3 changed files with 15 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
io_uring/io_uring.c
View File

@ -2757,8 +2757,8 @@ static void io_rings_free(struct io_ring_ctx *ctx)
ctx->sq_sqes = NULL;
}
int rings_size(unsigned int flags, unsigned int sq_entries,
unsigned int cq_entries, struct io_rings_layout *rl)
static int rings_size(unsigned int flags, unsigned int sq_entries,
unsigned int cq_entries, struct io_rings_layout *rl)
{
struct io_rings *rings;
size_t sqe_size;
@ -3353,10 +3353,11 @@ bool io_is_uring_fops(struct file *file)
}
static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
struct io_uring_params *p)
struct io_ctx_config *config)
{
struct io_uring_params *p = &config->p;
struct io_rings_layout *rl = &config->layout;
struct io_uring_region_desc rd;
struct io_rings_layout __rl, *rl = &__rl;
struct io_rings *rings;
int ret;
@ -3364,10 +3365,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
ctx->sq_entries = p->sq_entries;
ctx->cq_entries = p->cq_entries;
ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);
if (ret)
return ret;
memset(&rd, 0, sizeof(rd));
rd.size = PAGE_ALIGN(rl->rings_size);
if (ctx->flags & IORING_SETUP_NO_MMAP) {
@ -3378,7 +3375,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
if (ret)
return ret;
ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
@ -3560,6 +3556,14 @@ int io_prepare_config(struct io_ctx_config *config)
if (ret)
return ret;
ret = rings_size(p->flags, p->sq_entries, p->cq_entries,
&config->layout);
if (ret)
return ret;
if (!(p->flags & IORING_SETUP_NO_SQARRAY))
p->sq_off.array = config->layout.sq_array_offset;
return 0;
}
@ -3632,13 +3636,10 @@ static __cold int io_uring_create(struct io_ctx_config *config)
mmgrab(current->mm);
ctx->mm_account = current->mm;
ret = io_allocate_scq_urings(ctx, p);
ret = io_allocate_scq_urings(ctx, config);
if (ret)
goto err;
if (!(p->flags & IORING_SETUP_NO_SQARRAY))
p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
ret = io_sq_offload_create(ctx, p);
if (ret)
goto err;

3
io_uring/io_uring.h
View File

@ -27,6 +27,7 @@ struct io_rings_layout {
struct io_ctx_config {
struct io_uring_params p;
struct io_rings_layout layout;
struct io_uring_params __user *uptr;
};
@ -147,8 +148,6 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
#define IORING_MAX_ENTRIES 32768
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
int rings_size(unsigned int flags, unsigned int sq_entries,
unsigned int cq_entries, struct io_rings_layout *rl);
int io_prepare_config(struct io_ctx_config *config);
bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);

4
io_uring/register.c
View File

@ -423,10 +423,6 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
if (unlikely(ret))
return ret;
ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);
if (ret)
return ret;
memset(&rd, 0, sizeof(rd));
rd.size = PAGE_ALIGN(rl->rings_size);
if (p->flags & IORING_SETUP_NO_MMAP) {