io_uring: pre-calculate scq layout

Move ring layouts calculations into io_prepare_config(), so that more
misconfiguration checking can be done earlier before creating a ctx.
It also deduplicates some code with ring resizing. And as a bonus, now
it initialises params->sq_off.array, which is closer to all other user
offset init, and also applies it to ring resizing, which was previously
missing it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 58e0c0ece6f16a233648c02df9df51348907b9bb..ea5d9e26a10f09a6ecd889ea5a303b9e0460aa5c 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2757,8 +2757,8 @@ static void io_rings_free(struct io_ring_ctx *ctx)
        ctx->sq_sqes = NULL;
 }
 
-int rings_size(unsigned int flags, unsigned int sq_entries,
-               unsigned int cq_entries, struct io_rings_layout *rl)
+static int rings_size(unsigned int flags, unsigned int sq_entries,
+                     unsigned int cq_entries, struct io_rings_layout *rl)
 {
        struct io_rings *rings;
        size_t sqe_size;
@@ -3353,10 +3353,11 @@ bool io_is_uring_fops(struct file *file)
 }
 
 static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
-                                        struct io_uring_params *p)
+                                        struct io_ctx_config *config)
 {
+       struct io_uring_params *p = &config->p;
+       struct io_rings_layout *rl = &config->layout;
        struct io_uring_region_desc rd;
-       struct io_rings_layout __rl, *rl = &__rl;
        struct io_rings *rings;
        int ret;
 
@@ -3364,10 +3365,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
        ctx->sq_entries = p->sq_entries;
        ctx->cq_entries = p->cq_entries;
 
-       ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);
-       if (ret)
-               return ret;
-
        memset(&rd, 0, sizeof(rd));
        rd.size = PAGE_ALIGN(rl->rings_size);
        if (ctx->flags & IORING_SETUP_NO_MMAP) {
@@ -3378,7 +3375,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
        if (ret)
                return ret;
        ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
-
        if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
                ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
 
@@ -3560,6 +3556,14 @@ int io_prepare_config(struct io_ctx_config *config)
        if (ret)
                return ret;
 
+       ret = rings_size(p->flags, p->sq_entries, p->cq_entries,
+                        &config->layout);
+       if (ret)
+               return ret;
+
+       if (!(p->flags & IORING_SETUP_NO_SQARRAY))
+               p->sq_off.array = config->layout.sq_array_offset;
+
        return 0;
 }
 
@@ -3632,13 +3636,10 @@ static __cold int io_uring_create(struct io_ctx_config *config)
        mmgrab(current->mm);
        ctx->mm_account = current->mm;
 
-       ret = io_allocate_scq_urings(ctx, p);
+       ret = io_allocate_scq_urings(ctx, config);
        if (ret)
                goto err;
 
-       if (!(p->flags & IORING_SETUP_NO_SQARRAY))
-               p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
-
        ret = io_sq_offload_create(ctx, p);
        if (ret)
                goto err;

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 5e544c2d27c85f63b7b400b06e10d457c53e1183..a790c16854d32dcacccca7e6a91b23c61757914f 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -27,6 +27,7 @@ struct io_rings_layout {
 
 struct io_ctx_config {
        struct io_uring_params p;
+       struct io_rings_layout layout;
        struct io_uring_params __user *uptr;
 };
 
@@ -147,8 +148,6 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 #define IORING_MAX_ENTRIES     32768
 #define IORING_MAX_CQ_ENTRIES  (2 * IORING_MAX_ENTRIES)
 
-int rings_size(unsigned int flags, unsigned int sq_entries,
-               unsigned int cq_entries, struct io_rings_layout *rl);
 int io_prepare_config(struct io_ctx_config *config);
 
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);

diff --git a/io_uring/register.c b/io_uring/register.c
index fa245c87978a36253feb8962d16f038dff91208f..334a457da3f79bd956a57a3e243b6cf99d6206ba 100644 (file)
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -423,10 +423,6 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
        if (unlikely(ret))
                return ret;
 
-       ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);
-       if (ret)
-               return ret;
-
        memset(&rd, 0, sizeof(rd));
        rd.size = PAGE_ALIGN(rl->rings_size);
        if (p->flags & IORING_SETUP_NO_MMAP) {