io_uring: keep ring laoyut in a structure

Add a structure keeping SQ/CQ sizes and offsets. For now it only records
data previously returned from rings_size and the SQ size.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 40dfb851d46b2013e56b2c641cefd9bdc4c09fee..58e0c0ece6f16a233648c02df9df51348907b9bb 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2757,47 +2757,57 @@ static void io_rings_free(struct io_ring_ctx *ctx)
        ctx->sq_sqes = NULL;
 }
 
-unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
-                        unsigned int cq_entries, size_t *sq_offset)
+int rings_size(unsigned int flags, unsigned int sq_entries,
+               unsigned int cq_entries, struct io_rings_layout *rl)
 {
        struct io_rings *rings;
+       size_t sqe_size;
        size_t off;
 
-       *sq_offset = SIZE_MAX;
-
        if (flags & IORING_SETUP_CQE_MIXED) {
                if (cq_entries < 2)
-                       return SIZE_MAX;
+                       return -EOVERFLOW;
        }
        if (flags & IORING_SETUP_SQE_MIXED) {
                if (sq_entries < 2)
-                       return SIZE_MAX;
+                       return -EOVERFLOW;
        }
 
+       rl->sq_array_offset = SIZE_MAX;
+
+       sqe_size = sizeof(struct io_uring_sqe);
+       if (flags & IORING_SETUP_SQE128)
+               sqe_size *= 2;
+
+       rl->sq_size = array_size(sqe_size, sq_entries);
+       if (rl->sq_size == SIZE_MAX)
+               return -EOVERFLOW;
+
        off = struct_size(rings, cqes, cq_entries);
        if (flags & IORING_SETUP_CQE32)
                off = size_mul(off, 2);
        if (off == SIZE_MAX)
-               return SIZE_MAX;
+               return -EOVERFLOW;
 
 #ifdef CONFIG_SMP
        off = ALIGN(off, SMP_CACHE_BYTES);
        if (off == 0)
-               return SIZE_MAX;
+               return -EOVERFLOW;
 #endif
 
        if (!(flags & IORING_SETUP_NO_SQARRAY)) {
                size_t sq_array_size;
 
-               *sq_offset = off;
+               rl->sq_array_offset = off;
 
                sq_array_size = array_size(sizeof(u32), sq_entries);
                off = size_add(off, sq_array_size);
                if (off == SIZE_MAX)
-                       return SIZE_MAX;
+                       return -EOVERFLOW;
        }
 
-       return off;
+       rl->rings_size = off;
+       return 0;
 }
 
 static __cold void __io_req_caches_free(struct io_ring_ctx *ctx)
@@ -3346,28 +3356,20 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
                                         struct io_uring_params *p)
 {
        struct io_uring_region_desc rd;
+       struct io_rings_layout __rl, *rl = &__rl;
        struct io_rings *rings;
-       size_t sq_array_offset;
-       size_t sq_size, cq_size, sqe_size;
        int ret;
 
        /* make sure these are sane, as we already accounted them */
        ctx->sq_entries = p->sq_entries;
        ctx->cq_entries = p->cq_entries;
 
-       sqe_size = sizeof(struct io_uring_sqe);
-       if (p->flags & IORING_SETUP_SQE128)
-               sqe_size *= 2;
-       sq_size = array_size(sqe_size, p->sq_entries);
-       if (sq_size == SIZE_MAX)
-               return -EOVERFLOW;
-       cq_size = rings_size(ctx->flags, p->sq_entries, p->cq_entries,
-                         &sq_array_offset);
-       if (cq_size == SIZE_MAX)
-               return -EOVERFLOW;
+       ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);
+       if (ret)
+               return ret;
 
        memset(&rd, 0, sizeof(rd));
-       rd.size = PAGE_ALIGN(cq_size);
+       rd.size = PAGE_ALIGN(rl->rings_size);
        if (ctx->flags & IORING_SETUP_NO_MMAP) {
                rd.user_addr = p->cq_off.user_addr;
                rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -3378,10 +3380,10 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
        ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
 
        if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-               ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
+               ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
 
        memset(&rd, 0, sizeof(rd));
-       rd.size = PAGE_ALIGN(sq_size);
+       rd.size = PAGE_ALIGN(rl->sq_size);
        if (ctx->flags & IORING_SETUP_NO_MMAP) {
                rd.user_addr = p->sq_off.user_addr;
                rd.flags |= IORING_MEM_REGION_TYPE_USER;

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index d8bc44acb9fa718c6d468958ff2415e9b98b888f..5e544c2d27c85f63b7b400b06e10d457c53e1183 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -17,6 +17,14 @@
 #include <trace/events/io_uring.h>
 #endif
 
+struct io_rings_layout {
+       /* size of CQ + headers + SQ offset array */
+       size_t rings_size;
+       size_t sq_size;
+
+       size_t sq_array_offset;
+};
+
 struct io_ctx_config {
        struct io_uring_params p;
        struct io_uring_params __user *uptr;
@@ -139,8 +147,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 #define IORING_MAX_ENTRIES     32768
 #define IORING_MAX_CQ_ENTRIES  (2 * IORING_MAX_ENTRIES)
 
-unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
-                        unsigned int cq_entries, size_t *sq_offset);
+int rings_size(unsigned int flags, unsigned int sq_entries,
+               unsigned int cq_entries, struct io_rings_layout *rl);
 int io_prepare_config(struct io_ctx_config *config);
 
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);

diff --git a/io_uring/register.c b/io_uring/register.c
index 6b0024c20ce7be1246a1e5a2de41d8c6de331207..fa245c87978a36253feb8962d16f038dff91208f 100644 (file)
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -401,9 +401,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
        struct io_ctx_config config;
        struct io_uring_region_desc rd;
        struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
-       size_t size, sq_array_offset;
        unsigned i, tail, old_head;
        struct io_uring_params *p = &config.p;
+       struct io_rings_layout __rl, *rl = &__rl;
        int ret;
 
        memset(&config, 0, sizeof(config));
@@ -423,13 +423,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
        if (unlikely(ret))
                return ret;
 
-       size = rings_size(p->flags, p->sq_entries, p->cq_entries,
-                               &sq_array_offset);
-       if (size == SIZE_MAX)
-               return -EOVERFLOW;
+       ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);
+       if (ret)
+               return ret;
 
        memset(&rd, 0, sizeof(rd));
-       rd.size = PAGE_ALIGN(size);
+       rd.size = PAGE_ALIGN(rl->rings_size);
        if (p->flags & IORING_SETUP_NO_MMAP) {
                rd.user_addr = p->cq_off.user_addr;
                rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -458,17 +457,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
                return -EFAULT;
        }
 
-       if (p->flags & IORING_SETUP_SQE128)
-               size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);
-       else
-               size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
-       if (size == SIZE_MAX) {
-               io_register_free_rings(ctx, &n);
-               return -EOVERFLOW;
-       }
-
        memset(&rd, 0, sizeof(rd));
-       rd.size = PAGE_ALIGN(size);
+       rd.size = PAGE_ALIGN(rl->sq_size);
        if (p->flags & IORING_SETUP_NO_MMAP) {
                rd.user_addr = p->sq_off.user_addr;
                rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -551,7 +541,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 
        /* all done, store old pointers and assign new ones */
        if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-               ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset);
+               ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
 
        ctx->sq_entries = p->sq_entries;
        ctx->cq_entries = p->cq_entries;