io_uring/uring_cmd: avoid double indirect call in task work dispatch

io_uring task work dispatch makes an indirect call to struct io_kiocb's
io_task_work.func field to allow running arbitrary task work functions.
In the uring_cmd case, this calls io_uring_cmd_work(), which immediately
makes another indirect call to struct io_uring_cmd's task_work_cb field.
Change the uring_cmd task work callbacks to functions whose signatures
match io_req_tw_func_t. Add a function io_uring_cmd_from_tw() to convert
from the task work's struct io_tw_req argument to struct io_uring_cmd *.
Define a constant IO_URING_CMD_TASK_WORK_ISSUE_FLAGS to avoid
manufacturing issue_flags in the uring_cmd task work callbacks. Now
uring_cmd task work dispatch makes a single indirect call to the
uring_cmd implementation's callback. This also allows removing the
task_work_cb field from struct io_uring_cmd, freeing up 8 bytes for
future storage.
Since fuse_uring_send_in_task() now has access to the io_tw_token_t,
check its cancel field directly instead of relying on the
IO_URING_F_TASK_DEAD issue flag.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/ioctl.c b/block/ioctl.c
index d7489a56b33c347d40b3f8aa43a5154b1442b10b..4ed17c5a4accfc48570c8483e85835fdec66e4f9 100644 (file)
--- a/block/ioctl.c
+++ b/block/ioctl.c
@@ -769,14 +769,16 @@ struct blk_iou_cmd {
        bool nowait;
 };
 
-static void blk_cmd_complete(struct io_uring_cmd *cmd, unsigned int issue_flags)
+static void blk_cmd_complete(struct io_tw_req tw_req, io_tw_token_t tw)
 {
+       struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
        struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd);
 
        if (bic->res == -EAGAIN && bic->nowait)
                io_uring_cmd_issue_blocking(cmd);
        else
-               io_uring_cmd_done(cmd, bic->res, issue_flags);
+               io_uring_cmd_done(cmd, bic->res,
+                                 IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
 }
 
 static void bio_cmd_bio_end_io(struct bio *bio)

diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 0c74a41a67530013ecd978d460cfca11e4995fa9..e0c601128efaac75c0b1d53b0247e3442efe5937 100644 (file)
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -1302,10 +1302,9 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req,
        return true;
 }
 
-static void ublk_dispatch_req(struct ublk_queue *ubq,
-                             struct request *req,
-                             unsigned int issue_flags)
+static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req)
 {
+       unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
        int tag = req->tag;
        struct ublk_io *io = &ubq->ios[tag];
 
@@ -1348,13 +1347,13 @@ static void ublk_dispatch_req(struct ublk_queue *ubq,
                ublk_complete_io_cmd(io, req, UBLK_IO_RES_OK, issue_flags);
 }
 
-static void ublk_cmd_tw_cb(struct io_uring_cmd *cmd,
-                          unsigned int issue_flags)
+static void ublk_cmd_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw)
 {
+       struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
        struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
        struct ublk_queue *ubq = pdu->ubq;
 
-       ublk_dispatch_req(ubq, pdu->req, issue_flags);
+       ublk_dispatch_req(ubq, pdu->req);
 }
 
 static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
@@ -1366,9 +1365,9 @@ static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
        io_uring_cmd_complete_in_task(cmd, ublk_cmd_tw_cb);
 }
 
-static void ublk_cmd_list_tw_cb(struct io_uring_cmd *cmd,
-               unsigned int issue_flags)
+static void ublk_cmd_list_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw)
 {
+       struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
        struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
        struct request *rq = pdu->req_list;
        struct request *next;
@@ -1376,7 +1375,7 @@ static void ublk_cmd_list_tw_cb(struct io_uring_cmd *cmd,
        do {
                next = rq->rq_next;
                rq->rq_next = NULL;
-               ublk_dispatch_req(rq->mq_hctx->driver_data, rq, issue_flags);
+               ublk_dispatch_req(rq->mq_hctx->driver_data, rq);
                rq = next;
        } while (rq);
 }
@@ -2523,9 +2522,10 @@ static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,
        return NULL;
 }
 
-static void ublk_ch_uring_cmd_cb(struct io_uring_cmd *cmd,
-               unsigned int issue_flags)
+static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)
 {
+       unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
+       struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
        int ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
 
        if (ret != -EIOCBQUEUED)

diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index c212fa952c0f4c66b5e025b47498c598bc668dde..4fa8400a5627ab51ac86221ed5714adaa15b0af3 100644 (file)
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -398,14 +398,15 @@ static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu(
        return io_uring_cmd_to_pdu(ioucmd, struct nvme_uring_cmd_pdu);
 }
 
-static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd,
-                              unsigned issue_flags)
+static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
 {
+       struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req);
        struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
 
        if (pdu->bio)
                blk_rq_unmap_user(pdu->bio);
-       io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, issue_flags);
+       io_uring_cmd_done32(ioucmd, pdu->status, pdu->result,
+                           IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
 }
 
 static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,

diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index 8cb7d5a462ef793e1789c95091c1cfb15060ba45..3171d9df0246cf8a12be988f171ea8917836b27e 100644 (file)
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -4649,8 +4649,9 @@ struct io_btrfs_cmd {
        struct btrfs_uring_priv *priv;
 };
 
-static void btrfs_uring_read_finished(struct io_uring_cmd *cmd, unsigned int issue_flags)
+static void btrfs_uring_read_finished(struct io_tw_req tw_req, io_tw_token_t tw)
 {
+       struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
        struct io_btrfs_cmd *bc = io_uring_cmd_to_pdu(cmd, struct io_btrfs_cmd);
        struct btrfs_uring_priv *priv = bc->priv;
        struct btrfs_inode *inode = BTRFS_I(file_inode(priv->iocb.ki_filp));
@@ -4695,7 +4696,7 @@ static void btrfs_uring_read_finished(struct io_uring_cmd *cmd, unsigned int iss
        btrfs_unlock_extent(io_tree, priv->start, priv->lockend, &priv->cached_state);
        btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
 
-       io_uring_cmd_done(cmd, ret, issue_flags);
+       io_uring_cmd_done(cmd, ret, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
        add_rchar(current, ret);
 
        for (index = 0; index < priv->nr_pages; index++)

diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index f6b12aebb8bbe7d255980593b75b5fb5af9c669e..f8c93dc4576801abdd0b81a43d8356aee0095361 100644 (file)
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -1209,14 +1209,15 @@ static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd,
  * User buffers are not mapped yet - the application does not have permission
  * to write to it - this has to be executed in ring task context.
  */
-static void fuse_uring_send_in_task(struct io_uring_cmd *cmd,
-                                   unsigned int issue_flags)
+static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw)
 {
+       unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
+       struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
        struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd);
        struct fuse_ring_queue *queue = ent->queue;
        int err;
 
-       if (!(issue_flags & IO_URING_F_TASK_DEAD)) {
+       if (!tw.cancel) {
                err = fuse_uring_prepare_send(ent, ent->fuse_req);
                if (err) {
                        fuse_uring_next_fuse_req(ent, queue, issue_flags);

diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h
index 7509025b40711f286193f2d68cdd63e82719b787..375fd048c4cbc06965eca9964d825d58e2cc98aa 100644 (file)
--- a/include/linux/io_uring/cmd.h
+++ b/include/linux/io_uring/cmd.h
@@ -11,17 +11,13 @@
 /* io_uring_cmd is being issued again */
 #define IORING_URING_CMD_REISSUE       (1U << 31)
 
-typedef void (*io_uring_cmd_tw_t)(struct io_uring_cmd *cmd,
-                                 unsigned issue_flags);
-
 struct io_uring_cmd {
        struct file     *file;
        const struct io_uring_sqe *sqe;
-       /* callback to defer completions to task context */
-       io_uring_cmd_tw_t task_work_cb;
        u32             cmd_op;
        u32             flags;
        u8              pdu[32]; /* available inline for free use */
+       u8              unused[8];
 };
 
 static inline const void *io_uring_sqe_cmd(const struct io_uring_sqe *sqe)
@@ -60,7 +56,7 @@ void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
                         unsigned issue_flags, bool is_cqe32);
 
 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
-                           io_uring_cmd_tw_t task_work_cb,
+                           io_req_tw_func_t task_work_cb,
                            unsigned flags);
 
 /*
@@ -109,7 +105,7 @@ static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret,
 {
 }
 static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
-                           io_uring_cmd_tw_t task_work_cb, unsigned flags)
+                           io_req_tw_func_t task_work_cb, unsigned flags)
 {
 }
 static inline void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
@@ -132,15 +128,23 @@ static inline bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,
 }
 #endif
 
+static inline struct io_uring_cmd *io_uring_cmd_from_tw(struct io_tw_req tw_req)
+{
+       return io_kiocb_to_cmd(tw_req.req, struct io_uring_cmd);
+}
+
+/* task_work executor checks the deferred list completion */
+#define IO_URING_CMD_TASK_WORK_ISSUE_FLAGS IO_URING_F_COMPLETE_DEFER
+
 /* users must follow the IOU_F_TWQ_LAZY_WAKE semantics */
 static inline void io_uring_cmd_do_in_task_lazy(struct io_uring_cmd *ioucmd,
-                       io_uring_cmd_tw_t task_work_cb)
+                       io_req_tw_func_t task_work_cb)
 {
        __io_uring_cmd_do_in_task(ioucmd, task_work_cb, IOU_F_TWQ_LAZY_WAKE);
 }
 
 static inline void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
-                       io_uring_cmd_tw_t task_work_cb)
+                       io_req_tw_func_t task_work_cb)
 {
        __io_uring_cmd_do_in_task(ioucmd, task_work_cb, 0);
 }

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index f064a438ce433edccafaa481950de9a85280267b..92780764d5faf4e8525109d17412eae27bd63ea6 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -39,7 +39,6 @@ enum io_uring_cmd_flags {
        /* set when uring wants to cancel a previously issued command */
        IO_URING_F_CANCEL               = (1 << 11),
        IO_URING_F_COMPAT               = (1 << 12),
-       IO_URING_F_TASK_DEAD            = (1 << 13),
 };
 
 struct io_wq_work_node {

diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index c09b99e91c86e9be56549641451b7df94cd6e2a0..197474911f0463d471ed4e4ceedbe5d2992f65d5 100644 (file)
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -113,21 +113,8 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
 }
 EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
 
-static void io_uring_cmd_work(struct io_tw_req tw_req, io_tw_token_t tw)
-{
-       struct io_kiocb *req = tw_req.req;
-       struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
-       unsigned int flags = IO_URING_F_COMPLETE_DEFER;
-
-       if (unlikely(tw.cancel))
-               flags |= IO_URING_F_TASK_DEAD;
-
-       /* task_work executor checks the deffered list completion */
-       ioucmd->task_work_cb(ioucmd, flags);
-}
-
 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
-                       io_uring_cmd_tw_t task_work_cb,
+                       io_req_tw_func_t task_work_cb,
                        unsigned flags)
 {
        struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
@@ -135,8 +122,7 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
        if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
                return;
 
-       ioucmd->task_work_cb = task_work_cb;
-       req->io_task_work.func = io_uring_cmd_work;
+       req->io_task_work.func = task_work_cb;
        __io_req_task_work_add(req, flags);
 }
 EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);