io_uring: unify task_work cancelation checks

Rather than do per-tw checking, which needs to dip into the task_struct
for checking flags, do it upfront before running task_work. This places
a 'cancel' member in io_tw_token_t, which is assigned before running
task_work for that given ctx.

This is both more efficient in doing it upfront rather than for every
task_work, and it means that io_should_terminate_tw() can be made
private in io_uring.c rather than need to be called by various
callbacks of task_work.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index c2ea6280901dca8316c702c816498cd89be7919e..25ee982eb4350666b1993e5e98feaa346720c4cc 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -474,6 +474,7 @@ struct io_ring_ctx {
  * ONLY core io_uring.c should instantiate this struct.
  */
 struct io_tw_state {
+       bool cancel;
 };
 /* Alias to use in code that doesn't instantiate struct io_tw_state */
 typedef struct io_tw_state io_tw_token_t;

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 820ef05276667e74c259723bf9f3c605cf9d0505..c397118da85e27e142941a833c7a9a2ac7a4fe18 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -265,6 +265,20 @@ static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref)
        complete(&ctx->ref_comp);
 }
 
+/*
+ * Terminate the request if either of these conditions are true:
+ *
+ * 1) It's being executed by the original task, but that task is marked
+ *    with PF_EXITING as it's exiting.
+ * 2) PF_KTHREAD is set, in which case the invoker of the task_work is
+ *    our fallback task_work.
+ * 3) The ring has been closed and is going away.
+ */
+static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx)
+{
+       return (current->flags & (PF_EXITING | PF_KTHREAD)) || percpu_ref_is_dying(&ctx->refs);
+}
+
 static __cold void io_fallback_req_func(struct work_struct *work)
 {
        struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
@@ -275,8 +289,10 @@ static __cold void io_fallback_req_func(struct work_struct *work)
 
        percpu_ref_get(&ctx->refs);
        mutex_lock(&ctx->uring_lock);
-       llist_for_each_entry_safe(req, tmp, node, io_task_work.node)
+       llist_for_each_entry_safe(req, tmp, node, io_task_work.node) {
+               ts.cancel = io_should_terminate_tw(req->ctx);
                req->io_task_work.func(req, ts);
+       }
        io_submit_flush_completions(ctx);
        mutex_unlock(&ctx->uring_lock);
        percpu_ref_put(&ctx->refs);
@@ -1147,6 +1163,7 @@ struct llist_node *io_handle_tw_list(struct llist_node *node,
                        ctx = req->ctx;
                        mutex_lock(&ctx->uring_lock);
                        percpu_ref_get(&ctx->refs);
+                       ts.cancel = io_should_terminate_tw(ctx);
                }
                INDIRECT_CALL_2(req->io_task_work.func,
                                io_poll_task_func, io_req_rw_complete,
@@ -1205,11 +1222,6 @@ struct llist_node *tctx_task_work_run(struct io_uring_task *tctx,
 {
        struct llist_node *node;
 
-       if (unlikely(current->flags & PF_EXITING)) {
-               io_fallback_tw(tctx, true);
-               return NULL;
-       }
-
        node = llist_del_all(&tctx->task_list);
        if (node) {
                node = llist_reverse_order(node);
@@ -1399,6 +1411,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,
        if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
                atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
 again:
+       tw.cancel = io_should_terminate_tw(ctx);
        min_events -= ret;
        ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);
        if (ctx->retry_llist.first)
@@ -1458,7 +1471,7 @@ void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)
        struct io_ring_ctx *ctx = req->ctx;
 
        io_tw_lock(ctx, tw);
-       if (unlikely(io_should_terminate_tw(ctx)))
+       if (unlikely(tw.cancel))
                io_req_defer_failed(req, -EFAULT);
        else if (req->flags & REQ_F_FORCE_ASYNC)
                io_queue_iowq(req);

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 46d9141d772a71463bd2cbc0bb7c550786226790..78777bf1ea4b99f51452b97eccdbf5f213e4d27e 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -558,19 +558,6 @@ static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx)
                      ctx->submitter_task == current);
 }
 
-/*
- * Terminate the request if either of these conditions are true:
- *
- * 1) It's being executed by the original task, but that task is marked
- *    with PF_EXITING as it's exiting.
- * 2) PF_KTHREAD is set, in which case the invoker of the task_work is
- *    our fallback task_work.
- */
-static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx)
-{
-       return (current->flags & (PF_KTHREAD | PF_EXITING)) || percpu_ref_is_dying(&ctx->refs);
-}
-
 static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res)
 {
        io_req_set_res(req, res, 0);

diff --git a/io_uring/poll.c b/io_uring/poll.c
index b9681d0f9f1314bae28e227cb24418b56a26cf36..c403e751841a24f521bc6135f1e435bd56065d24 100644 (file)
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -224,7 +224,7 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
 {
        int v;
 
-       if (unlikely(io_should_terminate_tw(req->ctx)))
+       if (unlikely(tw.cancel))
                return -ECANCELED;
 
        do {

diff --git a/io_uring/timeout.c b/io_uring/timeout.c
index 17e3aab0af3676aeb90ecafbf78a0c216340744e..444142ba9d0457b6cd64b3f6903fbc3139360bad 100644 (file)
--- a/io_uring/timeout.c
+++ b/io_uring/timeout.c
@@ -324,7 +324,7 @@ static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw)
        int ret;
 
        if (prev) {
-               if (!io_should_terminate_tw(req->ctx)) {
+               if (!tw.cancel) {
                        struct io_cancel_data cd = {
                                .ctx            = req->ctx,
                                .data           = prev->cqe.user_data,

diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index d1e3ba62ee8e814a94058475398d81698d050c2b..1225f8124e4bb2e5ad3d88bdfcd8ae7e4e282bbe 100644 (file)
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -118,7 +118,7 @@ static void io_uring_cmd_work(struct io_kiocb *req, io_tw_token_t tw)
        struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
        unsigned int flags = IO_URING_F_COMPLETE_DEFER;
 
-       if (io_should_terminate_tw(req->ctx))
+       if (unlikely(tw.cancel))
                flags |= IO_URING_F_TASK_DEAD;
 
        /* task_work executor checks the deffered list completion */