io_uring/waitid: have io_waitid_complete() remove wait queue entry

Both callers of this need the entry potentially removed, so shift the
removal into the completion side and kill it from the two callers.

While at it, add a helper for removing the wait_queue_entry based
on the passed in io_kiocb.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/waitid.c b/io_uring/waitid.c
index f25110fb1b1225abea26c4f35f012813291ba5a3..ebe3769c54dc8e6fa5f2ba1a9b73cfcaa8f1dd56 100644 (file)
--- a/io_uring/waitid.c
+++ b/io_uring/waitid.c
@@ -109,6 +109,22 @@ static int io_waitid_finish(struct io_kiocb *req, int ret)
        return ret;
 }
 
+static void io_waitid_remove_wq(struct io_kiocb *req)
+{
+       struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
+       struct wait_queue_head *head;
+
+       head = READ_ONCE(iw->head);
+       if (head) {
+               struct io_waitid_async *iwa = req->async_data;
+
+               iw->head = NULL;
+               spin_lock_irq(&head->lock);
+               list_del_init(&iwa->wo.child_wait.entry);
+               spin_unlock_irq(&head->lock);
+       }
+}
+
 static void io_waitid_complete(struct io_kiocb *req, int ret)
 {
        struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
@@ -119,6 +135,7 @@ static void io_waitid_complete(struct io_kiocb *req, int ret)
        lockdep_assert_held(&req->ctx->uring_lock);
 
        hlist_del_init(&req->hash_node);
+       io_waitid_remove_wq(req);
 
        ret = io_waitid_finish(req, ret);
        if (ret < 0)
@@ -129,7 +146,8 @@ static void io_waitid_complete(struct io_kiocb *req, int ret)
 static bool __io_waitid_cancel(struct io_kiocb *req)
 {
        struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
-       struct io_waitid_async *iwa = req->async_data;
+
+       lockdep_assert_held(&req->ctx->uring_lock);
 
        /*
         * Mark us canceled regardless of ownership. This will prevent a
@@ -141,9 +159,6 @@ static bool __io_waitid_cancel(struct io_kiocb *req)
        if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
                return false;
 
-       spin_lock_irq(&iw->head->lock);
-       list_del_init(&iwa->wo.child_wait.entry);
-       spin_unlock_irq(&iw->head->lock);
        io_waitid_complete(req, -ECANCELED);
        io_req_queue_tw_complete(req, -ECANCELED);
        return true;
@@ -209,8 +224,7 @@ static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw)
                                io_waitid_drop_issue_ref(req);
                                return;
                        }
-
-                       remove_wait_queue(iw->head, &iwa->wo.child_wait);
+                       /* fall through to complete, will kill waitqueue */
                }
        }