srcu: Create an srcu_expedite_current() function

This commit creates an srcu_expedite_current() function that expedites
the current (and possibly the next) SRCU grace period for the specified
srcu_struct structure.  This functionality will be inherited by RCU
Tasks Trace courtesy of its mapping to SRCU fast.

If the current SRCU grace period is already waiting, that wait will
complete before the expediting takes effect.  If there is no SRCU grace
period in flight, this function might well create one.

[ paulmck: Apply Zqiang feedback for PREEMPT_RT use. ]

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: <bpf@vger.kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>

diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 51ce25f07930ee1e7dbbbc5bdae7b0e4336006af..3bfbd44cb1b37dec38ef655c0dc665524890c05a 100644 (file)
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -103,6 +103,7 @@ static inline void srcu_barrier(struct srcu_struct *ssp)
        synchronize_srcu(ssp);
 }
 
+static inline void srcu_expedite_current(struct srcu_struct *ssp) { }
 #define srcu_check_read_flavor(ssp, read_flavor) do { } while (0)
 #define srcu_check_read_flavor_force(ssp, read_flavor) do { } while (0)
 

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 42098e0fa0b7dd19c870c3956403c14792abd785..93ad18acd6d02dfce291acf9067acee84caa9afb 100644 (file)
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -42,6 +42,8 @@ struct srcu_data {
        struct timer_list delay_work;           /* Delay for CB invoking */
        struct work_struct work;                /* Context for CB invoking. */
        struct rcu_head srcu_barrier_head;      /* For srcu_barrier() use. */
+       struct rcu_head srcu_ec_head;           /* For srcu_expedite_current() use. */
+       int srcu_ec_state;                      /*  State for srcu_expedite_current(). */
        struct srcu_node *mynode;               /* Leaf srcu_node. */
        unsigned long grpmask;                  /* Mask for leaf srcu_node */
                                                /*  ->srcu_data_have_cbs[]. */
@@ -135,6 +137,11 @@ struct srcu_struct {
 #define SRCU_STATE_SCAN1       1
 #define SRCU_STATE_SCAN2       2
 
+/* Values for srcu_expedite_current() state (->srcu_ec_state). */
+#define SRCU_EC_IDLE           0
+#define SRCU_EC_PENDING                1
+#define SRCU_EC_REPOST         2
+
 /*
  * Values for initializing gp sequence fields. Higher values allow wrap arounds to
  * occur earlier.
@@ -210,6 +217,7 @@ struct srcu_struct {
 int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);
 void synchronize_srcu_expedited(struct srcu_struct *ssp);
 void srcu_barrier(struct srcu_struct *ssp);
+void srcu_expedite_current(struct srcu_struct *ssp);
 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf);
 
 // Converts a per-CPU pointer to an ->srcu_ctrs[] array element to that

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 1ff94b76d91f15dfae233add80e2cbd63694c06f..38b440b0b0c80be22a5a4ff6b09d8485982b1e68 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -1688,6 +1688,64 @@ void srcu_barrier(struct srcu_struct *ssp)
 }
 EXPORT_SYMBOL_GPL(srcu_barrier);
 
+/* Callback for srcu_expedite_current() usage. */
+static void srcu_expedite_current_cb(struct rcu_head *rhp)
+{
+       unsigned long flags;
+       bool needcb = false;
+       struct srcu_data *sdp = container_of(rhp, struct srcu_data, srcu_ec_head);
+
+       spin_lock_irqsave_sdp_contention(sdp, &flags);
+       if (sdp->srcu_ec_state == SRCU_EC_IDLE) {
+               WARN_ON_ONCE(1);
+       } else if (sdp->srcu_ec_state == SRCU_EC_PENDING) {
+               sdp->srcu_ec_state = SRCU_EC_IDLE;
+       } else {
+               WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST);
+               sdp->srcu_ec_state = SRCU_EC_PENDING;
+               needcb = true;
+       }
+       spin_unlock_irqrestore_rcu_node(sdp, flags);
+       // If needed, requeue ourselves as an expedited SRCU callback.
+       if (needcb)
+               __call_srcu(sdp->ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false);
+}
+
+/**
+ * srcu_expedite_current - Expedite the current SRCU grace period
+ * @ssp: srcu_struct to expedite.
+ *
+ * Cause the current SRCU grace period to become expedited.  The grace
+ * period following the current one might also be expedited.  If there is
+ * no current grace period, one might be created.  If the current grace
+ * period is currently sleeping, that sleep will complete before expediting
+ * will take effect.
+ */
+void srcu_expedite_current(struct srcu_struct *ssp)
+{
+       unsigned long flags;
+       bool needcb = false;
+       struct srcu_data *sdp;
+
+       migrate_disable();
+       sdp = this_cpu_ptr(ssp->sda);
+       spin_lock_irqsave_sdp_contention(sdp, &flags);
+       if (sdp->srcu_ec_state == SRCU_EC_IDLE) {
+               sdp->srcu_ec_state = SRCU_EC_PENDING;
+               needcb = true;
+       } else if (sdp->srcu_ec_state == SRCU_EC_PENDING) {
+               sdp->srcu_ec_state = SRCU_EC_REPOST;
+       } else {
+               WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST);
+       }
+       spin_unlock_irqrestore_rcu_node(sdp, flags);
+       // If needed, queue an expedited SRCU callback.
+       if (needcb)
+               __call_srcu(ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false);
+       migrate_enable();
+}
+EXPORT_SYMBOL_GPL(srcu_expedite_current);
+
 /**
  * srcu_batches_completed - return batches completed.
  * @ssp: srcu_struct on which to report batch completion.