srcu: Create a DEFINE_SRCU_FAST()

This commit creates DEFINE_SRCU_FAST() and DEFINE_STATIC_SRCU_FAST()
macros that are similar to DEFINE_SRCU() and DEFINE_STATIC_SRCU(),
but which create srcu_struct structures that are usable only by readers
initiated by srcu_read_lock_fast() and friends.

This commit does make DEFINE_SRCU_FAST() available to modules, in which
case the per-CPU srcu_data structures are not created at compile time, but
rather at module-load time.  This means that the >srcu_reader_flavor field
of the srcu_data structure is not available.  Therefore,
this commit instead creates an ->srcu_reader_flavor field in the
srcu_struct structure, adds arguments to the DEFINE_SRCU()-related
macros to initialize this new field, and extends the checks in the
__srcu_check_read_flavor() function to include this new field.

This commit also allows dynamically allocated srcu_struct structure
to be marked for SRCU-fast readers.  It does so by defining a new
init_srcu_struct_fast() function that marks the specified srcu_struct
structure for use by srcu_read_lock_fast() and friends.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: <bpf@vger.kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>

diff --git a/include/linux/notifier.h b/include/linux/notifier.h
index b42e647349688248d8e45e90f997eaca9954c009..01b6c9d9956f902cf6d150c765ee2e668481630e 100644 (file)
--- a/include/linux/notifier.h
+++ b/include/linux/notifier.h
@@ -109,7 +109,7 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh);
                .mutex = __MUTEX_INITIALIZER(name.mutex),       \
                .head = NULL,                                   \
                .srcuu = __SRCU_USAGE_INIT(name.srcuu),         \
-               .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \
+               .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu, 0), \
        }
 
 #define ATOMIC_NOTIFIER_HEAD(name)                             \

diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index ada65b58bc4c5fdc57446c10919270199dda129c..26de47820c58cd66ec61b2443607ba797d7d4395 100644 (file)
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -25,8 +25,10 @@ struct srcu_struct;
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 
-int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
-                      struct lock_class_key *key);
+int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key);
+#ifndef CONFIG_TINY_SRCU
+int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key);
+#endif // #ifndef CONFIG_TINY_SRCU
 
 #define init_srcu_struct(ssp) \
 ({ \
@@ -35,10 +37,20 @@ int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
        __init_srcu_struct((ssp), #ssp, &__srcu_key); \
 })
 
+#define init_srcu_struct_fast(ssp) \
+({ \
+       static struct lock_class_key __srcu_key; \
+       \
+       __init_srcu_struct_fast((ssp), #ssp, &__srcu_key); \
+})
+
 #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name },
 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 
 int init_srcu_struct(struct srcu_struct *ssp);
+#ifndef CONFIG_TINY_SRCU
+int init_srcu_struct_fast(struct srcu_struct *ssp);
+#endif // #ifndef CONFIG_TINY_SRCU
 
 #define __SRCU_DEP_MAP_INIT(srcu_name)
 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */

diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 3bfbd44cb1b37dec38ef655c0dc665524890c05a..92e6ab53398fc00f452b98cf7703c69e4a5ea706 100644 (file)
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -31,7 +31,7 @@ struct srcu_struct {
 
 void srcu_drive_gp(struct work_struct *wp);
 
-#define __SRCU_STRUCT_INIT(name, __ignored, ___ignored)                        \
+#define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, ____ignored)   \
 {                                                                      \
        .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq),        \
        .srcu_cb_tail = &name.srcu_cb_head,                             \
@@ -44,13 +44,20 @@ void srcu_drive_gp(struct work_struct *wp);
  * Tree SRCU, which needs some per-CPU data.
  */
 #define DEFINE_SRCU(name) \
-       struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name)
+       struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
 #define DEFINE_STATIC_SRCU(name) \
-       static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name)
+       static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
+#define DEFINE_SRCU_FAST(name) DEFINE_SRCU(name)
+#define DEFINE_STATIC_SRCU_FAST(name) \
+       static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
 
 // Dummy structure for srcu_notifier_head.
 struct srcu_usage { };
 #define __SRCU_USAGE_INIT(name) { }
+#define __init_srcu_struct_fast __init_srcu_struct
+#ifndef CONFIG_DEBUG_LOCK_ALLOC
+#define init_srcu_struct_fast init_srcu_struct
+#endif // #ifndef CONFIG_DEBUG_LOCK_ALLOC
 
 void synchronize_srcu(struct srcu_struct *ssp);
 

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 93ad18acd6d02dfce291acf9067acee84caa9afb..7ff4a11bc5a34ecee399f10663b5c3d274f6e313 100644 (file)
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -104,6 +104,7 @@ struct srcu_usage {
 struct srcu_struct {
        struct srcu_ctr __percpu *srcu_ctrp;
        struct srcu_data __percpu *sda;         /* Per-CPU srcu_data array. */
+       u8 srcu_reader_flavor;
        struct lockdep_map dep_map;
        struct srcu_usage *srcu_sup;            /* Update-side data. */
 };
@@ -162,20 +163,21 @@ struct srcu_struct {
        .work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0),                                 \
 }
 
-#define __SRCU_STRUCT_INIT_COMMON(name, usage_name)                                            \
+#define __SRCU_STRUCT_INIT_COMMON(name, usage_name, fast)                                      \
        .srcu_sup = &usage_name,                                                                \
+       .srcu_reader_flavor = fast,                                                             \
        __SRCU_DEP_MAP_INIT(name)
 
-#define __SRCU_STRUCT_INIT_MODULE(name, usage_name)                                            \
+#define __SRCU_STRUCT_INIT_MODULE(name, usage_name, fast)                                      \
 {                                                                                              \
-       __SRCU_STRUCT_INIT_COMMON(name, usage_name)                                             \
+       __SRCU_STRUCT_INIT_COMMON(name, usage_name, fast)                                       \
 }
 
-#define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name)                                                \
+#define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name, fast)                                  \
 {                                                                                              \
        .sda = &pcpu_name,                                                                      \
        .srcu_ctrp = &pcpu_name.srcu_ctrs[0],                                                   \
-       __SRCU_STRUCT_INIT_COMMON(name, usage_name)                                             \
+       __SRCU_STRUCT_INIT_COMMON(name, usage_name, fast)                                               \
 }
 
 /*
@@ -196,23 +198,29 @@ struct srcu_struct {
  *     init_srcu_struct(&my_srcu);
  *
  * See include/linux/percpu-defs.h for the rules on per-CPU variables.
+ *
+ * DEFINE_SRCU_FAST() creates an srcu_struct and associated structures
+ * whose readers must be of the SRCU-fast variety.
  */
 #ifdef MODULE
-# define __DEFINE_SRCU(name, is_static)                                                                \
+# define __DEFINE_SRCU(name, fast, is_static)                                                  \
        static struct srcu_usage name##_srcu_usage = __SRCU_USAGE_INIT(name##_srcu_usage);      \
-       is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name, name##_srcu_usage); \
+       is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name, name##_srcu_usage,  \
+                                                                     fast);                    \
        extern struct srcu_struct * const __srcu_struct_##name;                                 \
        struct srcu_struct * const __srcu_struct_##name                                         \
                __section("___srcu_struct_ptrs") = &name
 #else
-# define __DEFINE_SRCU(name, is_static)                                                                \
+# define __DEFINE_SRCU(name, fast, is_static)                                                  \
        static DEFINE_PER_CPU(struct srcu_data, name##_srcu_data);                              \
        static struct srcu_usage name##_srcu_usage = __SRCU_USAGE_INIT(name##_srcu_usage);      \
        is_static struct srcu_struct name =                                                     \
-               __SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data)
+               __SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data, fast)
 #endif
-#define DEFINE_SRCU(name)              __DEFINE_SRCU(name, /* not static */)
-#define DEFINE_STATIC_SRCU(name)       __DEFINE_SRCU(name, static)
+#define DEFINE_SRCU(name)              __DEFINE_SRCU(name, 0, /* not static */)
+#define DEFINE_STATIC_SRCU(name)       __DEFINE_SRCU(name, 0, static)
+#define DEFINE_SRCU_FAST(name)         __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, /* not static */)
+#define DEFINE_STATIC_SRCU_FAST(name)  __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, static)
 
 int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);
 void synchronize_srcu_expedited(struct srcu_struct *ssp);

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 38b440b0b0c80be22a5a4ff6b09d8485982b1e68..9869a13b8763421c162e58f9dd5a2acf43a03cd6 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -286,16 +286,29 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 
-int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
-                      struct lock_class_key *key)
+static int
+__init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock_class_key *key)
 {
        /* Don't re-initialize a lock while it is held. */
        debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));
        lockdep_init_map(&ssp->dep_map, name, key, 0);
        return init_srcu_struct_fields(ssp, false);
 }
+
+int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key)
+{
+       ssp->srcu_reader_flavor = 0;
+       return __init_srcu_struct_common(ssp, name, key);
+}
 EXPORT_SYMBOL_GPL(__init_srcu_struct);
 
+int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key)
+{
+       ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST;
+       return __init_srcu_struct_common(ssp, name, key);
+}
+EXPORT_SYMBOL_GPL(__init_srcu_struct_fast);
+
 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 
 /**
@@ -308,10 +321,26 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct);
  */
 int init_srcu_struct(struct srcu_struct *ssp)
 {
+       ssp->srcu_reader_flavor = 0;
        return init_srcu_struct_fields(ssp, false);
 }
 EXPORT_SYMBOL_GPL(init_srcu_struct);
 
+/**
+ * init_srcu_struct_fast - initialize a fast-reader sleep-RCU structure
+ * @ssp: structure to initialize.
+ *
+ * Must invoke this on a given srcu_struct before passing that srcu_struct
+ * to any other function.  Each srcu_struct represents a separate domain
+ * of SRCU protection.
+ */
+int init_srcu_struct_fast(struct srcu_struct *ssp)
+{
+       ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST;
+       return init_srcu_struct_fields(ssp, false);
+}
+EXPORT_SYMBOL_GPL(init_srcu_struct_fast);
+
 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 
 /*
@@ -734,6 +763,9 @@ void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)
 
        sdp = raw_cpu_ptr(ssp->sda);
        old_read_flavor = READ_ONCE(sdp->srcu_reader_flavor);
+       WARN_ON_ONCE(ssp->srcu_reader_flavor && read_flavor != ssp->srcu_reader_flavor);
+       WARN_ON_ONCE(old_read_flavor && ssp->srcu_reader_flavor &&
+                    old_read_flavor != ssp->srcu_reader_flavor);
        if (!old_read_flavor) {
                old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor);
                if (!old_read_flavor)