sched_ext: Use shorter slice in bypass mode

There have been reported cases of bypass mode not making forward progress fast
enough. The 20ms default slice is unnecessarily long for bypass mode where the
primary goal is ensuring all tasks can make forward progress.

Introduce SCX_SLICE_BYPASS set to 5ms and make the scheduler automatically
switch to it when entering bypass mode. Also make the bypass slice value
tunable through the slice_bypass_us module parameter (adjustable between 100us
and 100ms) to make it easier to test whether slice durations are a factor in
problem cases.

v3: Use READ_ONCE/WRITE_ONCE for scx_slice_dfl access (Dan).

v2: Removed slice_dfl_us module parameter. Fixed typos (Andrea).

Reviewed-by: Emil Tsalapatis <emil@etsalapatis.com>
Reviewed-by: Andrea Righi <arighi@nvidia.com>
Cc: Dan Schatzberg <schatzberg.dan@gmail.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index eb776b094d36e93a9dba41966d932e7086164993..60285c3d07cf61c6ac241a6118f4983af02c4494 100644 (file)
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -17,7 +17,18 @@
 enum scx_public_consts {
        SCX_OPS_NAME_LEN        = 128,
 
+       /*
+        * %SCX_SLICE_DFL is used to refill slices when the BPF scheduler misses
+        * to set the slice for a task that is selected for execution.
+        * %SCX_EV_REFILL_SLICE_DFL counts the number of times the default slice
+        * refill has been triggered.
+        *
+        * %SCX_SLICE_BYPASS is used as the slice for all tasks in the bypass
+        * mode. As making forward progress for all tasks is the main goal of
+        * the bypass mode, a shorter slice is used.
+        */
        SCX_SLICE_DFL           = 20 * 1000000, /* 20ms */
+       SCX_SLICE_BYPASS        =  5 * 1000000, /*  5ms */
        SCX_SLICE_INF           = U64_MAX,      /* infinite, implies nohz */
 };
 

diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index 652a364e9e4cbc61aef76bcb9a12138899151ad3..1a9b28dd0961b9ed401a9d5ebc1468642113d164 100644 (file)
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -143,6 +143,32 @@ static struct scx_dump_data scx_dump_data = {
 /* /sys/kernel/sched_ext interface */
 static struct kset *scx_kset;
 
+/*
+ * Parameters that can be adjusted through /sys/module/sched_ext/parameters.
+ * There usually is no reason to modify these as normal scheduler operation
+ * shouldn't be affected by them. The knobs are primarily for debugging.
+ */
+static u64 scx_slice_dfl = SCX_SLICE_DFL;
+static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC;
+
+static int set_slice_us(const char *val, const struct kernel_param *kp)
+{
+       return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC);
+}
+
+static const struct kernel_param_ops slice_us_param_ops = {
+       .set = set_slice_us,
+       .get = param_get_uint,
+};
+
+#undef MODULE_PARAM_PREFIX
+#define MODULE_PARAM_PREFIX    "sched_ext."
+
+module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600);
+MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)");
+
+#undef MODULE_PARAM_PREFIX
+
 #define CREATE_TRACE_POINTS
 #include <trace/events/sched_ext.h>
 
@@ -919,7 +945,7 @@ static void dsq_mod_nr(struct scx_dispatch_q *dsq, s32 delta)
 
 static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p)
 {
-       p->scx.slice = SCX_SLICE_DFL;
+       p->scx.slice = READ_ONCE(scx_slice_dfl);
        __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);
 }
 
@@ -2896,7 +2922,7 @@ void init_scx_entity(struct sched_ext_entity *scx)
        INIT_LIST_HEAD(&scx->runnable_node);
        scx->runnable_at = jiffies;
        scx->ddsp_dsq_id = SCX_DSQ_INVALID;
-       scx->slice = SCX_SLICE_DFL;
+       scx->slice = READ_ONCE(scx_slice_dfl);
 }
 
 void scx_pre_fork(struct task_struct *p)
@@ -3774,6 +3800,7 @@ static void scx_bypass(bool bypass)
                WARN_ON_ONCE(scx_bypass_depth <= 0);
                if (scx_bypass_depth != 1)
                        goto unlock;
+               WRITE_ONCE(scx_slice_dfl, scx_slice_bypass_us * NSEC_PER_USEC);
                bypass_timestamp = ktime_get_ns();
                if (sch)
                        scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1);
@@ -3782,6 +3809,7 @@ static void scx_bypass(bool bypass)
                WARN_ON_ONCE(scx_bypass_depth < 0);
                if (scx_bypass_depth != 0)
                        goto unlock;
+               WRITE_ONCE(scx_slice_dfl, SCX_SLICE_DFL);
                if (sch)
                        scx_add_event(sch, SCX_EV_BYPASS_DURATION,
                                      ktime_get_ns() - bypass_timestamp);
@@ -4780,7 +4808,7 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
                        queue_flags |= DEQUEUE_CLASS;
 
                scoped_guard (sched_change, p, queue_flags) {
-                       p->scx.slice = SCX_SLICE_DFL;
+                       p->scx.slice = READ_ONCE(scx_slice_dfl);
                        p->sched_class = new_class;
                }
        }