UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

sched/isolation: Split housekeeping cpumask per isolation features 

To prepare for supporting each housekeeping feature toward cpuset, split
the global housekeeping cpumask per HK_TYPE_* entry.

This will later allow, for example, to runtime modify the cpulist passed
through "isolcpus=", "nohz_full=" and "rcu_nocbs=" kernel boot
parameters.

Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Juri Lelli <juri.lelli@redhat.com>
Reviewed-by: Phil Auld <pauld@redhat.com>
Link: https://lore.kernel.org/r/20220207155910.527133-9-frederic@kernel.org
This commit is contained in:
Frederic Weisbecker 2022-02-07 16:59:10 +01:00 committed by Peter Zijlstra
parent 65e53f869e
commit ed3b362d54
1 changed files with 62 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
kernel/sched/isolation.c
View File

@ -23,12 +23,17 @@ enum hk_flags {
DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
EXPORT_SYMBOL_GPL(housekeeping_overridden); EXPORT_SYMBOL_GPL(housekeeping_overridden);
static cpumask_var_t housekeeping_mask;
static unsigned int housekeeping_flags; struct housekeeping {
cpumask_var_t cpumasks[HK_TYPE_MAX];
unsigned long flags;
};
static struct housekeeping housekeeping;
bool housekeeping_enabled(enum hk_type type) bool housekeeping_enabled(enum hk_type type)
{ {
return !!(housekeeping_flags & BIT(type)); return !!(housekeeping.flags & BIT(type));
} }
EXPORT_SYMBOL_GPL(housekeeping_enabled); EXPORT_SYMBOL_GPL(housekeeping_enabled);
@ -37,12 +42,12 @@ int housekeeping_any_cpu(enum hk_type type)
int cpu; int cpu;
if (static_branch_unlikely(&housekeeping_overridden)) { if (static_branch_unlikely(&housekeeping_overridden)) {
if (housekeeping_flags & BIT(type)) { if (housekeeping.flags & BIT(type)) {
cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id()); cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
if (cpu < nr_cpu_ids) if (cpu < nr_cpu_ids)
return cpu; return cpu;
return cpumask_any_and(housekeeping_mask, cpu_online_mask); return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
} }
} }
return smp_processor_id(); return smp_processor_id();
@ -52,8 +57,8 @@ EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
const struct cpumask *housekeeping_cpumask(enum hk_type type) const struct cpumask *housekeeping_cpumask(enum hk_type type)
{ {
if (static_branch_unlikely(&housekeeping_overridden)) if (static_branch_unlikely(&housekeeping_overridden))
if (housekeeping_flags & BIT(type)) if (housekeeping.flags & BIT(type))
return housekeeping_mask; return housekeeping.cpumasks[type];
return cpu_possible_mask; return cpu_possible_mask;
} }
EXPORT_SYMBOL_GPL(housekeeping_cpumask); EXPORT_SYMBOL_GPL(housekeeping_cpumask);
@ -61,40 +66,53 @@ EXPORT_SYMBOL_GPL(housekeeping_cpumask);
void housekeeping_affine(struct task_struct *t, enum hk_type type) void housekeeping_affine(struct task_struct *t, enum hk_type type)
{ {
if (static_branch_unlikely(&housekeeping_overridden)) if (static_branch_unlikely(&housekeeping_overridden))
if (housekeeping_flags & BIT(type)) if (housekeeping.flags & BIT(type))
set_cpus_allowed_ptr(t, housekeeping_mask); set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
} }
EXPORT_SYMBOL_GPL(housekeeping_affine); EXPORT_SYMBOL_GPL(housekeeping_affine);
bool housekeeping_test_cpu(int cpu, enum hk_type type) bool housekeeping_test_cpu(int cpu, enum hk_type type)
{ {
if (static_branch_unlikely(&housekeeping_overridden)) if (static_branch_unlikely(&housekeeping_overridden))
if (housekeeping_flags & BIT(type)) if (housekeeping.flags & BIT(type))
return cpumask_test_cpu(cpu, housekeeping_mask); return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
return true; return true;
} }
EXPORT_SYMBOL_GPL(housekeeping_test_cpu); EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
void __init housekeeping_init(void) void __init housekeeping_init(void)
{ {
if (!housekeeping_flags) enum hk_type type;
if (!housekeeping.flags)
return; return;
static_branch_enable(&housekeeping_overridden); static_branch_enable(&housekeeping_overridden);
if (housekeeping_flags & HK_FLAG_TICK) if (housekeeping.flags & HK_FLAG_TICK)
sched_tick_offload_init(); sched_tick_offload_init();
/* We need at least one CPU to handle housekeeping work */ for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); /* We need at least one CPU to handle housekeeping work */
WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
}
} }
static int __init housekeeping_setup(char *str, enum hk_flags flags) static void __init housekeeping_setup_type(enum hk_type type,
cpumask_var_t housekeeping_staging)
{
alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
cpumask_copy(housekeeping.cpumasks[type],
housekeeping_staging);
}
static int __init housekeeping_setup(char *str, unsigned long flags)
{ {
cpumask_var_t non_housekeeping_mask, housekeeping_staging; cpumask_var_t non_housekeeping_mask, housekeeping_staging;
int err = 0; int err = 0;
if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) { if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
pr_warn("Housekeeping: nohz unsupported." pr_warn("Housekeeping: nohz unsupported."
" Build with CONFIG_NO_HZ_FULL\n"); " Build with CONFIG_NO_HZ_FULL\n");
@ -115,26 +133,41 @@ static int __init housekeeping_setup(char *str, enum hk_flags flags)
if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) { if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) {
__cpumask_set_cpu(smp_processor_id(), housekeeping_staging); __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
if (!housekeeping_flags) { if (!housekeeping.flags) {
pr_warn("Housekeeping: must include one present CPU, " pr_warn("Housekeeping: must include one present CPU, "
"using boot CPU:%d\n", smp_processor_id()); "using boot CPU:%d\n", smp_processor_id());
} }
} }
if (!housekeeping_flags) { if (!housekeeping.flags) {
alloc_bootmem_cpumask_var(&housekeeping_mask); /* First setup call ("nohz_full=" or "isolcpus=") */
cpumask_copy(housekeeping_mask, housekeeping_staging); enum hk_type type;
for_each_set_bit(type, &flags, HK_TYPE_MAX)
housekeeping_setup_type(type, housekeeping_staging);
} else { } else {
if (!cpumask_equal(housekeeping_staging, housekeeping_mask)) { /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); enum hk_type type;
goto free_housekeeping_staging; unsigned long iter_flags = flags & housekeeping.flags;
for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
if (!cpumask_equal(housekeeping_staging,
housekeeping.cpumasks[type])) {
pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
goto free_housekeeping_staging;
}
} }
iter_flags = flags & ~housekeeping.flags;
for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
housekeeping_setup_type(type, housekeeping_staging);
} }
if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
tick_nohz_full_setup(non_housekeeping_mask); tick_nohz_full_setup(non_housekeeping_mask);
housekeeping_flags |= flags; housekeeping.flags |= flags;
err = 1; err = 1;
free_housekeeping_staging: free_housekeeping_staging:
@ -147,7 +180,7 @@ free_non_housekeeping_mask:
static int __init housekeeping_nohz_full_setup(char *str) static int __init housekeeping_nohz_full_setup(char *str)
{ {
unsigned int flags; unsigned long flags;
flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
HK_FLAG_MISC | HK_FLAG_KTHREAD; HK_FLAG_MISC | HK_FLAG_KTHREAD;
@ -158,7 +191,7 @@ __setup("nohz_full=", housekeeping_nohz_full_setup);
static int __init housekeeping_isolcpus_setup(char *str) static int __init housekeeping_isolcpus_setup(char *str)
{ {
unsigned int flags = 0; unsigned long flags = 0;
bool illegal = false; bool illegal = false;
char *par; char *par;
int len; int len;