hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'sched-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull scheduler changes from Ingo Molnar:
 "Bigger changes:

   - sched/idle restructuring: they are WIP preparation for deeper
     integration between the scheduler and idle state selection, by
     Nicolas Pitre.

   - add NUMA scheduling pseudo-interleaving, by Rik van Riel.

   - optimize cgroup context switches, by Peter Zijlstra.

   - RT scheduling enhancements, by Thomas Gleixner.

  The rest is smaller changes, non-urgnt fixes and cleanups"

* 'sched-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (68 commits)
  sched: Clean up the task_hot() function
  sched: Remove double calculation in fix_small_imbalance()
  sched: Fix broken setscheduler()
  sparc64, sched: Remove unused sparc64_multi_core
  sched: Remove unused mc_capable() and smt_capable()
  sched/numa: Move task_numa_free() to __put_task_struct()
  sched/fair: Fix endless loop in idle_balance()
  sched/core: Fix endless loop in pick_next_task()
  sched/fair: Push down check for high priority class task into idle_balance()
  sched/rt: Fix picking RT and DL tasks from empty queue
  trace: Replace hardcoding of 19 with MAX_NICE
  sched: Guarantee task priority in pick_next_task()
  sched/idle: Remove stale old file
  sched: Put rq's sched_avg under CONFIG_FAIR_GROUP_SCHED
  cpuidle/arm64: Remove redundant cpuidle_idle_call()
  cpuidle/powernv: Remove redundant cpuidle_idle_call()
  sched, nohz: Exclude isolated cores from load balancing
  sched: Fix select_task_rq_fair() description comments
  workqueue: Replace hardcoding of -20 and 19 with MIN_NICE and MAX_NICE
  sys: Replace hardcoding of -20 and 19 with MIN_NICE and MAX_NICE
  ...
This commit is contained in:
Linus Torvalds 2014-03-31 11:21:19 -07:00
parent 8c292f1174 6037dd1a49
commit 971eae7c99
45 changed files with 913 additions and 541 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Documentation/sysctl/kernel.txt
View File

@ -442,8 +442,7 @@ feature should be disabled. Otherwise, if the system overhead from the
feature is too high then the rate the kernel samples for NUMA hinting feature is too high then the rate the kernel samples for NUMA hinting
faults may be controlled by the numa_balancing_scan_period_min_ms, faults may be controlled by the numa_balancing_scan_period_min_ms,
numa_balancing_scan_delay_ms, numa_balancing_scan_period_max_ms, numa_balancing_scan_delay_ms, numa_balancing_scan_period_max_ms,
numa_balancing_scan_size_mb, numa_balancing_settle_count sysctls and numa_balancing_scan_size_mb, and numa_balancing_settle_count sysctls.
numa_balancing_migrate_deferred.
============================================================== ==============================================================
@ -484,13 +483,6 @@ rate for each task.
numa_balancing_scan_size_mb is how many megabytes worth of pages are numa_balancing_scan_size_mb is how many megabytes worth of pages are
scanned for a given scan. scanned for a given scan.
numa_balancing_migrate_deferred is how many page migrations get skipped
unconditionally, after a page migration is skipped because a page is shared
with other tasks. This reduces page migration overhead, and determines
how much stronger the "move task near its memory" policy scheduler becomes,
versus the "move memory near its task" memory management policy, for workloads
with shared memory.
============================================================== ==============================================================
osrelease, ostype & version: osrelease, ostype & version:

3
arch/arm/include/asm/topology.h
View File

@ -20,9 +20,6 @@ extern struct cputopo_arm cpu_topology[NR_CPUS];
#define topology_core_cpumask(cpu) (&cpu_topology[cpu].core_sibling) #define topology_core_cpumask(cpu) (&cpu_topology[cpu].core_sibling)
#define topology_thread_cpumask(cpu) (&cpu_topology[cpu].thread_sibling) #define topology_thread_cpumask(cpu) (&cpu_topology[cpu].thread_sibling)
#define mc_capable() (cpu_topology[0].socket_id != -1)
#define smt_capable() (cpu_topology[0].thread_id != -1)
void init_cpu_topology(void); void init_cpu_topology(void);
void store_cpu_topology(unsigned int cpuid); void store_cpu_topology(unsigned int cpuid);
const struct cpumask *cpu_coregroup_mask(int cpu); const struct cpumask *cpu_coregroup_mask(int cpu);

16
arch/arm/kernel/process.c
View File

@ -30,7 +30,6 @@
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <linux/random.h> #include <linux/random.h>
#include <linux/hw_breakpoint.h> #include <linux/hw_breakpoint.h>
#include <linux/cpuidle.h>
#include <linux/leds.h> #include <linux/leds.h>
#include <linux/reboot.h> #include <linux/reboot.h>
@ -133,7 +132,11 @@ EXPORT_SYMBOL_GPL(arm_pm_restart);
void (*arm_pm_idle)(void); void (*arm_pm_idle)(void);
static void default_idle(void) /*
* Called from the core idle loop.
*/
void arch_cpu_idle(void)
{ {
if (arm_pm_idle) if (arm_pm_idle)
arm_pm_idle(); arm_pm_idle();
@ -167,15 +170,6 @@ void arch_cpu_idle_dead(void)
} }
#endif #endif
/*
* Called from the core idle loop.
*/
void arch_cpu_idle(void)
{
if (cpuidle_idle_call())
default_idle();
}
/* /*
* Called by kexec, immediately prior to machine_kexec(). * Called by kexec, immediately prior to machine_kexec().
* *

7
arch/arm64/kernel/process.c
View File

@ -33,7 +33,6 @@
#include <linux/kallsyms.h> #include <linux/kallsyms.h>
#include <linux/init.h> #include <linux/init.h>
#include <linux/cpu.h> #include <linux/cpu.h>
#include <linux/cpuidle.h>
#include <linux/elfcore.h> #include <linux/elfcore.h>
#include <linux/pm.h> #include <linux/pm.h>
#include <linux/tick.h> #include <linux/tick.h>
@ -94,10 +93,8 @@ void arch_cpu_idle(void)
* This should do all the clock switching and wait for interrupt * This should do all the clock switching and wait for interrupt
* tricks * tricks
*/ */
if (cpuidle_idle_call()) { cpu_do_idle();
cpu_do_idle(); local_irq_enable();
local_irq_enable();
}
} }
#ifdef CONFIG_HOTPLUG_CPU #ifdef CONFIG_HOTPLUG_CPU

1
arch/ia64/include/asm/topology.h
View File

@ -77,7 +77,6 @@ void build_cpu_to_node_map(void);
#define topology_core_id(cpu) (cpu_data(cpu)->core_id) #define topology_core_id(cpu) (cpu_data(cpu)->core_id)
#define topology_core_cpumask(cpu) (&cpu_core_map[cpu]) #define topology_core_cpumask(cpu) (&cpu_core_map[cpu])
#define topology_thread_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu)) #define topology_thread_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu))
#define smt_capable() (smp_num_siblings > 1)
#endif #endif
extern void arch_fix_phys_package_id(int num, u32 slot); extern void arch_fix_phys_package_id(int num, u32 slot);

4
arch/mips/include/asm/topology.h
View File

@ -10,8 +10,4 @@
#include <topology.h> #include <topology.h>
#ifdef CONFIG_SMP
#define smt_capable() (smp_num_siblings > 1)
#endif
#endif /* __ASM_TOPOLOGY_H */ #endif /* __ASM_TOPOLOGY_H */

1
arch/powerpc/include/asm/topology.h
View File

@ -99,7 +99,6 @@ static inline int prrn_is_enabled(void)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
#include <asm/cputable.h> #include <asm/cputable.h>
#define smt_capable() (cpu_has_feature(CPU_FTR_SMT))
#ifdef CONFIG_PPC64 #ifdef CONFIG_PPC64
#include <asm/smp.h> #include <asm/smp.h>

1
arch/powerpc/platforms/cell/spufs/sched.c
View File

@ -83,7 +83,6 @@ static struct timer_list spuloadavg_timer;
#define MIN_SPU_TIMESLICE max(5 * HZ / (1000 * SPUSCHED_TICK), 1) #define MIN_SPU_TIMESLICE max(5 * HZ / (1000 * SPUSCHED_TICK), 1)
#define DEF_SPU_TIMESLICE (100 * HZ / (1000 * SPUSCHED_TICK)) #define DEF_SPU_TIMESLICE (100 * HZ / (1000 * SPUSCHED_TICK))
#define MAX_USER_PRIO (MAX_PRIO - MAX_RT_PRIO)
#define SCALE_PRIO(x, prio) \ #define SCALE_PRIO(x, prio) \
max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_SPU_TIMESLICE) max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_SPU_TIMESLICE)

13
arch/powerpc/platforms/powernv/setup.c
View File

@ -26,7 +26,6 @@
#include <linux/of_fdt.h> #include <linux/of_fdt.h>
#include <linux/interrupt.h> #include <linux/interrupt.h>
#include <linux/bug.h> #include <linux/bug.h>
#include <linux/cpuidle.h>
#include <linux/pci.h> #include <linux/pci.h>
#include <asm/machdep.h> #include <asm/machdep.h>
@ -225,16 +224,6 @@ static int __init pnv_probe(void)
return 1; return 1;
} }
void powernv_idle(void)
{
/* Hook to cpuidle framework if available, else
* call on default platform idle code
*/
if (cpuidle_idle_call()) {
power7_idle();
}
}
define_machine(powernv) { define_machine(powernv) {
.name = "PowerNV", .name = "PowerNV",
.probe = pnv_probe, .probe = pnv_probe,
@ -244,7 +233,7 @@ define_machine(powernv) {
.show_cpuinfo = pnv_show_cpuinfo, .show_cpuinfo = pnv_show_cpuinfo,
.progress = pnv_progress, .progress = pnv_progress,
.machine_shutdown = pnv_shutdown, .machine_shutdown = pnv_shutdown,
.power_save = powernv_idle, .power_save = power7_idle,
.calibrate_decr = generic_calibrate_decr, .calibrate_decr = generic_calibrate_decr,
.dma_set_mask = pnv_dma_set_mask, .dma_set_mask = pnv_dma_set_mask,
#ifdef CONFIG_KEXEC #ifdef CONFIG_KEXEC

34
arch/powerpc/platforms/pseries/setup.c
View File

@ -39,7 +39,6 @@
#include <linux/irq.h> #include <linux/irq.h>
#include <linux/seq_file.h> #include <linux/seq_file.h>
#include <linux/root_dev.h> #include <linux/root_dev.h>
#include <linux/cpuidle.h>
#include <linux/of.h> #include <linux/of.h>
#include <linux/kexec.h> #include <linux/kexec.h>
@ -356,29 +355,24 @@ early_initcall(alloc_dispatch_log_kmem_cache);
static void pseries_lpar_idle(void) static void pseries_lpar_idle(void)
{ {
/* This would call on the cpuidle framework, and the back-end pseries /*
* driver to go to idle states * Default handler to go into low thread priority and possibly
* low power mode by cedeing processor to hypervisor
*/ */
if (cpuidle_idle_call()) {
/* On error, execute default handler
* to go into low thread priority and possibly
* low power mode by cedeing processor to hypervisor
*/
/* Indicate to hypervisor that we are idle. */ /* Indicate to hypervisor that we are idle. */
get_lppaca()->idle = 1; get_lppaca()->idle = 1;
/* /*
* Yield the processor to the hypervisor. We return if * Yield the processor to the hypervisor. We return if
* an external interrupt occurs (which are driven prior * an external interrupt occurs (which are driven prior
* to returning here) or if a prod occurs from another * to returning here) or if a prod occurs from another
* processor. When returning here, external interrupts * processor. When returning here, external interrupts
* are enabled. * are enabled.
*/ */
cede_processor(); cede_processor();
get_lppaca()->idle = 0; get_lppaca()->idle = 0;
}
} }
/* /*

4
arch/sh/kernel/idle.c
View File

@ -16,7 +16,6 @@
#include <linux/thread_info.h> #include <linux/thread_info.h>
#include <linux/irqflags.h> #include <linux/irqflags.h>
#include <linux/smp.h> #include <linux/smp.h>
#include <linux/cpuidle.h>
#include <linux/atomic.h> #include <linux/atomic.h>
#include <asm/pgalloc.h> #include <asm/pgalloc.h>
#include <asm/smp.h> #include <asm/smp.h>
@ -40,8 +39,7 @@ void arch_cpu_idle_dead(void)
void arch_cpu_idle(void) void arch_cpu_idle(void)
{ {
if (cpuidle_idle_call()) sh_idle();
sh_idle();
} }
void __init select_idle_routine(void) void __init select_idle_routine(void)

1
arch/sparc/include/asm/smp_64.h
View File

@ -32,7 +32,6 @@
DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); DECLARE_PER_CPU(cpumask_t, cpu_sibling_map);
extern cpumask_t cpu_core_map[NR_CPUS]; extern cpumask_t cpu_core_map[NR_CPUS];
extern int sparc64_multi_core;
extern void arch_send_call_function_single_ipi(int cpu); extern void arch_send_call_function_single_ipi(int cpu);
extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);

2
arch/sparc/include/asm/topology_64.h
View File

@ -42,8 +42,6 @@ static inline int pcibus_to_node(struct pci_bus *pbus)
#define topology_core_id(cpu) (cpu_data(cpu).core_id) #define topology_core_id(cpu) (cpu_data(cpu).core_id)
#define topology_core_cpumask(cpu) (&cpu_core_map[cpu]) #define topology_core_cpumask(cpu) (&cpu_core_map[cpu])
#define topology_thread_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu)) #define topology_thread_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu))
#define mc_capable() (sparc64_multi_core)
#define smt_capable() (sparc64_multi_core)
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
extern cpumask_t cpu_core_map[NR_CPUS]; extern cpumask_t cpu_core_map[NR_CPUS];

4
arch/sparc/kernel/mdesc.c
View File

@ -896,10 +896,6 @@ void mdesc_fill_in_cpu_data(cpumask_t *mask)
mdesc_iterate_over_cpus(fill_in_one_cpu, NULL, mask); mdesc_iterate_over_cpus(fill_in_one_cpu, NULL, mask);
#ifdef CONFIG_SMP
sparc64_multi_core = 1;
#endif
hp = mdesc_grab(); hp = mdesc_grab();
set_core_ids(hp); set_core_ids(hp);

3
arch/sparc/kernel/prom_64.c
View File

@ -555,9 +555,6 @@ static void *fill_in_one_cpu(struct device_node *dp, int cpuid, int arg)
cpu_data(cpuid).core_id = portid + 1; cpu_data(cpuid).core_id = portid + 1;
cpu_data(cpuid).proc_id = portid; cpu_data(cpuid).proc_id = portid;
#ifdef CONFIG_SMP
sparc64_multi_core = 1;
#endif
} else { } else {
cpu_data(cpuid).dcache_size = cpu_data(cpuid).dcache_size =
of_getintprop_default(dp, "dcache-size", 16 * 1024); of_getintprop_default(dp, "dcache-size", 16 * 1024);

2
arch/sparc/kernel/smp_64.c
View File

@ -53,8 +53,6 @@
#include "cpumap.h" #include "cpumap.h"
int sparc64_multi_core __read_mostly;
DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE; DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE;
cpumask_t cpu_core_map[NR_CPUS] __read_mostly = cpumask_t cpu_core_map[NR_CPUS] __read_mostly =
{ [0 ... NR_CPUS-1] = CPU_MASK_NONE }; { [0 ... NR_CPUS-1] = CPU_MASK_NONE };

6
arch/x86/include/asm/topology.h
View File

@ -134,12 +134,6 @@ static inline void arch_fix_phys_package_id(int num, u32 slot)
struct pci_bus; struct pci_bus;
void x86_pci_root_bus_resources(int bus, struct list_head *resources); void x86_pci_root_bus_resources(int bus, struct list_head *resources);
#ifdef CONFIG_SMP
#define mc_capable() ((boot_cpu_data.x86_max_cores > 1) && \
(cpumask_weight(cpu_core_mask(0)) != nr_cpu_ids))
#define smt_capable() (smp_num_siblings > 1)
#endif
#ifdef CONFIG_NUMA #ifdef CONFIG_NUMA
extern int get_mp_bus_to_node(int busnum); extern int get_mp_bus_to_node(int busnum);
extern void set_mp_bus_to_node(int busnum, int node); extern void set_mp_bus_to_node(int busnum, int node);

5
arch/x86/kernel/process.c
View File

@ -298,10 +298,7 @@ void arch_cpu_idle_dead(void)
*/ */
void arch_cpu_idle(void) void arch_cpu_idle(void)
{ {
if (cpuidle_idle_call()) x86_idle();
x86_idle();
else
local_irq_enable();
} }
/* /*

5
drivers/cpuidle/cpuidle-powernv.c
View File

@ -14,6 +14,7 @@
#include <asm/machdep.h> #include <asm/machdep.h>
#include <asm/firmware.h> #include <asm/firmware.h>
#include <asm/runlatch.h>
struct cpuidle_driver powernv_idle_driver = { struct cpuidle_driver powernv_idle_driver = {
.name = "powernv_idle", .name = "powernv_idle",
@ -30,12 +31,14 @@ static int snooze_loop(struct cpuidle_device *dev,
local_irq_enable(); local_irq_enable();
set_thread_flag(TIF_POLLING_NRFLAG); set_thread_flag(TIF_POLLING_NRFLAG);
ppc64_runlatch_off();
while (!need_resched()) { while (!need_resched()) {
HMT_low(); HMT_low();
HMT_very_low(); HMT_very_low();
} }
HMT_medium(); HMT_medium();
ppc64_runlatch_on();
clear_thread_flag(TIF_POLLING_NRFLAG); clear_thread_flag(TIF_POLLING_NRFLAG);
smp_mb(); smp_mb();
return index; return index;
@ -45,7 +48,9 @@ static int nap_loop(struct cpuidle_device *dev,
struct cpuidle_driver *drv, struct cpuidle_driver *drv,
int index) int index)
{ {
ppc64_runlatch_off();
power7_idle(); power7_idle();
ppc64_runlatch_on();
return index; return index;
} }

6
drivers/cpuidle/cpuidle-pseries.c
View File

@ -17,6 +17,7 @@
#include <asm/reg.h> #include <asm/reg.h>
#include <asm/machdep.h> #include <asm/machdep.h>
#include <asm/firmware.h> #include <asm/firmware.h>
#include <asm/runlatch.h>
#include <asm/plpar_wrappers.h> #include <asm/plpar_wrappers.h>
struct cpuidle_driver pseries_idle_driver = { struct cpuidle_driver pseries_idle_driver = {
@ -29,6 +30,7 @@ static struct cpuidle_state *cpuidle_state_table;
static inline void idle_loop_prolog(unsigned long *in_purr) static inline void idle_loop_prolog(unsigned long *in_purr)
{ {
ppc64_runlatch_off();
*in_purr = mfspr(SPRN_PURR); *in_purr = mfspr(SPRN_PURR);
/* /*
* Indicate to the HV that we are idle. Now would be * Indicate to the HV that we are idle. Now would be
@ -45,6 +47,10 @@ static inline void idle_loop_epilog(unsigned long in_purr)
wait_cycles += mfspr(SPRN_PURR) - in_purr; wait_cycles += mfspr(SPRN_PURR) - in_purr;
get_lppaca()->wait_state_cycles = cpu_to_be64(wait_cycles); get_lppaca()->wait_state_cycles = cpu_to_be64(wait_cycles);
get_lppaca()->idle = 0; get_lppaca()->idle = 0;
if (irqs_disabled())
local_irq_enable();
ppc64_runlatch_on();
} }
static int snooze_loop(struct cpuidle_device *dev, static int snooze_loop(struct cpuidle_device *dev,

44
include/linux/sched.h
View File

@ -3,6 +3,8 @@
#include <uapi/linux/sched.h> #include <uapi/linux/sched.h>
#include <linux/sched/prio.h>
struct sched_param { struct sched_param {
int sched_priority; int sched_priority;
@ -1077,6 +1079,7 @@ struct sched_entity {
#endif #endif
#ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_FAIR_GROUP_SCHED
int depth;
struct sched_entity *parent; struct sched_entity *parent;
/* rq on which this entity is (to be) queued: */ /* rq on which this entity is (to be) queued: */
struct cfs_rq *cfs_rq; struct cfs_rq *cfs_rq;
@ -1460,6 +1463,9 @@ struct task_struct {
struct mutex perf_event_mutex; struct mutex perf_event_mutex;
struct list_head perf_event_list; struct list_head perf_event_list;
#endif #endif
#ifdef CONFIG_DEBUG_PREEMPT
unsigned long preempt_disable_ip;
#endif
#ifdef CONFIG_NUMA #ifdef CONFIG_NUMA
struct mempolicy *mempolicy; /* Protected by alloc_lock */ struct mempolicy *mempolicy; /* Protected by alloc_lock */
short il_next; short il_next;
@ -1470,9 +1476,10 @@ struct task_struct {
unsigned int numa_scan_period; unsigned int numa_scan_period;
unsigned int numa_scan_period_max; unsigned int numa_scan_period_max;
int numa_preferred_nid; int numa_preferred_nid;
int numa_migrate_deferred;
unsigned long numa_migrate_retry; unsigned long numa_migrate_retry;
u64 node_stamp; /* migration stamp */ u64 node_stamp; /* migration stamp */
u64 last_task_numa_placement;
u64 last_sum_exec_runtime;
struct callback_head numa_work; struct callback_head numa_work;
struct list_head numa_entry; struct list_head numa_entry;
@ -1483,15 +1490,22 @@ struct task_struct {
* Scheduling placement decisions are made based on the these counts. * Scheduling placement decisions are made based on the these counts.
* The values remain static for the duration of a PTE scan * The values remain static for the duration of a PTE scan
*/ */
unsigned long *numa_faults; unsigned long *numa_faults_memory;
unsigned long total_numa_faults; unsigned long total_numa_faults;
/* /*
* numa_faults_buffer records faults per node during the current * numa_faults_buffer records faults per node during the current
* scan window. When the scan completes, the counts in numa_faults * scan window. When the scan completes, the counts in
* decay and these values are copied. * numa_faults_memory decay and these values are copied.
*/ */
unsigned long *numa_faults_buffer; unsigned long *numa_faults_buffer_memory;
/*
* Track the nodes the process was running on when a NUMA hinting
* fault was incurred.
*/
unsigned long *numa_faults_cpu;
unsigned long *numa_faults_buffer_cpu;
/* /*
* numa_faults_locality tracks if faults recorded during the last * numa_faults_locality tracks if faults recorded during the last
@ -1596,8 +1610,8 @@ extern void task_numa_fault(int last_node, int node, int pages, int flags);
extern pid_t task_numa_group_id(struct task_struct *p); extern pid_t task_numa_group_id(struct task_struct *p);
extern void set_numabalancing_state(bool enabled); extern void set_numabalancing_state(bool enabled);
extern void task_numa_free(struct task_struct *p); extern void task_numa_free(struct task_struct *p);
extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page,
extern unsigned int sysctl_numa_balancing_migrate_deferred; int src_nid, int dst_cpu);
#else #else
static inline void task_numa_fault(int last_node, int node, int pages, static inline void task_numa_fault(int last_node, int node, int pages,
int flags) int flags)
@ -1613,6 +1627,11 @@ static inline void set_numabalancing_state(bool enabled)
static inline void task_numa_free(struct task_struct *p) static inline void task_numa_free(struct task_struct *p)
{ {
} }
static inline bool should_numa_migrate_memory(struct task_struct *p,
struct page *page, int src_nid, int dst_cpu)
{
return true;
}
#endif #endif
static inline struct pid *task_pid(struct task_struct *task) static inline struct pid *task_pid(struct task_struct *task)
@ -2080,7 +2099,16 @@ static inline void sched_autogroup_exit(struct signal_struct *sig) { }
extern bool yield_to(struct task_struct *p, bool preempt); extern bool yield_to(struct task_struct *p, bool preempt);
extern void set_user_nice(struct task_struct *p, long nice); extern void set_user_nice(struct task_struct *p, long nice);
extern int task_prio(const struct task_struct *p); extern int task_prio(const struct task_struct *p);
extern int task_nice(const struct task_struct *p); /**
* task_nice - return the nice value of a given task.
* @p: the task in question.
*
* Return: The nice value [ -20 ... 0 ... 19 ].
*/
static inline int task_nice(const struct task_struct *p)
{
return PRIO_TO_NICE((p)->static_prio);
}
extern int can_nice(const struct task_struct *p, const int nice); extern int can_nice(const struct task_struct *p, const int nice);
extern int task_curr(const struct task_struct *p); extern int task_curr(const struct task_struct *p);
extern int idle_cpu(int cpu); extern int idle_cpu(int cpu);

44
include/linux/sched/prio.h Normal file
View File

@ -0,0 +1,44 @@
#ifndef _SCHED_PRIO_H
#define _SCHED_PRIO_H
#define MAX_NICE 19
#define MIN_NICE -20
#define NICE_WIDTH (MAX_NICE - MIN_NICE + 1)
/*
* Priority of a process goes from 0..MAX_PRIO-1, valid RT
* priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH
* tasks are in the range MAX_RT_PRIO..MAX_PRIO-1. Priority
* values are inverted: lower p->prio value means higher priority.
*
* The MAX_USER_RT_PRIO value allows the actual maximum
* RT priority to be separate from the value exported to
* user-space. This allows kernel threads to set their
* priority to a value higher than any user task. Note:
* MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO.
*/
#define MAX_USER_RT_PRIO 100
#define MAX_RT_PRIO MAX_USER_RT_PRIO
#define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH)
#define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2)
/*
* Convert user-nice values [ -20 ... 0 ... 19 ]
* to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
* and back.
*/
#define NICE_TO_PRIO(nice) ((nice) + DEFAULT_PRIO)
#define PRIO_TO_NICE(prio) ((prio) - DEFAULT_PRIO)
/*
* 'User priority' is the nice value converted to something we
* can work with better when scaling various scheduler parameters,
* it's a [ 0 ... 39 ] range.
*/
#define USER_PRIO(p) ((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
#endif /* _SCHED_PRIO_H */

26
include/linux/sched/rt.h
View File

@ -1,24 +1,7 @@
#ifndef _SCHED_RT_H #ifndef _SCHED_RT_H
#define _SCHED_RT_H #define _SCHED_RT_H
/* #include <linux/sched/prio.h>
* Priority of a process goes from 0..MAX_PRIO-1, valid RT
* priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH
* tasks are in the range MAX_RT_PRIO..MAX_PRIO-1. Priority
* values are inverted: lower p->prio value means higher priority.
*
* The MAX_USER_RT_PRIO value allows the actual maximum
* RT priority to be separate from the value exported to
* user-space. This allows kernel threads to set their
* priority to a value higher than any user task. Note:
* MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO.
*/
#define MAX_USER_RT_PRIO 100
#define MAX_RT_PRIO MAX_USER_RT_PRIO
#define MAX_PRIO (MAX_RT_PRIO + 40)
#define DEFAULT_PRIO (MAX_RT_PRIO + 20)
static inline int rt_prio(int prio) static inline int rt_prio(int prio)
{ {
@ -35,6 +18,7 @@ static inline int rt_task(struct task_struct *p)
#ifdef CONFIG_RT_MUTEXES #ifdef CONFIG_RT_MUTEXES
extern int rt_mutex_getprio(struct task_struct *p); extern int rt_mutex_getprio(struct task_struct *p);
extern void rt_mutex_setprio(struct task_struct *p, int prio); extern void rt_mutex_setprio(struct task_struct *p, int prio);
extern int rt_mutex_check_prio(struct task_struct *task, int newprio);
extern struct task_struct *rt_mutex_get_top_task(struct task_struct *task); extern struct task_struct *rt_mutex_get_top_task(struct task_struct *task);
extern void rt_mutex_adjust_pi(struct task_struct *p); extern void rt_mutex_adjust_pi(struct task_struct *p);
static inline bool tsk_is_pi_blocked(struct task_struct *tsk) static inline bool tsk_is_pi_blocked(struct task_struct *tsk)
@ -46,6 +30,12 @@ static inline int rt_mutex_getprio(struct task_struct *p)
{ {
return p->normal_prio; return p->normal_prio;
} }
static inline int rt_mutex_check_prio(struct task_struct *task, int newprio)
{
return 0;
}
static inline struct task_struct *rt_mutex_get_top_task(struct task_struct *task) static inline struct task_struct *rt_mutex_get_top_task(struct task_struct *task)
{ {
return NULL; return NULL;

1
kernel/Makefile
View File

@ -22,7 +22,6 @@ obj-y += sched/
obj-y += locking/ obj-y += locking/
obj-y += power/ obj-y += power/
obj-y += printk/ obj-y += printk/
obj-y += cpu/
obj-y += irq/ obj-y += irq/
obj-y += rcu/ obj-y += rcu/

1
kernel/cpu/Makefile
View File

@ -1 +0,0 @@
obj-y = idle.o

1
kernel/fork.c
View File

@ -237,6 +237,7 @@ void __put_task_struct(struct task_struct *tsk)
WARN_ON(atomic_read(&tsk->usage)); WARN_ON(atomic_read(&tsk->usage));
WARN_ON(tsk == current); WARN_ON(tsk == current);
task_numa_free(tsk);
security_task_free(tsk); security_task_free(tsk);
exit_creds(tsk); exit_creds(tsk);
delayacct_tsk_free(tsk); delayacct_tsk_free(tsk);

12
kernel/locking/rtmutex.c
View File

@ -212,6 +212,18 @@ struct task_struct *rt_mutex_get_top_task(struct task_struct *task)
return task_top_pi_waiter(task)->task; return task_top_pi_waiter(task)->task;
} }
/*
* Called by sched_setscheduler() to check whether the priority change
* is overruled by a possible priority boosting.
*/
int rt_mutex_check_prio(struct task_struct *task, int newprio)
{
if (!task_has_pi_waiters(task))
return 0;
return task_top_pi_waiter(task)->task->prio <= newprio;
}
/* /*
* Adjust the priority of a task, after its pi_waiters got modified. * Adjust the priority of a task, after its pi_waiters got modified.
* *

8
kernel/rcu/rcutorture.c
View File

@ -696,7 +696,7 @@ rcu_torture_writer(void *arg)
static DEFINE_TORTURE_RANDOM(rand); static DEFINE_TORTURE_RANDOM(rand);
VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
set_user_nice(current, 19); set_user_nice(current, MAX_NICE);
do { do {
schedule_timeout_uninterruptible(1); schedule_timeout_uninterruptible(1);
@ -759,7 +759,7 @@ rcu_torture_fakewriter(void *arg)
DEFINE_TORTURE_RANDOM(rand); DEFINE_TORTURE_RANDOM(rand);
VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
set_user_nice(current, 19); set_user_nice(current, MAX_NICE);
do { do {
schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); schedule_timeout_uninterruptible(1 + torture_random(&rand)%10);
@ -872,7 +872,7 @@ rcu_torture_reader(void *arg)
unsigned long long ts; unsigned long long ts;
VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); VERBOSE_TOROUT_STRING("rcu_torture_reader task started");
set_user_nice(current, 19); set_user_nice(current, MAX_NICE);
if (irqreader && cur_ops->irq_capable) if (irqreader && cur_ops->irq_capable)
setup_timer_on_stack(&t, rcu_torture_timer, 0); setup_timer_on_stack(&t, rcu_torture_timer, 0);
@ -1161,7 +1161,7 @@ static int rcu_torture_barrier_cbs(void *arg)
init_rcu_head_on_stack(&rcu); init_rcu_head_on_stack(&rcu);
VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started");
set_user_nice(current, 19); set_user_nice(current, MAX_NICE);
do { do {
wait_event(barrier_cbs_wq[myid], wait_event(barrier_cbs_wq[myid],
(newphase = (newphase =

2
kernel/sched/Makefile
View File

@ -13,7 +13,7 @@ endif
obj-y += core.o proc.o clock.o cputime.o obj-y += core.o proc.o clock.o cputime.o
obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o
obj-y += wait.o completion.o obj-y += wait.o completion.o idle.o
obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o
obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o
obj-$(CONFIG_SCHEDSTATS) += stats.o obj-$(CONFIG_SCHEDSTATS) += stats.o

2
kernel/sched/auto_group.c
View File

@ -203,7 +203,7 @@ int proc_sched_autogroup_set_nice(struct task_struct *p, int nice)
struct autogroup *ag; struct autogroup *ag;
int err; int err;
if (nice < -20 || nice > 19) if (nice < MIN_NICE || nice > MAX_NICE)
return -EINVAL; return -EINVAL;
err = security_task_setnice(current, nice); err = security_task_setnice(current, nice);

207
kernel/sched/core.c
View File

@ -1745,8 +1745,10 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0; p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
p->numa_scan_period = sysctl_numa_balancing_scan_delay; p->numa_scan_period = sysctl_numa_balancing_scan_delay;
p->numa_work.next = &p->numa_work; p->numa_work.next = &p->numa_work;
p->numa_faults = NULL; p->numa_faults_memory = NULL;
p->numa_faults_buffer = NULL; p->numa_faults_buffer_memory = NULL;
p->last_task_numa_placement = 0;
p->last_sum_exec_runtime = 0;
INIT_LIST_HEAD(&p->numa_entry); INIT_LIST_HEAD(&p->numa_entry);
p->numa_group = NULL; p->numa_group = NULL;
@ -2149,8 +2151,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
if (mm) if (mm)
mmdrop(mm); mmdrop(mm);
if (unlikely(prev_state == TASK_DEAD)) { if (unlikely(prev_state == TASK_DEAD)) {
task_numa_free(prev);
if (prev->sched_class->task_dead) if (prev->sched_class->task_dead)
prev->sched_class->task_dead(prev); prev->sched_class->task_dead(prev);
@ -2167,13 +2167,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
/* assumes rq->lock is held */
static inline void pre_schedule(struct rq *rq, struct task_struct *prev)
{
if (prev->sched_class->pre_schedule)
prev->sched_class->pre_schedule(rq, prev);
}
/* rq->lock is NOT held, but preemption is disabled */ /* rq->lock is NOT held, but preemption is disabled */
static inline void post_schedule(struct rq *rq) static inline void post_schedule(struct rq *rq)
{ {
@ -2191,10 +2184,6 @@ static inline void post_schedule(struct rq *rq)
#else #else
static inline void pre_schedule(struct rq *rq, struct task_struct *p)
{
}
static inline void post_schedule(struct rq *rq) static inline void post_schedule(struct rq *rq)
{ {
} }
@ -2510,8 +2499,13 @@ void __kprobes preempt_count_add(int val)
DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
PREEMPT_MASK - 10); PREEMPT_MASK - 10);
#endif #endif
if (preempt_count() == val) if (preempt_count() == val) {
trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); unsigned long ip = get_parent_ip(CALLER_ADDR1);
#ifdef CONFIG_DEBUG_PREEMPT
current->preempt_disable_ip = ip;
#endif
trace_preempt_off(CALLER_ADDR0, ip);
}
} }
EXPORT_SYMBOL(preempt_count_add); EXPORT_SYMBOL(preempt_count_add);
@ -2554,6 +2548,13 @@ static noinline void __schedule_bug(struct task_struct *prev)
print_modules(); print_modules();
if (irqs_disabled()) if (irqs_disabled())
print_irqtrace_events(prev); print_irqtrace_events(prev);
#ifdef CONFIG_DEBUG_PREEMPT
if (in_atomic_preempt_off()) {
pr_err("Preemption disabled at:");
print_ip_sym(current->preempt_disable_ip);
pr_cont("\n");
}
#endif
dump_stack(); dump_stack();
add_taint(TAINT_WARN, LOCKDEP_STILL_OK); add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
} }
@ -2577,36 +2578,34 @@ static inline void schedule_debug(struct task_struct *prev)
schedstat_inc(this_rq(), sched_count); schedstat_inc(this_rq(), sched_count);
} }
static void put_prev_task(struct rq *rq, struct task_struct *prev)
{
if (prev->on_rq || rq->skip_clock_update < 0)
update_rq_clock(rq);
prev->sched_class->put_prev_task(rq, prev);
}
/* /*
* Pick up the highest-prio task: * Pick up the highest-prio task:
*/ */
static inline struct task_struct * static inline struct task_struct *
pick_next_task(struct rq *rq) pick_next_task(struct rq *rq, struct task_struct *prev)
{ {
const struct sched_class *class; const struct sched_class *class = &fair_sched_class;
struct task_struct *p; struct task_struct *p;
/* /*
* Optimization: we know that if all tasks are in * Optimization: we know that if all tasks are in
* the fair class we can call that function directly: * the fair class we can call that function directly:
*/ */
if (likely(rq->nr_running == rq->cfs.h_nr_running)) { if (likely(prev->sched_class == class &&
p = fair_sched_class.pick_next_task(rq); rq->nr_running == rq->cfs.h_nr_running)) {
if (likely(p)) p = fair_sched_class.pick_next_task(rq, prev);
if (likely(p && p != RETRY_TASK))
return p; return p;
} }
again:
for_each_class(class) { for_each_class(class) {
p = class->pick_next_task(rq); p = class->pick_next_task(rq, prev);
if (p) if (p) {
if (unlikely(p == RETRY_TASK))
goto again;
return p; return p;
}
} }
BUG(); /* the idle class will always have a runnable task */ BUG(); /* the idle class will always have a runnable task */
@ -2700,13 +2699,10 @@ need_resched:
switch_count = &prev->nvcsw; switch_count = &prev->nvcsw;
} }
pre_schedule(rq, prev); if (prev->on_rq || rq->skip_clock_update < 0)
update_rq_clock(rq);
if (unlikely(!rq->nr_running)) next = pick_next_task(rq, prev);
idle_balance(cpu, rq);
put_prev_task(rq, prev);
next = pick_next_task(rq);
clear_tsk_need_resched(prev); clear_tsk_need_resched(prev);
clear_preempt_need_resched(); clear_preempt_need_resched();
rq->skip_clock_update = 0; rq->skip_clock_update = 0;
@ -2908,7 +2904,8 @@ EXPORT_SYMBOL(sleep_on_timeout);
* This function changes the 'effective' priority of a task. It does * This function changes the 'effective' priority of a task. It does
* not touch ->normal_prio like __setscheduler(). * not touch ->normal_prio like __setscheduler().
* *
* Used by the rt_mutex code to implement priority inheritance logic. * Used by the rt_mutex code to implement priority inheritance
* logic. Call site only calls if the priority of the task changed.
*/ */
void rt_mutex_setprio(struct task_struct *p, int prio) void rt_mutex_setprio(struct task_struct *p, int prio)
{ {
@ -2998,7 +2995,7 @@ void set_user_nice(struct task_struct *p, long nice)
unsigned long flags; unsigned long flags;
struct rq *rq; struct rq *rq;
if (TASK_NICE(p) == nice || nice < -20 || nice > 19) if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
return; return;
/* /*
* We have to be careful, if called from sys_setpriority(), * We have to be careful, if called from sys_setpriority(),
@ -3076,11 +3073,11 @@ SYSCALL_DEFINE1(nice, int, increment)
if (increment > 40) if (increment > 40)
increment = 40; increment = 40;
nice = TASK_NICE(current) + increment; nice = task_nice(current) + increment;
if (nice < -20) if (nice < MIN_NICE)
nice = -20; nice = MIN_NICE;
if (nice > 19) if (nice > MAX_NICE)
nice = 19; nice = MAX_NICE;
if (increment < 0 && !can_nice(current, nice)) if (increment < 0 && !can_nice(current, nice))
return -EPERM; return -EPERM;
@ -3108,18 +3105,6 @@ int task_prio(const struct task_struct *p)
return p->prio - MAX_RT_PRIO; return p->prio - MAX_RT_PRIO;
} }
/**
* task_nice - return the nice value of a given task.
* @p: the task in question.
*
* Return: The nice value [ -20 ... 0 ... 19 ].
*/
int task_nice(const struct task_struct *p)
{
return TASK_NICE(p);
}
EXPORT_SYMBOL(task_nice);
/** /**
* idle_cpu - is a given cpu idle currently? * idle_cpu - is a given cpu idle currently?
* @cpu: the processor in question. * @cpu: the processor in question.
@ -3189,9 +3174,8 @@ __setparam_dl(struct task_struct *p, const struct sched_attr *attr)
dl_se->dl_new = 1; dl_se->dl_new = 1;
} }
/* Actually do priority change: must hold pi & rq lock. */ static void __setscheduler_params(struct task_struct *p,
static void __setscheduler(struct rq *rq, struct task_struct *p, const struct sched_attr *attr)
const struct sched_attr *attr)
{ {
int policy = attr->sched_policy; int policy = attr->sched_policy;
@ -3211,9 +3195,21 @@ static void __setscheduler(struct rq *rq, struct task_struct *p,
* getparam()/getattr() don't report silly values for !rt tasks. * getparam()/getattr() don't report silly values for !rt tasks.
*/ */
p->rt_priority = attr->sched_priority; p->rt_priority = attr->sched_priority;
p->normal_prio = normal_prio(p); p->normal_prio = normal_prio(p);
p->prio = rt_mutex_getprio(p); set_load_weight(p);
}
/* Actually do priority change: must hold pi & rq lock. */
static void __setscheduler(struct rq *rq, struct task_struct *p,
const struct sched_attr *attr)
{
__setscheduler_params(p, attr);
/*
* If we get here, there was no pi waiters boosting the
* task. It is safe to use the normal prio.
*/
p->prio = normal_prio(p);
if (dl_prio(p->prio)) if (dl_prio(p->prio))
p->sched_class = &dl_sched_class; p->sched_class = &dl_sched_class;
@ -3221,8 +3217,6 @@ static void __setscheduler(struct rq *rq, struct task_struct *p,
p->sched_class = &rt_sched_class; p->sched_class = &rt_sched_class;
else else
p->sched_class = &fair_sched_class; p->sched_class = &fair_sched_class;
set_load_weight(p);
} }
static void static void
@ -3275,6 +3269,8 @@ static int __sched_setscheduler(struct task_struct *p,
const struct sched_attr *attr, const struct sched_attr *attr,
bool user) bool user)
{ {
int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
MAX_RT_PRIO - 1 - attr->sched_priority;
int retval, oldprio, oldpolicy = -1, on_rq, running; int retval, oldprio, oldpolicy = -1, on_rq, running;
int policy = attr->sched_policy; int policy = attr->sched_policy;
unsigned long flags; unsigned long flags;
@ -3319,7 +3315,7 @@ recheck:
*/ */
if (user && !capable(CAP_SYS_NICE)) { if (user && !capable(CAP_SYS_NICE)) {
if (fair_policy(policy)) { if (fair_policy(policy)) {
if (attr->sched_nice < TASK_NICE(p) && if (attr->sched_nice < task_nice(p) &&
!can_nice(p, attr->sched_nice)) !can_nice(p, attr->sched_nice))
return -EPERM; return -EPERM;
} }
@ -3352,7 +3348,7 @@ recheck:
* SCHED_NORMAL if the RLIMIT_NICE would normally permit it. * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
*/ */
if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) { if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) {
if (!can_nice(p, TASK_NICE(p))) if (!can_nice(p, task_nice(p)))
return -EPERM; return -EPERM;
} }
@ -3389,16 +3385,18 @@ recheck:
} }
/* /*
* If not changing anything there's no need to proceed further: * If not changing anything there's no need to proceed further,
* but store a possible modification of reset_on_fork.
*/ */
if (unlikely(policy == p->policy)) { if (unlikely(policy == p->policy)) {
if (fair_policy(policy) && attr->sched_nice != TASK_NICE(p)) if (fair_policy(policy) && attr->sched_nice != task_nice(p))
goto change; goto change;
if (rt_policy(policy) && attr->sched_priority != p->rt_priority) if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
goto change; goto change;
if (dl_policy(policy)) if (dl_policy(policy))
goto change; goto change;
p->sched_reset_on_fork = reset_on_fork;
task_rq_unlock(rq, p, &flags); task_rq_unlock(rq, p, &flags);
return 0; return 0;
} }
@ -3452,6 +3450,24 @@ change:
return -EBUSY; return -EBUSY;
} }
p->sched_reset_on_fork = reset_on_fork;
oldprio = p->prio;
/*
* Special case for priority boosted tasks.
*
* If the new priority is lower or equal (user space view)
* than the current (boosted) priority, we just store the new
* normal parameters and do not touch the scheduler class and
* the runqueue. This will be done when the task deboost
* itself.
*/
if (rt_mutex_check_prio(p, newprio)) {
__setscheduler_params(p, attr);
task_rq_unlock(rq, p, &flags);
return 0;
}
on_rq = p->on_rq; on_rq = p->on_rq;
running = task_current(rq, p); running = task_current(rq, p);
if (on_rq) if (on_rq)
@ -3459,16 +3475,18 @@ change:
if (running) if (running)
p->sched_class->put_prev_task(rq, p); p->sched_class->put_prev_task(rq, p);
p->sched_reset_on_fork = reset_on_fork;
oldprio = p->prio;
prev_class = p->sched_class; prev_class = p->sched_class;
__setscheduler(rq, p, attr); __setscheduler(rq, p, attr);
if (running) if (running)
p->sched_class->set_curr_task(rq); p->sched_class->set_curr_task(rq);
if (on_rq) if (on_rq) {
enqueue_task(rq, p, 0); /*
* We enqueue to tail when the priority of a task is
* increased (user space view).
*/
enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0);
}
check_class_changed(rq, p, prev_class, oldprio); check_class_changed(rq, p, prev_class, oldprio);
task_rq_unlock(rq, p, &flags); task_rq_unlock(rq, p, &flags);
@ -3624,7 +3642,7 @@ static int sched_copy_attr(struct sched_attr __user *uattr,
* XXX: do we want to be lenient like existing syscalls; or do we want * XXX: do we want to be lenient like existing syscalls; or do we want
* to be strict and return an error on out-of-bounds values? * to be strict and return an error on out-of-bounds values?
*/ */
attr->sched_nice = clamp(attr->sched_nice, -20, 19); attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
out: out:
return ret; return ret;
@ -3845,7 +3863,7 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
else if (task_has_rt_policy(p)) else if (task_has_rt_policy(p))
attr.sched_priority = p->rt_priority; attr.sched_priority = p->rt_priority;
else else
attr.sched_nice = TASK_NICE(p); attr.sched_nice = task_nice(p);
rcu_read_unlock(); rcu_read_unlock();
@ -4483,6 +4501,7 @@ void init_idle(struct task_struct *idle, int cpu)
rcu_read_unlock(); rcu_read_unlock();
rq->curr = rq->idle = idle; rq->curr = rq->idle = idle;
idle->on_rq = 1;
#if defined(CONFIG_SMP) #if defined(CONFIG_SMP)
idle->on_cpu = 1; idle->on_cpu = 1;
#endif #endif
@ -4721,6 +4740,22 @@ static void calc_load_migrate(struct rq *rq)
atomic_long_add(delta, &calc_load_tasks); atomic_long_add(delta, &calc_load_tasks);
} }
static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
{
}
static const struct sched_class fake_sched_class = {
.put_prev_task = put_prev_task_fake,
};
static struct task_struct fake_task = {
/*
* Avoid pull_{rt,dl}_task()
*/
.prio = MAX_PRIO + 1,
.sched_class = &fake_sched_class,
};
/* /*
* Migrate all tasks from the rq, sleeping tasks will be migrated by * Migrate all tasks from the rq, sleeping tasks will be migrated by
* try_to_wake_up()->select_task_rq(). * try_to_wake_up()->select_task_rq().
@ -4761,7 +4796,7 @@ static void migrate_tasks(unsigned int dead_cpu)
if (rq->nr_running == 1) if (rq->nr_running == 1)
break; break;
next = pick_next_task(rq); next = pick_next_task(rq, &fake_task);
BUG_ON(!next); BUG_ON(!next);
next->sched_class->put_prev_task(rq, next); next->sched_class->put_prev_task(rq, next);
@ -4851,7 +4886,7 @@ set_table_entry(struct ctl_table *entry,
static struct ctl_table * static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd) sd_alloc_ctl_domain_table(struct sched_domain *sd)
{ {
struct ctl_table *table = sd_alloc_ctl_entry(13); struct ctl_table *table = sd_alloc_ctl_entry(14);
if (table == NULL) if (table == NULL)
return NULL; return NULL;
@ -4879,9 +4914,12 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd)
sizeof(int), 0644, proc_dointvec_minmax, false); sizeof(int), 0644, proc_dointvec_minmax, false);
set_table_entry(&table[10], "flags", &sd->flags, set_table_entry(&table[10], "flags", &sd->flags,
sizeof(int), 0644, proc_dointvec_minmax, false); sizeof(int), 0644, proc_dointvec_minmax, false);
set_table_entry(&table[11], "name", sd->name, set_table_entry(&table[11], "max_newidle_lb_cost",
&sd->max_newidle_lb_cost,
sizeof(long), 0644, proc_doulongvec_minmax, false);
set_table_entry(&table[12], "name", sd->name,
CORENAME_MAX_SIZE, 0444, proc_dostring, false); CORENAME_MAX_SIZE, 0444, proc_dostring, false);
/* &table[12] is terminator */ /* &table[13] is terminator */
return table; return table;
} }
@ -6858,7 +6896,6 @@ void __init sched_init(void)
rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
#ifdef CONFIG_RT_GROUP_SCHED #ifdef CONFIG_RT_GROUP_SCHED
INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
#endif #endif
@ -6947,7 +6984,8 @@ void __might_sleep(const char *file, int line, int preempt_offset)
static unsigned long prev_jiffy; /* ratelimiting */ static unsigned long prev_jiffy; /* ratelimiting */
rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */ rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) || if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
!is_idle_task(current)) ||
system_state != SYSTEM_RUNNING || oops_in_progress) system_state != SYSTEM_RUNNING || oops_in_progress)
return; return;
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
@ -6965,6 +7003,13 @@ void __might_sleep(const char *file, int line, int preempt_offset)
debug_show_held_locks(current); debug_show_held_locks(current);
if (irqs_disabled()) if (irqs_disabled())
print_irqtrace_events(current); print_irqtrace_events(current);
#ifdef CONFIG_DEBUG_PREEMPT
if (!preempt_count_equals(preempt_offset)) {
pr_err("Preemption disabled at:");
print_ip_sym(current->preempt_disable_ip);
pr_cont("\n");
}
#endif
dump_stack(); dump_stack();
} }
EXPORT_SYMBOL(__might_sleep); EXPORT_SYMBOL(__might_sleep);
@ -7018,7 +7063,7 @@ void normalize_rt_tasks(void)
* Renice negative nice level userspace * Renice negative nice level userspace
* tasks back to 0: * tasks back to 0:
*/ */
if (TASK_NICE(p) < 0 && p->mm) if (task_nice(p) < 0 && p->mm)
set_user_nice(p, 0); set_user_nice(p, 0);
continue; continue;
} }

4
kernel/sched/cputime.c
View File

@ -142,7 +142,7 @@ void account_user_time(struct task_struct *p, cputime_t cputime,
p->utimescaled += cputime_scaled; p->utimescaled += cputime_scaled;
account_group_user_time(p, cputime); account_group_user_time(p, cputime);
index = (TASK_NICE(p) > 0) ? CPUTIME_NICE : CPUTIME_USER; index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER;
/* Add user time to cpustat. */ /* Add user time to cpustat. */
task_group_account_field(p, index, (__force u64) cputime); task_group_account_field(p, index, (__force u64) cputime);
@ -169,7 +169,7 @@ static void account_guest_time(struct task_struct *p, cputime_t cputime,
p->gtime += cputime; p->gtime += cputime;
/* Add guest time to cpustat. */ /* Add guest time to cpustat. */
if (TASK_NICE(p) > 0) { if (task_nice(p) > 0) {
cpustat[CPUTIME_NICE] += (__force u64) cputime; cpustat[CPUTIME_NICE] += (__force u64) cputime;
cpustat[CPUTIME_GUEST_NICE] += (__force u64) cputime; cpustat[CPUTIME_GUEST_NICE] += (__force u64) cputime;
} else { } else {

56
kernel/sched/deadline.c
View File

@ -210,6 +210,16 @@ static inline int has_pushable_dl_tasks(struct rq *rq)
static int push_dl_task(struct rq *rq); static int push_dl_task(struct rq *rq);
static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev)
{
return dl_task(prev);
}
static inline void set_post_schedule(struct rq *rq)
{
rq->post_schedule = has_pushable_dl_tasks(rq);
}
#else #else
static inline static inline
@ -232,6 +242,19 @@ void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
{ {
} }
static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev)
{
return false;
}
static inline int pull_dl_task(struct rq *rq)
{
return 0;
}
static inline void set_post_schedule(struct rq *rq)
{
}
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags); static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags);
@ -586,8 +609,8 @@ static void update_curr_dl(struct rq *rq)
* approach need further study. * approach need further study.
*/ */
delta_exec = rq_clock_task(rq) - curr->se.exec_start; delta_exec = rq_clock_task(rq) - curr->se.exec_start;
if (unlikely((s64)delta_exec < 0)) if (unlikely((s64)delta_exec <= 0))
delta_exec = 0; return;
schedstat_set(curr->se.statistics.exec_max, schedstat_set(curr->se.statistics.exec_max,
max(curr->se.statistics.exec_max, delta_exec)); max(curr->se.statistics.exec_max, delta_exec));
@ -942,6 +965,8 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)
resched_task(rq->curr); resched_task(rq->curr);
} }
static int pull_dl_task(struct rq *this_rq);
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
/* /*
@ -988,7 +1013,7 @@ static struct sched_dl_entity *pick_next_dl_entity(struct rq *rq,
return rb_entry(left, struct sched_dl_entity, rb_node); return rb_entry(left, struct sched_dl_entity, rb_node);
} }
struct task_struct *pick_next_task_dl(struct rq *rq) struct task_struct *pick_next_task_dl(struct rq *rq, struct task_struct *prev)
{ {
struct sched_dl_entity *dl_se; struct sched_dl_entity *dl_se;
struct task_struct *p; struct task_struct *p;
@ -996,9 +1021,20 @@ struct task_struct *pick_next_task_dl(struct rq *rq)
dl_rq = &rq->dl; dl_rq = &rq->dl;
if (need_pull_dl_task(rq, prev))
pull_dl_task(rq);
/*
* When prev is DL, we may throttle it in put_prev_task().
* So, we update time before we check for dl_nr_running.
*/
if (prev->sched_class == &dl_sched_class)
update_curr_dl(rq);
if (unlikely(!dl_rq->dl_nr_running)) if (unlikely(!dl_rq->dl_nr_running))
return NULL; return NULL;
put_prev_task(rq, prev);
dl_se = pick_next_dl_entity(rq, dl_rq); dl_se = pick_next_dl_entity(rq, dl_rq);
BUG_ON(!dl_se); BUG_ON(!dl_se);
@ -1013,9 +1049,7 @@ struct task_struct *pick_next_task_dl(struct rq *rq)
start_hrtick_dl(rq, p); start_hrtick_dl(rq, p);
#endif #endif
#ifdef CONFIG_SMP set_post_schedule(rq);
rq->post_schedule = has_pushable_dl_tasks(rq);
#endif /* CONFIG_SMP */
return p; return p;
} }
@ -1424,13 +1458,6 @@ skip:
return ret; return ret;
} }
static void pre_schedule_dl(struct rq *rq, struct task_struct *prev)
{
/* Try to pull other tasks here */
if (dl_task(prev))
pull_dl_task(rq);
}
static void post_schedule_dl(struct rq *rq) static void post_schedule_dl(struct rq *rq)
{ {
push_dl_tasks(rq); push_dl_tasks(rq);
@ -1558,7 +1585,7 @@ static void switched_to_dl(struct rq *rq, struct task_struct *p)
if (unlikely(p->dl.dl_throttled)) if (unlikely(p->dl.dl_throttled))
return; return;
if (p->on_rq || rq->curr != p) { if (p->on_rq && rq->curr != p) {
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
if (rq->dl.overloaded && push_dl_task(rq) && rq != task_rq(p)) if (rq->dl.overloaded && push_dl_task(rq) && rq != task_rq(p))
/* Only reschedule if pushing failed */ /* Only reschedule if pushing failed */
@ -1623,7 +1650,6 @@ const struct sched_class dl_sched_class = {
.set_cpus_allowed = set_cpus_allowed_dl, .set_cpus_allowed = set_cpus_allowed_dl,
.rq_online = rq_online_dl, .rq_online = rq_online_dl,
.rq_offline = rq_offline_dl, .rq_offline = rq_offline_dl,
.pre_schedule = pre_schedule_dl,
.post_schedule = post_schedule_dl, .post_schedule = post_schedule_dl,
.task_woken = task_woken_dl, .task_woken = task_woken_dl,
#endif #endif

7
kernel/sched/debug.c
View File

@ -321,6 +321,7 @@ do { \
P(sched_goidle); P(sched_goidle);
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
P64(avg_idle); P64(avg_idle);
P64(max_idle_balance_cost);
#endif #endif
P(ttwu_count); P(ttwu_count);
@ -533,15 +534,15 @@ static void sched_show_numa(struct task_struct *p, struct seq_file *m)
unsigned long nr_faults = -1; unsigned long nr_faults = -1;
int cpu_current, home_node; int cpu_current, home_node;
if (p->numa_faults) if (p->numa_faults_memory)
nr_faults = p->numa_faults[2*node + i]; nr_faults = p->numa_faults_memory[2*node + i];
cpu_current = !i ? (task_node(p) == node) : cpu_current = !i ? (task_node(p) == node) :
(pol && node_isset(node, pol->v.nodes)); (pol && node_isset(node, pol->v.nodes));
home_node = (p->numa_preferred_nid == node); home_node = (p->numa_preferred_nid == node);
SEQ_printf(m, "numa_faults, %d, %d, %d, %d, %ld\n", SEQ_printf(m, "numa_faults_memory, %d, %d, %d, %d, %ld\n",
i, node, cpu_current, home_node, nr_faults); i, node, cpu_current, home_node, nr_faults);
} }
} }

600
kernel/sched/fair.c
View File

File diff suppressed because it is too large Load Diff

7
kernel/cpu/idle.c → kernel/sched/idle.c
View File

@ -3,6 +3,7 @@
*/ */
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/cpu.h> #include <linux/cpu.h>
#include <linux/cpuidle.h>
#include <linux/tick.h> #include <linux/tick.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/stackprotector.h> #include <linux/stackprotector.h>
@ -95,8 +96,10 @@ static void cpu_idle_loop(void)
if (!current_clr_polling_and_test()) { if (!current_clr_polling_and_test()) {
stop_critical_timings(); stop_critical_timings();
rcu_idle_enter(); rcu_idle_enter();
arch_cpu_idle(); if (cpuidle_idle_call())
WARN_ON_ONCE(irqs_disabled()); arch_cpu_idle();
if (WARN_ON_ONCE(irqs_disabled()))
local_irq_enable();
rcu_idle_exit(); rcu_idle_exit();
start_critical_timings(); start_critical_timings();
} else { } else {

25
kernel/sched/idle_task.c
View File

@ -13,18 +13,8 @@ select_task_rq_idle(struct task_struct *p, int cpu, int sd_flag, int flags)
{ {
return task_cpu(p); /* IDLE tasks as never migrated */ return task_cpu(p); /* IDLE tasks as never migrated */
} }
static void pre_schedule_idle(struct rq *rq, struct task_struct *prev)
{
idle_exit_fair(rq);
rq_last_tick_reset(rq);
}
static void post_schedule_idle(struct rq *rq)
{
idle_enter_fair(rq);
}
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
/* /*
* Idle tasks are unconditionally rescheduled: * Idle tasks are unconditionally rescheduled:
*/ */
@ -33,13 +23,12 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl
resched_task(rq->idle); resched_task(rq->idle);
} }
static struct task_struct *pick_next_task_idle(struct rq *rq) static struct task_struct *
pick_next_task_idle(struct rq *rq, struct task_struct *prev)
{ {
put_prev_task(rq, prev);
schedstat_inc(rq, sched_goidle); schedstat_inc(rq, sched_goidle);
#ifdef CONFIG_SMP
/* Trigger the post schedule to do an idle_enter for CFS */
rq->post_schedule = 1;
#endif
return rq->idle; return rq->idle;
} }
@ -58,6 +47,8 @@ dequeue_task_idle(struct rq *rq, struct task_struct *p, int flags)
static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) static void put_prev_task_idle(struct rq *rq, struct task_struct *prev)
{ {
idle_exit_fair(rq);
rq_last_tick_reset(rq);
} }
static void task_tick_idle(struct rq *rq, struct task_struct *curr, int queued) static void task_tick_idle(struct rq *rq, struct task_struct *curr, int queued)
@ -101,8 +92,6 @@ const struct sched_class idle_sched_class = {
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
.select_task_rq = select_task_rq_idle, .select_task_rq = select_task_rq_idle,
.pre_schedule = pre_schedule_idle,
.post_schedule = post_schedule_idle,
#endif #endif
.set_curr_task = set_curr_task_idle, .set_curr_task = set_curr_task_idle,

102
kernel/sched/rt.c
View File

@ -229,6 +229,14 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static int pull_rt_task(struct rq *this_rq);
static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
{
/* Try to pull RT tasks here if we lower this rq's prio */
return rq->rt.highest_prio.curr > prev->prio;
}
static inline int rt_overloaded(struct rq *rq) static inline int rt_overloaded(struct rq *rq)
{ {
return atomic_read(&rq->rd->rto_count); return atomic_read(&rq->rd->rto_count);
@ -315,6 +323,15 @@ static inline int has_pushable_tasks(struct rq *rq)
return !plist_head_empty(&rq->rt.pushable_tasks); return !plist_head_empty(&rq->rt.pushable_tasks);
} }
static inline void set_post_schedule(struct rq *rq)
{
/*
* We detect this state here so that we can avoid taking the RQ
* lock again later if there is no need to push
*/
rq->post_schedule = has_pushable_tasks(rq);
}
static void enqueue_pushable_task(struct rq *rq, struct task_struct *p) static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
{ {
plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
@ -359,6 +376,19 @@ void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{ {
} }
static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
{
return false;
}
static inline int pull_rt_task(struct rq *this_rq)
{
return 0;
}
static inline void set_post_schedule(struct rq *rq)
{
}
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
static inline int on_rt_rq(struct sched_rt_entity *rt_se) static inline int on_rt_rq(struct sched_rt_entity *rt_se)
@ -440,11 +470,6 @@ static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
dequeue_rt_entity(rt_se); dequeue_rt_entity(rt_se);
} }
static inline int rt_rq_throttled(struct rt_rq *rt_rq)
{
return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
}
static int rt_se_boosted(struct sched_rt_entity *rt_se) static int rt_se_boosted(struct sched_rt_entity *rt_se)
{ {
struct rt_rq *rt_rq = group_rt_rq(rt_se); struct rt_rq *rt_rq = group_rt_rq(rt_se);
@ -515,11 +540,6 @@ static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
{ {
} }
static inline int rt_rq_throttled(struct rt_rq *rt_rq)
{
return rt_rq->rt_throttled;
}
static inline const struct cpumask *sched_rt_period_mask(void) static inline const struct cpumask *sched_rt_period_mask(void)
{ {
return cpu_online_mask; return cpu_online_mask;
@ -1318,15 +1338,7 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)
{ {
struct sched_rt_entity *rt_se; struct sched_rt_entity *rt_se;
struct task_struct *p; struct task_struct *p;
struct rt_rq *rt_rq; struct rt_rq *rt_rq = &rq->rt;
rt_rq = &rq->rt;
if (!rt_rq->rt_nr_running)
return NULL;
if (rt_rq_throttled(rt_rq))
return NULL;
do { do {
rt_se = pick_next_rt_entity(rq, rt_rq); rt_se = pick_next_rt_entity(rq, rt_rq);
@ -1340,21 +1352,45 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)
return p; return p;
} }
static struct task_struct *pick_next_task_rt(struct rq *rq) static struct task_struct *
pick_next_task_rt(struct rq *rq, struct task_struct *prev)
{ {
struct task_struct *p = _pick_next_task_rt(rq); struct task_struct *p;
struct rt_rq *rt_rq = &rq->rt;
if (need_pull_rt_task(rq, prev)) {
pull_rt_task(rq);
/*
* pull_rt_task() can drop (and re-acquire) rq->lock; this
* means a dl task can slip in, in which case we need to
* re-start task selection.
*/
if (unlikely(rq->dl.dl_nr_running))
return RETRY_TASK;
}
/*
* We may dequeue prev's rt_rq in put_prev_task().
* So, we update time before rt_nr_running check.
*/
if (prev->sched_class == &rt_sched_class)
update_curr_rt(rq);
if (!rt_rq->rt_nr_running)
return NULL;
if (rt_rq_throttled(rt_rq))
return NULL;
put_prev_task(rq, prev);
p = _pick_next_task_rt(rq);
/* The running task is never eligible for pushing */ /* The running task is never eligible for pushing */
if (p) if (p)
dequeue_pushable_task(rq, p); dequeue_pushable_task(rq, p);
#ifdef CONFIG_SMP set_post_schedule(rq);
/*
* We detect this state here so that we can avoid taking the RQ
* lock again later if there is no need to push
*/
rq->post_schedule = has_pushable_tasks(rq);
#endif
return p; return p;
} }
@ -1724,13 +1760,6 @@ skip:
return ret; return ret;
} }
static void pre_schedule_rt(struct rq *rq, struct task_struct *prev)
{
/* Try to pull RT tasks here if we lower this rq's prio */
if (rq->rt.highest_prio.curr > prev->prio)
pull_rt_task(rq);
}
static void post_schedule_rt(struct rq *rq) static void post_schedule_rt(struct rq *rq)
{ {
push_rt_tasks(rq); push_rt_tasks(rq);
@ -1833,7 +1862,7 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p)
resched_task(rq->curr); resched_task(rq->curr);
} }
void init_sched_rt_class(void) void __init init_sched_rt_class(void)
{ {
unsigned int i; unsigned int i;
@ -2007,7 +2036,6 @@ const struct sched_class rt_sched_class = {
.set_cpus_allowed = set_cpus_allowed_rt, .set_cpus_allowed = set_cpus_allowed_rt,
.rq_online = rq_online_rt, .rq_online = rq_online_rt,
.rq_offline = rq_offline_rt, .rq_offline = rq_offline_rt,
.pre_schedule = pre_schedule_rt,
.post_schedule = post_schedule_rt, .post_schedule = post_schedule_rt,
.task_woken = task_woken_rt, .task_woken = task_woken_rt,
.switched_from = switched_from_rt, .switched_from = switched_from_rt,

65
kernel/sched/sched.h
View File

@ -23,24 +23,6 @@ extern atomic_long_t calc_load_tasks;
extern long calc_load_fold_active(struct rq *this_rq); extern long calc_load_fold_active(struct rq *this_rq);
extern void update_cpu_load_active(struct rq *this_rq); extern void update_cpu_load_active(struct rq *this_rq);
/*
* Convert user-nice values [ -20 ... 0 ... 19 ]
* to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
* and back.
*/
#define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20)
#define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20)
#define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio)
/*
* 'User priority' is the nice value converted to something we
* can work with better when scaling various scheduler parameters,
* it's a [ 0 ... 39 ] range.
*/
#define USER_PRIO(p) ((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
/* /*
* Helpers for converting nanosecond timing to jiffy resolution * Helpers for converting nanosecond timing to jiffy resolution
*/ */
@ -441,6 +423,18 @@ struct rt_rq {
#endif #endif
}; };
#ifdef CONFIG_RT_GROUP_SCHED
static inline int rt_rq_throttled(struct rt_rq *rt_rq)
{
return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
}
#else
static inline int rt_rq_throttled(struct rt_rq *rt_rq)
{
return rt_rq->rt_throttled;
}
#endif
/* Deadline class' related fields in a runqueue */ /* Deadline class' related fields in a runqueue */
struct dl_rq { struct dl_rq {
/* runqueue is an rbtree, ordered by deadline */ /* runqueue is an rbtree, ordered by deadline */
@ -558,11 +552,9 @@ struct rq {
#ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_FAIR_GROUP_SCHED
/* list of leaf cfs_rq on this cpu: */ /* list of leaf cfs_rq on this cpu: */
struct list_head leaf_cfs_rq_list; struct list_head leaf_cfs_rq_list;
#endif /* CONFIG_FAIR_GROUP_SCHED */
#ifdef CONFIG_RT_GROUP_SCHED struct sched_avg avg;
struct list_head leaf_rt_rq_list; #endif /* CONFIG_FAIR_GROUP_SCHED */
#endif
/* /*
* This is part of a global counter where only the total sum * This is part of a global counter where only the total sum
@ -651,8 +643,6 @@ struct rq {
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
struct llist_head wake_list; struct llist_head wake_list;
#endif #endif
struct sched_avg avg;
}; };
static inline int cpu_of(struct rq *rq) static inline int cpu_of(struct rq *rq)
@ -1112,6 +1102,8 @@ static const u32 prio_to_wmult[40] = {
#define DEQUEUE_SLEEP 1 #define DEQUEUE_SLEEP 1
#define RETRY_TASK ((void *)-1UL)
struct sched_class { struct sched_class {
const struct sched_class *next; const struct sched_class *next;
@ -1122,14 +1114,22 @@ struct sched_class {
void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags); void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags);
struct task_struct * (*pick_next_task) (struct rq *rq); /*
* It is the responsibility of the pick_next_task() method that will
* return the next task to call put_prev_task() on the @prev task or
* something equivalent.
*
* May return RETRY_TASK when it finds a higher prio class has runnable
* tasks.
*/
struct task_struct * (*pick_next_task) (struct rq *rq,
struct task_struct *prev);
void (*put_prev_task) (struct rq *rq, struct task_struct *p); void (*put_prev_task) (struct rq *rq, struct task_struct *p);
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
int (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags); int (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags);
void (*migrate_task_rq)(struct task_struct *p, int next_cpu); void (*migrate_task_rq)(struct task_struct *p, int next_cpu);
void (*pre_schedule) (struct rq *this_rq, struct task_struct *task);
void (*post_schedule) (struct rq *this_rq); void (*post_schedule) (struct rq *this_rq);
void (*task_waking) (struct task_struct *task); void (*task_waking) (struct task_struct *task);
void (*task_woken) (struct rq *this_rq, struct task_struct *task); void (*task_woken) (struct rq *this_rq, struct task_struct *task);
@ -1159,6 +1159,11 @@ struct sched_class {
#endif #endif
}; };
static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
{
prev->sched_class->put_prev_task(rq, prev);
}
#define sched_class_highest (&stop_sched_class) #define sched_class_highest (&stop_sched_class)
#define for_each_class(class) \ #define for_each_class(class) \
for (class = sched_class_highest; class; class = class->next) for (class = sched_class_highest; class; class = class->next)
@ -1175,16 +1180,14 @@ extern const struct sched_class idle_sched_class;
extern void update_group_power(struct sched_domain *sd, int cpu); extern void update_group_power(struct sched_domain *sd, int cpu);
extern void trigger_load_balance(struct rq *rq); extern void trigger_load_balance(struct rq *rq);
extern void idle_balance(int this_cpu, struct rq *this_rq);
extern void idle_enter_fair(struct rq *this_rq); extern void idle_enter_fair(struct rq *this_rq);
extern void idle_exit_fair(struct rq *this_rq); extern void idle_exit_fair(struct rq *this_rq);
#else /* CONFIG_SMP */ #else
static inline void idle_balance(int cpu, struct rq *rq) static inline void idle_enter_fair(struct rq *rq) { }
{ static inline void idle_exit_fair(struct rq *rq) { }
}
#endif #endif

15
kernel/sched/stop_task.c
View File

@ -23,16 +23,19 @@ check_preempt_curr_stop(struct rq *rq, struct task_struct *p, int flags)
/* we're never preempted */ /* we're never preempted */
} }
static struct task_struct *pick_next_task_stop(struct rq *rq) static struct task_struct *
pick_next_task_stop(struct rq *rq, struct task_struct *prev)
{ {
struct task_struct *stop = rq->stop; struct task_struct *stop = rq->stop;
if (stop && stop->on_rq) { if (!stop || !stop->on_rq)
stop->se.exec_start = rq_clock_task(rq); return NULL;
return stop;
}
return NULL; put_prev_task(rq, prev);
stop->se.exec_start = rq_clock_task(rq);
return stop;
} }
static void static void

8
kernel/sys.c
View File

@ -174,10 +174,10 @@ SYSCALL_DEFINE3(setpriority, int, which, int, who, int, niceval)
/* normalize: avoid signed division (rounding problems) */ /* normalize: avoid signed division (rounding problems) */
error = -ESRCH; error = -ESRCH;
if (niceval < -20) if (niceval < MIN_NICE)
niceval = -20; niceval = MIN_NICE;
if (niceval > 19) if (niceval > MAX_NICE)
niceval = 19; niceval = MAX_NICE;
rcu_read_lock(); rcu_read_lock();
read_lock(&tasklist_lock); read_lock(&tasklist_lock);

7
kernel/sysctl.c
View File

@ -385,13 +385,6 @@ static struct ctl_table kern_table[] = {
.mode = 0644, .mode = 0644,
.proc_handler = proc_dointvec, .proc_handler = proc_dointvec,
}, },
{
.procname = "numa_balancing_migrate_deferred",
.data = &sysctl_numa_balancing_migrate_deferred,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec,
},
{ {
.procname = "numa_balancing", .procname = "numa_balancing",
.data = NULL, /* filled in by handler */ .data = NULL, /* filled in by handler */

6
kernel/trace/ring_buffer_benchmark.c
View File

@ -40,8 +40,8 @@ static int write_iteration = 50;
module_param(write_iteration, uint, 0644); module_param(write_iteration, uint, 0644);
MODULE_PARM_DESC(write_iteration, "# of writes between timestamp readings"); MODULE_PARM_DESC(write_iteration, "# of writes between timestamp readings");
static int producer_nice = 19; static int producer_nice = MAX_NICE;
static int consumer_nice = 19; static int consumer_nice = MAX_NICE;
static int producer_fifo = -1; static int producer_fifo = -1;
static int consumer_fifo = -1; static int consumer_fifo = -1;
@ -308,7 +308,7 @@ static void ring_buffer_producer(void)
/* Let the user know that the test is running at low priority */ /* Let the user know that the test is running at low priority */
if (producer_fifo < 0 && consumer_fifo < 0 && if (producer_fifo < 0 && consumer_fifo < 0 &&
producer_nice == 19 && consumer_nice == 19) producer_nice == MAX_NICE && consumer_nice == MAX_NICE)
trace_printk("WARNING!!! This test is running at lowest priority.\n"); trace_printk("WARNING!!! This test is running at lowest priority.\n");
trace_printk("Time: %lld (usecs)\n", time); trace_printk("Time: %lld (usecs)\n", time);

2
kernel/workqueue.c
View File

@ -3225,7 +3225,7 @@ static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
return -ENOMEM; return -ENOMEM;
if (sscanf(buf, "%d", &attrs->nice) == 1 && if (sscanf(buf, "%d", &attrs->nice) == 1 &&
attrs->nice >= -20 && attrs->nice <= 19) attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
ret = apply_workqueue_attrs(wq, attrs); ret = apply_workqueue_attrs(wq, attrs);
else else
ret = -EINVAL; ret = -EINVAL;

74
mm/mempolicy.c
View File

@ -2301,35 +2301,6 @@ static void sp_free(struct sp_node *n)
kmem_cache_free(sn_cache, n); kmem_cache_free(sn_cache, n);
} }
#ifdef CONFIG_NUMA_BALANCING
static bool numa_migrate_deferred(struct task_struct *p, int last_cpupid)
{
/* Never defer a private fault */
if (cpupid_match_pid(p, last_cpupid))
return false;
if (p->numa_migrate_deferred) {
p->numa_migrate_deferred--;
return true;
}
return false;
}
static inline void defer_numa_migrate(struct task_struct *p)
{
p->numa_migrate_deferred = sysctl_numa_balancing_migrate_deferred;
}
#else
static inline bool numa_migrate_deferred(struct task_struct *p, int last_cpupid)
{
return false;
}
static inline void defer_numa_migrate(struct task_struct *p)
{
}
#endif /* CONFIG_NUMA_BALANCING */
/** /**
* mpol_misplaced - check whether current page node is valid in policy * mpol_misplaced - check whether current page node is valid in policy
* *
@ -2403,52 +2374,9 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
/* Migrate the page towards the node whose CPU is referencing it */ /* Migrate the page towards the node whose CPU is referencing it */
if (pol->flags & MPOL_F_MORON) { if (pol->flags & MPOL_F_MORON) {
int last_cpupid;
int this_cpupid;
polnid = thisnid; polnid = thisnid;
this_cpupid = cpu_pid_to_cpupid(thiscpu, current->pid);
/* if (!should_numa_migrate_memory(current, page, curnid, thiscpu))
* Multi-stage node selection is used in conjunction
* with a periodic migration fault to build a temporal
* task<->page relation. By using a two-stage filter we
* remove short/unlikely relations.
*
* Using P(p) ~ n_p / n_t as per frequentist
* probability, we can equate a task's usage of a
* particular page (n_p) per total usage of this
* page (n_t) (in a given time-span) to a probability.
*
* Our periodic faults will sample this probability and
* getting the same result twice in a row, given these
* samples are fully independent, is then given by
* P(n)^2, provided our sample period is sufficiently
* short compared to the usage pattern.
*
* This quadric squishes small probabilities, making
* it less likely we act on an unlikely task<->page
* relation.
*/
last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
if (!cpupid_pid_unset(last_cpupid) && cpupid_to_nid(last_cpupid) != thisnid) {
/* See sysctl_numa_balancing_migrate_deferred comment */
if (!cpupid_match_pid(current, last_cpupid))
defer_numa_migrate(current);
goto out;
}
/*
* The quadratic filter above reduces extraneous migration
* of shared pages somewhat. This code reduces it even more,
* reducing the overhead of page migrations of shared pages.
* This makes workloads with shared pages rely more on
* "move task near its memory", and less on "move memory
* towards its task", which is exactly what we want.
*/
if (numa_migrate_deferred(current, last_cpupid))
goto out; goto out;
} }