sched/clock, x86: Use a static_key for sched_clock_stable
In order to avoid the runtime condition and variable load turn sched_clock_stable into a static_key. Also provide a shorter implementation of local_clock() and cpu_clock(int) when sched_clock_stable==1. MAINLINE PRE POST sched_clock_stable: 1 1 1 (cold) sched_clock: 329841 221876 215295 (cold) local_clock: 301773 234692 220773 (warm) sched_clock: 38375 25602 25659 (warm) local_clock: 100371 33265 27242 (warm) rdtsc: 27340 24214 24208 sched_clock_stable: 0 0 0 (cold) sched_clock: 382634 235941 237019 (cold) local_clock: 396890 297017 294819 (warm) sched_clock: 38194 25233 25609 (warm) local_clock: 143452 71234 71232 (warm) rdtsc: 27345 24245 24243 Signed-off-by: Peter Zijlstra <peterz@infradead.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Andrew Morton <akpm@linux-foundation.org> Link: http://lkml.kernel.org/n/tip-eummbdechzz37mwmpags1gjr@git.kernel.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
ef08f0fff8
commit
35af99e646
|
@ -487,7 +487,7 @@ static void early_init_amd(struct cpuinfo_x86 *c)
|
|||
set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
|
||||
set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
|
||||
if (!check_tsc_unstable())
|
||||
sched_clock_stable = 1;
|
||||
set_sched_clock_stable();
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
|
|
|
@ -93,7 +93,7 @@ static void early_init_intel(struct cpuinfo_x86 *c)
|
|||
set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
|
||||
set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
|
||||
if (!check_tsc_unstable())
|
||||
sched_clock_stable = 1;
|
||||
set_sched_clock_stable();
|
||||
}
|
||||
|
||||
/* Penwell and Cloverview have the TSC which doesn't sleep on S3 */
|
||||
|
|
|
@ -1890,7 +1890,7 @@ void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
|
|||
userpg->cap_user_rdpmc = x86_pmu.attr_rdpmc;
|
||||
userpg->pmc_width = x86_pmu.cntval_bits;
|
||||
|
||||
if (!sched_clock_stable)
|
||||
if (!sched_clock_stable())
|
||||
return;
|
||||
|
||||
data = cyc2ns_read_begin();
|
||||
|
|
|
@ -822,7 +822,7 @@ static unsigned long long cyc2ns_suspend;
|
|||
|
||||
void tsc_save_sched_clock_state(void)
|
||||
{
|
||||
if (!sched_clock_stable)
|
||||
if (!sched_clock_stable())
|
||||
return;
|
||||
|
||||
cyc2ns_suspend = sched_clock();
|
||||
|
@ -842,7 +842,7 @@ void tsc_restore_sched_clock_state(void)
|
|||
unsigned long flags;
|
||||
int cpu;
|
||||
|
||||
if (!sched_clock_stable)
|
||||
if (!sched_clock_stable())
|
||||
return;
|
||||
|
||||
local_irq_save(flags);
|
||||
|
@ -984,7 +984,7 @@ void mark_tsc_unstable(char *reason)
|
|||
{
|
||||
if (!tsc_unstable) {
|
||||
tsc_unstable = 1;
|
||||
sched_clock_stable = 0;
|
||||
clear_sched_clock_stable();
|
||||
disable_sched_clock_irqtime();
|
||||
pr_info("Marking TSC unstable due to %s\n", reason);
|
||||
/* Change only the rating, when not registered */
|
||||
|
|
|
@ -1994,7 +1994,9 @@ static inline void sched_clock_idle_wakeup_event(u64 delta_ns)
|
|||
* but then during bootup it turns out that sched_clock()
|
||||
* is reliable after all:
|
||||
*/
|
||||
extern int sched_clock_stable;
|
||||
extern int sched_clock_stable(void);
|
||||
extern void set_sched_clock_stable(void);
|
||||
extern void clear_sched_clock_stable(void);
|
||||
|
||||
extern void sched_clock_tick(void);
|
||||
extern void sched_clock_idle_sleep_event(void);
|
||||
|
|
|
@ -58,6 +58,7 @@
|
|||
#include <linux/percpu.h>
|
||||
#include <linux/ktime.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/static_key.h>
|
||||
|
||||
/*
|
||||
* Scheduler clock - returns current time in nanosec units.
|
||||
|
@ -74,7 +75,27 @@ EXPORT_SYMBOL_GPL(sched_clock);
|
|||
__read_mostly int sched_clock_running;
|
||||
|
||||
#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
|
||||
__read_mostly int sched_clock_stable;
|
||||
static struct static_key __sched_clock_stable = STATIC_KEY_INIT;
|
||||
|
||||
int sched_clock_stable(void)
|
||||
{
|
||||
if (static_key_false(&__sched_clock_stable))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
void set_sched_clock_stable(void)
|
||||
{
|
||||
if (!sched_clock_stable())
|
||||
static_key_slow_dec(&__sched_clock_stable);
|
||||
}
|
||||
|
||||
void clear_sched_clock_stable(void)
|
||||
{
|
||||
/* XXX worry about clock continuity */
|
||||
if (sched_clock_stable())
|
||||
static_key_slow_inc(&__sched_clock_stable);
|
||||
}
|
||||
|
||||
struct sched_clock_data {
|
||||
u64 tick_raw;
|
||||
|
@ -234,7 +255,7 @@ u64 sched_clock_cpu(int cpu)
|
|||
struct sched_clock_data *scd;
|
||||
u64 clock;
|
||||
|
||||
if (sched_clock_stable)
|
||||
if (sched_clock_stable())
|
||||
return sched_clock();
|
||||
|
||||
if (unlikely(!sched_clock_running))
|
||||
|
@ -257,7 +278,7 @@ void sched_clock_tick(void)
|
|||
struct sched_clock_data *scd;
|
||||
u64 now, now_gtod;
|
||||
|
||||
if (sched_clock_stable)
|
||||
if (sched_clock_stable())
|
||||
return;
|
||||
|
||||
if (unlikely(!sched_clock_running))
|
||||
|
@ -308,7 +329,10 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
|
|||
*/
|
||||
u64 cpu_clock(int cpu)
|
||||
{
|
||||
return sched_clock_cpu(cpu);
|
||||
if (static_key_false(&__sched_clock_stable))
|
||||
return sched_clock_cpu(cpu);
|
||||
|
||||
return sched_clock();
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -320,7 +344,10 @@ u64 cpu_clock(int cpu)
|
|||
*/
|
||||
u64 local_clock(void)
|
||||
{
|
||||
return sched_clock_cpu(raw_smp_processor_id());
|
||||
if (static_key_false(&__sched_clock_stable))
|
||||
return sched_clock_cpu(raw_smp_processor_id());
|
||||
|
||||
return sched_clock();
|
||||
}
|
||||
|
||||
#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
|
||||
|
@ -340,12 +367,12 @@ u64 sched_clock_cpu(int cpu)
|
|||
|
||||
u64 cpu_clock(int cpu)
|
||||
{
|
||||
return sched_clock_cpu(cpu);
|
||||
return sched_clock();
|
||||
}
|
||||
|
||||
u64 local_clock(void)
|
||||
{
|
||||
return sched_clock_cpu(0);
|
||||
return sched_clock();
|
||||
}
|
||||
|
||||
#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
|
||||
|
|
|
@ -371,7 +371,7 @@ static void sched_debug_header(struct seq_file *m)
|
|||
PN(cpu_clk);
|
||||
P(jiffies);
|
||||
#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
|
||||
P(sched_clock_stable);
|
||||
P(sched_clock_stable());
|
||||
#endif
|
||||
#undef PN
|
||||
#undef P
|
||||
|
|
|
@ -177,7 +177,7 @@ static bool can_stop_full_tick(void)
|
|||
* TODO: kick full dynticks CPUs when
|
||||
* sched_clock_stable is set.
|
||||
*/
|
||||
if (!sched_clock_stable) {
|
||||
if (!sched_clock_stable()) {
|
||||
trace_tick_stop(0, "unstable sched clock\n");
|
||||
/*
|
||||
* Don't allow the user to think they can get
|
||||
|
|
|
@ -2558,7 +2558,7 @@ rb_reserve_next_event(struct ring_buffer *buffer,
|
|||
if (unlikely(test_time_stamp(delta))) {
|
||||
int local_clock_stable = 1;
|
||||
#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
|
||||
local_clock_stable = sched_clock_stable;
|
||||
local_clock_stable = sched_clock_stable();
|
||||
#endif
|
||||
WARN_ONCE(delta > (1ULL << 59),
|
||||
KERN_WARNING "Delta way too big! %llu ts=%llu write stamp = %llu\n%s",
|
||||
|
|
Loading…
Reference in New Issue