2018-11-01 02:21:09 +08:00
// SPDX-License-Identifier: GPL-2.0+
2006-06-26 15:25:05 +08:00
/*
* This file contains the functions which manage clocksource drivers .
*
* Copyright ( C ) 2004 , 2005 IBM , John Stultz ( johnstul @ us . ibm . com )
*/
2015-05-26 02:49:55 +08:00
# define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
2011-12-15 07:28:51 +08:00
# include <linux/device.h>
2006-06-26 15:25:05 +08:00
# include <linux/clocksource.h>
# include <linux/init.h>
# include <linux/module.h>
[PATCH] kernel/time/clocksource.c needs struct task_struct on m68k
kernel/time/clocksource.c needs struct task_struct on m68k.
Because it uses spin_unlock_irq(), which, on m68k, uses hardirq_count(), which
uses preempt_count(), which needs to dereference struct task_struct, we
have to include sched.h. Because it would cause a loop inclusion, we
cannot include sched.h in any other of asm-m68k/system.h,
linux/thread_info.h, linux/hardirq.h, which leaves this ugly include in
a C file as the only simple solution.
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Roman Zippel <zippel@linux-m68k.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: <stable@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-10 17:43:43 +08:00
# include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
2007-02-16 17:28:03 +08:00
# include <linux/tick.h>
2009-08-18 23:09:42 +08:00
# include <linux/kthread.h>
2021-05-28 03:01:21 +08:00
# include <linux/prandom.h>
# include <linux/cpu.h>
2006-06-26 15:25:05 +08:00
2015-03-25 20:07:37 +08:00
# include "tick-internal.h"
2014-07-17 05:05:10 +08:00
# include "timekeeping_internal.h"
2013-04-26 04:31:50 +08:00
2009-11-11 22:05:29 +08:00
/**
* clocks_calc_mult_shift - calculate mult / shift factors for scaled math of clocks
* @ mult : pointer to mult variable
* @ shift : pointer to shift variable
* @ from : frequency to convert from
* @ to : frequency to convert to
2011-01-12 01:18:12 +08:00
* @ maxsec : guaranteed runtime conversion range in seconds
2009-11-11 22:05:29 +08:00
*
* The function evaluates the shift / mult pair for the scaled math
* operations of clocksources and clockevents .
*
* @ to and @ from are frequency values in HZ . For clock sources @ to is
* NSEC_PER_SEC = = 1 GHz and @ from is the counter frequency . For clock
* event @ to is the counter frequency and @ from is NSEC_PER_SEC .
*
2011-01-12 01:18:12 +08:00
* The @ maxsec conversion range argument controls the time frame in
2009-11-11 22:05:29 +08:00
* seconds which must be covered by the runtime conversion with the
* calculated mult and shift factors . This guarantees that no 64 bit
* overflow happens when the input value of the conversion is
* multiplied with the calculated mult factor . Larger ranges may
2021-03-23 05:39:03 +08:00
* reduce the conversion accuracy by choosing smaller mult and shift
2009-11-11 22:05:29 +08:00
* factors .
*/
void
2011-01-12 01:18:12 +08:00
clocks_calc_mult_shift ( u32 * mult , u32 * shift , u32 from , u32 to , u32 maxsec )
2009-11-11 22:05:29 +08:00
{
u64 tmp ;
u32 sft , sftacc = 32 ;
/*
* Calculate the shift factor which is limiting the conversion
* range :
*/
2011-01-12 01:18:12 +08:00
tmp = ( ( u64 ) maxsec * from ) > > 32 ;
2009-11-11 22:05:29 +08:00
while ( tmp ) {
tmp > > = 1 ;
sftacc - - ;
}
/*
* Find the conversion shift / mult pair which has the best
* accuracy and fits the maxsec conversion range :
*/
for ( sft = 32 ; sft > 0 ; sft - - ) {
tmp = ( u64 ) to < < sft ;
2010-12-17 03:03:27 +08:00
tmp + = from / 2 ;
2009-11-11 22:05:29 +08:00
do_div ( tmp , from ) ;
if ( ( tmp > > sftacc ) = = 0 )
break ;
}
* mult = tmp ;
* shift = sft ;
}
2016-12-07 08:00:43 +08:00
EXPORT_SYMBOL_GPL ( clocks_calc_mult_shift ) ;
2009-11-11 22:05:29 +08:00
2006-06-26 15:25:05 +08:00
/*[Clocksource internal variables]---------
* curr_clocksource :
2009-08-14 21:47:21 +08:00
* currently selected clocksource .
2018-07-17 15:55:16 +08:00
* suspend_clocksource :
* used to calculate the suspend time .
2006-06-26 15:25:05 +08:00
* clocksource_list :
* linked list with the registered clocksources
2009-08-14 21:47:30 +08:00
* clocksource_mutex :
* protects manipulations to curr_clocksource and the clocksource_list
2006-06-26 15:25:05 +08:00
* override_name :
* Name of the user - specified clocksource .
*/
2009-08-14 21:47:21 +08:00
static struct clocksource * curr_clocksource ;
2018-07-17 15:55:16 +08:00
static struct clocksource * suspend_clocksource ;
2006-06-26 15:25:05 +08:00
static LIST_HEAD ( clocksource_list ) ;
2009-08-14 21:47:30 +08:00
static DEFINE_MUTEX ( clocksource_mutex ) ;
2013-04-26 04:31:45 +08:00
static char override_name [ CS_NAME_LEN ] ;
2009-09-15 01:49:02 +08:00
static int finished_booting ;
2018-07-17 15:55:16 +08:00
static u64 suspend_start ;
2006-06-26 15:25:05 +08:00
2022-12-07 11:36:10 +08:00
/*
* Interval : 0.5 sec .
*/
# define WATCHDOG_INTERVAL (HZ >> 1)
clocksource: Reduce clocksource-skew threshold
Currently, WATCHDOG_THRESHOLD is set to detect a 62.5-millisecond skew in
a 500-millisecond WATCHDOG_INTERVAL. This requires that clocks be skewed
by more than 12.5% in order to be marked unstable. Except that a clock
that is skewed by that much is probably destroying unsuspecting software
right and left. And given that there are now checks for false-positive
skews due to delays between reading the two clocks, it should be possible
to greatly decrease WATCHDOG_THRESHOLD, at least for fine-grained clocks
such as TSC.
Therefore, add a new uncertainty_margin field to the clocksource structure
that contains the maximum uncertainty in nanoseconds for the corresponding
clock. This field may be initialized manually, as it is for
clocksource_tsc_early and clocksource_jiffies, which is copied to
refined_jiffies. If the field is not initialized manually, it will be
computed at clock-registry time as the period of the clock in question
based on the scale and freq parameters to __clocksource_update_freq_scale()
function. If either of those two parameters are zero, the
tens-of-milliseconds WATCHDOG_THRESHOLD is used as a cowardly alternative
to dividing by zero. No matter how the uncertainty_margin field is
calculated, it is bounded below by twice WATCHDOG_MAX_SKEW, that is, by 100
microseconds.
Note that manually initialized uncertainty_margin fields are not adjusted,
but there is a WARN_ON_ONCE() that triggers if any such field is less than
twice WATCHDOG_MAX_SKEW. This WARN_ON_ONCE() is intended to discourage
production use of the one-nanosecond uncertainty_margin values that are
used to test the clock-skew code itself.
The actual clock-skew check uses the sum of the uncertainty_margin fields
of the two clocksource structures being compared. Integer overflow is
avoided because the largest computed value of the uncertainty_margin
fields is one billion (10^9), and double that value fits into an
unsigned int. However, if someone manually specifies (say) UINT_MAX,
they will get what they deserve.
Note that the refined_jiffies uncertainty_margin field is initialized to
TICK_NSEC, which means that skew checks involving this clocksource will
be sufficently forgiving. In a similar vein, the clocksource_tsc_early
uncertainty_margin field is initialized to 32*NSEC_PER_MSEC, which
replicates the current behavior and allows custom setting if needed
in order to address the rare skews detected for this clocksource in
current mainline.
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-4-paulmck@kernel.org
2021-05-28 03:01:22 +08:00
/*
* Threshold : 0.0312 s , when doubled : 0.0625 s .
* Also a default for cs - > uncertainty_margin when registering clocks .
*/
# define WATCHDOG_THRESHOLD (NSEC_PER_SEC >> 5)
/*
* Maximum permissible delay between two readouts of the watchdog
* clocksource surrounding a read of the clocksource being validated .
* This delay could be due to SMIs , NMIs , or to VCPU preemptions . Used as
* a lower bound for cs - > uncertainty_margin values when registering clocks .
2022-12-07 11:36:10 +08:00
*
* The default of 500 parts per million is based on NTP ' s limits .
* If a clocksource is good enough for NTP , it is good enough for us !
clocksource: Reduce clocksource-skew threshold
Currently, WATCHDOG_THRESHOLD is set to detect a 62.5-millisecond skew in
a 500-millisecond WATCHDOG_INTERVAL. This requires that clocks be skewed
by more than 12.5% in order to be marked unstable. Except that a clock
that is skewed by that much is probably destroying unsuspecting software
right and left. And given that there are now checks for false-positive
skews due to delays between reading the two clocks, it should be possible
to greatly decrease WATCHDOG_THRESHOLD, at least for fine-grained clocks
such as TSC.
Therefore, add a new uncertainty_margin field to the clocksource structure
that contains the maximum uncertainty in nanoseconds for the corresponding
clock. This field may be initialized manually, as it is for
clocksource_tsc_early and clocksource_jiffies, which is copied to
refined_jiffies. If the field is not initialized manually, it will be
computed at clock-registry time as the period of the clock in question
based on the scale and freq parameters to __clocksource_update_freq_scale()
function. If either of those two parameters are zero, the
tens-of-milliseconds WATCHDOG_THRESHOLD is used as a cowardly alternative
to dividing by zero. No matter how the uncertainty_margin field is
calculated, it is bounded below by twice WATCHDOG_MAX_SKEW, that is, by 100
microseconds.
Note that manually initialized uncertainty_margin fields are not adjusted,
but there is a WARN_ON_ONCE() that triggers if any such field is less than
twice WATCHDOG_MAX_SKEW. This WARN_ON_ONCE() is intended to discourage
production use of the one-nanosecond uncertainty_margin values that are
used to test the clock-skew code itself.
The actual clock-skew check uses the sum of the uncertainty_margin fields
of the two clocksource structures being compared. Integer overflow is
avoided because the largest computed value of the uncertainty_margin
fields is one billion (10^9), and double that value fits into an
unsigned int. However, if someone manually specifies (say) UINT_MAX,
they will get what they deserve.
Note that the refined_jiffies uncertainty_margin field is initialized to
TICK_NSEC, which means that skew checks involving this clocksource will
be sufficently forgiving. In a similar vein, the clocksource_tsc_early
uncertainty_margin field is initialized to 32*NSEC_PER_MSEC, which
replicates the current behavior and allows custom setting if needed
in order to address the rare skews detected for this clocksource in
current mainline.
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-4-paulmck@kernel.org
2021-05-28 03:01:22 +08:00
*/
2021-12-06 11:38:15 +08:00
# ifdef CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US
# define MAX_SKEW_USEC CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US
# else
2022-12-07 11:36:10 +08:00
# define MAX_SKEW_USEC (125 * WATCHDOG_INTERVAL / HZ)
2021-12-06 11:38:15 +08:00
# endif
# define WATCHDOG_MAX_SKEW (MAX_SKEW_USEC * NSEC_PER_USEC)
clocksource: Reduce clocksource-skew threshold
Currently, WATCHDOG_THRESHOLD is set to detect a 62.5-millisecond skew in
a 500-millisecond WATCHDOG_INTERVAL. This requires that clocks be skewed
by more than 12.5% in order to be marked unstable. Except that a clock
that is skewed by that much is probably destroying unsuspecting software
right and left. And given that there are now checks for false-positive
skews due to delays between reading the two clocks, it should be possible
to greatly decrease WATCHDOG_THRESHOLD, at least for fine-grained clocks
such as TSC.
Therefore, add a new uncertainty_margin field to the clocksource structure
that contains the maximum uncertainty in nanoseconds for the corresponding
clock. This field may be initialized manually, as it is for
clocksource_tsc_early and clocksource_jiffies, which is copied to
refined_jiffies. If the field is not initialized manually, it will be
computed at clock-registry time as the period of the clock in question
based on the scale and freq parameters to __clocksource_update_freq_scale()
function. If either of those two parameters are zero, the
tens-of-milliseconds WATCHDOG_THRESHOLD is used as a cowardly alternative
to dividing by zero. No matter how the uncertainty_margin field is
calculated, it is bounded below by twice WATCHDOG_MAX_SKEW, that is, by 100
microseconds.
Note that manually initialized uncertainty_margin fields are not adjusted,
but there is a WARN_ON_ONCE() that triggers if any such field is less than
twice WATCHDOG_MAX_SKEW. This WARN_ON_ONCE() is intended to discourage
production use of the one-nanosecond uncertainty_margin values that are
used to test the clock-skew code itself.
The actual clock-skew check uses the sum of the uncertainty_margin fields
of the two clocksource structures being compared. Integer overflow is
avoided because the largest computed value of the uncertainty_margin
fields is one billion (10^9), and double that value fits into an
unsigned int. However, if someone manually specifies (say) UINT_MAX,
they will get what they deserve.
Note that the refined_jiffies uncertainty_margin field is initialized to
TICK_NSEC, which means that skew checks involving this clocksource will
be sufficently forgiving. In a similar vein, the clocksource_tsc_early
uncertainty_margin field is initialized to 32*NSEC_PER_MSEC, which
replicates the current behavior and allows custom setting if needed
in order to address the rare skews detected for this clocksource in
current mainline.
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-4-paulmck@kernel.org
2021-05-28 03:01:22 +08:00
2007-02-16 17:27:43 +08:00
# ifdef CONFIG_CLOCKSOURCE_WATCHDOG
2009-09-11 21:33:05 +08:00
static void clocksource_watchdog_work ( struct work_struct * work ) ;
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
static void clocksource_select ( void ) ;
2009-09-11 21:33:05 +08:00
2007-02-16 17:27:43 +08:00
static LIST_HEAD ( watchdog_list ) ;
static struct clocksource * watchdog ;
static struct timer_list watchdog_timer ;
2009-09-11 21:33:05 +08:00
static DECLARE_WORK ( watchdog_work , clocksource_watchdog_work ) ;
2007-02-16 17:27:43 +08:00
static DEFINE_SPINLOCK ( watchdog_lock ) ;
2009-08-14 21:47:24 +08:00
static int watchdog_running ;
2011-09-12 19:32:23 +08:00
static atomic_t watchdog_reset_pending ;
2007-05-09 17:35:15 +08:00
2019-05-24 18:33:39 +08:00
static inline void clocksource_watchdog_lock ( unsigned long * flags )
2018-04-23 23:28:55 +08:00
{
spin_lock_irqsave ( & watchdog_lock , * flags ) ;
}
2019-05-24 18:33:39 +08:00
static inline void clocksource_watchdog_unlock ( unsigned long * flags )
2018-04-23 23:28:55 +08:00
{
spin_unlock_irqrestore ( & watchdog_lock , * flags ) ;
}
2018-09-05 16:41:58 +08:00
static int clocksource_watchdog_kthread ( void * data ) ;
static void __clocksource_change_rating ( struct clocksource * cs , int rating ) ;
static void clocksource_watchdog_work ( struct work_struct * work )
{
/*
* We cannot directly run clocksource_watchdog_kthread ( ) here , because
* clocksource_select ( ) calls timekeeping_notify ( ) which uses
* stop_machine ( ) . One cannot use stop_machine ( ) from a workqueue ( ) due
* lock inversions wrt CPU hotplug .
*
* Also , we only ever run this work once or twice during the lifetime
* of the kernel , so there is no point in creating a more permanent
* kthread for this .
*
* If kthread_run fails the next watchdog scan over the
* watchdog_list will find the unstable clock again .
*/
kthread_run ( clocksource_watchdog_kthread , NULL , " kwatchdog " ) ;
}
2009-08-29 02:25:24 +08:00
static void __clocksource_unstable ( struct clocksource * cs )
2007-02-16 17:27:43 +08:00
{
cs - > flags & = ~ ( CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG ) ;
2009-08-14 21:47:25 +08:00
cs - > flags | = CLOCK_SOURCE_UNSTABLE ;
2016-12-15 18:44:28 +08:00
2018-04-30 18:00:13 +08:00
/*
2018-09-05 16:41:58 +08:00
* If the clocksource is registered clocksource_watchdog_kthread ( ) will
2018-04-30 18:00:13 +08:00
* re - rate and re - select .
*/
if ( list_empty ( & cs - > list ) ) {
cs - > rating = 0 ;
2018-04-23 23:28:55 +08:00
return ;
2018-04-30 18:00:13 +08:00
}
2018-04-23 23:28:55 +08:00
2016-12-15 18:44:28 +08:00
if ( cs - > mark_unstable )
cs - > mark_unstable ( cs ) ;
2018-09-05 16:41:58 +08:00
/* kick clocksource_watchdog_kthread() */
2009-09-15 01:49:02 +08:00
if ( finished_booting )
schedule_work ( & watchdog_work ) ;
2007-02-16 17:27:43 +08:00
}
2009-08-29 02:25:24 +08:00
/**
* clocksource_mark_unstable - mark clocksource unstable via watchdog
* @ cs : clocksource to be marked unstable
*
2018-04-30 18:00:14 +08:00
* This function is called by the x86 TSC code to mark clocksources as unstable ;
2018-09-05 16:41:58 +08:00
* it defers demotion and re - selection to a kthread .
2009-08-29 02:25:24 +08:00
*/
void clocksource_mark_unstable ( struct clocksource * cs )
{
unsigned long flags ;
spin_lock_irqsave ( & watchdog_lock , flags ) ;
if ( ! ( cs - > flags & CLOCK_SOURCE_UNSTABLE ) ) {
2018-04-23 23:28:55 +08:00
if ( ! list_empty ( & cs - > list ) & & list_empty ( & cs - > wd_list ) )
2009-08-29 02:25:24 +08:00
list_add ( & cs - > wd_list , & watchdog_list ) ;
__clocksource_unstable ( cs ) ;
}
spin_unlock_irqrestore ( & watchdog_lock , flags ) ;
}
2021-11-19 03:14:37 +08:00
ulong max_cswd_read_retries = 2 ;
clocksource: Retry clock read if long delays detected
When the clocksource watchdog marks a clock as unstable, this might be due
to that clock being unstable or it might be due to delays that happen to
occur between the reads of the two clocks. Yes, interrupts are disabled
across those two reads, but there are no shortage of things that can delay
interrupts-disabled regions of code ranging from SMI handlers to vCPU
preemption. It would be good to have some indication as to why the clock
was marked unstable.
Therefore, re-read the watchdog clock on either side of the read from the
clock under test. If the watchdog clock shows an excessive time delta
between its pair of reads, the reads are retried.
The maximum number of retries is specified by a new kernel boot parameter
clocksource.max_cswd_read_retries, which defaults to three, that is, up to
four reads, one initial and up to three retries. If more than one retry
was required, a message is printed on the console (the occasional single
retry is expected behavior, especially in guest OSes). If the maximum
number of retries is exceeded, the clock under test will be marked
unstable. However, the probability of this happening due to various sorts
of delays is quite small. In addition, the reason (clock-read delays) for
the unstable marking will be apparent.
Reported-by: Chris Mason <clm@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-1-paulmck@kernel.org
2021-05-28 03:01:19 +08:00
module_param ( max_cswd_read_retries , ulong , 0644 ) ;
clocksource: Provide kernel module to test clocksource watchdog
When the clocksource watchdog marks a clock as unstable, this might
be due to that clock being unstable or it might be due to delays that
happen to occur between the reads of the two clocks. It would be good
to have a way of testing the clocksource watchdog's ability to
distinguish between these two causes of clock skew and instability.
Therefore, provide a new clocksource-wdtest module selected by a new
TEST_CLOCKSOURCE_WATCHDOG Kconfig option. This module has a single module
parameter named "holdoff" that provides the number of seconds of delay
before testing should start, which defaults to zero when built as a module
and to 10 seconds when built directly into the kernel. Very large systems
that boot slowly may need to increase the value of this module parameter.
This module uses hand-crafted clocksource structures to do its testing,
thus avoiding messing up timing for the rest of the kernel and for user
applications. This module first verifies that the ->uncertainty_margin
field of the clocksource structures are set sanely. It then tests the
delay-detection capability of the clocksource watchdog, increasing the
number of consecutive delays injected, first provoking console messages
complaining about the delays and finally forcing a clock-skew event.
Unexpected test results cause at least one WARN_ON_ONCE() console splat.
If there are no splats, the test has passed. Finally, it fuzzes the
value returned from a clocksource to test the clocksource watchdog's
ability to detect time skew.
This module checks the state of its clocksource after each test, and
uses WARN_ON_ONCE() to emit a console splat if there are any failures.
This should enable all types of test frameworks to detect any such
failures.
This facility is intended for diagnostic use only, and should be avoided
on production systems.
Reported-by: Chris Mason <clm@fb.com>
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-5-paulmck@kernel.org
2021-05-28 03:01:23 +08:00
EXPORT_SYMBOL_GPL ( max_cswd_read_retries ) ;
2021-05-28 03:01:21 +08:00
static int verify_n_cpus = 8 ;
module_param ( verify_n_cpus , int , 0644 ) ;
clocksource: Retry clock read if long delays detected
When the clocksource watchdog marks a clock as unstable, this might be due
to that clock being unstable or it might be due to delays that happen to
occur between the reads of the two clocks. Yes, interrupts are disabled
across those two reads, but there are no shortage of things that can delay
interrupts-disabled regions of code ranging from SMI handlers to vCPU
preemption. It would be good to have some indication as to why the clock
was marked unstable.
Therefore, re-read the watchdog clock on either side of the read from the
clock under test. If the watchdog clock shows an excessive time delta
between its pair of reads, the reads are retried.
The maximum number of retries is specified by a new kernel boot parameter
clocksource.max_cswd_read_retries, which defaults to three, that is, up to
four reads, one initial and up to three retries. If more than one retry
was required, a message is printed on the console (the occasional single
retry is expected behavior, especially in guest OSes). If the maximum
number of retries is exceeded, the clock under test will be marked
unstable. However, the probability of this happening due to various sorts
of delays is quite small. In addition, the reason (clock-read delays) for
the unstable marking will be apparent.
Reported-by: Chris Mason <clm@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-1-paulmck@kernel.org
2021-05-28 03:01:19 +08:00
clocksource: Avoid accidental unstable marking of clocksources
Since commit db3a34e17433 ("clocksource: Retry clock read if long delays
detected") and commit 2e27e793e280 ("clocksource: Reduce clocksource-skew
threshold"), it is found that tsc clocksource fallback to hpet can
sometimes happen on both Intel and AMD systems especially when they are
running stressful benchmarking workloads. Of the 23 systems tested with
a v5.14 kernel, 10 of them have switched to hpet clock source during
the test run.
The result of falling back to hpet is a drastic reduction of performance
when running benchmarks. For example, the fio performance tests can
drop up to 70% whereas the iperf3 performance can drop up to 80%.
4 hpet fallbacks happened during bootup. They were:
[ 8.749399] clocksource: timekeeping watchdog on CPU13: hpet read-back delay of 263750ns, attempt 4, marking unstable
[ 12.044610] clocksource: timekeeping watchdog on CPU19: hpet read-back delay of 186166ns, attempt 4, marking unstable
[ 17.336941] clocksource: timekeeping watchdog on CPU28: hpet read-back delay of 182291ns, attempt 4, marking unstable
[ 17.518565] clocksource: timekeeping watchdog on CPU34: hpet read-back delay of 252196ns, attempt 4, marking unstable
Other fallbacks happen when the systems were running stressful
benchmarks. For example:
[ 2685.867873] clocksource: timekeeping watchdog on CPU117: hpet read-back delay of 57269ns, attempt 4, marking unstable
[46215.471228] clocksource: timekeeping watchdog on CPU8: hpet read-back delay of 61460ns, attempt 4, marking unstable
Commit 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold"),
changed the skew margin from 100us to 50us. I think this is too small
and can easily be exceeded when running some stressful workloads on a
thermally stressed system. So it is switched back to 100us.
Even a maximum skew margin of 100us may be too small in for some systems
when booting up especially if those systems are under thermal stress. To
eliminate the case that the large skew is due to the system being too
busy slowing down the reading of both the watchdog and the clocksource,
an extra consecutive read of watchdog clock is being done to check this.
The consecutive watchdog read delay is compared against
WATCHDOG_MAX_SKEW/2. If the delay exceeds the limit, we assume that
the system is just too busy. A warning will be printed to the console
and the clock skew check is skipped for this round.
Fixes: db3a34e17433 ("clocksource: Retry clock read if long delays detected")
Fixes: 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold")
Signed-off-by: Waiman Long <longman@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-19 03:14:36 +08:00
enum wd_read_status {
WD_READ_SUCCESS ,
WD_READ_UNSTABLE ,
WD_READ_SKIP
} ;
static enum wd_read_status cs_watchdog_read ( struct clocksource * cs , u64 * csnow , u64 * wdnow )
clocksource: Retry clock read if long delays detected
When the clocksource watchdog marks a clock as unstable, this might be due
to that clock being unstable or it might be due to delays that happen to
occur between the reads of the two clocks. Yes, interrupts are disabled
across those two reads, but there are no shortage of things that can delay
interrupts-disabled regions of code ranging from SMI handlers to vCPU
preemption. It would be good to have some indication as to why the clock
was marked unstable.
Therefore, re-read the watchdog clock on either side of the read from the
clock under test. If the watchdog clock shows an excessive time delta
between its pair of reads, the reads are retried.
The maximum number of retries is specified by a new kernel boot parameter
clocksource.max_cswd_read_retries, which defaults to three, that is, up to
four reads, one initial and up to three retries. If more than one retry
was required, a message is printed on the console (the occasional single
retry is expected behavior, especially in guest OSes). If the maximum
number of retries is exceeded, the clock under test will be marked
unstable. However, the probability of this happening due to various sorts
of delays is quite small. In addition, the reason (clock-read delays) for
the unstable marking will be apparent.
Reported-by: Chris Mason <clm@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-1-paulmck@kernel.org
2021-05-28 03:01:19 +08:00
{
unsigned int nretries ;
clocksource: Avoid accidental unstable marking of clocksources
Since commit db3a34e17433 ("clocksource: Retry clock read if long delays
detected") and commit 2e27e793e280 ("clocksource: Reduce clocksource-skew
threshold"), it is found that tsc clocksource fallback to hpet can
sometimes happen on both Intel and AMD systems especially when they are
running stressful benchmarking workloads. Of the 23 systems tested with
a v5.14 kernel, 10 of them have switched to hpet clock source during
the test run.
The result of falling back to hpet is a drastic reduction of performance
when running benchmarks. For example, the fio performance tests can
drop up to 70% whereas the iperf3 performance can drop up to 80%.
4 hpet fallbacks happened during bootup. They were:
[ 8.749399] clocksource: timekeeping watchdog on CPU13: hpet read-back delay of 263750ns, attempt 4, marking unstable
[ 12.044610] clocksource: timekeeping watchdog on CPU19: hpet read-back delay of 186166ns, attempt 4, marking unstable
[ 17.336941] clocksource: timekeeping watchdog on CPU28: hpet read-back delay of 182291ns, attempt 4, marking unstable
[ 17.518565] clocksource: timekeeping watchdog on CPU34: hpet read-back delay of 252196ns, attempt 4, marking unstable
Other fallbacks happen when the systems were running stressful
benchmarks. For example:
[ 2685.867873] clocksource: timekeeping watchdog on CPU117: hpet read-back delay of 57269ns, attempt 4, marking unstable
[46215.471228] clocksource: timekeeping watchdog on CPU8: hpet read-back delay of 61460ns, attempt 4, marking unstable
Commit 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold"),
changed the skew margin from 100us to 50us. I think this is too small
and can easily be exceeded when running some stressful workloads on a
thermally stressed system. So it is switched back to 100us.
Even a maximum skew margin of 100us may be too small in for some systems
when booting up especially if those systems are under thermal stress. To
eliminate the case that the large skew is due to the system being too
busy slowing down the reading of both the watchdog and the clocksource,
an extra consecutive read of watchdog clock is being done to check this.
The consecutive watchdog read delay is compared against
WATCHDOG_MAX_SKEW/2. If the delay exceeds the limit, we assume that
the system is just too busy. A warning will be printed to the console
and the clock skew check is skipped for this round.
Fixes: db3a34e17433 ("clocksource: Retry clock read if long delays detected")
Fixes: 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold")
Signed-off-by: Waiman Long <longman@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-19 03:14:36 +08:00
u64 wd_end , wd_end2 , wd_delta ;
int64_t wd_delay , wd_seq_delay ;
clocksource: Retry clock read if long delays detected
When the clocksource watchdog marks a clock as unstable, this might be due
to that clock being unstable or it might be due to delays that happen to
occur between the reads of the two clocks. Yes, interrupts are disabled
across those two reads, but there are no shortage of things that can delay
interrupts-disabled regions of code ranging from SMI handlers to vCPU
preemption. It would be good to have some indication as to why the clock
was marked unstable.
Therefore, re-read the watchdog clock on either side of the read from the
clock under test. If the watchdog clock shows an excessive time delta
between its pair of reads, the reads are retried.
The maximum number of retries is specified by a new kernel boot parameter
clocksource.max_cswd_read_retries, which defaults to three, that is, up to
four reads, one initial and up to three retries. If more than one retry
was required, a message is printed on the console (the occasional single
retry is expected behavior, especially in guest OSes). If the maximum
number of retries is exceeded, the clock under test will be marked
unstable. However, the probability of this happening due to various sorts
of delays is quite small. In addition, the reason (clock-read delays) for
the unstable marking will be apparent.
Reported-by: Chris Mason <clm@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-1-paulmck@kernel.org
2021-05-28 03:01:19 +08:00
for ( nretries = 0 ; nretries < = max_cswd_read_retries ; nretries + + ) {
local_irq_disable ( ) ;
* wdnow = watchdog - > read ( watchdog ) ;
* csnow = cs - > read ( cs ) ;
wd_end = watchdog - > read ( watchdog ) ;
clocksource: Avoid accidental unstable marking of clocksources
Since commit db3a34e17433 ("clocksource: Retry clock read if long delays
detected") and commit 2e27e793e280 ("clocksource: Reduce clocksource-skew
threshold"), it is found that tsc clocksource fallback to hpet can
sometimes happen on both Intel and AMD systems especially when they are
running stressful benchmarking workloads. Of the 23 systems tested with
a v5.14 kernel, 10 of them have switched to hpet clock source during
the test run.
The result of falling back to hpet is a drastic reduction of performance
when running benchmarks. For example, the fio performance tests can
drop up to 70% whereas the iperf3 performance can drop up to 80%.
4 hpet fallbacks happened during bootup. They were:
[ 8.749399] clocksource: timekeeping watchdog on CPU13: hpet read-back delay of 263750ns, attempt 4, marking unstable
[ 12.044610] clocksource: timekeeping watchdog on CPU19: hpet read-back delay of 186166ns, attempt 4, marking unstable
[ 17.336941] clocksource: timekeeping watchdog on CPU28: hpet read-back delay of 182291ns, attempt 4, marking unstable
[ 17.518565] clocksource: timekeeping watchdog on CPU34: hpet read-back delay of 252196ns, attempt 4, marking unstable
Other fallbacks happen when the systems were running stressful
benchmarks. For example:
[ 2685.867873] clocksource: timekeeping watchdog on CPU117: hpet read-back delay of 57269ns, attempt 4, marking unstable
[46215.471228] clocksource: timekeeping watchdog on CPU8: hpet read-back delay of 61460ns, attempt 4, marking unstable
Commit 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold"),
changed the skew margin from 100us to 50us. I think this is too small
and can easily be exceeded when running some stressful workloads on a
thermally stressed system. So it is switched back to 100us.
Even a maximum skew margin of 100us may be too small in for some systems
when booting up especially if those systems are under thermal stress. To
eliminate the case that the large skew is due to the system being too
busy slowing down the reading of both the watchdog and the clocksource,
an extra consecutive read of watchdog clock is being done to check this.
The consecutive watchdog read delay is compared against
WATCHDOG_MAX_SKEW/2. If the delay exceeds the limit, we assume that
the system is just too busy. A warning will be printed to the console
and the clock skew check is skipped for this round.
Fixes: db3a34e17433 ("clocksource: Retry clock read if long delays detected")
Fixes: 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold")
Signed-off-by: Waiman Long <longman@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-19 03:14:36 +08:00
wd_end2 = watchdog - > read ( watchdog ) ;
clocksource: Retry clock read if long delays detected
When the clocksource watchdog marks a clock as unstable, this might be due
to that clock being unstable or it might be due to delays that happen to
occur between the reads of the two clocks. Yes, interrupts are disabled
across those two reads, but there are no shortage of things that can delay
interrupts-disabled regions of code ranging from SMI handlers to vCPU
preemption. It would be good to have some indication as to why the clock
was marked unstable.
Therefore, re-read the watchdog clock on either side of the read from the
clock under test. If the watchdog clock shows an excessive time delta
between its pair of reads, the reads are retried.
The maximum number of retries is specified by a new kernel boot parameter
clocksource.max_cswd_read_retries, which defaults to three, that is, up to
four reads, one initial and up to three retries. If more than one retry
was required, a message is printed on the console (the occasional single
retry is expected behavior, especially in guest OSes). If the maximum
number of retries is exceeded, the clock under test will be marked
unstable. However, the probability of this happening due to various sorts
of delays is quite small. In addition, the reason (clock-read delays) for
the unstable marking will be apparent.
Reported-by: Chris Mason <clm@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-1-paulmck@kernel.org
2021-05-28 03:01:19 +08:00
local_irq_enable ( ) ;
wd_delta = clocksource_delta ( wd_end , * wdnow , watchdog - > mask ) ;
wd_delay = clocksource_cyc2ns ( wd_delta , watchdog - > mult ,
watchdog - > shift ) ;
if ( wd_delay < = WATCHDOG_MAX_SKEW ) {
if ( nretries > 1 | | nretries > = max_cswd_read_retries ) {
pr_warn ( " timekeeping watchdog on CPU%d: %s retried %d times before success \n " ,
smp_processor_id ( ) , watchdog - > name , nretries ) ;
}
clocksource: Avoid accidental unstable marking of clocksources
Since commit db3a34e17433 ("clocksource: Retry clock read if long delays
detected") and commit 2e27e793e280 ("clocksource: Reduce clocksource-skew
threshold"), it is found that tsc clocksource fallback to hpet can
sometimes happen on both Intel and AMD systems especially when they are
running stressful benchmarking workloads. Of the 23 systems tested with
a v5.14 kernel, 10 of them have switched to hpet clock source during
the test run.
The result of falling back to hpet is a drastic reduction of performance
when running benchmarks. For example, the fio performance tests can
drop up to 70% whereas the iperf3 performance can drop up to 80%.
4 hpet fallbacks happened during bootup. They were:
[ 8.749399] clocksource: timekeeping watchdog on CPU13: hpet read-back delay of 263750ns, attempt 4, marking unstable
[ 12.044610] clocksource: timekeeping watchdog on CPU19: hpet read-back delay of 186166ns, attempt 4, marking unstable
[ 17.336941] clocksource: timekeeping watchdog on CPU28: hpet read-back delay of 182291ns, attempt 4, marking unstable
[ 17.518565] clocksource: timekeeping watchdog on CPU34: hpet read-back delay of 252196ns, attempt 4, marking unstable
Other fallbacks happen when the systems were running stressful
benchmarks. For example:
[ 2685.867873] clocksource: timekeeping watchdog on CPU117: hpet read-back delay of 57269ns, attempt 4, marking unstable
[46215.471228] clocksource: timekeeping watchdog on CPU8: hpet read-back delay of 61460ns, attempt 4, marking unstable
Commit 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold"),
changed the skew margin from 100us to 50us. I think this is too small
and can easily be exceeded when running some stressful workloads on a
thermally stressed system. So it is switched back to 100us.
Even a maximum skew margin of 100us may be too small in for some systems
when booting up especially if those systems are under thermal stress. To
eliminate the case that the large skew is due to the system being too
busy slowing down the reading of both the watchdog and the clocksource,
an extra consecutive read of watchdog clock is being done to check this.
The consecutive watchdog read delay is compared against
WATCHDOG_MAX_SKEW/2. If the delay exceeds the limit, we assume that
the system is just too busy. A warning will be printed to the console
and the clock skew check is skipped for this round.
Fixes: db3a34e17433 ("clocksource: Retry clock read if long delays detected")
Fixes: 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold")
Signed-off-by: Waiman Long <longman@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-19 03:14:36 +08:00
return WD_READ_SUCCESS ;
clocksource: Retry clock read if long delays detected
When the clocksource watchdog marks a clock as unstable, this might be due
to that clock being unstable or it might be due to delays that happen to
occur between the reads of the two clocks. Yes, interrupts are disabled
across those two reads, but there are no shortage of things that can delay
interrupts-disabled regions of code ranging from SMI handlers to vCPU
preemption. It would be good to have some indication as to why the clock
was marked unstable.
Therefore, re-read the watchdog clock on either side of the read from the
clock under test. If the watchdog clock shows an excessive time delta
between its pair of reads, the reads are retried.
The maximum number of retries is specified by a new kernel boot parameter
clocksource.max_cswd_read_retries, which defaults to three, that is, up to
four reads, one initial and up to three retries. If more than one retry
was required, a message is printed on the console (the occasional single
retry is expected behavior, especially in guest OSes). If the maximum
number of retries is exceeded, the clock under test will be marked
unstable. However, the probability of this happening due to various sorts
of delays is quite small. In addition, the reason (clock-read delays) for
the unstable marking will be apparent.
Reported-by: Chris Mason <clm@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-1-paulmck@kernel.org
2021-05-28 03:01:19 +08:00
}
clocksource: Avoid accidental unstable marking of clocksources
Since commit db3a34e17433 ("clocksource: Retry clock read if long delays
detected") and commit 2e27e793e280 ("clocksource: Reduce clocksource-skew
threshold"), it is found that tsc clocksource fallback to hpet can
sometimes happen on both Intel and AMD systems especially when they are
running stressful benchmarking workloads. Of the 23 systems tested with
a v5.14 kernel, 10 of them have switched to hpet clock source during
the test run.
The result of falling back to hpet is a drastic reduction of performance
when running benchmarks. For example, the fio performance tests can
drop up to 70% whereas the iperf3 performance can drop up to 80%.
4 hpet fallbacks happened during bootup. They were:
[ 8.749399] clocksource: timekeeping watchdog on CPU13: hpet read-back delay of 263750ns, attempt 4, marking unstable
[ 12.044610] clocksource: timekeeping watchdog on CPU19: hpet read-back delay of 186166ns, attempt 4, marking unstable
[ 17.336941] clocksource: timekeeping watchdog on CPU28: hpet read-back delay of 182291ns, attempt 4, marking unstable
[ 17.518565] clocksource: timekeeping watchdog on CPU34: hpet read-back delay of 252196ns, attempt 4, marking unstable
Other fallbacks happen when the systems were running stressful
benchmarks. For example:
[ 2685.867873] clocksource: timekeeping watchdog on CPU117: hpet read-back delay of 57269ns, attempt 4, marking unstable
[46215.471228] clocksource: timekeeping watchdog on CPU8: hpet read-back delay of 61460ns, attempt 4, marking unstable
Commit 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold"),
changed the skew margin from 100us to 50us. I think this is too small
and can easily be exceeded when running some stressful workloads on a
thermally stressed system. So it is switched back to 100us.
Even a maximum skew margin of 100us may be too small in for some systems
when booting up especially if those systems are under thermal stress. To
eliminate the case that the large skew is due to the system being too
busy slowing down the reading of both the watchdog and the clocksource,
an extra consecutive read of watchdog clock is being done to check this.
The consecutive watchdog read delay is compared against
WATCHDOG_MAX_SKEW/2. If the delay exceeds the limit, we assume that
the system is just too busy. A warning will be printed to the console
and the clock skew check is skipped for this round.
Fixes: db3a34e17433 ("clocksource: Retry clock read if long delays detected")
Fixes: 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold")
Signed-off-by: Waiman Long <longman@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-19 03:14:36 +08:00
/*
* Now compute delay in consecutive watchdog read to see if
* there is too much external interferences that cause
* significant delay in reading both clocksource and watchdog .
*
* If consecutive WD read - back delay > WATCHDOG_MAX_SKEW / 2 ,
* report system busy , reinit the watchdog and skip the current
* watchdog test .
*/
wd_delta = clocksource_delta ( wd_end2 , wd_end , watchdog - > mask ) ;
wd_seq_delay = clocksource_cyc2ns ( wd_delta , watchdog - > mult , watchdog - > shift ) ;
if ( wd_seq_delay > WATCHDOG_MAX_SKEW / 2 )
goto skip_test ;
clocksource: Retry clock read if long delays detected
When the clocksource watchdog marks a clock as unstable, this might be due
to that clock being unstable or it might be due to delays that happen to
occur between the reads of the two clocks. Yes, interrupts are disabled
across those two reads, but there are no shortage of things that can delay
interrupts-disabled regions of code ranging from SMI handlers to vCPU
preemption. It would be good to have some indication as to why the clock
was marked unstable.
Therefore, re-read the watchdog clock on either side of the read from the
clock under test. If the watchdog clock shows an excessive time delta
between its pair of reads, the reads are retried.
The maximum number of retries is specified by a new kernel boot parameter
clocksource.max_cswd_read_retries, which defaults to three, that is, up to
four reads, one initial and up to three retries. If more than one retry
was required, a message is printed on the console (the occasional single
retry is expected behavior, especially in guest OSes). If the maximum
number of retries is exceeded, the clock under test will be marked
unstable. However, the probability of this happening due to various sorts
of delays is quite small. In addition, the reason (clock-read delays) for
the unstable marking will be apparent.
Reported-by: Chris Mason <clm@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-1-paulmck@kernel.org
2021-05-28 03:01:19 +08:00
}
2022-12-14 05:57:28 +08:00
pr_warn ( " timekeeping watchdog on CPU%d: wd-%s-wd excessive read-back delay of %lldns vs. limit of %ldns, wd-wd read-back delay only %lldns, attempt %d, marking %s unstable \n " ,
smp_processor_id ( ) , cs - > name , wd_delay , WATCHDOG_MAX_SKEW , wd_seq_delay , nretries , cs - > name ) ;
clocksource: Avoid accidental unstable marking of clocksources
Since commit db3a34e17433 ("clocksource: Retry clock read if long delays
detected") and commit 2e27e793e280 ("clocksource: Reduce clocksource-skew
threshold"), it is found that tsc clocksource fallback to hpet can
sometimes happen on both Intel and AMD systems especially when they are
running stressful benchmarking workloads. Of the 23 systems tested with
a v5.14 kernel, 10 of them have switched to hpet clock source during
the test run.
The result of falling back to hpet is a drastic reduction of performance
when running benchmarks. For example, the fio performance tests can
drop up to 70% whereas the iperf3 performance can drop up to 80%.
4 hpet fallbacks happened during bootup. They were:
[ 8.749399] clocksource: timekeeping watchdog on CPU13: hpet read-back delay of 263750ns, attempt 4, marking unstable
[ 12.044610] clocksource: timekeeping watchdog on CPU19: hpet read-back delay of 186166ns, attempt 4, marking unstable
[ 17.336941] clocksource: timekeeping watchdog on CPU28: hpet read-back delay of 182291ns, attempt 4, marking unstable
[ 17.518565] clocksource: timekeeping watchdog on CPU34: hpet read-back delay of 252196ns, attempt 4, marking unstable
Other fallbacks happen when the systems were running stressful
benchmarks. For example:
[ 2685.867873] clocksource: timekeeping watchdog on CPU117: hpet read-back delay of 57269ns, attempt 4, marking unstable
[46215.471228] clocksource: timekeeping watchdog on CPU8: hpet read-back delay of 61460ns, attempt 4, marking unstable
Commit 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold"),
changed the skew margin from 100us to 50us. I think this is too small
and can easily be exceeded when running some stressful workloads on a
thermally stressed system. So it is switched back to 100us.
Even a maximum skew margin of 100us may be too small in for some systems
when booting up especially if those systems are under thermal stress. To
eliminate the case that the large skew is due to the system being too
busy slowing down the reading of both the watchdog and the clocksource,
an extra consecutive read of watchdog clock is being done to check this.
The consecutive watchdog read delay is compared against
WATCHDOG_MAX_SKEW/2. If the delay exceeds the limit, we assume that
the system is just too busy. A warning will be printed to the console
and the clock skew check is skipped for this round.
Fixes: db3a34e17433 ("clocksource: Retry clock read if long delays detected")
Fixes: 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold")
Signed-off-by: Waiman Long <longman@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-19 03:14:36 +08:00
return WD_READ_UNSTABLE ;
skip_test :
pr_info ( " timekeeping watchdog on CPU%d: %s wd-wd read-back delay of %lldns \n " ,
smp_processor_id ( ) , watchdog - > name , wd_seq_delay ) ;
pr_info ( " wd-%s-wd read-back delay of %lldns, clock-skew test skipped! \n " ,
cs - > name , wd_delay ) ;
return WD_READ_SKIP ;
clocksource: Retry clock read if long delays detected
When the clocksource watchdog marks a clock as unstable, this might be due
to that clock being unstable or it might be due to delays that happen to
occur between the reads of the two clocks. Yes, interrupts are disabled
across those two reads, but there are no shortage of things that can delay
interrupts-disabled regions of code ranging from SMI handlers to vCPU
preemption. It would be good to have some indication as to why the clock
was marked unstable.
Therefore, re-read the watchdog clock on either side of the read from the
clock under test. If the watchdog clock shows an excessive time delta
between its pair of reads, the reads are retried.
The maximum number of retries is specified by a new kernel boot parameter
clocksource.max_cswd_read_retries, which defaults to three, that is, up to
four reads, one initial and up to three retries. If more than one retry
was required, a message is printed on the console (the occasional single
retry is expected behavior, especially in guest OSes). If the maximum
number of retries is exceeded, the clock under test will be marked
unstable. However, the probability of this happening due to various sorts
of delays is quite small. In addition, the reason (clock-read delays) for
the unstable marking will be apparent.
Reported-by: Chris Mason <clm@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-1-paulmck@kernel.org
2021-05-28 03:01:19 +08:00
}
2021-05-28 03:01:20 +08:00
static u64 csnow_mid ;
static cpumask_t cpus_ahead ;
static cpumask_t cpus_behind ;
2021-05-28 03:01:21 +08:00
static cpumask_t cpus_chosen ;
static void clocksource_verify_choose_cpus ( void )
{
int cpu , i , n = verify_n_cpus ;
if ( n < 0 ) {
/* Check all of the CPUs. */
cpumask_copy ( & cpus_chosen , cpu_online_mask ) ;
cpumask_clear_cpu ( smp_processor_id ( ) , & cpus_chosen ) ;
return ;
}
/* If no checking desired, or no other CPU to check, leave. */
cpumask_clear ( & cpus_chosen ) ;
if ( n = = 0 | | num_online_cpus ( ) < = 1 )
return ;
/* Make sure to select at least one CPU other than the current CPU. */
2021-08-15 05:17:05 +08:00
cpu = cpumask_first ( cpu_online_mask ) ;
2021-05-28 03:01:21 +08:00
if ( cpu = = smp_processor_id ( ) )
cpu = cpumask_next ( cpu , cpu_online_mask ) ;
if ( WARN_ON_ONCE ( cpu > = nr_cpu_ids ) )
return ;
cpumask_set_cpu ( cpu , & cpus_chosen ) ;
/* Force a sane value for the boot parameter. */
if ( n > nr_cpu_ids )
n = nr_cpu_ids ;
/*
* Randomly select the specified number of CPUs . If the same
* CPU is selected multiple times , that CPU is checked only once ,
* and no replacement CPU is selected . This gracefully handles
* situations where verify_n_cpus is greater than the number of
* CPUs that are currently online .
*/
for ( i = 1 ; i < n ; i + + ) {
2022-10-10 10:44:02 +08:00
cpu = get_random_u32_below ( nr_cpu_ids ) ;
2021-05-28 03:01:21 +08:00
cpu = cpumask_next ( cpu - 1 , cpu_online_mask ) ;
if ( cpu > = nr_cpu_ids )
2021-08-15 05:17:05 +08:00
cpu = cpumask_first ( cpu_online_mask ) ;
2021-05-28 03:01:21 +08:00
if ( ! WARN_ON_ONCE ( cpu > = nr_cpu_ids ) )
cpumask_set_cpu ( cpu , & cpus_chosen ) ;
}
/* Don't verify ourselves. */
cpumask_clear_cpu ( smp_processor_id ( ) , & cpus_chosen ) ;
}
2021-05-28 03:01:20 +08:00
static void clocksource_verify_one_cpu ( void * csin )
{
struct clocksource * cs = ( struct clocksource * ) csin ;
csnow_mid = cs - > read ( cs ) ;
}
clocksource: Provide kernel module to test clocksource watchdog
When the clocksource watchdog marks a clock as unstable, this might
be due to that clock being unstable or it might be due to delays that
happen to occur between the reads of the two clocks. It would be good
to have a way of testing the clocksource watchdog's ability to
distinguish between these two causes of clock skew and instability.
Therefore, provide a new clocksource-wdtest module selected by a new
TEST_CLOCKSOURCE_WATCHDOG Kconfig option. This module has a single module
parameter named "holdoff" that provides the number of seconds of delay
before testing should start, which defaults to zero when built as a module
and to 10 seconds when built directly into the kernel. Very large systems
that boot slowly may need to increase the value of this module parameter.
This module uses hand-crafted clocksource structures to do its testing,
thus avoiding messing up timing for the rest of the kernel and for user
applications. This module first verifies that the ->uncertainty_margin
field of the clocksource structures are set sanely. It then tests the
delay-detection capability of the clocksource watchdog, increasing the
number of consecutive delays injected, first provoking console messages
complaining about the delays and finally forcing a clock-skew event.
Unexpected test results cause at least one WARN_ON_ONCE() console splat.
If there are no splats, the test has passed. Finally, it fuzzes the
value returned from a clocksource to test the clocksource watchdog's
ability to detect time skew.
This module checks the state of its clocksource after each test, and
uses WARN_ON_ONCE() to emit a console splat if there are any failures.
This should enable all types of test frameworks to detect any such
failures.
This facility is intended for diagnostic use only, and should be avoided
on production systems.
Reported-by: Chris Mason <clm@fb.com>
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-5-paulmck@kernel.org
2021-05-28 03:01:23 +08:00
void clocksource_verify_percpu ( struct clocksource * cs )
2021-05-28 03:01:20 +08:00
{
int64_t cs_nsec , cs_nsec_max = 0 , cs_nsec_min = LLONG_MAX ;
u64 csnow_begin , csnow_end ;
int cpu , testcpu ;
s64 delta ;
2021-05-28 03:01:21 +08:00
if ( verify_n_cpus = = 0 )
return ;
2021-05-28 03:01:20 +08:00
cpumask_clear ( & cpus_ahead ) ;
cpumask_clear ( & cpus_behind ) ;
2021-08-03 22:16:17 +08:00
cpus_read_lock ( ) ;
2021-05-28 03:01:20 +08:00
preempt_disable ( ) ;
2021-05-28 03:01:21 +08:00
clocksource_verify_choose_cpus ( ) ;
2022-02-11 06:49:07 +08:00
if ( cpumask_empty ( & cpus_chosen ) ) {
2021-05-28 03:01:21 +08:00
preempt_enable ( ) ;
2021-08-03 22:16:17 +08:00
cpus_read_unlock ( ) ;
2021-05-28 03:01:21 +08:00
pr_warn ( " Not enough CPUs to check clocksource '%s'. \n " , cs - > name ) ;
return ;
}
2021-05-28 03:01:20 +08:00
testcpu = smp_processor_id ( ) ;
2021-05-28 03:01:21 +08:00
pr_warn ( " Checking clocksource %s synchronization from CPU %d to CPUs %*pbl. \n " , cs - > name , testcpu , cpumask_pr_args ( & cpus_chosen ) ) ;
for_each_cpu ( cpu , & cpus_chosen ) {
2021-05-28 03:01:20 +08:00
if ( cpu = = testcpu )
continue ;
csnow_begin = cs - > read ( cs ) ;
smp_call_function_single ( cpu , clocksource_verify_one_cpu , cs , 1 ) ;
csnow_end = cs - > read ( cs ) ;
delta = ( s64 ) ( ( csnow_mid - csnow_begin ) & cs - > mask ) ;
if ( delta < 0 )
cpumask_set_cpu ( cpu , & cpus_behind ) ;
delta = ( csnow_end - csnow_mid ) & cs - > mask ;
if ( delta < 0 )
cpumask_set_cpu ( cpu , & cpus_ahead ) ;
delta = clocksource_delta ( csnow_end , csnow_begin , cs - > mask ) ;
cs_nsec = clocksource_cyc2ns ( delta , cs - > mult , cs - > shift ) ;
if ( cs_nsec > cs_nsec_max )
cs_nsec_max = cs_nsec ;
if ( cs_nsec < cs_nsec_min )
cs_nsec_min = cs_nsec ;
}
preempt_enable ( ) ;
2021-08-03 22:16:17 +08:00
cpus_read_unlock ( ) ;
2021-05-28 03:01:20 +08:00
if ( ! cpumask_empty ( & cpus_ahead ) )
pr_warn ( " CPUs %*pbl ahead of CPU %d for clocksource %s. \n " ,
cpumask_pr_args ( & cpus_ahead ) , testcpu , cs - > name ) ;
if ( ! cpumask_empty ( & cpus_behind ) )
pr_warn ( " CPUs %*pbl behind CPU %d for clocksource %s. \n " ,
cpumask_pr_args ( & cpus_behind ) , testcpu , cs - > name ) ;
if ( ! cpumask_empty ( & cpus_ahead ) | | ! cpumask_empty ( & cpus_behind ) )
pr_warn ( " CPU %d check durations %lldns - %lldns for clocksource %s. \n " ,
testcpu , cs_nsec_min , cs_nsec_max , cs - > name ) ;
}
clocksource: Provide kernel module to test clocksource watchdog
When the clocksource watchdog marks a clock as unstable, this might
be due to that clock being unstable or it might be due to delays that
happen to occur between the reads of the two clocks. It would be good
to have a way of testing the clocksource watchdog's ability to
distinguish between these two causes of clock skew and instability.
Therefore, provide a new clocksource-wdtest module selected by a new
TEST_CLOCKSOURCE_WATCHDOG Kconfig option. This module has a single module
parameter named "holdoff" that provides the number of seconds of delay
before testing should start, which defaults to zero when built as a module
and to 10 seconds when built directly into the kernel. Very large systems
that boot slowly may need to increase the value of this module parameter.
This module uses hand-crafted clocksource structures to do its testing,
thus avoiding messing up timing for the rest of the kernel and for user
applications. This module first verifies that the ->uncertainty_margin
field of the clocksource structures are set sanely. It then tests the
delay-detection capability of the clocksource watchdog, increasing the
number of consecutive delays injected, first provoking console messages
complaining about the delays and finally forcing a clock-skew event.
Unexpected test results cause at least one WARN_ON_ONCE() console splat.
If there are no splats, the test has passed. Finally, it fuzzes the
value returned from a clocksource to test the clocksource watchdog's
ability to detect time skew.
This module checks the state of its clocksource after each test, and
uses WARN_ON_ONCE() to emit a console splat if there are any failures.
This should enable all types of test frameworks to detect any such
failures.
This facility is intended for diagnostic use only, and should be avoided
on production systems.
Reported-by: Chris Mason <clm@fb.com>
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-5-paulmck@kernel.org
2021-05-28 03:01:23 +08:00
EXPORT_SYMBOL_GPL ( clocksource_verify_percpu ) ;
2021-05-28 03:01:20 +08:00
treewide: setup_timer() -> timer_setup()
This converts all remaining cases of the old setup_timer() API into using
timer_setup(), where the callback argument is the structure already
holding the struct timer_list. These should have no behavioral changes,
since they just change which pointer is passed into the callback with
the same available pointers after conversion. It handles the following
examples, in addition to some other variations.
Casting from unsigned long:
void my_callback(unsigned long data)
{
struct something *ptr = (struct something *)data;
...
}
...
setup_timer(&ptr->my_timer, my_callback, ptr);
and forced object casts:
void my_callback(struct something *ptr)
{
...
}
...
setup_timer(&ptr->my_timer, my_callback, (unsigned long)ptr);
become:
void my_callback(struct timer_list *t)
{
struct something *ptr = from_timer(ptr, t, my_timer);
...
}
...
timer_setup(&ptr->my_timer, my_callback, 0);
Direct function assignments:
void my_callback(unsigned long data)
{
struct something *ptr = (struct something *)data;
...
}
...
ptr->my_timer.function = my_callback;
have a temporary cast added, along with converting the args:
void my_callback(struct timer_list *t)
{
struct something *ptr = from_timer(ptr, t, my_timer);
...
}
...
ptr->my_timer.function = (TIMER_FUNC_TYPE)my_callback;
And finally, callbacks without a data assignment:
void my_callback(unsigned long data)
{
...
}
...
setup_timer(&ptr->my_timer, my_callback, 0);
have their argument renamed to verify they're unused during conversion:
void my_callback(struct timer_list *unused)
{
...
}
...
timer_setup(&ptr->my_timer, my_callback, 0);
The conversion is done with the following Coccinelle script:
spatch --very-quiet --all-includes --include-headers \
-I ./arch/x86/include -I ./arch/x86/include/generated \
-I ./include -I ./arch/x86/include/uapi \
-I ./arch/x86/include/generated/uapi -I ./include/uapi \
-I ./include/generated/uapi --include ./include/linux/kconfig.h \
--dir . \
--cocci-file ~/src/data/timer_setup.cocci
@fix_address_of@
expression e;
@@
setup_timer(
-&(e)
+&e
, ...)
// Update any raw setup_timer() usages that have a NULL callback, but
// would otherwise match change_timer_function_usage, since the latter
// will update all function assignments done in the face of a NULL
// function initialization in setup_timer().
@change_timer_function_usage_NULL@
expression _E;
identifier _timer;
type _cast_data;
@@
(
-setup_timer(&_E->_timer, NULL, _E);
+timer_setup(&_E->_timer, NULL, 0);
|
-setup_timer(&_E->_timer, NULL, (_cast_data)_E);
+timer_setup(&_E->_timer, NULL, 0);
|
-setup_timer(&_E._timer, NULL, &_E);
+timer_setup(&_E._timer, NULL, 0);
|
-setup_timer(&_E._timer, NULL, (_cast_data)&_E);
+timer_setup(&_E._timer, NULL, 0);
)
@change_timer_function_usage@
expression _E;
identifier _timer;
struct timer_list _stl;
identifier _callback;
type _cast_func, _cast_data;
@@
(
-setup_timer(&_E->_timer, _callback, _E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, &_callback, _E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, _callback, (_cast_data)_E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, &_callback, (_cast_data)_E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, (_cast_func)_callback, _E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, (_cast_func)&_callback, _E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, (_cast_func)_callback, (_cast_data)_E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, (_cast_func)&_callback, (_cast_data)_E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E._timer, _callback, (_cast_data)_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, _callback, (_cast_data)&_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, &_callback, (_cast_data)_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, &_callback, (_cast_data)&_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, (_cast_func)_callback, (_cast_data)_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, (_cast_func)_callback, (_cast_data)&_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, (_cast_func)&_callback, (_cast_data)_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, (_cast_func)&_callback, (_cast_data)&_E);
+timer_setup(&_E._timer, _callback, 0);
|
_E->_timer@_stl.function = _callback;
|
_E->_timer@_stl.function = &_callback;
|
_E->_timer@_stl.function = (_cast_func)_callback;
|
_E->_timer@_stl.function = (_cast_func)&_callback;
|
_E._timer@_stl.function = _callback;
|
_E._timer@_stl.function = &_callback;
|
_E._timer@_stl.function = (_cast_func)_callback;
|
_E._timer@_stl.function = (_cast_func)&_callback;
)
// callback(unsigned long arg)
@change_callback_handle_cast
depends on change_timer_function_usage@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._timer;
type _origtype;
identifier _origarg;
type _handletype;
identifier _handle;
@@
void _callback(
-_origtype _origarg
+struct timer_list *t
)
{
(
... when != _origarg
_handletype *_handle =
-(_handletype *)_origarg;
+from_timer(_handle, t, _timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle =
-(void *)_origarg;
+from_timer(_handle, t, _timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle;
... when != _handle
_handle =
-(_handletype *)_origarg;
+from_timer(_handle, t, _timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle;
... when != _handle
_handle =
-(void *)_origarg;
+from_timer(_handle, t, _timer);
... when != _origarg
)
}
// callback(unsigned long arg) without existing variable
@change_callback_handle_cast_no_arg
depends on change_timer_function_usage &&
!change_callback_handle_cast@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._timer;
type _origtype;
identifier _origarg;
type _handletype;
@@
void _callback(
-_origtype _origarg
+struct timer_list *t
)
{
+ _handletype *_origarg = from_timer(_origarg, t, _timer);
+
... when != _origarg
- (_handletype *)_origarg
+ _origarg
... when != _origarg
}
// Avoid already converted callbacks.
@match_callback_converted
depends on change_timer_function_usage &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg@
identifier change_timer_function_usage._callback;
identifier t;
@@
void _callback(struct timer_list *t)
{ ... }
// callback(struct something *handle)
@change_callback_handle_arg
depends on change_timer_function_usage &&
!match_callback_converted &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._timer;
type _handletype;
identifier _handle;
@@
void _callback(
-_handletype *_handle
+struct timer_list *t
)
{
+ _handletype *_handle = from_timer(_handle, t, _timer);
...
}
// If change_callback_handle_arg ran on an empty function, remove
// the added handler.
@unchange_callback_handle_arg
depends on change_timer_function_usage &&
change_callback_handle_arg@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._timer;
type _handletype;
identifier _handle;
identifier t;
@@
void _callback(struct timer_list *t)
{
- _handletype *_handle = from_timer(_handle, t, _timer);
}
// We only want to refactor the setup_timer() data argument if we've found
// the matching callback. This undoes changes in change_timer_function_usage.
@unchange_timer_function_usage
depends on change_timer_function_usage &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg &&
!change_callback_handle_arg@
expression change_timer_function_usage._E;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type change_timer_function_usage._cast_data;
@@
(
-timer_setup(&_E->_timer, _callback, 0);
+setup_timer(&_E->_timer, _callback, (_cast_data)_E);
|
-timer_setup(&_E._timer, _callback, 0);
+setup_timer(&_E._timer, _callback, (_cast_data)&_E);
)
// If we fixed a callback from a .function assignment, fix the
// assignment cast now.
@change_timer_function_assignment
depends on change_timer_function_usage &&
(change_callback_handle_cast ||
change_callback_handle_cast_no_arg ||
change_callback_handle_arg)@
expression change_timer_function_usage._E;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type _cast_func;
typedef TIMER_FUNC_TYPE;
@@
(
_E->_timer.function =
-_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_timer.function =
-&_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_timer.function =
-(_cast_func)_callback;
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_timer.function =
-(_cast_func)&_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._timer.function =
-_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._timer.function =
-&_callback;
+(TIMER_FUNC_TYPE)_callback
;
|
_E._timer.function =
-(_cast_func)_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._timer.function =
-(_cast_func)&_callback
+(TIMER_FUNC_TYPE)_callback
;
)
// Sometimes timer functions are called directly. Replace matched args.
@change_timer_function_calls
depends on change_timer_function_usage &&
(change_callback_handle_cast ||
change_callback_handle_cast_no_arg ||
change_callback_handle_arg)@
expression _E;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type _cast_data;
@@
_callback(
(
-(_cast_data)_E
+&_E->_timer
|
-(_cast_data)&_E
+&_E._timer
|
-_E
+&_E->_timer
)
)
// If a timer has been configured without a data argument, it can be
// converted without regard to the callback argument, since it is unused.
@match_timer_function_unused_data@
expression _E;
identifier _timer;
identifier _callback;
@@
(
-setup_timer(&_E->_timer, _callback, 0);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, _callback, 0L);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, _callback, 0UL);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E._timer, _callback, 0);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, _callback, 0L);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, _callback, 0UL);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_timer, _callback, 0);
+timer_setup(&_timer, _callback, 0);
|
-setup_timer(&_timer, _callback, 0L);
+timer_setup(&_timer, _callback, 0);
|
-setup_timer(&_timer, _callback, 0UL);
+timer_setup(&_timer, _callback, 0);
|
-setup_timer(_timer, _callback, 0);
+timer_setup(_timer, _callback, 0);
|
-setup_timer(_timer, _callback, 0L);
+timer_setup(_timer, _callback, 0);
|
-setup_timer(_timer, _callback, 0UL);
+timer_setup(_timer, _callback, 0);
)
@change_callback_unused_data
depends on match_timer_function_unused_data@
identifier match_timer_function_unused_data._callback;
type _origtype;
identifier _origarg;
@@
void _callback(
-_origtype _origarg
+struct timer_list *unused
)
{
... when != _origarg
}
Signed-off-by: Kees Cook <keescook@chromium.org>
2017-10-17 05:43:17 +08:00
static void clocksource_watchdog ( struct timer_list * unused )
2007-02-16 17:27:43 +08:00
{
2016-12-22 03:32:01 +08:00
u64 csnow , wdnow , cslast , wdlast , delta ;
2011-09-12 19:32:23 +08:00
int next_cpu , reset_pending ;
clocksource: Retry clock read if long delays detected
When the clocksource watchdog marks a clock as unstable, this might be due
to that clock being unstable or it might be due to delays that happen to
occur between the reads of the two clocks. Yes, interrupts are disabled
across those two reads, but there are no shortage of things that can delay
interrupts-disabled regions of code ranging from SMI handlers to vCPU
preemption. It would be good to have some indication as to why the clock
was marked unstable.
Therefore, re-read the watchdog clock on either side of the read from the
clock under test. If the watchdog clock shows an excessive time delta
between its pair of reads, the reads are retried.
The maximum number of retries is specified by a new kernel boot parameter
clocksource.max_cswd_read_retries, which defaults to three, that is, up to
four reads, one initial and up to three retries. If more than one retry
was required, a message is printed on the console (the occasional single
retry is expected behavior, especially in guest OSes). If the maximum
number of retries is exceeded, the clock under test will be marked
unstable. However, the probability of this happening due to various sorts
of delays is quite small. In addition, the reason (clock-read delays) for
the unstable marking will be apparent.
Reported-by: Chris Mason <clm@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-1-paulmck@kernel.org
2021-05-28 03:01:19 +08:00
int64_t wd_nsec , cs_nsec ;
struct clocksource * cs ;
clocksource: Avoid accidental unstable marking of clocksources
Since commit db3a34e17433 ("clocksource: Retry clock read if long delays
detected") and commit 2e27e793e280 ("clocksource: Reduce clocksource-skew
threshold"), it is found that tsc clocksource fallback to hpet can
sometimes happen on both Intel and AMD systems especially when they are
running stressful benchmarking workloads. Of the 23 systems tested with
a v5.14 kernel, 10 of them have switched to hpet clock source during
the test run.
The result of falling back to hpet is a drastic reduction of performance
when running benchmarks. For example, the fio performance tests can
drop up to 70% whereas the iperf3 performance can drop up to 80%.
4 hpet fallbacks happened during bootup. They were:
[ 8.749399] clocksource: timekeeping watchdog on CPU13: hpet read-back delay of 263750ns, attempt 4, marking unstable
[ 12.044610] clocksource: timekeeping watchdog on CPU19: hpet read-back delay of 186166ns, attempt 4, marking unstable
[ 17.336941] clocksource: timekeeping watchdog on CPU28: hpet read-back delay of 182291ns, attempt 4, marking unstable
[ 17.518565] clocksource: timekeeping watchdog on CPU34: hpet read-back delay of 252196ns, attempt 4, marking unstable
Other fallbacks happen when the systems were running stressful
benchmarks. For example:
[ 2685.867873] clocksource: timekeeping watchdog on CPU117: hpet read-back delay of 57269ns, attempt 4, marking unstable
[46215.471228] clocksource: timekeeping watchdog on CPU8: hpet read-back delay of 61460ns, attempt 4, marking unstable
Commit 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold"),
changed the skew margin from 100us to 50us. I think this is too small
and can easily be exceeded when running some stressful workloads on a
thermally stressed system. So it is switched back to 100us.
Even a maximum skew margin of 100us may be too small in for some systems
when booting up especially if those systems are under thermal stress. To
eliminate the case that the large skew is due to the system being too
busy slowing down the reading of both the watchdog and the clocksource,
an extra consecutive read of watchdog clock is being done to check this.
The consecutive watchdog read delay is compared against
WATCHDOG_MAX_SKEW/2. If the delay exceeds the limit, we assume that
the system is just too busy. A warning will be printed to the console
and the clock skew check is skipped for this round.
Fixes: db3a34e17433 ("clocksource: Retry clock read if long delays detected")
Fixes: 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold")
Signed-off-by: Waiman Long <longman@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-19 03:14:36 +08:00
enum wd_read_status read_ret ;
clocksource: Reduce clocksource-skew threshold
Currently, WATCHDOG_THRESHOLD is set to detect a 62.5-millisecond skew in
a 500-millisecond WATCHDOG_INTERVAL. This requires that clocks be skewed
by more than 12.5% in order to be marked unstable. Except that a clock
that is skewed by that much is probably destroying unsuspecting software
right and left. And given that there are now checks for false-positive
skews due to delays between reading the two clocks, it should be possible
to greatly decrease WATCHDOG_THRESHOLD, at least for fine-grained clocks
such as TSC.
Therefore, add a new uncertainty_margin field to the clocksource structure
that contains the maximum uncertainty in nanoseconds for the corresponding
clock. This field may be initialized manually, as it is for
clocksource_tsc_early and clocksource_jiffies, which is copied to
refined_jiffies. If the field is not initialized manually, it will be
computed at clock-registry time as the period of the clock in question
based on the scale and freq parameters to __clocksource_update_freq_scale()
function. If either of those two parameters are zero, the
tens-of-milliseconds WATCHDOG_THRESHOLD is used as a cowardly alternative
to dividing by zero. No matter how the uncertainty_margin field is
calculated, it is bounded below by twice WATCHDOG_MAX_SKEW, that is, by 100
microseconds.
Note that manually initialized uncertainty_margin fields are not adjusted,
but there is a WARN_ON_ONCE() that triggers if any such field is less than
twice WATCHDOG_MAX_SKEW. This WARN_ON_ONCE() is intended to discourage
production use of the one-nanosecond uncertainty_margin values that are
used to test the clock-skew code itself.
The actual clock-skew check uses the sum of the uncertainty_margin fields
of the two clocksource structures being compared. Integer overflow is
avoided because the largest computed value of the uncertainty_margin
fields is one billion (10^9), and double that value fits into an
unsigned int. However, if someone manually specifies (say) UINT_MAX,
they will get what they deserve.
Note that the refined_jiffies uncertainty_margin field is initialized to
TICK_NSEC, which means that skew checks involving this clocksource will
be sufficently forgiving. In a similar vein, the clocksource_tsc_early
uncertainty_margin field is initialized to 32*NSEC_PER_MSEC, which
replicates the current behavior and allows custom setting if needed
in order to address the rare skews detected for this clocksource in
current mainline.
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-4-paulmck@kernel.org
2021-05-28 03:01:22 +08:00
u32 md ;
2007-02-16 17:27:43 +08:00
spin_lock ( & watchdog_lock ) ;
2009-08-14 21:47:24 +08:00
if ( ! watchdog_running )
goto out ;
2007-02-16 17:27:43 +08:00
2011-09-12 19:32:23 +08:00
reset_pending = atomic_read ( & watchdog_reset_pending ) ;
2009-08-14 21:47:25 +08:00
list_for_each_entry ( cs , & watchdog_list , wd_list ) {
/* Clocksource already marked unstable? */
2009-08-18 23:09:42 +08:00
if ( cs - > flags & CLOCK_SOURCE_UNSTABLE ) {
2009-09-15 01:49:02 +08:00
if ( finished_booting )
schedule_work ( & watchdog_work ) ;
2009-08-14 21:47:25 +08:00
continue ;
2009-08-18 23:09:42 +08:00
}
2009-08-14 21:47:25 +08:00
clocksource: Avoid accidental unstable marking of clocksources
Since commit db3a34e17433 ("clocksource: Retry clock read if long delays
detected") and commit 2e27e793e280 ("clocksource: Reduce clocksource-skew
threshold"), it is found that tsc clocksource fallback to hpet can
sometimes happen on both Intel and AMD systems especially when they are
running stressful benchmarking workloads. Of the 23 systems tested with
a v5.14 kernel, 10 of them have switched to hpet clock source during
the test run.
The result of falling back to hpet is a drastic reduction of performance
when running benchmarks. For example, the fio performance tests can
drop up to 70% whereas the iperf3 performance can drop up to 80%.
4 hpet fallbacks happened during bootup. They were:
[ 8.749399] clocksource: timekeeping watchdog on CPU13: hpet read-back delay of 263750ns, attempt 4, marking unstable
[ 12.044610] clocksource: timekeeping watchdog on CPU19: hpet read-back delay of 186166ns, attempt 4, marking unstable
[ 17.336941] clocksource: timekeeping watchdog on CPU28: hpet read-back delay of 182291ns, attempt 4, marking unstable
[ 17.518565] clocksource: timekeeping watchdog on CPU34: hpet read-back delay of 252196ns, attempt 4, marking unstable
Other fallbacks happen when the systems were running stressful
benchmarks. For example:
[ 2685.867873] clocksource: timekeeping watchdog on CPU117: hpet read-back delay of 57269ns, attempt 4, marking unstable
[46215.471228] clocksource: timekeeping watchdog on CPU8: hpet read-back delay of 61460ns, attempt 4, marking unstable
Commit 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold"),
changed the skew margin from 100us to 50us. I think this is too small
and can easily be exceeded when running some stressful workloads on a
thermally stressed system. So it is switched back to 100us.
Even a maximum skew margin of 100us may be too small in for some systems
when booting up especially if those systems are under thermal stress. To
eliminate the case that the large skew is due to the system being too
busy slowing down the reading of both the watchdog and the clocksource,
an extra consecutive read of watchdog clock is being done to check this.
The consecutive watchdog read delay is compared against
WATCHDOG_MAX_SKEW/2. If the delay exceeds the limit, we assume that
the system is just too busy. A warning will be printed to the console
and the clock skew check is skipped for this round.
Fixes: db3a34e17433 ("clocksource: Retry clock read if long delays detected")
Fixes: 2e27e793e280 ("clocksource: Reduce clocksource-skew threshold")
Signed-off-by: Waiman Long <longman@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-19 03:14:36 +08:00
read_ret = cs_watchdog_read ( cs , & csnow , & wdnow ) ;
if ( read_ret ! = WD_READ_SUCCESS ) {
if ( read_ret = = WD_READ_UNSTABLE )
/* Clock readout unreliable, so give it up. */
__clocksource_unstable ( cs ) ;
clocksource: Retry clock read if long delays detected
When the clocksource watchdog marks a clock as unstable, this might be due
to that clock being unstable or it might be due to delays that happen to
occur between the reads of the two clocks. Yes, interrupts are disabled
across those two reads, but there are no shortage of things that can delay
interrupts-disabled regions of code ranging from SMI handlers to vCPU
preemption. It would be good to have some indication as to why the clock
was marked unstable.
Therefore, re-read the watchdog clock on either side of the read from the
clock under test. If the watchdog clock shows an excessive time delta
between its pair of reads, the reads are retried.
The maximum number of retries is specified by a new kernel boot parameter
clocksource.max_cswd_read_retries, which defaults to three, that is, up to
four reads, one initial and up to three retries. If more than one retry
was required, a message is printed on the console (the occasional single
retry is expected behavior, especially in guest OSes). If the maximum
number of retries is exceeded, the clock under test will be marked
unstable. However, the probability of this happening due to various sorts
of delays is quite small. In addition, the reason (clock-read delays) for
the unstable marking will be apparent.
Reported-by: Chris Mason <clm@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-1-paulmck@kernel.org
2021-05-28 03:01:19 +08:00
continue ;
}
2007-05-09 17:35:15 +08:00
2009-08-14 21:47:22 +08:00
/* Clocksource initialized ? */
2011-09-12 19:32:23 +08:00
if ( ! ( cs - > flags & CLOCK_SOURCE_WATCHDOG ) | |
atomic_read ( & watchdog_reset_pending ) ) {
2009-08-14 21:47:22 +08:00
cs - > flags | = CLOCK_SOURCE_WATCHDOG ;
2011-06-16 22:22:08 +08:00
cs - > wd_last = wdnow ;
cs - > cs_last = csnow ;
2007-05-09 17:35:15 +08:00
continue ;
}
2014-07-17 05:05:10 +08:00
delta = clocksource_delta ( wdnow , cs - > wd_last , watchdog - > mask ) ;
wd_nsec = clocksource_cyc2ns ( delta , watchdog - > mult ,
watchdog - > shift ) ;
2011-06-16 22:22:08 +08:00
2014-07-17 05:05:10 +08:00
delta = clocksource_delta ( csnow , cs - > cs_last , cs - > mask ) ;
cs_nsec = clocksource_cyc2ns ( delta , cs - > mult , cs - > shift ) ;
2015-03-12 12:16:36 +08:00
wdlast = cs - > wd_last ; /* save these in case we print them */
cslast = cs - > cs_last ;
2011-06-16 22:22:08 +08:00
cs - > cs_last = csnow ;
cs - > wd_last = wdnow ;
2011-09-12 19:32:23 +08:00
if ( atomic_read ( & watchdog_reset_pending ) )
continue ;
2011-06-16 22:22:08 +08:00
/* Check the deviation from the watchdog clocksource. */
clocksource: Reduce clocksource-skew threshold
Currently, WATCHDOG_THRESHOLD is set to detect a 62.5-millisecond skew in
a 500-millisecond WATCHDOG_INTERVAL. This requires that clocks be skewed
by more than 12.5% in order to be marked unstable. Except that a clock
that is skewed by that much is probably destroying unsuspecting software
right and left. And given that there are now checks for false-positive
skews due to delays between reading the two clocks, it should be possible
to greatly decrease WATCHDOG_THRESHOLD, at least for fine-grained clocks
such as TSC.
Therefore, add a new uncertainty_margin field to the clocksource structure
that contains the maximum uncertainty in nanoseconds for the corresponding
clock. This field may be initialized manually, as it is for
clocksource_tsc_early and clocksource_jiffies, which is copied to
refined_jiffies. If the field is not initialized manually, it will be
computed at clock-registry time as the period of the clock in question
based on the scale and freq parameters to __clocksource_update_freq_scale()
function. If either of those two parameters are zero, the
tens-of-milliseconds WATCHDOG_THRESHOLD is used as a cowardly alternative
to dividing by zero. No matter how the uncertainty_margin field is
calculated, it is bounded below by twice WATCHDOG_MAX_SKEW, that is, by 100
microseconds.
Note that manually initialized uncertainty_margin fields are not adjusted,
but there is a WARN_ON_ONCE() that triggers if any such field is less than
twice WATCHDOG_MAX_SKEW. This WARN_ON_ONCE() is intended to discourage
production use of the one-nanosecond uncertainty_margin values that are
used to test the clock-skew code itself.
The actual clock-skew check uses the sum of the uncertainty_margin fields
of the two clocksource structures being compared. Integer overflow is
avoided because the largest computed value of the uncertainty_margin
fields is one billion (10^9), and double that value fits into an
unsigned int. However, if someone manually specifies (say) UINT_MAX,
they will get what they deserve.
Note that the refined_jiffies uncertainty_margin field is initialized to
TICK_NSEC, which means that skew checks involving this clocksource will
be sufficently forgiving. In a similar vein, the clocksource_tsc_early
uncertainty_margin field is initialized to 32*NSEC_PER_MSEC, which
replicates the current behavior and allows custom setting if needed
in order to address the rare skews detected for this clocksource in
current mainline.
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-4-paulmck@kernel.org
2021-05-28 03:01:22 +08:00
md = cs - > uncertainty_margin + watchdog - > uncertainty_margin ;
if ( abs ( cs_nsec - wd_nsec ) > md ) {
2022-12-14 08:42:15 +08:00
u64 cs_wd_msec ;
u64 wd_msec ;
u32 wd_rem ;
2015-09-10 17:01:56 +08:00
pr_warn ( " timekeeping watchdog on CPU%d: Marking clocksource '%s' as unstable because the skew is too large: \n " ,
smp_processor_id ( ) , cs - > name ) ;
2021-05-28 03:01:24 +08:00
pr_warn ( " '%s' wd_nsec: %lld wd_now: %llx wd_last: %llx mask: %llx \n " ,
watchdog - > name , wd_nsec , wdnow , wdlast , watchdog - > mask ) ;
pr_warn ( " '%s' cs_nsec: %lld cs_now: %llx cs_last: %llx mask: %llx \n " ,
cs - > name , cs_nsec , csnow , cslast , cs - > mask ) ;
2022-12-14 08:42:15 +08:00
cs_wd_msec = div_u64_rem ( cs_nsec - wd_nsec , 1000U * 1000U , & wd_rem ) ;
wd_msec = div_u64_rem ( wd_nsec , 1000U * 1000U , & wd_rem ) ;
pr_warn ( " Clocksource '%s' skewed %lld ns (%lld ms) over watchdog '%s' interval of %lld ns (%lld ms) \n " ,
cs - > name , cs_nsec - wd_nsec , cs_wd_msec , watchdog - > name , wd_nsec , wd_msec ) ;
2021-05-28 03:01:21 +08:00
if ( curr_clocksource = = cs )
pr_warn ( " '%s' is current clocksource. \n " , cs - > name ) ;
else if ( curr_clocksource )
pr_warn ( " '%s' (not '%s') is current clocksource. \n " , curr_clocksource - > name , cs - > name ) ;
else
pr_warn ( " No current clocksource. \n " ) ;
2015-03-12 12:16:36 +08:00
__clocksource_unstable ( cs ) ;
2009-08-14 21:47:22 +08:00
continue ;
}
2017-04-21 18:14:13 +08:00
if ( cs = = curr_clocksource & & cs - > tick_stable )
cs - > tick_stable ( cs ) ;
2009-08-14 21:47:22 +08:00
if ( ! ( cs - > flags & CLOCK_SOURCE_VALID_FOR_HRES ) & &
( cs - > flags & CLOCK_SOURCE_IS_CONTINUOUS ) & &
( watchdog - > flags & CLOCK_SOURCE_IS_CONTINUOUS ) ) {
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
/* Mark it valid for high-res. */
2009-08-14 21:47:22 +08:00
cs - > flags | = CLOCK_SOURCE_VALID_FOR_HRES ;
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
/*
* clocksource_done_booting ( ) will sort it if
* finished_booting is not set yet .
*/
if ( ! finished_booting )
continue ;
2009-08-14 21:47:22 +08:00
/*
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
* If this is not the current clocksource let
* the watchdog thread reselect it . Due to the
* change to high res this clocksource might
* be preferred now . If it is the current
* clocksource let the tick code know about
* that change .
2009-08-14 21:47:22 +08:00
*/
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
if ( cs ! = curr_clocksource ) {
cs - > flags | = CLOCK_SOURCE_RESELECT ;
schedule_work ( & watchdog_work ) ;
} else {
tick_clock_notify ( ) ;
}
2007-02-16 17:27:43 +08:00
}
}
2011-09-12 19:32:23 +08:00
/*
* We only clear the watchdog_reset_pending , when we did a
* full cycle through all clocksources .
*/
if ( reset_pending )
atomic_dec ( & watchdog_reset_pending ) ;
2009-08-14 21:47:25 +08:00
/*
* Cycle through CPUs to check if the CPUs stay synchronized
* to each other .
*/
next_cpu = cpumask_next ( raw_smp_processor_id ( ) , cpu_online_mask ) ;
if ( next_cpu > = nr_cpu_ids )
next_cpu = cpumask_first ( cpu_online_mask ) ;
2020-02-01 00:08:59 +08:00
/*
* Arm timer if not already pending : could race with concurrent
* pair clocksource_stop_watchdog ( ) clocksource_start_watchdog ( ) .
*/
if ( ! timer_pending ( & watchdog_timer ) ) {
watchdog_timer . expires + = WATCHDOG_INTERVAL ;
add_timer_on ( & watchdog_timer , next_cpu ) ;
}
2009-08-14 21:47:24 +08:00
out :
2007-02-16 17:27:43 +08:00
spin_unlock ( & watchdog_lock ) ;
}
2009-08-14 21:47:23 +08:00
2009-08-14 21:47:24 +08:00
static inline void clocksource_start_watchdog ( void )
{
if ( watchdog_running | | ! watchdog | | list_empty ( & watchdog_list ) )
return ;
treewide: setup_timer() -> timer_setup()
This converts all remaining cases of the old setup_timer() API into using
timer_setup(), where the callback argument is the structure already
holding the struct timer_list. These should have no behavioral changes,
since they just change which pointer is passed into the callback with
the same available pointers after conversion. It handles the following
examples, in addition to some other variations.
Casting from unsigned long:
void my_callback(unsigned long data)
{
struct something *ptr = (struct something *)data;
...
}
...
setup_timer(&ptr->my_timer, my_callback, ptr);
and forced object casts:
void my_callback(struct something *ptr)
{
...
}
...
setup_timer(&ptr->my_timer, my_callback, (unsigned long)ptr);
become:
void my_callback(struct timer_list *t)
{
struct something *ptr = from_timer(ptr, t, my_timer);
...
}
...
timer_setup(&ptr->my_timer, my_callback, 0);
Direct function assignments:
void my_callback(unsigned long data)
{
struct something *ptr = (struct something *)data;
...
}
...
ptr->my_timer.function = my_callback;
have a temporary cast added, along with converting the args:
void my_callback(struct timer_list *t)
{
struct something *ptr = from_timer(ptr, t, my_timer);
...
}
...
ptr->my_timer.function = (TIMER_FUNC_TYPE)my_callback;
And finally, callbacks without a data assignment:
void my_callback(unsigned long data)
{
...
}
...
setup_timer(&ptr->my_timer, my_callback, 0);
have their argument renamed to verify they're unused during conversion:
void my_callback(struct timer_list *unused)
{
...
}
...
timer_setup(&ptr->my_timer, my_callback, 0);
The conversion is done with the following Coccinelle script:
spatch --very-quiet --all-includes --include-headers \
-I ./arch/x86/include -I ./arch/x86/include/generated \
-I ./include -I ./arch/x86/include/uapi \
-I ./arch/x86/include/generated/uapi -I ./include/uapi \
-I ./include/generated/uapi --include ./include/linux/kconfig.h \
--dir . \
--cocci-file ~/src/data/timer_setup.cocci
@fix_address_of@
expression e;
@@
setup_timer(
-&(e)
+&e
, ...)
// Update any raw setup_timer() usages that have a NULL callback, but
// would otherwise match change_timer_function_usage, since the latter
// will update all function assignments done in the face of a NULL
// function initialization in setup_timer().
@change_timer_function_usage_NULL@
expression _E;
identifier _timer;
type _cast_data;
@@
(
-setup_timer(&_E->_timer, NULL, _E);
+timer_setup(&_E->_timer, NULL, 0);
|
-setup_timer(&_E->_timer, NULL, (_cast_data)_E);
+timer_setup(&_E->_timer, NULL, 0);
|
-setup_timer(&_E._timer, NULL, &_E);
+timer_setup(&_E._timer, NULL, 0);
|
-setup_timer(&_E._timer, NULL, (_cast_data)&_E);
+timer_setup(&_E._timer, NULL, 0);
)
@change_timer_function_usage@
expression _E;
identifier _timer;
struct timer_list _stl;
identifier _callback;
type _cast_func, _cast_data;
@@
(
-setup_timer(&_E->_timer, _callback, _E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, &_callback, _E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, _callback, (_cast_data)_E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, &_callback, (_cast_data)_E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, (_cast_func)_callback, _E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, (_cast_func)&_callback, _E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, (_cast_func)_callback, (_cast_data)_E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, (_cast_func)&_callback, (_cast_data)_E);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E._timer, _callback, (_cast_data)_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, _callback, (_cast_data)&_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, &_callback, (_cast_data)_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, &_callback, (_cast_data)&_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, (_cast_func)_callback, (_cast_data)_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, (_cast_func)_callback, (_cast_data)&_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, (_cast_func)&_callback, (_cast_data)_E);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, (_cast_func)&_callback, (_cast_data)&_E);
+timer_setup(&_E._timer, _callback, 0);
|
_E->_timer@_stl.function = _callback;
|
_E->_timer@_stl.function = &_callback;
|
_E->_timer@_stl.function = (_cast_func)_callback;
|
_E->_timer@_stl.function = (_cast_func)&_callback;
|
_E._timer@_stl.function = _callback;
|
_E._timer@_stl.function = &_callback;
|
_E._timer@_stl.function = (_cast_func)_callback;
|
_E._timer@_stl.function = (_cast_func)&_callback;
)
// callback(unsigned long arg)
@change_callback_handle_cast
depends on change_timer_function_usage@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._timer;
type _origtype;
identifier _origarg;
type _handletype;
identifier _handle;
@@
void _callback(
-_origtype _origarg
+struct timer_list *t
)
{
(
... when != _origarg
_handletype *_handle =
-(_handletype *)_origarg;
+from_timer(_handle, t, _timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle =
-(void *)_origarg;
+from_timer(_handle, t, _timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle;
... when != _handle
_handle =
-(_handletype *)_origarg;
+from_timer(_handle, t, _timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle;
... when != _handle
_handle =
-(void *)_origarg;
+from_timer(_handle, t, _timer);
... when != _origarg
)
}
// callback(unsigned long arg) without existing variable
@change_callback_handle_cast_no_arg
depends on change_timer_function_usage &&
!change_callback_handle_cast@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._timer;
type _origtype;
identifier _origarg;
type _handletype;
@@
void _callback(
-_origtype _origarg
+struct timer_list *t
)
{
+ _handletype *_origarg = from_timer(_origarg, t, _timer);
+
... when != _origarg
- (_handletype *)_origarg
+ _origarg
... when != _origarg
}
// Avoid already converted callbacks.
@match_callback_converted
depends on change_timer_function_usage &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg@
identifier change_timer_function_usage._callback;
identifier t;
@@
void _callback(struct timer_list *t)
{ ... }
// callback(struct something *handle)
@change_callback_handle_arg
depends on change_timer_function_usage &&
!match_callback_converted &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._timer;
type _handletype;
identifier _handle;
@@
void _callback(
-_handletype *_handle
+struct timer_list *t
)
{
+ _handletype *_handle = from_timer(_handle, t, _timer);
...
}
// If change_callback_handle_arg ran on an empty function, remove
// the added handler.
@unchange_callback_handle_arg
depends on change_timer_function_usage &&
change_callback_handle_arg@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._timer;
type _handletype;
identifier _handle;
identifier t;
@@
void _callback(struct timer_list *t)
{
- _handletype *_handle = from_timer(_handle, t, _timer);
}
// We only want to refactor the setup_timer() data argument if we've found
// the matching callback. This undoes changes in change_timer_function_usage.
@unchange_timer_function_usage
depends on change_timer_function_usage &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg &&
!change_callback_handle_arg@
expression change_timer_function_usage._E;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type change_timer_function_usage._cast_data;
@@
(
-timer_setup(&_E->_timer, _callback, 0);
+setup_timer(&_E->_timer, _callback, (_cast_data)_E);
|
-timer_setup(&_E._timer, _callback, 0);
+setup_timer(&_E._timer, _callback, (_cast_data)&_E);
)
// If we fixed a callback from a .function assignment, fix the
// assignment cast now.
@change_timer_function_assignment
depends on change_timer_function_usage &&
(change_callback_handle_cast ||
change_callback_handle_cast_no_arg ||
change_callback_handle_arg)@
expression change_timer_function_usage._E;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type _cast_func;
typedef TIMER_FUNC_TYPE;
@@
(
_E->_timer.function =
-_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_timer.function =
-&_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_timer.function =
-(_cast_func)_callback;
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_timer.function =
-(_cast_func)&_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._timer.function =
-_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._timer.function =
-&_callback;
+(TIMER_FUNC_TYPE)_callback
;
|
_E._timer.function =
-(_cast_func)_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._timer.function =
-(_cast_func)&_callback
+(TIMER_FUNC_TYPE)_callback
;
)
// Sometimes timer functions are called directly. Replace matched args.
@change_timer_function_calls
depends on change_timer_function_usage &&
(change_callback_handle_cast ||
change_callback_handle_cast_no_arg ||
change_callback_handle_arg)@
expression _E;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type _cast_data;
@@
_callback(
(
-(_cast_data)_E
+&_E->_timer
|
-(_cast_data)&_E
+&_E._timer
|
-_E
+&_E->_timer
)
)
// If a timer has been configured without a data argument, it can be
// converted without regard to the callback argument, since it is unused.
@match_timer_function_unused_data@
expression _E;
identifier _timer;
identifier _callback;
@@
(
-setup_timer(&_E->_timer, _callback, 0);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, _callback, 0L);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E->_timer, _callback, 0UL);
+timer_setup(&_E->_timer, _callback, 0);
|
-setup_timer(&_E._timer, _callback, 0);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, _callback, 0L);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_E._timer, _callback, 0UL);
+timer_setup(&_E._timer, _callback, 0);
|
-setup_timer(&_timer, _callback, 0);
+timer_setup(&_timer, _callback, 0);
|
-setup_timer(&_timer, _callback, 0L);
+timer_setup(&_timer, _callback, 0);
|
-setup_timer(&_timer, _callback, 0UL);
+timer_setup(&_timer, _callback, 0);
|
-setup_timer(_timer, _callback, 0);
+timer_setup(_timer, _callback, 0);
|
-setup_timer(_timer, _callback, 0L);
+timer_setup(_timer, _callback, 0);
|
-setup_timer(_timer, _callback, 0UL);
+timer_setup(_timer, _callback, 0);
)
@change_callback_unused_data
depends on match_timer_function_unused_data@
identifier match_timer_function_unused_data._callback;
type _origtype;
identifier _origarg;
@@
void _callback(
-_origtype _origarg
+struct timer_list *unused
)
{
... when != _origarg
}
Signed-off-by: Kees Cook <keescook@chromium.org>
2017-10-17 05:43:17 +08:00
timer_setup ( & watchdog_timer , clocksource_watchdog , 0 ) ;
2009-08-14 21:47:24 +08:00
watchdog_timer . expires = jiffies + WATCHDOG_INTERVAL ;
add_timer_on ( & watchdog_timer , cpumask_first ( cpu_online_mask ) ) ;
watchdog_running = 1 ;
}
static inline void clocksource_stop_watchdog ( void )
{
if ( ! watchdog_running | | ( watchdog & & ! list_empty ( & watchdog_list ) ) )
return ;
del_timer ( & watchdog_timer ) ;
watchdog_running = 0 ;
}
2009-08-14 21:47:23 +08:00
static inline void clocksource_reset_watchdog ( void )
{
struct clocksource * cs ;
list_for_each_entry ( cs , & watchdog_list , wd_list )
cs - > flags & = ~ CLOCK_SOURCE_WATCHDOG ;
}
2007-05-09 17:35:15 +08:00
static void clocksource_resume_watchdog ( void )
{
2011-09-12 19:32:23 +08:00
atomic_inc ( & watchdog_reset_pending ) ;
2007-05-09 17:35:15 +08:00
}
2009-08-14 21:47:24 +08:00
static void clocksource_enqueue_watchdog ( struct clocksource * cs )
2007-02-16 17:27:43 +08:00
{
2018-04-30 18:00:11 +08:00
INIT_LIST_HEAD ( & cs - > wd_list ) ;
2007-02-16 17:27:43 +08:00
if ( cs - > flags & CLOCK_SOURCE_MUST_VERIFY ) {
2009-08-14 21:47:24 +08:00
/* cs is a clocksource to be watched. */
2007-02-16 17:27:43 +08:00
list_add ( & cs - > wd_list , & watchdog_list ) ;
2009-08-14 21:47:24 +08:00
cs - > flags & = ~ CLOCK_SOURCE_WATCHDOG ;
2007-03-25 20:42:51 +08:00
} else {
2009-08-14 21:47:24 +08:00
/* cs is a watchdog. */
2007-03-25 20:42:51 +08:00
if ( cs - > flags & CLOCK_SOURCE_IS_CONTINUOUS )
2007-02-16 17:27:43 +08:00
cs - > flags | = CLOCK_SOURCE_VALID_FOR_HRES ;
2016-01-23 01:31:53 +08:00
}
}
static void clocksource_select_watchdog ( bool fallback )
{
struct clocksource * cs , * old_wd ;
unsigned long flags ;
spin_lock_irqsave ( & watchdog_lock , flags ) ;
/* save current watchdog */
old_wd = watchdog ;
if ( fallback )
watchdog = NULL ;
list_for_each_entry ( cs , & clocksource_list , list ) {
/* cs is a clocksource to be watched. */
if ( cs - > flags & CLOCK_SOURCE_MUST_VERIFY )
continue ;
/* Skip current if we were requested for a fallback. */
if ( fallback & & cs = = old_wd )
continue ;
2009-08-14 21:47:24 +08:00
/* Pick the best watchdog. */
2016-01-23 01:31:53 +08:00
if ( ! watchdog | | cs - > rating > watchdog - > rating )
2007-02-16 17:27:43 +08:00
watchdog = cs ;
}
2016-01-23 01:31:53 +08:00
/* If we failed to find a fallback restore the old one. */
if ( ! watchdog )
watchdog = old_wd ;
/* If we changed the watchdog we need to reset cycles. */
if ( watchdog ! = old_wd )
clocksource_reset_watchdog ( ) ;
2009-08-14 21:47:24 +08:00
/* Check if the watchdog timer needs to be started. */
clocksource_start_watchdog ( ) ;
2007-02-16 17:27:43 +08:00
spin_unlock_irqrestore ( & watchdog_lock , flags ) ;
}
2009-08-14 21:47:24 +08:00
static void clocksource_dequeue_watchdog ( struct clocksource * cs )
{
2013-04-26 04:31:46 +08:00
if ( cs ! = watchdog ) {
if ( cs - > flags & CLOCK_SOURCE_MUST_VERIFY ) {
/* cs is a watched clocksource. */
list_del_init ( & cs - > wd_list ) ;
/* Check if the watchdog timer needs to be stopped. */
clocksource_stop_watchdog ( ) ;
2009-08-14 21:47:24 +08:00
}
}
}
2018-09-05 16:41:58 +08:00
static int __clocksource_watchdog_kthread ( void )
2009-08-14 21:47:25 +08:00
{
struct clocksource * cs , * tmp ;
unsigned long flags ;
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
int select = 0 ;
2009-08-14 21:47:25 +08:00
2021-05-28 03:01:20 +08:00
/* Do any required per-CPU skew verification. */
if ( curr_clocksource & &
curr_clocksource - > flags & CLOCK_SOURCE_UNSTABLE & &
curr_clocksource - > flags & CLOCK_SOURCE_VERIFY_PERCPU )
clocksource_verify_percpu ( curr_clocksource ) ;
2009-08-14 21:47:25 +08:00
spin_lock_irqsave ( & watchdog_lock , flags ) ;
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
list_for_each_entry_safe ( cs , tmp , & watchdog_list , wd_list ) {
2009-08-14 21:47:25 +08:00
if ( cs - > flags & CLOCK_SOURCE_UNSTABLE ) {
list_del_init ( & cs - > wd_list ) ;
2018-04-23 23:28:55 +08:00
__clocksource_change_rating ( cs , 0 ) ;
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
select = 1 ;
}
if ( cs - > flags & CLOCK_SOURCE_RESELECT ) {
cs - > flags & = ~ CLOCK_SOURCE_RESELECT ;
select = 1 ;
2009-08-14 21:47:25 +08:00
}
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
}
2009-08-14 21:47:25 +08:00
/* Check if the watchdog timer needs to be stopped. */
clocksource_stop_watchdog ( ) ;
2009-08-15 19:20:42 +08:00
spin_unlock_irqrestore ( & watchdog_lock , flags ) ;
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
return select ;
}
2018-09-05 16:41:58 +08:00
static int clocksource_watchdog_kthread ( void * data )
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
{
mutex_lock ( & clocksource_mutex ) ;
2018-09-05 16:41:58 +08:00
if ( __clocksource_watchdog_kthread ( ) )
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
clocksource_select ( ) ;
2009-08-19 17:26:09 +08:00
mutex_unlock ( & clocksource_mutex ) ;
2018-09-05 16:41:58 +08:00
return 0 ;
2009-08-14 21:47:25 +08:00
}
2013-04-26 04:31:46 +08:00
static bool clocksource_is_watchdog ( struct clocksource * cs )
{
return cs = = watchdog ;
}
2009-08-14 21:47:24 +08:00
# else /* CONFIG_CLOCKSOURCE_WATCHDOG */
static void clocksource_enqueue_watchdog ( struct clocksource * cs )
2007-02-16 17:27:43 +08:00
{
if ( cs - > flags & CLOCK_SOURCE_IS_CONTINUOUS )
cs - > flags | = CLOCK_SOURCE_VALID_FOR_HRES ;
}
2007-05-09 17:35:15 +08:00
2016-01-23 01:31:53 +08:00
static void clocksource_select_watchdog ( bool fallback ) { }
2009-08-14 21:47:24 +08:00
static inline void clocksource_dequeue_watchdog ( struct clocksource * cs ) { }
2007-05-09 17:35:15 +08:00
static inline void clocksource_resume_watchdog ( void ) { }
2018-09-05 16:41:58 +08:00
static inline int __clocksource_watchdog_kthread ( void ) { return 0 ; }
2013-04-26 04:31:46 +08:00
static bool clocksource_is_watchdog ( struct clocksource * cs ) { return false ; }
2013-02-23 04:08:56 +08:00
void clocksource_mark_unstable ( struct clocksource * cs ) { }
2009-08-14 21:47:24 +08:00
2018-05-17 03:59:43 +08:00
static inline void clocksource_watchdog_lock ( unsigned long * flags ) { }
static inline void clocksource_watchdog_unlock ( unsigned long * flags ) { }
2018-04-23 23:28:55 +08:00
2009-08-14 21:47:24 +08:00
# endif /* CONFIG_CLOCKSOURCE_WATCHDOG */
2007-02-16 17:27:43 +08:00
2018-07-17 15:55:16 +08:00
static bool clocksource_is_suspend ( struct clocksource * cs )
{
return cs = = suspend_clocksource ;
}
static void __clocksource_suspend_select ( struct clocksource * cs )
{
/*
* Skip the clocksource which will be stopped in suspend state .
*/
if ( ! ( cs - > flags & CLOCK_SOURCE_SUSPEND_NONSTOP ) )
return ;
/*
* The nonstop clocksource can be selected as the suspend clocksource to
* calculate the suspend time , so it should not supply suspend / resume
* interfaces to suspend the nonstop clocksource when system suspends .
*/
if ( cs - > suspend | | cs - > resume ) {
pr_warn ( " Nonstop clocksource %s should not supply suspend/resume interfaces \n " ,
cs - > name ) ;
}
/* Pick the best rating. */
if ( ! suspend_clocksource | | cs - > rating > suspend_clocksource - > rating )
suspend_clocksource = cs ;
}
/**
* clocksource_suspend_select - Select the best clocksource for suspend timing
* @ fallback : if select a fallback clocksource
*/
static void clocksource_suspend_select ( bool fallback )
{
struct clocksource * cs , * old_suspend ;
old_suspend = suspend_clocksource ;
if ( fallback )
suspend_clocksource = NULL ;
list_for_each_entry ( cs , & clocksource_list , list ) {
/* Skip current if we were requested for a fallback. */
if ( fallback & & cs = = old_suspend )
continue ;
__clocksource_suspend_select ( cs ) ;
}
}
/**
* clocksource_start_suspend_timing - Start measuring the suspend timing
* @ cs : current clocksource from timekeeping
* @ start_cycles : current cycles from timekeeping
*
* This function will save the start cycle values of suspend timer to calculate
* the suspend time when resuming system .
*
* This function is called late in the suspend process from timekeeping_suspend ( ) ,
2021-03-23 05:39:03 +08:00
* that means processes are frozen , non - boot cpus and interrupts are disabled
2018-07-17 15:55:16 +08:00
* now . It is therefore possible to start the suspend timer without taking the
* clocksource mutex .
*/
void clocksource_start_suspend_timing ( struct clocksource * cs , u64 start_cycles )
{
if ( ! suspend_clocksource )
return ;
/*
* If current clocksource is the suspend timer , we should use the
* tkr_mono . cycle_last value as suspend_start to avoid same reading
* from suspend timer .
*/
if ( clocksource_is_suspend ( cs ) ) {
suspend_start = start_cycles ;
return ;
}
if ( suspend_clocksource - > enable & &
suspend_clocksource - > enable ( suspend_clocksource ) ) {
pr_warn_once ( " Failed to enable the non-suspend-able clocksource. \n " ) ;
return ;
}
suspend_start = suspend_clocksource - > read ( suspend_clocksource ) ;
}
/**
* clocksource_stop_suspend_timing - Stop measuring the suspend timing
* @ cs : current clocksource from timekeeping
* @ cycle_now : current cycles from timekeeping
*
* This function will calculate the suspend time from suspend timer .
*
* Returns nanoseconds since suspend started , 0 if no usable suspend clocksource .
*
* This function is called early in the resume process from timekeeping_resume ( ) ,
* that means there is only one cpu , no processes are running and the interrupts
* are disabled . It is therefore possible to stop the suspend timer without
* taking the clocksource mutex .
*/
u64 clocksource_stop_suspend_timing ( struct clocksource * cs , u64 cycle_now )
{
u64 now , delta , nsec = 0 ;
if ( ! suspend_clocksource )
return 0 ;
/*
* If current clocksource is the suspend timer , we should use the
* tkr_mono . cycle_last value from timekeeping as current cycle to
* avoid same reading from suspend timer .
*/
if ( clocksource_is_suspend ( cs ) )
now = cycle_now ;
else
now = suspend_clocksource - > read ( suspend_clocksource ) ;
if ( now > suspend_start ) {
delta = clocksource_delta ( now , suspend_start ,
suspend_clocksource - > mask ) ;
nsec = mul_u64_u32_shr ( delta , suspend_clocksource - > mult ,
suspend_clocksource - > shift ) ;
}
/*
* Disable the suspend timer to save power if current clocksource is
* not the suspend timer .
*/
if ( ! clocksource_is_suspend ( cs ) & & suspend_clocksource - > disable )
suspend_clocksource - > disable ( suspend_clocksource ) ;
return nsec ;
}
2010-02-03 06:41:41 +08:00
/**
* clocksource_suspend - suspend the clocksource ( s )
*/
void clocksource_suspend ( void )
{
struct clocksource * cs ;
list_for_each_entry_reverse ( cs , & clocksource_list , list )
if ( cs - > suspend )
cs - > suspend ( cs ) ;
}
2007-05-09 17:35:15 +08:00
/**
* clocksource_resume - resume the clocksource ( s )
*/
void clocksource_resume ( void )
{
2007-10-19 14:39:58 +08:00
struct clocksource * cs ;
2007-05-09 17:35:15 +08:00
2009-08-14 21:47:30 +08:00
list_for_each_entry ( cs , & clocksource_list , list )
2007-05-09 17:35:15 +08:00
if ( cs - > resume )
2010-02-03 06:41:39 +08:00
cs - > resume ( cs ) ;
2007-05-09 17:35:15 +08:00
clocksource_resume_watchdog ( ) ;
}
2008-02-16 04:55:54 +08:00
/**
* clocksource_touch_watchdog - Update watchdog
*
* Update the watchdog after exception contexts such as kgdb so as not
2010-01-26 19:51:10 +08:00
* to incorrectly trip the watchdog . This might fail when the kernel
* was stopped in code which holds watchdog_lock .
2008-02-16 04:55:54 +08:00
*/
void clocksource_touch_watchdog ( void )
{
clocksource_resume_watchdog ( ) ;
}
2011-11-01 05:06:35 +08:00
/**
* clocksource_max_adjustment - Returns max adjustment amount
* @ cs : Pointer to clocksource
*
*/
static u32 clocksource_max_adjustment ( struct clocksource * cs )
{
u64 ret ;
/*
2012-03-15 11:28:56 +08:00
* We won ' t try to correct for more than 11 % adjustments ( 110 , 000 ppm ) ,
2011-11-01 05:06:35 +08:00
*/
ret = ( u64 ) cs - > mult * 11 ;
do_div ( ret , 100 ) ;
return ( u32 ) ret ;
}
2009-08-19 01:45:10 +08:00
/**
2013-07-19 07:21:14 +08:00
* clocks_calc_max_nsecs - Returns maximum nanoseconds that can be converted
* @ mult : cycle to nanosecond multiplier
* @ shift : cycle to nanosecond divisor ( power of two )
* @ maxadj : maximum adjustment value to mult ( ~ 11 % )
* @ mask : bitmask for two ' s complement subtraction of non 64 bit counters
2015-03-12 12:16:31 +08:00
* @ max_cyc : maximum cycle value before potential overflow ( does not include
* any safety margin )
2015-03-12 12:16:30 +08:00
*
2015-04-02 11:34:39 +08:00
* NOTE : This function includes a safety margin of 50 % , in other words , we
* return half the number of nanoseconds the hardware counter can technically
* cover . This is done so that we can potentially detect problems caused by
* delayed timers or bad hardware , which might result in time intervals that
2015-08-25 14:42:53 +08:00
* are larger than what the math used can handle without overflows .
2009-08-19 01:45:10 +08:00
*/
2015-03-12 12:16:31 +08:00
u64 clocks_calc_max_nsecs ( u32 mult , u32 shift , u32 maxadj , u64 mask , u64 * max_cyc )
2009-08-19 01:45:10 +08:00
{
u64 max_nsecs , max_cycles ;
/*
* Calculate the maximum number of cycles that we can pass to the
2015-03-12 12:16:29 +08:00
* cyc2ns ( ) function without overflowing a 64 - bit result .
2009-08-19 01:45:10 +08:00
*/
2015-03-12 12:16:29 +08:00
max_cycles = ULLONG_MAX ;
do_div ( max_cycles , mult + maxadj ) ;
2009-08-19 01:45:10 +08:00
/*
* The actual maximum number of cycles we can defer the clocksource is
2013-07-19 07:21:14 +08:00
* determined by the minimum of max_cycles and mask .
2011-11-01 05:06:35 +08:00
* Note : Here we subtract the maxadj to make sure we don ' t sleep for
* too long if there ' s a large negative adjustment .
2009-08-19 01:45:10 +08:00
*/
2013-07-19 07:21:14 +08:00
max_cycles = min ( max_cycles , mask ) ;
max_nsecs = clocksource_cyc2ns ( max_cycles , mult - maxadj , shift ) ;
2015-03-12 12:16:31 +08:00
/* return the max_cycles value as well if requested */
if ( max_cyc )
* max_cyc = max_cycles ;
2015-03-12 12:16:30 +08:00
/* Return 50% of the actual maximum, so we can detect bad values */
max_nsecs > > = 1 ;
2013-07-19 07:21:14 +08:00
return max_nsecs ;
}
/**
2015-03-12 12:16:31 +08:00
* clocksource_update_max_deferment - Updates the clocksource max_idle_ns & max_cycles
* @ cs : Pointer to clocksource to be updated
2013-07-19 07:21:14 +08:00
*
*/
2015-03-12 12:16:31 +08:00
static inline void clocksource_update_max_deferment ( struct clocksource * cs )
2013-07-19 07:21:14 +08:00
{
2015-03-12 12:16:31 +08:00
cs - > max_idle_ns = clocks_calc_max_nsecs ( cs - > mult , cs - > shift ,
cs - > maxadj , cs - > mask ,
& cs - > max_cycles ) ;
2009-08-19 01:45:10 +08:00
}
2013-04-26 04:31:45 +08:00
static struct clocksource * clocksource_find_best ( bool oneshot , bool skipcur )
2013-04-26 04:31:43 +08:00
{
struct clocksource * cs ;
if ( ! finished_booting | | list_empty ( & clocksource_list ) )
return NULL ;
/*
* We pick the clocksource with the highest rating . If oneshot
* mode is active , we pick the highres valid clocksource with
* the best rating .
*/
list_for_each_entry ( cs , & clocksource_list , list ) {
2013-04-26 04:31:45 +08:00
if ( skipcur & & cs = = curr_clocksource )
continue ;
2013-04-26 04:31:43 +08:00
if ( oneshot & & ! ( cs - > flags & CLOCK_SOURCE_VALID_FOR_HRES ) )
continue ;
return cs ;
}
return NULL ;
}
2013-04-26 04:31:45 +08:00
static void __clocksource_select ( bool skipcur )
2006-06-26 15:25:05 +08:00
{
2013-04-26 04:31:43 +08:00
bool oneshot = tick_oneshot_mode_active ( ) ;
2009-08-14 21:47:21 +08:00
struct clocksource * best , * cs ;
2007-02-16 17:27:43 +08:00
2013-04-26 04:31:43 +08:00
/* Find the best suitable clocksource */
2013-04-26 04:31:45 +08:00
best = clocksource_find_best ( oneshot , skipcur ) ;
2013-04-26 04:31:43 +08:00
if ( ! best )
2009-08-14 21:47:21 +08:00
return ;
2013-04-26 04:31:43 +08:00
2018-01-17 14:01:28 +08:00
if ( ! strlen ( override_name ) )
goto found ;
2009-08-14 21:47:21 +08:00
/* Check for the override clocksource. */
list_for_each_entry ( cs , & clocksource_list , list ) {
2013-04-26 04:31:45 +08:00
if ( skipcur & & cs = = curr_clocksource )
continue ;
2009-08-14 21:47:21 +08:00
if ( strcmp ( cs - > name , override_name ) ! = 0 )
continue ;
/*
* Check to make sure we don ' t switch to a non - highres
* capable clocksource if the tick code is in oneshot
* mode ( highres or nohz )
*/
2013-04-26 04:31:43 +08:00
if ( ! ( cs - > flags & CLOCK_SOURCE_VALID_FOR_HRES ) & & oneshot ) {
2009-08-14 21:47:21 +08:00
/* Override clocksource cannot be used. */
2016-08-07 00:07:30 +08:00
if ( cs - > flags & CLOCK_SOURCE_UNSTABLE ) {
pr_warn ( " Override clocksource %s is unstable and not HRT compatible - cannot switch while in HRT/NOHZ mode \n " ,
cs - > name ) ;
override_name [ 0 ] = 0 ;
} else {
/*
* The override cannot be currently verified .
* Deferring to let the watchdog check .
*/
pr_info ( " Override clocksource %s is not currently HRT compatible - deferring \n " ,
cs - > name ) ;
}
2009-08-14 21:47:21 +08:00
} else
/* Override clocksource can be used. */
best = cs ;
break ;
}
2013-04-26 04:31:44 +08:00
2018-01-17 14:01:28 +08:00
found :
2013-04-26 04:31:44 +08:00
if ( curr_clocksource ! = best & & ! timekeeping_notify ( best ) ) {
pr_info ( " Switched to clocksource %s \n " , best - > name ) ;
2009-08-14 21:47:30 +08:00
curr_clocksource = best ;
}
2009-08-14 21:47:21 +08:00
}
2006-06-26 15:25:05 +08:00
2013-04-26 04:31:45 +08:00
/**
* clocksource_select - Select the best clocksource available
*
* Private function . Must hold clocksource_mutex when called .
*
* Select the clocksource with the best rating , or the clocksource ,
* which is selected by userspace override .
*/
static void clocksource_select ( void )
{
2015-09-23 19:19:19 +08:00
__clocksource_select ( false ) ;
2013-04-26 04:31:45 +08:00
}
2013-04-26 04:31:46 +08:00
static void clocksource_select_fallback ( void )
{
2015-09-23 19:19:19 +08:00
__clocksource_select ( true ) ;
2013-04-26 04:31:46 +08:00
}
2009-08-14 21:47:30 +08:00
/*
* clocksource_done_booting - Called near the end of core bootup
*
* Hack to avoid lots of clocksource churn at boot time .
* We use fs_initcall because we want this to start before
* device_initcall but after subsys_initcall .
*/
static int __init clocksource_done_booting ( void )
{
2010-03-02 04:34:43 +08:00
mutex_lock ( & clocksource_mutex ) ;
curr_clocksource = clocksource_default_clock ( ) ;
2009-08-14 21:47:30 +08:00
finished_booting = 1 ;
2009-09-15 01:49:02 +08:00
/*
* Run the watchdog first to eliminate unstable clock sources
*/
2018-09-05 16:41:58 +08:00
__clocksource_watchdog_kthread ( ) ;
2009-08-14 21:47:30 +08:00
clocksource_select ( ) ;
2009-09-15 01:51:11 +08:00
mutex_unlock ( & clocksource_mutex ) ;
2009-08-14 21:47:30 +08:00
return 0 ;
}
fs_initcall ( clocksource_done_booting ) ;
2007-02-16 17:27:33 +08:00
/*
* Enqueue the clocksource sorted by rating
2006-06-26 15:25:05 +08:00
*/
2009-08-14 21:47:21 +08:00
static void clocksource_enqueue ( struct clocksource * cs )
2006-06-26 15:25:05 +08:00
{
2009-08-14 21:47:21 +08:00
struct list_head * entry = & clocksource_list ;
struct clocksource * tmp ;
2007-02-16 17:27:33 +08:00
2016-04-25 17:20:28 +08:00
list_for_each_entry ( tmp , & clocksource_list , list ) {
2007-02-16 17:27:33 +08:00
/* Keep track of the place, where to insert */
2016-04-25 17:20:28 +08:00
if ( tmp - > rating < cs - > rating )
break ;
entry = & tmp - > list ;
}
2009-08-14 21:47:21 +08:00
list_add ( & cs - > list , entry ) ;
2006-06-26 15:25:05 +08:00
}
2010-05-08 09:07:38 +08:00
/**
2015-03-12 12:16:40 +08:00
* __clocksource_update_freq_scale - Used update clocksource with new freq
2011-12-19 17:13:19 +08:00
* @ cs : clocksource to be registered
2010-05-08 09:07:38 +08:00
* @ scale : Scale factor multiplied against freq to get clocksource hz
* @ freq : clocksource frequency ( cycles per second ) divided by scale
*
2010-07-14 08:56:28 +08:00
* This should only be called from the clocksource - > enable ( ) method .
2010-05-08 09:07:38 +08:00
*
* This * SHOULD NOT * be called directly ! Please use the
2015-03-12 12:16:40 +08:00
* __clocksource_update_freq_hz ( ) or __clocksource_update_freq_khz ( ) helper
* functions .
2010-05-08 09:07:38 +08:00
*/
2015-03-12 12:16:40 +08:00
void __clocksource_update_freq_scale ( struct clocksource * cs , u32 scale , u32 freq )
2010-05-08 09:07:38 +08:00
{
2011-05-20 16:50:52 +08:00
u64 sec ;
2015-03-12 12:16:37 +08:00
2010-05-08 09:07:38 +08:00
/*
2015-03-12 12:16:37 +08:00
* Default clocksources are * special * and self - define their mult / shift .
* But , you ' re not special , so you should specify a freq value .
2010-05-08 09:07:38 +08:00
*/
2015-03-12 12:16:37 +08:00
if ( freq ) {
/*
* Calc the maximum number of seconds which we can run before
* wrapping around . For clocksources which have a mask > 32 - bit
* we need to limit the max sleep time to have a good
* conversion precision . 10 minutes is still a reasonable
* amount . That results in a shift value of 24 for a
* clocksource with mask > = 40 - bit and f > = 4 GHz . That maps to
* ~ 0.06 ppm granularity for NTP .
*/
sec = cs - > mask ;
do_div ( sec , freq ) ;
do_div ( sec , scale ) ;
if ( ! sec )
sec = 1 ;
else if ( sec > 600 & & cs - > mask > UINT_MAX )
sec = 600 ;
clocks_calc_mult_shift ( & cs - > mult , & cs - > shift , freq ,
NSEC_PER_SEC / scale , sec * scale ) ;
}
clocksource: Reduce clocksource-skew threshold
Currently, WATCHDOG_THRESHOLD is set to detect a 62.5-millisecond skew in
a 500-millisecond WATCHDOG_INTERVAL. This requires that clocks be skewed
by more than 12.5% in order to be marked unstable. Except that a clock
that is skewed by that much is probably destroying unsuspecting software
right and left. And given that there are now checks for false-positive
skews due to delays between reading the two clocks, it should be possible
to greatly decrease WATCHDOG_THRESHOLD, at least for fine-grained clocks
such as TSC.
Therefore, add a new uncertainty_margin field to the clocksource structure
that contains the maximum uncertainty in nanoseconds for the corresponding
clock. This field may be initialized manually, as it is for
clocksource_tsc_early and clocksource_jiffies, which is copied to
refined_jiffies. If the field is not initialized manually, it will be
computed at clock-registry time as the period of the clock in question
based on the scale and freq parameters to __clocksource_update_freq_scale()
function. If either of those two parameters are zero, the
tens-of-milliseconds WATCHDOG_THRESHOLD is used as a cowardly alternative
to dividing by zero. No matter how the uncertainty_margin field is
calculated, it is bounded below by twice WATCHDOG_MAX_SKEW, that is, by 100
microseconds.
Note that manually initialized uncertainty_margin fields are not adjusted,
but there is a WARN_ON_ONCE() that triggers if any such field is less than
twice WATCHDOG_MAX_SKEW. This WARN_ON_ONCE() is intended to discourage
production use of the one-nanosecond uncertainty_margin values that are
used to test the clock-skew code itself.
The actual clock-skew check uses the sum of the uncertainty_margin fields
of the two clocksource structures being compared. Integer overflow is
avoided because the largest computed value of the uncertainty_margin
fields is one billion (10^9), and double that value fits into an
unsigned int. However, if someone manually specifies (say) UINT_MAX,
they will get what they deserve.
Note that the refined_jiffies uncertainty_margin field is initialized to
TICK_NSEC, which means that skew checks involving this clocksource will
be sufficently forgiving. In a similar vein, the clocksource_tsc_early
uncertainty_margin field is initialized to 32*NSEC_PER_MSEC, which
replicates the current behavior and allows custom setting if needed
in order to address the rare skews detected for this clocksource in
current mainline.
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Feng Tang <feng.tang@intel.com>
Link: https://lore.kernel.org/r/20210527190124.440372-4-paulmck@kernel.org
2021-05-28 03:01:22 +08:00
/*
* If the uncertainty margin is not specified , calculate it .
* If both scale and freq are non - zero , calculate the clock
* period , but bound below at 2 * WATCHDOG_MAX_SKEW . However ,
* if either of scale or freq is zero , be very conservative and
* take the tens - of - milliseconds WATCHDOG_THRESHOLD value for the
* uncertainty margin . Allow stupidly small uncertainty margins
* to be specified by the caller for testing purposes , but warn
* to discourage production use of this capability .
*/
if ( scale & & freq & & ! cs - > uncertainty_margin ) {
cs - > uncertainty_margin = NSEC_PER_SEC / ( scale * freq ) ;
if ( cs - > uncertainty_margin < 2 * WATCHDOG_MAX_SKEW )
cs - > uncertainty_margin = 2 * WATCHDOG_MAX_SKEW ;
} else if ( ! cs - > uncertainty_margin ) {
cs - > uncertainty_margin = WATCHDOG_THRESHOLD ;
}
WARN_ON_ONCE ( cs - > uncertainty_margin < 2 * WATCHDOG_MAX_SKEW ) ;
2011-11-01 05:06:35 +08:00
/*
2015-03-12 12:16:30 +08:00
* Ensure clocksources that have large ' mult ' values don ' t overflow
* when adjusted .
2011-11-01 05:06:35 +08:00
*/
cs - > maxadj = clocksource_max_adjustment ( cs ) ;
2015-03-12 12:16:37 +08:00
while ( freq & & ( ( cs - > mult + cs - > maxadj < cs - > mult )
| | ( cs - > mult - cs - > maxadj > cs - > mult ) ) ) {
2011-11-01 05:06:35 +08:00
cs - > mult > > = 1 ;
cs - > shift - - ;
cs - > maxadj = clocksource_max_adjustment ( cs ) ;
}
2015-03-12 12:16:37 +08:00
/*
* Only warn for * special * clocksources that self - define
* their mult / shift values and don ' t specify a freq .
*/
WARN_ONCE ( cs - > mult + cs - > maxadj < cs - > mult ,
" timekeeping: Clocksource %s might overflow on 11%% adjustment \n " ,
cs - > name ) ;
2015-03-12 12:16:31 +08:00
clocksource_update_max_deferment ( cs ) ;
2015-03-12 12:16:39 +08:00
2015-05-26 02:49:55 +08:00
pr_info ( " %s: mask: 0x%llx max_cycles: 0x%llx, max_idle_ns: %lld ns \n " ,
cs - > name , cs - > mask , cs - > max_cycles , cs - > max_idle_ns ) ;
2010-07-14 08:56:28 +08:00
}
2015-03-12 12:16:40 +08:00
EXPORT_SYMBOL_GPL ( __clocksource_update_freq_scale ) ;
2010-07-14 08:56:28 +08:00
/**
* __clocksource_register_scale - Used to install new clocksources
2011-12-19 17:13:19 +08:00
* @ cs : clocksource to be registered
2010-07-14 08:56:28 +08:00
* @ scale : Scale factor multiplied against freq to get clocksource hz
* @ freq : clocksource frequency ( cycles per second ) divided by scale
*
* Returns - EBUSY if registration fails , zero otherwise .
*
* This * SHOULD NOT * be called directly ! Please use the
* clocksource_register_hz ( ) or clocksource_register_khz helper functions .
*/
int __clocksource_register_scale ( struct clocksource * cs , u32 scale , u32 freq )
{
2018-04-23 23:28:55 +08:00
unsigned long flags ;
2010-07-14 08:56:28 +08:00
2018-09-17 20:45:34 +08:00
clocksource_arch_init ( cs ) ;
2020-12-09 14:09:27 +08:00
if ( WARN_ON_ONCE ( ( unsigned int ) cs - > id > = CSID_MAX ) )
cs - > id = CSID_GENERIC ;
2020-02-07 20:38:55 +08:00
if ( cs - > vdso_clock_mode < 0 | |
cs - > vdso_clock_mode > = VDSO_CLOCKMODE_MAX ) {
pr_warn ( " clocksource %s registered with invalid VDSO mode %d. Disabling VDSO support. \n " ,
cs - > name , cs - > vdso_clock_mode ) ;
cs - > vdso_clock_mode = VDSO_CLOCKMODE_NONE ;
}
tree-wide: fix comment/printk typos
"gadget", "through", "command", "maintain", "maintain", "controller", "address",
"between", "initiali[zs]e", "instead", "function", "select", "already",
"equal", "access", "management", "hierarchy", "registration", "interest",
"relative", "memory", "offset", "already",
Signed-off-by: Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
Signed-off-by: Jiri Kosina <jkosina@suse.cz>
2010-11-02 03:38:34 +08:00
/* Initialize mult/shift and max_idle_ns */
2015-03-12 12:16:40 +08:00
__clocksource_update_freq_scale ( cs , scale , freq ) ;
2010-05-08 09:07:38 +08:00
2014-09-18 22:59:07 +08:00
/* Add clocksource to the clocksource list */
2010-05-08 09:07:38 +08:00
mutex_lock ( & clocksource_mutex ) ;
2018-04-23 23:28:55 +08:00
clocksource_watchdog_lock ( & flags ) ;
2010-05-08 09:07:38 +08:00
clocksource_enqueue ( cs ) ;
clocksource_enqueue_watchdog ( cs ) ;
2018-04-23 23:28:55 +08:00
clocksource_watchdog_unlock ( & flags ) ;
2011-05-05 09:16:50 +08:00
clocksource_select ( ) ;
2016-01-23 01:31:53 +08:00
clocksource_select_watchdog ( false ) ;
2018-07-17 15:55:16 +08:00
__clocksource_suspend_select ( cs ) ;
2010-05-08 09:07:38 +08:00
mutex_unlock ( & clocksource_mutex ) ;
return 0 ;
}
EXPORT_SYMBOL_GPL ( __clocksource_register_scale ) ;
2009-08-19 17:26:09 +08:00
static void __clocksource_change_rating ( struct clocksource * cs , int rating )
{
list_del ( & cs - > list ) ;
cs - > rating = rating ;
clocksource_enqueue ( cs ) ;
}
2006-06-26 15:25:05 +08:00
/**
2007-02-16 17:27:33 +08:00
* clocksource_change_rating - Change the rating of a registered clocksource
2011-12-19 17:13:19 +08:00
* @ cs : clocksource to be changed
* @ rating : new rating
2006-06-26 15:25:05 +08:00
*/
2007-02-16 17:27:33 +08:00
void clocksource_change_rating ( struct clocksource * cs , int rating )
2006-06-26 15:25:05 +08:00
{
2018-04-23 23:28:55 +08:00
unsigned long flags ;
2009-08-14 21:47:30 +08:00
mutex_lock ( & clocksource_mutex ) ;
2018-04-23 23:28:55 +08:00
clocksource_watchdog_lock ( & flags ) ;
2009-08-19 17:26:09 +08:00
__clocksource_change_rating ( cs , rating ) ;
2018-04-23 23:28:55 +08:00
clocksource_watchdog_unlock ( & flags ) ;
clocksource: Reselect clocksource when watchdog validated high-res capability
Up to commit 5d33b883a (clocksource: Always verify highres capability)
we had no sanity check when selecting a clocksource, which prevented
that a non highres capable clocksource is used when the system already
switched to highres/nohz mode.
The new sanity check works as Alex and Tim found out. It prevents the
TSC from being used. This happens because on x86 the boot process
looks like this:
tsc_start_freqency_validation(TSC);
clocksource_register(HPET);
clocksource_done_booting();
clocksource_select()
Selects HPET which is valid for high-res
switch_to_highres();
clocksource_register(TSC);
TSC is not selected, because it is not yet
flagged as VALID_HIGH_RES
clocksource_watchdog()
Validates TSC for highres, but that does not make TSC
the current clocksource.
Before the sanity check was added, we installed TSC unvalidated which
worked most of the time. If the TSC was really detected as unstable,
then the unstable logic removed it and installed HPET again.
The sanity check is correct and needed. So the watchdog needs to kick
a reselection of the clocksource, when it qualifies TSC as a valid
high res clocksource.
To solve this, we mark the clocksource which got the flag
CLOCK_SOURCE_VALID_FOR_HRES set by the watchdog with an new flag
CLOCK_SOURCE_RESELECT and trigger the watchdog thread. The watchdog
thread evaluates the flag and invokes clocksource_select() when set.
To avoid that the clocksource_done_booting() code, which is about to
install the first real clocksource anyway, needs to go through
clocksource_select and tick_oneshot_notify() pointlessly, split out
the clocksource_watchdog_kthread() list walk code and invoke the
select/notify only when called from clocksource_watchdog_kthread().
So clocksource_done_booting() can utilize the same splitout code
without the select/notify invocation and the clocksource_mutex
unlock/relock dance.
Reported-and-tested-by: Alex Shi <alex.shi@intel.com>
Cc: Hans Peter Anvin <hpa@linux.intel.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Tested-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Davidlohr Bueso <davidlohr.bueso@hp.com>
Cc: John Stultz <john.stultz@linaro.org>
Link: http://lkml.kernel.org/r/alpine.DEB.2.02.1307042239150.11637@ionos.tec.linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-07-05 04:46:45 +08:00
clocksource_select ( ) ;
2016-01-23 01:31:53 +08:00
clocksource_select_watchdog ( false ) ;
2018-07-17 15:55:16 +08:00
clocksource_suspend_select ( false ) ;
2009-08-14 21:47:30 +08:00
mutex_unlock ( & clocksource_mutex ) ;
2006-06-26 15:25:05 +08:00
}
2009-08-14 21:47:24 +08:00
EXPORT_SYMBOL ( clocksource_change_rating ) ;
2006-06-26 15:25:05 +08:00
2013-04-26 04:31:46 +08:00
/*
* Unbind clocksource @ cs . Called with clocksource_mutex held
*/
static int clocksource_unbind ( struct clocksource * cs )
{
2018-04-23 23:28:55 +08:00
unsigned long flags ;
2016-01-23 01:31:53 +08:00
if ( clocksource_is_watchdog ( cs ) ) {
/* Select and try to install a replacement watchdog. */
clocksource_select_watchdog ( true ) ;
if ( clocksource_is_watchdog ( cs ) )
return - EBUSY ;
}
2013-04-26 04:31:46 +08:00
if ( cs = = curr_clocksource ) {
/* Select and try to install a replacement clock source */
clocksource_select_fallback ( ) ;
if ( curr_clocksource = = cs )
return - EBUSY ;
}
2018-04-23 23:28:55 +08:00
2018-07-17 15:55:16 +08:00
if ( clocksource_is_suspend ( cs ) ) {
/*
* Select and try to install a replacement suspend clocksource .
* If no replacement suspend clocksource , we will just let the
* clocksource go and have no suspend clocksource .
*/
clocksource_suspend_select ( true ) ;
}
2018-04-23 23:28:55 +08:00
clocksource_watchdog_lock ( & flags ) ;
2013-04-26 04:31:46 +08:00
clocksource_dequeue_watchdog ( cs ) ;
list_del_init ( & cs - > list ) ;
2018-04-23 23:28:55 +08:00
clocksource_watchdog_unlock ( & flags ) ;
2013-04-26 04:31:46 +08:00
return 0 ;
}
2008-01-30 20:30:02 +08:00
/**
* clocksource_unregister - remove a registered clocksource
2011-12-19 17:13:19 +08:00
* @ cs : clocksource to be unregistered
2008-01-30 20:30:02 +08:00
*/
2013-04-26 04:31:46 +08:00
int clocksource_unregister ( struct clocksource * cs )
2008-01-30 20:30:02 +08:00
{
2013-04-26 04:31:46 +08:00
int ret = 0 ;
2009-08-14 21:47:30 +08:00
mutex_lock ( & clocksource_mutex ) ;
2013-04-26 04:31:46 +08:00
if ( ! list_empty ( & cs - > list ) )
ret = clocksource_unbind ( cs ) ;
2009-08-14 21:47:30 +08:00
mutex_unlock ( & clocksource_mutex ) ;
2013-04-26 04:31:46 +08:00
return ret ;
2008-01-30 20:30:02 +08:00
}
2009-08-14 21:47:24 +08:00
EXPORT_SYMBOL ( clocksource_unregister ) ;
2008-01-30 20:30:02 +08:00
2006-12-10 18:21:30 +08:00
# ifdef CONFIG_SYSFS
2006-06-26 15:25:05 +08:00
/**
2018-01-17 14:01:29 +08:00
* current_clocksource_show - sysfs interface for current clocksource
2006-06-26 15:25:05 +08:00
* @ dev : unused
2011-12-19 17:13:19 +08:00
* @ attr : unused
2006-06-26 15:25:05 +08:00
* @ buf : char buffer to be filled with clocksource list
*
* Provides sysfs interface for listing current clocksource .
*/
2018-01-17 14:01:29 +08:00
static ssize_t current_clocksource_show ( struct device * dev ,
struct device_attribute * attr ,
char * buf )
2006-06-26 15:25:05 +08:00
{
2008-02-06 17:36:53 +08:00
ssize_t count = 0 ;
2006-06-26 15:25:05 +08:00
2009-08-14 21:47:30 +08:00
mutex_lock ( & clocksource_mutex ) ;
2008-02-06 17:36:53 +08:00
count = snprintf ( buf , PAGE_SIZE , " %s \n " , curr_clocksource - > name ) ;
2009-08-14 21:47:30 +08:00
mutex_unlock ( & clocksource_mutex ) ;
2006-06-26 15:25:05 +08:00
2008-02-06 17:36:53 +08:00
return count ;
2006-06-26 15:25:05 +08:00
}
2013-10-12 01:11:55 +08:00
ssize_t sysfs_get_uname ( const char * buf , char * dst , size_t cnt )
2013-04-26 04:31:45 +08:00
{
size_t ret = cnt ;
/* strings from sysfs write are not 0 terminated! */
if ( ! cnt | | cnt > = CS_NAME_LEN )
return - EINVAL ;
/* strip of \n: */
if ( buf [ cnt - 1 ] = = ' \n ' )
cnt - - ;
if ( cnt > 0 )
memcpy ( dst , buf , cnt ) ;
dst [ cnt ] = 0 ;
return ret ;
}
2006-06-26 15:25:05 +08:00
/**
2018-01-17 14:01:29 +08:00
* current_clocksource_store - interface for manually overriding clocksource
2006-06-26 15:25:05 +08:00
* @ dev : unused
2011-12-19 17:13:19 +08:00
* @ attr : unused
2006-06-26 15:25:05 +08:00
* @ buf : name of override clocksource
* @ count : length of buffer
*
* Takes input from sysfs interface for manually overriding the default
2009-10-06 18:42:51 +08:00
* clocksource selection .
2006-06-26 15:25:05 +08:00
*/
2018-01-17 14:01:29 +08:00
static ssize_t current_clocksource_store ( struct device * dev ,
struct device_attribute * attr ,
const char * buf , size_t count )
2006-06-26 15:25:05 +08:00
{
2013-09-12 18:28:54 +08:00
ssize_t ret ;
2006-06-26 15:25:05 +08:00
2009-08-14 21:47:30 +08:00
mutex_lock ( & clocksource_mutex ) ;
2006-06-26 15:25:05 +08:00
2013-04-26 04:31:50 +08:00
ret = sysfs_get_uname ( buf , override_name , count ) ;
2013-04-26 04:31:45 +08:00
if ( ret > = 0 )
clocksource_select ( ) ;
2006-06-26 15:25:05 +08:00
2009-08-14 21:47:30 +08:00
mutex_unlock ( & clocksource_mutex ) ;
2006-06-26 15:25:05 +08:00
return ret ;
}
2018-01-17 14:01:29 +08:00
static DEVICE_ATTR_RW ( current_clocksource ) ;
2006-06-26 15:25:05 +08:00
2013-04-26 04:31:46 +08:00
/**
2018-01-17 14:01:29 +08:00
* unbind_clocksource_store - interface for manually unbinding clocksource
2013-04-26 04:31:46 +08:00
* @ dev : unused
* @ attr : unused
* @ buf : unused
* @ count : length of buffer
*
* Takes input from sysfs interface for manually unbinding a clocksource .
*/
2018-01-17 14:01:29 +08:00
static ssize_t unbind_clocksource_store ( struct device * dev ,
2013-04-26 04:31:46 +08:00
struct device_attribute * attr ,
const char * buf , size_t count )
{
struct clocksource * cs ;
char name [ CS_NAME_LEN ] ;
2013-09-12 18:28:54 +08:00
ssize_t ret ;
2013-04-26 04:31:46 +08:00
2013-04-26 04:31:50 +08:00
ret = sysfs_get_uname ( buf , name , count ) ;
2013-04-26 04:31:46 +08:00
if ( ret < 0 )
return ret ;
ret = - ENODEV ;
mutex_lock ( & clocksource_mutex ) ;
list_for_each_entry ( cs , & clocksource_list , list ) {
if ( strcmp ( cs - > name , name ) )
continue ;
ret = clocksource_unbind ( cs ) ;
break ;
}
mutex_unlock ( & clocksource_mutex ) ;
return ret ? ret : count ;
}
2018-01-17 14:01:29 +08:00
static DEVICE_ATTR_WO ( unbind_clocksource ) ;
2013-04-26 04:31:46 +08:00
2006-06-26 15:25:05 +08:00
/**
2018-01-17 14:01:29 +08:00
* available_clocksource_show - sysfs interface for listing clocksource
2006-06-26 15:25:05 +08:00
* @ dev : unused
2011-12-19 17:13:19 +08:00
* @ attr : unused
2006-06-26 15:25:05 +08:00
* @ buf : char buffer to be filled with clocksource list
*
* Provides sysfs interface for listing registered clocksources
*/
2018-01-17 14:01:29 +08:00
static ssize_t available_clocksource_show ( struct device * dev ,
struct device_attribute * attr ,
char * buf )
2006-06-26 15:25:05 +08:00
{
2007-10-19 14:39:58 +08:00
struct clocksource * src ;
2008-02-06 17:36:53 +08:00
ssize_t count = 0 ;
2006-06-26 15:25:05 +08:00
2009-08-14 21:47:30 +08:00
mutex_lock ( & clocksource_mutex ) ;
2007-10-19 14:39:58 +08:00
list_for_each_entry ( src , & clocksource_list , list ) {
2009-06-12 17:29:27 +08:00
/*
* Don ' t show non - HRES clocksource if the tick code is
* in one shot mode ( highres = on or nohz = on )
*/
if ( ! tick_oneshot_mode_active ( ) | |
( src - > flags & CLOCK_SOURCE_VALID_FOR_HRES ) )
2009-01-22 13:53:22 +08:00
count + = snprintf ( buf + count ,
2008-02-06 17:36:53 +08:00
max ( ( ssize_t ) PAGE_SIZE - count , ( ssize_t ) 0 ) ,
" %s " , src - > name ) ;
2006-06-26 15:25:05 +08:00
}
2009-08-14 21:47:30 +08:00
mutex_unlock ( & clocksource_mutex ) ;
2006-06-26 15:25:05 +08:00
2008-02-06 17:36:53 +08:00
count + = snprintf ( buf + count ,
max ( ( ssize_t ) PAGE_SIZE - count , ( ssize_t ) 0 ) , " \n " ) ;
2006-06-26 15:25:05 +08:00
2008-02-06 17:36:53 +08:00
return count ;
2006-06-26 15:25:05 +08:00
}
2018-01-17 14:01:29 +08:00
static DEVICE_ATTR_RO ( available_clocksource ) ;
2006-06-26 15:25:05 +08:00
2018-01-17 14:01:30 +08:00
static struct attribute * clocksource_attrs [ ] = {
& dev_attr_current_clocksource . attr ,
& dev_attr_unbind_clocksource . attr ,
& dev_attr_available_clocksource . attr ,
NULL
} ;
ATTRIBUTE_GROUPS ( clocksource ) ;
2011-12-15 07:28:51 +08:00
static struct bus_type clocksource_subsys = {
2007-12-20 09:09:39 +08:00
. name = " clocksource " ,
2011-12-15 07:28:51 +08:00
. dev_name = " clocksource " ,
2006-06-26 15:25:05 +08:00
} ;
2011-12-15 07:28:51 +08:00
static struct device device_clocksource = {
2006-06-26 15:25:05 +08:00
. id = 0 ,
2011-12-15 07:28:51 +08:00
. bus = & clocksource_subsys ,
2018-01-17 14:01:30 +08:00
. groups = clocksource_groups ,
2006-06-26 15:25:05 +08:00
} ;
2006-06-26 15:25:06 +08:00
static int __init init_clocksource_sysfs ( void )
2006-06-26 15:25:05 +08:00
{
2011-12-15 07:28:51 +08:00
int error = subsys_system_register ( & clocksource_subsys , NULL ) ;
2006-06-26 15:25:05 +08:00
if ( ! error )
2011-12-15 07:28:51 +08:00
error = device_register ( & device_clocksource ) ;
2018-01-17 14:01:30 +08:00
2006-06-26 15:25:05 +08:00
return error ;
}
device_initcall ( init_clocksource_sysfs ) ;
2006-12-10 18:21:30 +08:00
# endif /* CONFIG_SYSFS */
2006-06-26 15:25:05 +08:00
/**
* boot_override_clocksource - boot clock override
* @ str : override name
*
* Takes a clocksource = boot argument and uses it
* as the clocksource override name .
*/
static int __init boot_override_clocksource ( char * str )
{
2009-08-14 21:47:30 +08:00
mutex_lock ( & clocksource_mutex ) ;
2006-06-26 15:25:05 +08:00
if ( str )
strlcpy ( override_name , str , sizeof ( override_name ) ) ;
2009-08-14 21:47:30 +08:00
mutex_unlock ( & clocksource_mutex ) ;
2006-06-26 15:25:05 +08:00
return 1 ;
}
__setup ( " clocksource= " , boot_override_clocksource ) ;
/**
* boot_override_clock - Compatibility layer for deprecated boot option
* @ str : override name
*
* DEPRECATED ! Takes a clock = boot argument and uses it
* as the clocksource override name
*/
static int __init boot_override_clock ( char * str )
{
2006-06-26 15:25:12 +08:00
if ( ! strcmp ( str , " pmtmr " ) ) {
2015-05-26 02:49:55 +08:00
pr_warn ( " clock=pmtmr is deprecated - use clocksource=acpi_pm \n " ) ;
2006-06-26 15:25:12 +08:00
return boot_override_clocksource ( " acpi_pm " ) ;
}
2015-05-26 02:49:55 +08:00
pr_warn ( " clock= boot option is deprecated - use clocksource=xyz \n " ) ;
2006-06-26 15:25:05 +08:00
return boot_override_clocksource ( str ) ;
}
__setup ( " clock= " , boot_override_clock ) ;
