2018-11-01 02:21:09 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
2006-01-10 12:52:32 +08:00
|
|
|
/*
|
2007-02-16 17:27:50 +08:00
|
|
|
* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
|
2007-02-16 17:28:03 +08:00
|
|
|
* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
|
2007-02-16 17:28:11 +08:00
|
|
|
* Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
|
2006-01-10 12:52:32 +08:00
|
|
|
*
|
|
|
|
* High-resolution kernel timers
|
|
|
|
*
|
2018-11-01 02:21:08 +08:00
|
|
|
* In contrast to the low-resolution timeout API, aka timer wheel,
|
|
|
|
* hrtimers provide finer resolution and accuracy depending on system
|
|
|
|
* configuration and capabilities.
|
2006-01-10 12:52:32 +08:00
|
|
|
*
|
|
|
|
* Started by: Thomas Gleixner and Ingo Molnar
|
|
|
|
*
|
|
|
|
* Credits:
|
2018-11-01 02:21:08 +08:00
|
|
|
* Based on the original timer wheel code
|
2006-01-10 12:52:32 +08:00
|
|
|
*
|
2006-02-01 19:05:13 +08:00
|
|
|
* Help, testing, suggestions, bugfixes, improvements were
|
|
|
|
* provided by:
|
|
|
|
*
|
|
|
|
* George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
|
|
|
|
* et. al.
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/cpu.h>
|
2011-05-24 02:51:41 +08:00
|
|
|
#include <linux/export.h>
|
2006-01-10 12:52:32 +08:00
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/hrtimer.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/syscalls.h>
|
|
|
|
#include <linux/interrupt.h>
|
2007-02-16 17:28:03 +08:00
|
|
|
#include <linux/tick.h>
|
2007-02-16 17:28:11 +08:00
|
|
|
#include <linux/err.h>
|
2008-04-30 15:55:04 +08:00
|
|
|
#include <linux/debugobjects.h>
|
2017-02-03 02:15:33 +08:00
|
|
|
#include <linux/sched/signal.h>
|
2013-02-07 23:46:59 +08:00
|
|
|
#include <linux/sched/sysctl.h>
|
2013-02-07 23:47:07 +08:00
|
|
|
#include <linux/sched/rt.h>
|
sched/deadline: Add SCHED_DEADLINE structures & implementation
Introduces the data structures, constants and symbols needed for
SCHED_DEADLINE implementation.
Core data structure of SCHED_DEADLINE are defined, along with their
initializers. Hooks for checking if a task belong to the new policy
are also added where they are needed.
Adds a scheduling class, in sched/dl.c and a new policy called
SCHED_DEADLINE. It is an implementation of the Earliest Deadline
First (EDF) scheduling algorithm, augmented with a mechanism (called
Constant Bandwidth Server, CBS) that makes it possible to isolate
the behaviour of tasks between each other.
The typical -deadline task will be made up of a computation phase
(instance) which is activated on a periodic or sporadic fashion. The
expected (maximum) duration of such computation is called the task's
runtime; the time interval by which each instance need to be completed
is called the task's relative deadline. The task's absolute deadline
is dynamically calculated as the time instant a task (better, an
instance) activates plus the relative deadline.
The EDF algorithms selects the task with the smallest absolute
deadline as the one to be executed first, while the CBS ensures each
task to run for at most its runtime every (relative) deadline
length time interval, avoiding any interference between different
tasks (bandwidth isolation).
Thanks to this feature, also tasks that do not strictly comply with
the computational model sketched above can effectively use the new
policy.
To summarize, this patch:
- introduces the data structures, constants and symbols needed;
- implements the core logic of the scheduling algorithm in the new
scheduling class file;
- provides all the glue code between the new scheduling class and
the core scheduler and refines the interactions between sched/dl
and the other existing scheduling classes.
Signed-off-by: Dario Faggioli <raistlin@linux.it>
Signed-off-by: Michael Trimarchi <michael@amarulasolutions.com>
Signed-off-by: Fabio Checconi <fchecconi@gmail.com>
Signed-off-by: Juri Lelli <juri.lelli@gmail.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1383831828-15501-4-git-send-email-juri.lelli@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-11-28 18:14:43 +08:00
|
|
|
#include <linux/sched/deadline.h>
|
2017-02-09 01:51:35 +08:00
|
|
|
#include <linux/sched/nohz.h>
|
2017-02-09 01:51:35 +08:00
|
|
|
#include <linux/sched/debug.h>
|
2009-04-16 14:46:41 +08:00
|
|
|
#include <linux/timer.h>
|
2013-05-07 07:50:19 +08:00
|
|
|
#include <linux/freezer.h>
|
2017-06-07 16:42:31 +08:00
|
|
|
#include <linux/compat.h>
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2016-12-25 03:46:01 +08:00
|
|
|
#include <linux/uaccess.h>
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2009-08-10 10:51:23 +08:00
|
|
|
#include <trace/events/timer.h>
|
|
|
|
|
2015-03-25 20:07:37 +08:00
|
|
|
#include "tick-internal.h"
|
2014-07-17 05:04:02 +08:00
|
|
|
|
2017-12-21 18:41:56 +08:00
|
|
|
/*
|
|
|
|
* Masks for selecting the soft and hard context timers from
|
|
|
|
* cpu_base->active
|
|
|
|
*/
|
|
|
|
#define MASK_SHIFT (HRTIMER_BASE_MONOTONIC_SOFT)
|
|
|
|
#define HRTIMER_ACTIVE_HARD ((1U << MASK_SHIFT) - 1)
|
|
|
|
#define HRTIMER_ACTIVE_SOFT (HRTIMER_ACTIVE_HARD << MASK_SHIFT)
|
|
|
|
#define HRTIMER_ACTIVE_ALL (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
|
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
/*
|
|
|
|
* The timer bases:
|
2006-02-01 19:05:11 +08:00
|
|
|
*
|
2015-08-25 14:42:53 +08:00
|
|
|
* There are more clockids than hrtimer bases. Thus, we index
|
2010-12-15 11:37:07 +08:00
|
|
|
* into the timer bases by the hrtimer_base_type enum. When trying
|
|
|
|
* to reach a base using a clockid, hrtimer_clockid_to_base()
|
|
|
|
* is used to convert from clockid to the proper hrtimer_base_type.
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
2007-02-16 17:28:11 +08:00
|
|
|
DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
hrtimer: Don't reinitialize a cpu_base lock on CPU_UP
The current code makes the assumption that a cpu_base lock won't be
held if the CPU corresponding to that cpu_base is offline, which isn't
always true.
If a hrtimer is not queued, then it will not be migrated by
migrate_hrtimers() when a CPU is offlined. Therefore, the hrtimer's
cpu_base may still point to a CPU which has subsequently gone offline
if the timer wasn't enqueued at the time the CPU went down.
Normally this wouldn't be a problem, but a cpu_base's lock is blindly
reinitialized each time a CPU is brought up. If a CPU is brought
online during the period that another thread is performing a hrtimer
operation on a stale hrtimer, then the lock will be reinitialized
under its feet, and a SPIN_BUG() like the following will be observed:
<0>[ 28.082085] BUG: spinlock already unlocked on CPU#0, swapper/0/0
<0>[ 28.087078] lock: 0xc4780b40, value 0x0 .magic: dead4ead, .owner: <none>/-1, .owner_cpu: -1
<4>[ 42.451150] [<c0014398>] (unwind_backtrace+0x0/0x120) from [<c0269220>] (do_raw_spin_unlock+0x44/0xdc)
<4>[ 42.460430] [<c0269220>] (do_raw_spin_unlock+0x44/0xdc) from [<c071b5bc>] (_raw_spin_unlock+0x8/0x30)
<4>[ 42.469632] [<c071b5bc>] (_raw_spin_unlock+0x8/0x30) from [<c00a9ce0>] (__hrtimer_start_range_ns+0x1e4/0x4f8)
<4>[ 42.479521] [<c00a9ce0>] (__hrtimer_start_range_ns+0x1e4/0x4f8) from [<c00aa014>] (hrtimer_start+0x20/0x28)
<4>[ 42.489247] [<c00aa014>] (hrtimer_start+0x20/0x28) from [<c00e6190>] (rcu_idle_enter_common+0x1ac/0x320)
<4>[ 42.498709] [<c00e6190>] (rcu_idle_enter_common+0x1ac/0x320) from [<c00e6440>] (rcu_idle_enter+0xa0/0xb8)
<4>[ 42.508259] [<c00e6440>] (rcu_idle_enter+0xa0/0xb8) from [<c000f268>] (cpu_idle+0x24/0xf0)
<4>[ 42.516503] [<c000f268>] (cpu_idle+0x24/0xf0) from [<c06ed3c0>] (rest_init+0x88/0xa0)
<4>[ 42.524319] [<c06ed3c0>] (rest_init+0x88/0xa0) from [<c0c00978>] (start_kernel+0x3d0/0x434)
As an example, this particular crash occurred when hrtimer_start() was
executed on CPU #0. The code locked the hrtimer's current cpu_base
corresponding to CPU #1. CPU #0 then tried to switch the hrtimer's
cpu_base to an optimal CPU which was online. In this case, it selected
the cpu_base corresponding to CPU #3.
Before it could proceed, CPU #1 came online and reinitialized the
spinlock corresponding to its cpu_base. Thus now CPU #0 held a lock
which was reinitialized. When CPU #0 finally ended up unlocking the
old cpu_base corresponding to CPU #1 so that it could switch to CPU
#3, we hit this SPIN_BUG() above while in switch_hrtimer_base().
CPU #0 CPU #1
---- ----
... <offline>
hrtimer_start()
lock_hrtimer_base(base #1)
... init_hrtimers_cpu()
switch_hrtimer_base() ...
... raw_spin_lock_init(&cpu_base->lock)
raw_spin_unlock(&cpu_base->lock) ...
<spin_bug>
Solve this by statically initializing the lock.
Signed-off-by: Michael Bohan <mbohan@codeaurora.org>
Link: http://lkml.kernel.org/r/1363745965-23475-1-git-send-email-mbohan@codeaurora.org
Cc: stable@vger.kernel.org
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-03-20 10:19:25 +08:00
|
|
|
.lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
|
2007-02-16 17:27:50 +08:00
|
|
|
.clock_base =
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2007-02-16 17:27:50 +08:00
|
|
|
{
|
2011-05-20 19:05:15 +08:00
|
|
|
.index = HRTIMER_BASE_MONOTONIC,
|
|
|
|
.clockid = CLOCK_MONOTONIC,
|
2007-02-16 17:27:50 +08:00
|
|
|
.get_time = &ktime_get,
|
|
|
|
},
|
2011-05-21 05:14:04 +08:00
|
|
|
{
|
|
|
|
.index = HRTIMER_BASE_REALTIME,
|
|
|
|
.clockid = CLOCK_REALTIME,
|
|
|
|
.get_time = &ktime_get_real,
|
|
|
|
},
|
2018-04-25 21:33:38 +08:00
|
|
|
{
|
|
|
|
.index = HRTIMER_BASE_BOOTTIME,
|
|
|
|
.clockid = CLOCK_BOOTTIME,
|
|
|
|
.get_time = &ktime_get_boottime,
|
|
|
|
},
|
2013-01-22 09:00:11 +08:00
|
|
|
{
|
|
|
|
.index = HRTIMER_BASE_TAI,
|
|
|
|
.clockid = CLOCK_TAI,
|
|
|
|
.get_time = &ktime_get_clocktai,
|
|
|
|
},
|
2017-12-21 18:41:55 +08:00
|
|
|
{
|
|
|
|
.index = HRTIMER_BASE_MONOTONIC_SOFT,
|
|
|
|
.clockid = CLOCK_MONOTONIC,
|
|
|
|
.get_time = &ktime_get,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
.index = HRTIMER_BASE_REALTIME_SOFT,
|
|
|
|
.clockid = CLOCK_REALTIME,
|
|
|
|
.get_time = &ktime_get_real,
|
|
|
|
},
|
2018-04-25 21:33:38 +08:00
|
|
|
{
|
|
|
|
.index = HRTIMER_BASE_BOOTTIME_SOFT,
|
|
|
|
.clockid = CLOCK_BOOTTIME,
|
|
|
|
.get_time = &ktime_get_boottime,
|
|
|
|
},
|
2017-12-21 18:41:55 +08:00
|
|
|
{
|
|
|
|
.index = HRTIMER_BASE_TAI_SOFT,
|
|
|
|
.clockid = CLOCK_TAI,
|
|
|
|
.get_time = &ktime_get_clocktai,
|
|
|
|
},
|
2007-02-16 17:27:50 +08:00
|
|
|
}
|
2006-01-10 12:52:32 +08:00
|
|
|
};
|
|
|
|
|
2011-05-03 03:24:27 +08:00
|
|
|
static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {
|
2016-01-16 01:41:09 +08:00
|
|
|
/* Make sure we catch unsupported clockids */
|
|
|
|
[0 ... MAX_CLOCKS - 1] = HRTIMER_MAX_CLOCK_BASES,
|
|
|
|
|
2011-04-29 06:02:00 +08:00
|
|
|
[CLOCK_REALTIME] = HRTIMER_BASE_REALTIME,
|
|
|
|
[CLOCK_MONOTONIC] = HRTIMER_BASE_MONOTONIC,
|
2018-04-25 21:33:38 +08:00
|
|
|
[CLOCK_BOOTTIME] = HRTIMER_BASE_BOOTTIME,
|
2013-01-22 09:00:11 +08:00
|
|
|
[CLOCK_TAI] = HRTIMER_BASE_TAI,
|
2011-04-29 06:02:00 +08:00
|
|
|
};
|
2010-12-15 11:37:07 +08:00
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
/*
|
|
|
|
* Functions and macros which are different for UP/SMP systems are kept in a
|
|
|
|
* single place
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
2015-06-11 20:46:48 +08:00
|
|
|
/*
|
|
|
|
* We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()
|
|
|
|
* such that hrtimer_callback_running() can unconditionally dereference
|
|
|
|
* timer->base->cpu_base
|
|
|
|
*/
|
|
|
|
static struct hrtimer_cpu_base migration_cpu_base = {
|
2020-07-20 23:55:30 +08:00
|
|
|
.clock_base = { {
|
|
|
|
.cpu_base = &migration_cpu_base,
|
|
|
|
.seq = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq,
|
|
|
|
&migration_cpu_base.lock),
|
|
|
|
}, },
|
2015-06-11 20:46:48 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
#define migration_base migration_cpu_base.clock_base[0]
|
|
|
|
|
2019-09-04 22:55:27 +08:00
|
|
|
static inline bool is_migration_base(struct hrtimer_clock_base *base)
|
|
|
|
{
|
|
|
|
return base == &migration_base;
|
|
|
|
}
|
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
/*
|
|
|
|
* We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
|
|
|
|
* means that all timers which are tied to this base via timer->base are
|
|
|
|
* locked, and the base itself is locked too.
|
|
|
|
*
|
|
|
|
* So __run_timers/migrate_timers can safely modify all timers which could
|
|
|
|
* be found on the lists/queues.
|
|
|
|
*
|
|
|
|
* When the timer's base is locked, and the timer removed from list, it is
|
2015-06-11 20:46:48 +08:00
|
|
|
* possible to set timer->base = &migration_base and drop the lock: the timer
|
|
|
|
* remains locked.
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
2007-02-16 17:27:50 +08:00
|
|
|
static
|
|
|
|
struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
|
|
|
|
unsigned long *flags)
|
2023-06-21 15:59:28 +08:00
|
|
|
__acquires(&timer->base->lock)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2007-02-16 17:27:50 +08:00
|
|
|
struct hrtimer_clock_base *base;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
|
|
|
for (;;) {
|
2019-10-09 01:32:04 +08:00
|
|
|
base = READ_ONCE(timer->base);
|
2015-06-11 20:46:48 +08:00
|
|
|
if (likely(base != &migration_base)) {
|
2009-11-17 23:36:54 +08:00
|
|
|
raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
|
2006-01-10 12:52:32 +08:00
|
|
|
if (likely(base == timer->base))
|
|
|
|
return base;
|
|
|
|
/* The timer has migrated to another CPU: */
|
2009-11-17 23:36:54 +08:00
|
|
|
raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-07-10 20:57:05 +08:00
|
|
|
/*
|
2017-12-21 18:41:44 +08:00
|
|
|
* We do not migrate the timer when it is expiring before the next
|
|
|
|
* event on the target cpu. When high resolution is enabled, we cannot
|
|
|
|
* reprogram the target cpu hardware and we would cause it to fire
|
|
|
|
* late. To keep it simple, we handle the high resolution enabled and
|
|
|
|
* disabled case similar.
|
2009-07-10 20:57:05 +08:00
|
|
|
*
|
|
|
|
* Called with cpu_base->lock of target cpu held.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
|
|
|
|
{
|
|
|
|
ktime_t expires;
|
|
|
|
|
|
|
|
expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
|
2017-12-21 18:41:50 +08:00
|
|
|
return expires < new_base->cpu_base->expires_next;
|
2009-07-10 20:57:05 +08:00
|
|
|
}
|
|
|
|
|
2015-05-27 06:50:33 +08:00
|
|
|
static inline
|
|
|
|
struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
|
|
|
|
int pinned)
|
|
|
|
{
|
2018-01-15 06:30:51 +08:00
|
|
|
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
|
|
|
|
if (static_branch_likely(&timers_migration_enabled) && !pinned)
|
|
|
|
return &per_cpu(hrtimer_bases, get_nohz_timer_target());
|
|
|
|
#endif
|
2015-08-18 22:18:28 +08:00
|
|
|
return base;
|
2015-05-27 06:50:33 +08:00
|
|
|
}
|
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
/*
|
2015-08-18 22:18:29 +08:00
|
|
|
* We switch the timer base to a power-optimized selected CPU target,
|
|
|
|
* if:
|
|
|
|
* - NO_HZ_COMMON is enabled
|
|
|
|
* - timer migration is enabled
|
|
|
|
* - the timer callback is not running
|
|
|
|
* - the timer is not the first expiring timer on the new target
|
|
|
|
*
|
|
|
|
* If one of the above requirements is not fulfilled we move the timer
|
|
|
|
* to the current CPU or leave it on the previously assigned CPU if
|
|
|
|
* the timer callback is currently running.
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
2007-02-16 17:27:50 +08:00
|
|
|
static inline struct hrtimer_clock_base *
|
2009-04-16 14:43:26 +08:00
|
|
|
switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
|
|
|
|
int pinned)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2015-08-18 22:18:29 +08:00
|
|
|
struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;
|
2007-02-16 17:27:50 +08:00
|
|
|
struct hrtimer_clock_base *new_base;
|
2011-05-20 19:05:15 +08:00
|
|
|
int basenum = base->index;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2015-08-18 22:18:29 +08:00
|
|
|
this_cpu_base = this_cpu_ptr(&hrtimer_bases);
|
|
|
|
new_cpu_base = get_target_base(this_cpu_base, pinned);
|
2009-04-16 14:46:41 +08:00
|
|
|
again:
|
2010-12-15 11:37:07 +08:00
|
|
|
new_base = &new_cpu_base->clock_base[basenum];
|
2006-01-10 12:52:32 +08:00
|
|
|
|
|
|
|
if (base != new_base) {
|
|
|
|
/*
|
2009-07-10 20:57:05 +08:00
|
|
|
* We are trying to move timer to new_base.
|
2006-01-10 12:52:32 +08:00
|
|
|
* However we can't change timer's base while it is running,
|
|
|
|
* so we keep it on the same CPU. No hassle vs. reprogramming
|
|
|
|
* the event source in the high resolution case. The softirq
|
|
|
|
* code will take care of this when the timer function has
|
|
|
|
* completed. There is no conflict as we hold the lock until
|
|
|
|
* the timer is enqueued.
|
|
|
|
*/
|
2007-02-16 17:28:11 +08:00
|
|
|
if (unlikely(hrtimer_callback_running(timer)))
|
2006-01-10 12:52:32 +08:00
|
|
|
return base;
|
|
|
|
|
2015-06-11 20:46:48 +08:00
|
|
|
/* See the comment in lock_hrtimer_base() */
|
2019-10-09 01:32:04 +08:00
|
|
|
WRITE_ONCE(timer->base, &migration_base);
|
2009-11-17 23:36:54 +08:00
|
|
|
raw_spin_unlock(&base->cpu_base->lock);
|
|
|
|
raw_spin_lock(&new_base->cpu_base->lock);
|
2009-04-16 14:46:41 +08:00
|
|
|
|
2015-08-18 22:18:29 +08:00
|
|
|
if (new_cpu_base != this_cpu_base &&
|
2015-05-27 06:50:33 +08:00
|
|
|
hrtimer_check_target(timer, new_base)) {
|
2009-11-17 23:36:54 +08:00
|
|
|
raw_spin_unlock(&new_base->cpu_base->lock);
|
|
|
|
raw_spin_lock(&base->cpu_base->lock);
|
2015-08-18 22:18:29 +08:00
|
|
|
new_cpu_base = this_cpu_base;
|
2019-10-09 01:32:04 +08:00
|
|
|
WRITE_ONCE(timer->base, base);
|
2009-07-10 20:57:05 +08:00
|
|
|
goto again;
|
2009-04-16 14:46:41 +08:00
|
|
|
}
|
2019-10-09 01:32:04 +08:00
|
|
|
WRITE_ONCE(timer->base, new_base);
|
2014-04-30 16:43:10 +08:00
|
|
|
} else {
|
2015-08-18 22:18:29 +08:00
|
|
|
if (new_cpu_base != this_cpu_base &&
|
2015-05-27 06:50:33 +08:00
|
|
|
hrtimer_check_target(timer, new_base)) {
|
2015-08-18 22:18:29 +08:00
|
|
|
new_cpu_base = this_cpu_base;
|
2014-04-30 16:43:10 +08:00
|
|
|
goto again;
|
|
|
|
}
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
|
|
|
return new_base;
|
|
|
|
}
|
|
|
|
|
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
|
2019-09-04 22:55:27 +08:00
|
|
|
static inline bool is_migration_base(struct hrtimer_clock_base *base)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2007-02-16 17:27:50 +08:00
|
|
|
static inline struct hrtimer_clock_base *
|
2006-01-10 12:52:32 +08:00
|
|
|
lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
|
2023-06-21 15:59:28 +08:00
|
|
|
__acquires(&timer->base->cpu_base->lock)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2007-02-16 17:27:50 +08:00
|
|
|
struct hrtimer_clock_base *base = timer->base;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2009-11-17 23:36:54 +08:00
|
|
|
raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
|
|
|
return base;
|
|
|
|
}
|
|
|
|
|
2009-04-16 14:46:41 +08:00
|
|
|
# define switch_hrtimer_base(t, b, p) (b)
|
2006-01-10 12:52:32 +08:00
|
|
|
|
|
|
|
#endif /* !CONFIG_SMP */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Functions for the union type storage format of ktime_t which are
|
|
|
|
* too large for inlining:
|
|
|
|
*/
|
|
|
|
#if BITS_PER_LONG < 64
|
|
|
|
/*
|
|
|
|
* Divide a ktime value by a nanosecond value
|
|
|
|
*/
|
2015-05-09 04:47:23 +08:00
|
|
|
s64 __ktime_divns(const ktime_t kt, s64 div)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
|
|
|
int sft = 0;
|
2015-05-09 04:47:23 +08:00
|
|
|
s64 dclc;
|
|
|
|
u64 tmp;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2008-05-23 06:25:11 +08:00
|
|
|
dclc = ktime_to_ns(kt);
|
2015-05-09 04:47:23 +08:00
|
|
|
tmp = dclc < 0 ? -dclc : dclc;
|
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
/* Make sure the divisor is less than 2^32: */
|
|
|
|
while (div >> 32) {
|
|
|
|
sft++;
|
|
|
|
div >>= 1;
|
|
|
|
}
|
2015-05-09 04:47:23 +08:00
|
|
|
tmp >>= sft;
|
2020-01-30 21:08:51 +08:00
|
|
|
do_div(tmp, (u32) div);
|
2015-05-09 04:47:23 +08:00
|
|
|
return dclc < 0 ? -tmp : tmp;
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
2014-12-04 03:43:06 +08:00
|
|
|
EXPORT_SYMBOL_GPL(__ktime_divns);
|
2006-01-10 12:52:32 +08:00
|
|
|
#endif /* BITS_PER_LONG >= 64 */
|
|
|
|
|
2008-02-13 16:20:43 +08:00
|
|
|
/*
|
|
|
|
* Add two ktime values and do a safety check for overflow:
|
|
|
|
*/
|
|
|
|
ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
|
|
|
|
{
|
2016-08-13 07:37:04 +08:00
|
|
|
ktime_t res = ktime_add_unsafe(lhs, rhs);
|
2008-02-13 16:20:43 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We use KTIME_SEC_MAX here, the maximum timeout which we can
|
|
|
|
* return to user space in a timespec:
|
|
|
|
*/
|
2016-12-25 18:38:40 +08:00
|
|
|
if (res < 0 || res < lhs || res < rhs)
|
2008-02-13 16:20:43 +08:00
|
|
|
res = ktime_set(KTIME_SEC_MAX, 0);
|
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2009-05-28 21:21:24 +08:00
|
|
|
EXPORT_SYMBOL_GPL(ktime_add_safe);
|
|
|
|
|
2008-04-30 15:55:04 +08:00
|
|
|
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
|
|
|
|
|
2020-08-15 08:40:27 +08:00
|
|
|
static const struct debug_obj_descr hrtimer_debug_descr;
|
2008-04-30 15:55:04 +08:00
|
|
|
|
2011-03-07 16:58:33 +08:00
|
|
|
static void *hrtimer_debug_hint(void *addr)
|
|
|
|
{
|
|
|
|
return ((struct hrtimer *) addr)->function;
|
|
|
|
}
|
|
|
|
|
2008-04-30 15:55:04 +08:00
|
|
|
/*
|
|
|
|
* fixup_init is called when:
|
|
|
|
* - an active object is initialized
|
|
|
|
*/
|
2016-05-20 08:09:29 +08:00
|
|
|
static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)
|
2008-04-30 15:55:04 +08:00
|
|
|
{
|
|
|
|
struct hrtimer *timer = addr;
|
|
|
|
|
|
|
|
switch (state) {
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
hrtimer_cancel(timer);
|
|
|
|
debug_object_init(timer, &hrtimer_debug_descr);
|
2016-05-20 08:09:29 +08:00
|
|
|
return true;
|
2008-04-30 15:55:04 +08:00
|
|
|
default:
|
2016-05-20 08:09:29 +08:00
|
|
|
return false;
|
2008-04-30 15:55:04 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fixup_activate is called when:
|
|
|
|
* - an active object is activated
|
2016-05-20 08:09:41 +08:00
|
|
|
* - an unknown non-static object is activated
|
2008-04-30 15:55:04 +08:00
|
|
|
*/
|
2016-05-20 08:09:29 +08:00
|
|
|
static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
|
2008-04-30 15:55:04 +08:00
|
|
|
{
|
|
|
|
switch (state) {
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
WARN_ON(1);
|
2020-08-24 06:36:59 +08:00
|
|
|
fallthrough;
|
2008-04-30 15:55:04 +08:00
|
|
|
default:
|
2016-05-20 08:09:29 +08:00
|
|
|
return false;
|
2008-04-30 15:55:04 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fixup_free is called when:
|
|
|
|
* - an active object is freed
|
|
|
|
*/
|
2016-05-20 08:09:29 +08:00
|
|
|
static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
|
2008-04-30 15:55:04 +08:00
|
|
|
{
|
|
|
|
struct hrtimer *timer = addr;
|
|
|
|
|
|
|
|
switch (state) {
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
hrtimer_cancel(timer);
|
|
|
|
debug_object_free(timer, &hrtimer_debug_descr);
|
2016-05-20 08:09:29 +08:00
|
|
|
return true;
|
2008-04-30 15:55:04 +08:00
|
|
|
default:
|
2016-05-20 08:09:29 +08:00
|
|
|
return false;
|
2008-04-30 15:55:04 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-08-15 08:40:27 +08:00
|
|
|
static const struct debug_obj_descr hrtimer_debug_descr = {
|
2008-04-30 15:55:04 +08:00
|
|
|
.name = "hrtimer",
|
2011-03-07 16:58:33 +08:00
|
|
|
.debug_hint = hrtimer_debug_hint,
|
2008-04-30 15:55:04 +08:00
|
|
|
.fixup_init = hrtimer_fixup_init,
|
|
|
|
.fixup_activate = hrtimer_fixup_activate,
|
|
|
|
.fixup_free = hrtimer_fixup_free,
|
|
|
|
};
|
|
|
|
|
|
|
|
static inline void debug_hrtimer_init(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_init(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
static inline void debug_hrtimer_activate(struct hrtimer *timer,
|
|
|
|
enum hrtimer_mode mode)
|
2008-04-30 15:55:04 +08:00
|
|
|
{
|
|
|
|
debug_object_activate(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_deactivate(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode);
|
|
|
|
|
|
|
|
void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
debug_object_init_on_stack(timer, &hrtimer_debug_descr);
|
|
|
|
__hrtimer_init(timer, clock_id, mode);
|
|
|
|
}
|
2009-08-29 14:41:29 +08:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
|
2008-04-30 15:55:04 +08:00
|
|
|
|
2019-07-27 02:30:50 +08:00
|
|
|
static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
|
|
|
|
clockid_t clock_id, enum hrtimer_mode mode);
|
|
|
|
|
|
|
|
void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
|
|
|
|
clockid_t clock_id, enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr);
|
|
|
|
__hrtimer_init_sleeper(sl, clock_id, mode);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_init_sleeper_on_stack);
|
|
|
|
|
2008-04-30 15:55:04 +08:00
|
|
|
void destroy_hrtimer_on_stack(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_free(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
2016-05-27 08:21:05 +08:00
|
|
|
EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
|
2008-04-30 15:55:04 +08:00
|
|
|
|
|
|
|
#else
|
2017-12-21 18:41:57 +08:00
|
|
|
|
2008-04-30 15:55:04 +08:00
|
|
|
static inline void debug_hrtimer_init(struct hrtimer *timer) { }
|
2017-12-21 18:41:57 +08:00
|
|
|
static inline void debug_hrtimer_activate(struct hrtimer *timer,
|
|
|
|
enum hrtimer_mode mode) { }
|
2008-04-30 15:55:04 +08:00
|
|
|
static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
|
|
|
|
#endif
|
|
|
|
|
2009-08-10 10:51:23 +08:00
|
|
|
static inline void
|
|
|
|
debug_init(struct hrtimer *timer, clockid_t clockid,
|
|
|
|
enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
debug_hrtimer_init(timer);
|
|
|
|
trace_hrtimer_init(timer, clockid, mode);
|
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:38 +08:00
|
|
|
static inline void debug_activate(struct hrtimer *timer,
|
|
|
|
enum hrtimer_mode mode)
|
2009-08-10 10:51:23 +08:00
|
|
|
{
|
2017-12-21 18:41:57 +08:00
|
|
|
debug_hrtimer_activate(timer, mode);
|
2017-12-21 18:41:38 +08:00
|
|
|
trace_hrtimer_start(timer, mode);
|
2009-08-10 10:51:23 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_deactivate(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_hrtimer_deactivate(timer);
|
|
|
|
trace_hrtimer_cancel(timer);
|
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:39 +08:00
|
|
|
static struct hrtimer_clock_base *
|
|
|
|
__next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)
|
|
|
|
{
|
|
|
|
unsigned int idx;
|
|
|
|
|
|
|
|
if (!*active)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
idx = __ffs(*active);
|
|
|
|
*active &= ~(1U << idx);
|
|
|
|
|
|
|
|
return &cpu_base->clock_base[idx];
|
|
|
|
}
|
|
|
|
|
|
|
|
#define for_each_active_base(base, cpu_base, active) \
|
|
|
|
while ((base = __next_base((cpu_base), &(active))))
|
|
|
|
|
2017-12-21 18:41:53 +08:00
|
|
|
static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
|
2018-04-04 05:17:00 +08:00
|
|
|
const struct hrtimer *exclude,
|
2017-12-21 18:41:53 +08:00
|
|
|
unsigned int active,
|
|
|
|
ktime_t expires_next)
|
2015-01-21 04:24:10 +08:00
|
|
|
{
|
2017-12-21 18:41:39 +08:00
|
|
|
struct hrtimer_clock_base *base;
|
2017-12-21 18:41:53 +08:00
|
|
|
ktime_t expires;
|
2015-01-21 04:24:10 +08:00
|
|
|
|
2017-12-21 18:41:39 +08:00
|
|
|
for_each_active_base(base, cpu_base, active) {
|
2015-01-21 04:24:10 +08:00
|
|
|
struct timerqueue_node *next;
|
|
|
|
struct hrtimer *timer;
|
|
|
|
|
2015-04-15 05:08:41 +08:00
|
|
|
next = timerqueue_getnext(&base->active);
|
2015-01-21 04:24:10 +08:00
|
|
|
timer = container_of(next, struct hrtimer, node);
|
2018-04-04 05:17:00 +08:00
|
|
|
if (timer == exclude) {
|
|
|
|
/* Get to the next timer in the queue. */
|
2018-04-08 01:11:55 +08:00
|
|
|
next = timerqueue_iterate_next(next);
|
2018-04-04 05:17:00 +08:00
|
|
|
if (!next)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
timer = container_of(next, struct hrtimer, node);
|
|
|
|
}
|
2015-01-21 04:24:10 +08:00
|
|
|
expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
|
2016-12-25 18:38:40 +08:00
|
|
|
if (expires < expires_next) {
|
2015-01-21 04:24:10 +08:00
|
|
|
expires_next = expires;
|
2018-04-04 05:17:00 +08:00
|
|
|
|
|
|
|
/* Skip cpu_base update if a timer is being excluded. */
|
|
|
|
if (exclude)
|
|
|
|
continue;
|
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
if (timer->is_soft)
|
|
|
|
cpu_base->softirq_next_timer = timer;
|
|
|
|
else
|
|
|
|
cpu_base->next_timer = timer;
|
2015-04-15 05:08:49 +08:00
|
|
|
}
|
2015-01-21 04:24:10 +08:00
|
|
|
}
|
|
|
|
/*
|
|
|
|
* clock_was_set() might have changed base->offset of any of
|
|
|
|
* the clock bases so the result might be negative. Fix it up
|
|
|
|
* to prevent a false positive in clockevents_program_event().
|
|
|
|
*/
|
2016-12-25 18:38:40 +08:00
|
|
|
if (expires_next < 0)
|
|
|
|
expires_next = 0;
|
2015-01-21 04:24:10 +08:00
|
|
|
return expires_next;
|
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:56 +08:00
|
|
|
/*
|
2021-02-24 00:02:40 +08:00
|
|
|
* Recomputes cpu_base::*next_timer and returns the earliest expires_next
|
|
|
|
* but does not set cpu_base::*expires_next, that is done by
|
|
|
|
* hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating
|
|
|
|
* cpu_base::*expires_next right away, reprogramming logic would no longer
|
|
|
|
* work.
|
2017-12-21 18:41:56 +08:00
|
|
|
*
|
2017-12-21 18:41:57 +08:00
|
|
|
* When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
|
|
|
|
* those timers will get run whenever the softirq gets handled, at the end of
|
|
|
|
* hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.
|
|
|
|
*
|
|
|
|
* Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases.
|
|
|
|
* The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual
|
|
|
|
* softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD.
|
|
|
|
*
|
2017-12-21 18:41:56 +08:00
|
|
|
* @active_mask must be one of:
|
2017-12-21 18:41:57 +08:00
|
|
|
* - HRTIMER_ACTIVE_ALL,
|
2017-12-21 18:41:56 +08:00
|
|
|
* - HRTIMER_ACTIVE_SOFT, or
|
|
|
|
* - HRTIMER_ACTIVE_HARD.
|
|
|
|
*/
|
2017-12-21 18:41:57 +08:00
|
|
|
static ktime_t
|
|
|
|
__hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
|
2017-12-21 18:41:53 +08:00
|
|
|
{
|
2017-12-21 18:41:56 +08:00
|
|
|
unsigned int active;
|
2017-12-21 18:41:57 +08:00
|
|
|
struct hrtimer *next_timer = NULL;
|
2017-12-21 18:41:53 +08:00
|
|
|
ktime_t expires_next = KTIME_MAX;
|
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
|
|
|
|
active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
|
|
|
|
cpu_base->softirq_next_timer = NULL;
|
2018-04-04 05:17:00 +08:00
|
|
|
expires_next = __hrtimer_next_event_base(cpu_base, NULL,
|
|
|
|
active, KTIME_MAX);
|
2017-12-21 18:41:57 +08:00
|
|
|
|
|
|
|
next_timer = cpu_base->softirq_next_timer;
|
|
|
|
}
|
2017-12-21 18:41:53 +08:00
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
if (active_mask & HRTIMER_ACTIVE_HARD) {
|
|
|
|
active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
|
|
|
|
cpu_base->next_timer = next_timer;
|
2018-04-04 05:17:00 +08:00
|
|
|
expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,
|
|
|
|
expires_next);
|
2017-12-21 18:41:57 +08:00
|
|
|
}
|
2017-12-21 18:41:53 +08:00
|
|
|
|
|
|
|
return expires_next;
|
|
|
|
}
|
|
|
|
|
2021-02-24 00:02:40 +08:00
|
|
|
static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
|
|
|
|
{
|
|
|
|
ktime_t expires_next, soft = KTIME_MAX;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the soft interrupt has already been activated, ignore the
|
|
|
|
* soft bases. They will be handled in the already raised soft
|
|
|
|
* interrupt.
|
|
|
|
*/
|
|
|
|
if (!cpu_base->softirq_activated) {
|
|
|
|
soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
|
|
|
|
/*
|
|
|
|
* Update the soft expiry time. clock_settime() might have
|
|
|
|
* affected it.
|
|
|
|
*/
|
|
|
|
cpu_base->softirq_expires_next = soft;
|
|
|
|
}
|
|
|
|
|
|
|
|
expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);
|
|
|
|
/*
|
|
|
|
* If a softirq timer is expiring first, update cpu_base->next_timer
|
|
|
|
* and program the hardware with the soft expiry time.
|
|
|
|
*/
|
|
|
|
if (expires_next > soft) {
|
|
|
|
cpu_base->next_timer = cpu_base->softirq_next_timer;
|
|
|
|
expires_next = soft;
|
|
|
|
}
|
|
|
|
|
|
|
|
return expires_next;
|
|
|
|
}
|
|
|
|
|
2015-04-15 05:08:35 +08:00
|
|
|
static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
|
|
|
|
{
|
|
|
|
ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
|
2018-04-25 21:33:38 +08:00
|
|
|
ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
|
2015-04-15 05:08:35 +08:00
|
|
|
ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
|
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq,
|
2018-04-25 21:33:38 +08:00
|
|
|
offs_real, offs_boot, offs_tai);
|
2017-12-21 18:41:57 +08:00
|
|
|
|
|
|
|
base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
|
2018-04-25 21:33:38 +08:00
|
|
|
base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
|
2017-12-21 18:41:57 +08:00
|
|
|
base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
|
|
|
|
|
|
|
|
return now;
|
2015-04-15 05:08:35 +08:00
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:42 +08:00
|
|
|
/*
|
|
|
|
* Is the high resolution mode active ?
|
|
|
|
*/
|
|
|
|
static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
|
|
|
|
{
|
|
|
|
return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
|
|
|
|
cpu_base->hres_active : 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int hrtimer_hres_active(void)
|
|
|
|
{
|
|
|
|
return __hrtimer_hres_active(this_cpu_ptr(&hrtimer_bases));
|
|
|
|
}
|
|
|
|
|
2021-08-13 04:32:30 +08:00
|
|
|
static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
|
|
|
|
struct hrtimer *next_timer,
|
|
|
|
ktime_t expires_next)
|
2007-02-16 17:28:11 +08:00
|
|
|
{
|
2016-12-25 18:38:40 +08:00
|
|
|
cpu_base->expires_next = expires_next;
|
2009-09-02 11:03:33 +08:00
|
|
|
|
2014-04-30 06:55:02 +08:00
|
|
|
/*
|
2017-12-21 18:41:48 +08:00
|
|
|
* If hres is not active, hardware does not have to be
|
|
|
|
* reprogrammed yet.
|
|
|
|
*
|
2014-04-30 06:55:02 +08:00
|
|
|
* If a hang was detected in the last timer interrupt then we
|
|
|
|
* leave the hang delay active in the hardware. We want the
|
|
|
|
* system to make progress. That also prevents the following
|
|
|
|
* scenario:
|
|
|
|
* T1 expires 50ms from now
|
|
|
|
* T2 expires 5s from now
|
|
|
|
*
|
|
|
|
* T1 is removed, so this code is called and would reprogram
|
|
|
|
* the hardware to 5s from now. Any hrtimer_start after that
|
|
|
|
* will not reprogram the hardware due to hang_detected being
|
2021-03-23 05:39:03 +08:00
|
|
|
* set. So we'd effectively block all timers until the T2 event
|
2014-04-30 06:55:02 +08:00
|
|
|
* fires.
|
|
|
|
*/
|
2017-12-21 18:41:48 +08:00
|
|
|
if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
|
2014-04-30 06:55:02 +08:00
|
|
|
return;
|
|
|
|
|
2021-07-13 21:39:47 +08:00
|
|
|
tick_program_event(expires_next, 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Reprogram the event source with checking both queues for the
|
|
|
|
* next event
|
|
|
|
* Called with interrupts disabled and base->lock held
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
|
|
|
|
{
|
|
|
|
ktime_t expires_next;
|
|
|
|
|
|
|
|
expires_next = hrtimer_update_next_event(cpu_base);
|
|
|
|
|
2021-08-13 04:32:30 +08:00
|
|
|
if (skip_equal && expires_next == cpu_base->expires_next)
|
|
|
|
return;
|
|
|
|
|
|
|
|
__hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);
|
2007-02-16 17:28:11 +08:00
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:47 +08:00
|
|
|
/* High resolution timer related functions */
|
|
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
|
|
|
|
|
|
|
/*
|
|
|
|
* High resolution timer enabled ?
|
|
|
|
*/
|
|
|
|
static bool hrtimer_hres_enabled __read_mostly = true;
|
|
|
|
unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
|
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_resolution);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Enable / Disable high resolution mode
|
|
|
|
*/
|
|
|
|
static int __init setup_hrtimer_hres(char *str)
|
|
|
|
{
|
|
|
|
return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
__setup("highres=", setup_hrtimer_hres);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* hrtimer_high_res_enabled - query, if the highres mode is enabled
|
|
|
|
*/
|
|
|
|
static inline int hrtimer_is_hres_enabled(void)
|
|
|
|
{
|
|
|
|
return hrtimer_hres_enabled;
|
|
|
|
}
|
|
|
|
|
2021-07-13 21:39:49 +08:00
|
|
|
static void retrigger_next_event(void *arg);
|
2011-05-02 22:48:57 +08:00
|
|
|
|
2007-02-16 17:28:11 +08:00
|
|
|
/*
|
|
|
|
* Switch to high resolution mode
|
|
|
|
*/
|
2015-08-12 04:40:43 +08:00
|
|
|
static void hrtimer_switch_to_hres(void)
|
2007-02-16 17:28:11 +08:00
|
|
|
{
|
2015-04-15 05:08:51 +08:00
|
|
|
struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
|
2007-02-16 17:28:11 +08:00
|
|
|
|
|
|
|
if (tick_init_highres()) {
|
2018-07-12 22:41:18 +08:00
|
|
|
pr_warn("Could not switch to high resolution mode on CPU %u\n",
|
|
|
|
base->cpu);
|
2015-08-22 16:10:47 +08:00
|
|
|
return;
|
2007-02-16 17:28:11 +08:00
|
|
|
}
|
|
|
|
base->hres_active = 1;
|
2015-04-15 05:08:27 +08:00
|
|
|
hrtimer_resolution = HIGH_RES_NSEC;
|
2007-02-16 17:28:11 +08:00
|
|
|
|
|
|
|
tick_setup_sched_timer();
|
|
|
|
/* "Retrigger" the interrupt to get things going */
|
|
|
|
retrigger_next_event(NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
static inline int hrtimer_is_hres_enabled(void) { return 0; }
|
2015-08-12 04:40:43 +08:00
|
|
|
static inline void hrtimer_switch_to_hres(void) { }
|
2007-02-16 17:28:11 +08:00
|
|
|
|
|
|
|
#endif /* CONFIG_HIGH_RES_TIMERS */
|
2021-07-13 21:39:49 +08:00
|
|
|
/*
|
|
|
|
* Retrigger next event is called after clock was set with interrupts
|
|
|
|
* disabled through an SMP function call or directly from low level
|
|
|
|
* resume code.
|
|
|
|
*
|
|
|
|
* This is only invoked when:
|
|
|
|
* - CONFIG_HIGH_RES_TIMERS is enabled.
|
|
|
|
* - CONFIG_NOHZ_COMMON is enabled
|
|
|
|
*
|
|
|
|
* For the other cases this function is empty and because the call sites
|
|
|
|
* are optimized out it vanishes as well, i.e. no need for lots of
|
|
|
|
* #ifdeffery.
|
|
|
|
*/
|
|
|
|
static void retrigger_next_event(void *arg)
|
|
|
|
{
|
|
|
|
struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When high resolution mode or nohz is active, then the offsets of
|
|
|
|
* CLOCK_REALTIME/TAI/BOOTTIME have to be updated. Otherwise the
|
|
|
|
* next tick will take care of that.
|
|
|
|
*
|
|
|
|
* If high resolution mode is active then the next expiring timer
|
|
|
|
* must be reevaluated and the clock event device reprogrammed if
|
|
|
|
* necessary.
|
|
|
|
*
|
|
|
|
* In the NOHZ case the update of the offset and the reevaluation
|
|
|
|
* of the next expiring timer is enough. The return from the SMP
|
|
|
|
* function call will take care of the reprogramming in case the
|
|
|
|
* CPU was in a NOHZ idle sleep.
|
|
|
|
*/
|
|
|
|
if (!__hrtimer_hres_active(base) && !tick_nohz_active)
|
|
|
|
return;
|
|
|
|
|
|
|
|
raw_spin_lock(&base->lock);
|
|
|
|
hrtimer_update_base(base);
|
|
|
|
if (__hrtimer_hres_active(base))
|
|
|
|
hrtimer_force_reprogram(base, 0);
|
|
|
|
else
|
|
|
|
hrtimer_update_next_event(base);
|
|
|
|
raw_spin_unlock(&base->lock);
|
|
|
|
}
|
2007-02-16 17:28:11 +08:00
|
|
|
|
2017-12-21 18:41:46 +08:00
|
|
|
/*
|
|
|
|
* When a timer is enqueued and expires earlier than the already enqueued
|
|
|
|
* timers, we have to check, whether it expires earlier than the timer for
|
|
|
|
* which the clock event device was armed.
|
|
|
|
*
|
|
|
|
* Called with interrupts disabled and base->cpu_base.lock held
|
|
|
|
*/
|
2017-12-21 18:41:57 +08:00
|
|
|
static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
|
2017-12-21 18:41:46 +08:00
|
|
|
{
|
|
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
2017-12-21 18:41:51 +08:00
|
|
|
struct hrtimer_clock_base *base = timer->base;
|
2017-12-21 18:41:46 +08:00
|
|
|
ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
|
|
|
|
|
|
|
|
WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
|
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
/*
|
|
|
|
* CLOCK_REALTIME timer might be requested with an absolute
|
|
|
|
* expiry time which is less than base->offset. Set it to 0.
|
|
|
|
*/
|
|
|
|
if (expires < 0)
|
|
|
|
expires = 0;
|
|
|
|
|
|
|
|
if (timer->is_soft) {
|
|
|
|
/*
|
|
|
|
* soft hrtimer could be started on a remote CPU. In this
|
|
|
|
* case softirq_expires_next needs to be updated on the
|
|
|
|
* remote CPU. The soft hrtimer will not expire before the
|
|
|
|
* first hard hrtimer on the remote CPU -
|
|
|
|
* hrtimer_check_target() prevents this case.
|
|
|
|
*/
|
|
|
|
struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
|
|
|
|
|
|
|
|
if (timer_cpu_base->softirq_activated)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (!ktime_before(expires, timer_cpu_base->softirq_expires_next))
|
|
|
|
return;
|
|
|
|
|
|
|
|
timer_cpu_base->softirq_next_timer = timer;
|
|
|
|
timer_cpu_base->softirq_expires_next = expires;
|
|
|
|
|
|
|
|
if (!ktime_before(expires, timer_cpu_base->expires_next) ||
|
|
|
|
!reprogram)
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:46 +08:00
|
|
|
/*
|
|
|
|
* If the timer is not on the current cpu, we cannot reprogram
|
|
|
|
* the other cpus clock event device.
|
|
|
|
*/
|
|
|
|
if (base->cpu_base != cpu_base)
|
|
|
|
return;
|
|
|
|
|
2021-08-13 04:32:30 +08:00
|
|
|
if (expires >= cpu_base->expires_next)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the hrtimer interrupt is running, then it will reevaluate the
|
|
|
|
* clock bases and reprogram the clock event device.
|
|
|
|
*/
|
|
|
|
if (cpu_base->in_hrtirq)
|
|
|
|
return;
|
|
|
|
|
|
|
|
cpu_base->next_timer = timer;
|
|
|
|
|
|
|
|
__hrtimer_reprogram(cpu_base, timer, expires);
|
2017-12-21 18:41:46 +08:00
|
|
|
}
|
|
|
|
|
2021-07-13 21:39:55 +08:00
|
|
|
static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,
|
|
|
|
unsigned int active)
|
|
|
|
{
|
|
|
|
struct hrtimer_clock_base *base;
|
|
|
|
unsigned int seq;
|
|
|
|
ktime_t expires;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Update the base offsets unconditionally so the following
|
|
|
|
* checks whether the SMP function call is required works.
|
|
|
|
*
|
|
|
|
* The update is safe even when the remote CPU is in the hrtimer
|
|
|
|
* interrupt or the hrtimer soft interrupt and expiring affected
|
|
|
|
* bases. Either it will see the update before handling a base or
|
|
|
|
* it will see it when it finishes the processing and reevaluates
|
|
|
|
* the next expiring timer.
|
|
|
|
*/
|
|
|
|
seq = cpu_base->clock_was_set_seq;
|
|
|
|
hrtimer_update_base(cpu_base);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the sequence did not change over the update then the
|
|
|
|
* remote CPU already handled it.
|
|
|
|
*/
|
|
|
|
if (seq == cpu_base->clock_was_set_seq)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the remote CPU is currently handling an hrtimer interrupt, it
|
|
|
|
* will reevaluate the first expiring timer of all clock bases
|
|
|
|
* before reprogramming. Nothing to do here.
|
|
|
|
*/
|
|
|
|
if (cpu_base->in_hrtirq)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Walk the affected clock bases and check whether the first expiring
|
|
|
|
* timer in a clock base is moving ahead of the first expiring timer of
|
|
|
|
* @cpu_base. If so, the IPI must be invoked because per CPU clock
|
|
|
|
* event devices cannot be remotely reprogrammed.
|
|
|
|
*/
|
|
|
|
active &= cpu_base->active_bases;
|
|
|
|
|
|
|
|
for_each_active_base(base, cpu_base, active) {
|
|
|
|
struct timerqueue_node *next;
|
|
|
|
|
|
|
|
next = timerqueue_getnext(&base->active);
|
|
|
|
expires = ktime_sub(next->expires, base->offset);
|
|
|
|
if (expires < cpu_base->expires_next)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
/* Extra check for softirq clock bases */
|
|
|
|
if (base->clockid < HRTIMER_BASE_MONOTONIC_SOFT)
|
|
|
|
continue;
|
|
|
|
if (cpu_base->softirq_activated)
|
|
|
|
continue;
|
|
|
|
if (expires < cpu_base->softirq_expires_next)
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2011-05-02 22:48:57 +08:00
|
|
|
/*
|
2021-07-13 21:39:49 +08:00
|
|
|
* Clock was set. This might affect CLOCK_REALTIME, CLOCK_TAI and
|
|
|
|
* CLOCK_BOOTTIME (for late sleep time injection).
|
2011-05-02 22:48:57 +08:00
|
|
|
*
|
2021-07-13 21:39:49 +08:00
|
|
|
* This requires to update the offsets for these clocks
|
|
|
|
* vs. CLOCK_MONOTONIC. When high resolution timers are enabled, then this
|
|
|
|
* also requires to eventually reprogram the per CPU clock event devices
|
|
|
|
* when the change moves an affected timer ahead of the first expiring
|
|
|
|
* timer on that CPU. Obviously remote per CPU clock event devices cannot
|
|
|
|
* be reprogrammed. The other reason why an IPI has to be sent is when the
|
|
|
|
* system is in !HIGH_RES and NOHZ mode. The NOHZ mode updates the offsets
|
|
|
|
* in the tick, which obviously might be stopped, so this has to bring out
|
|
|
|
* the remote CPU which might sleep in idle to get this sorted.
|
2011-05-02 22:48:57 +08:00
|
|
|
*/
|
2021-07-13 21:39:53 +08:00
|
|
|
void clock_was_set(unsigned int bases)
|
2011-05-02 22:48:57 +08:00
|
|
|
{
|
2021-08-13 04:31:24 +08:00
|
|
|
struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases);
|
2021-07-13 21:39:54 +08:00
|
|
|
cpumask_var_t mask;
|
|
|
|
int cpu;
|
|
|
|
|
2021-08-13 04:31:24 +08:00
|
|
|
if (!__hrtimer_hres_active(cpu_base) && !tick_nohz_active)
|
2021-07-13 21:39:49 +08:00
|
|
|
goto out_timerfd;
|
|
|
|
|
2021-07-13 21:39:54 +08:00
|
|
|
if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
|
|
|
|
on_each_cpu(retrigger_next_event, NULL, 1);
|
|
|
|
goto out_timerfd;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Avoid interrupting CPUs if possible */
|
|
|
|
cpus_read_lock();
|
|
|
|
for_each_online_cpu(cpu) {
|
|
|
|
unsigned long flags;
|
|
|
|
|
2021-08-13 04:31:24 +08:00
|
|
|
cpu_base = &per_cpu(hrtimer_bases, cpu);
|
2021-07-13 21:39:54 +08:00
|
|
|
raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
2021-07-13 21:39:55 +08:00
|
|
|
|
|
|
|
if (update_needs_ipi(cpu_base, bases))
|
2021-07-13 21:39:54 +08:00
|
|
|
cpumask_set_cpu(cpu, mask);
|
2021-07-13 21:39:55 +08:00
|
|
|
|
2021-07-13 21:39:54 +08:00
|
|
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
preempt_disable();
|
|
|
|
smp_call_function_many(mask, retrigger_next_event, NULL, 1);
|
|
|
|
preempt_enable();
|
|
|
|
cpus_read_unlock();
|
|
|
|
free_cpumask_var(mask);
|
2021-07-13 21:39:49 +08:00
|
|
|
|
|
|
|
out_timerfd:
|
2011-05-20 22:18:50 +08:00
|
|
|
timerfd_clock_was_set();
|
2011-05-02 22:48:57 +08:00
|
|
|
}
|
|
|
|
|
2021-07-13 21:39:48 +08:00
|
|
|
static void clock_was_set_work(struct work_struct *work)
|
|
|
|
{
|
2021-07-13 21:39:53 +08:00
|
|
|
clock_was_set(CLOCK_SET_WALL);
|
2021-07-13 21:39:48 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static DECLARE_WORK(hrtimer_work, clock_was_set_work);
|
|
|
|
|
|
|
|
/*
|
2021-07-13 21:39:51 +08:00
|
|
|
* Called from timekeeping code to reprogram the hrtimer interrupt device
|
|
|
|
* on all cpus and to notify timerfd.
|
2021-07-13 21:39:48 +08:00
|
|
|
*/
|
|
|
|
void clock_was_set_delayed(void)
|
|
|
|
{
|
|
|
|
schedule_work(&hrtimer_work);
|
|
|
|
}
|
|
|
|
|
2011-05-02 22:48:57 +08:00
|
|
|
/*
|
2021-07-13 21:39:51 +08:00
|
|
|
* Called during resume either directly from via timekeeping_resume()
|
|
|
|
* or in the case of s2idle from tick_unfreeze() to ensure that the
|
|
|
|
* hrtimers are up to date.
|
2011-05-02 22:48:57 +08:00
|
|
|
*/
|
2021-07-13 21:39:51 +08:00
|
|
|
void hrtimers_resume_local(void)
|
2011-05-02 22:48:57 +08:00
|
|
|
{
|
2017-11-06 23:01:21 +08:00
|
|
|
lockdep_assert_irqs_disabled();
|
2013-07-05 18:09:18 +08:00
|
|
|
/* Retrigger on the local CPU */
|
2011-05-02 22:48:57 +08:00
|
|
|
retrigger_next_event(NULL);
|
|
|
|
}
|
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
/*
|
2007-10-20 07:56:53 +08:00
|
|
|
* Counterpart to lock_hrtimer_base above:
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
|
|
|
static inline
|
|
|
|
void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
|
2023-06-21 15:59:28 +08:00
|
|
|
__releases(&timer->base->cpu_base->lock)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2009-11-17 23:36:54 +08:00
|
|
|
raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_forward - forward the timer expiry
|
|
|
|
* @timer: hrtimer to forward
|
2006-03-26 17:38:06 +08:00
|
|
|
* @now: forward past this time
|
2006-01-10 12:52:32 +08:00
|
|
|
* @interval: the interval to forward
|
|
|
|
*
|
|
|
|
* Forward the timer expiry so it will expire in the future.
|
2006-01-17 06:58:55 +08:00
|
|
|
* Returns the number of overruns.
|
2015-04-14 05:02:22 +08:00
|
|
|
*
|
|
|
|
* Can be safely called from the callback function of @timer. If
|
|
|
|
* called from other contexts @timer must neither be enqueued nor
|
|
|
|
* running the callback and the caller needs to take care of
|
|
|
|
* serialization.
|
|
|
|
*
|
|
|
|
* Note: This only updates the timer expiry value and does not requeue
|
|
|
|
* the timer.
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
timerfd: new timerfd API
This is the new timerfd API as it is implemented by the following patch:
int timerfd_create(int clockid, int flags);
int timerfd_settime(int ufd, int flags,
const struct itimerspec *utmr,
struct itimerspec *otmr);
int timerfd_gettime(int ufd, struct itimerspec *otmr);
The timerfd_create() API creates an un-programmed timerfd fd. The "clockid"
parameter can be either CLOCK_MONOTONIC or CLOCK_REALTIME.
The timerfd_settime() API give new settings by the timerfd fd, by optionally
retrieving the previous expiration time (in case the "otmr" parameter is not
NULL).
The time value specified in "utmr" is absolute, if the TFD_TIMER_ABSTIME bit
is set in the "flags" parameter. Otherwise it's a relative time.
The timerfd_gettime() API returns the next expiration time of the timer, or
{0, 0} if the timerfd has not been set yet.
Like the previous timerfd API implementation, read(2) and poll(2) are
supported (with the same interface). Here's a simple test program I used to
exercise the new timerfd APIs:
http://www.xmailserver.org/timerfd-test2.c
[akpm@linux-foundation.org: coding-style cleanups]
[akpm@linux-foundation.org: fix ia64 build]
[akpm@linux-foundation.org: fix m68k build]
[akpm@linux-foundation.org: fix mips build]
[akpm@linux-foundation.org: fix alpha, arm, blackfin, cris, m68k, s390, sparc and sparc64 builds]
[heiko.carstens@de.ibm.com: fix s390]
[akpm@linux-foundation.org: fix powerpc build]
[akpm@linux-foundation.org: fix sparc64 more]
Signed-off-by: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Davide Libenzi <davidel@xmailserver.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 14:27:26 +08:00
|
|
|
u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
timerfd: new timerfd API
This is the new timerfd API as it is implemented by the following patch:
int timerfd_create(int clockid, int flags);
int timerfd_settime(int ufd, int flags,
const struct itimerspec *utmr,
struct itimerspec *otmr);
int timerfd_gettime(int ufd, struct itimerspec *otmr);
The timerfd_create() API creates an un-programmed timerfd fd. The "clockid"
parameter can be either CLOCK_MONOTONIC or CLOCK_REALTIME.
The timerfd_settime() API give new settings by the timerfd fd, by optionally
retrieving the previous expiration time (in case the "otmr" parameter is not
NULL).
The time value specified in "utmr" is absolute, if the TFD_TIMER_ABSTIME bit
is set in the "flags" parameter. Otherwise it's a relative time.
The timerfd_gettime() API returns the next expiration time of the timer, or
{0, 0} if the timerfd has not been set yet.
Like the previous timerfd API implementation, read(2) and poll(2) are
supported (with the same interface). Here's a simple test program I used to
exercise the new timerfd APIs:
http://www.xmailserver.org/timerfd-test2.c
[akpm@linux-foundation.org: coding-style cleanups]
[akpm@linux-foundation.org: fix ia64 build]
[akpm@linux-foundation.org: fix m68k build]
[akpm@linux-foundation.org: fix mips build]
[akpm@linux-foundation.org: fix alpha, arm, blackfin, cris, m68k, s390, sparc and sparc64 builds]
[heiko.carstens@de.ibm.com: fix s390]
[akpm@linux-foundation.org: fix powerpc build]
[akpm@linux-foundation.org: fix sparc64 more]
Signed-off-by: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Davide Libenzi <davidel@xmailserver.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 14:27:26 +08:00
|
|
|
u64 orun = 1;
|
2006-03-26 17:38:06 +08:00
|
|
|
ktime_t delta;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2008-09-02 06:02:30 +08:00
|
|
|
delta = ktime_sub(now, hrtimer_get_expires(timer));
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2016-12-25 18:38:40 +08:00
|
|
|
if (delta < 0)
|
2006-01-10 12:52:32 +08:00
|
|
|
return 0;
|
|
|
|
|
2014-05-20 21:49:48 +08:00
|
|
|
if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))
|
|
|
|
return 0;
|
|
|
|
|
2016-12-25 18:38:40 +08:00
|
|
|
if (interval < hrtimer_resolution)
|
|
|
|
interval = hrtimer_resolution;
|
2006-01-12 18:47:34 +08:00
|
|
|
|
2016-12-25 18:38:40 +08:00
|
|
|
if (unlikely(delta >= interval)) {
|
2006-03-26 17:38:11 +08:00
|
|
|
s64 incr = ktime_to_ns(interval);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
|
|
|
orun = ktime_divns(delta, incr);
|
2008-09-02 06:02:30 +08:00
|
|
|
hrtimer_add_expires_ns(timer, incr * orun);
|
2016-12-25 18:38:40 +08:00
|
|
|
if (hrtimer_get_expires_tv64(timer) > now)
|
2006-01-10 12:52:32 +08:00
|
|
|
return orun;
|
|
|
|
/*
|
|
|
|
* This (and the ktime_add() below) is the
|
|
|
|
* correction for exact:
|
|
|
|
*/
|
|
|
|
orun++;
|
|
|
|
}
|
2008-09-02 06:02:30 +08:00
|
|
|
hrtimer_add_expires(timer, interval);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
|
|
|
return orun;
|
|
|
|
}
|
2007-05-08 15:31:58 +08:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_forward);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* enqueue_hrtimer - internal function to (re)start a timer
|
|
|
|
*
|
|
|
|
* The timer is inserted in expiry order. Insertion into the
|
|
|
|
* red black tree is O(log(n)). Must hold the base lock.
|
2009-01-05 18:28:22 +08:00
|
|
|
*
|
|
|
|
* Returns 1 when the new timer is the leftmost timer in the tree.
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
2009-01-05 18:28:22 +08:00
|
|
|
static int enqueue_hrtimer(struct hrtimer *timer,
|
2017-12-21 18:41:38 +08:00
|
|
|
struct hrtimer_clock_base *base,
|
|
|
|
enum hrtimer_mode mode)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2017-12-21 18:41:38 +08:00
|
|
|
debug_activate(timer, mode);
|
2008-04-30 15:55:04 +08:00
|
|
|
|
2011-05-20 19:05:15 +08:00
|
|
|
base->cpu_base->active_bases |= 1 << base->index;
|
2007-02-16 17:28:11 +08:00
|
|
|
|
2019-11-07 01:48:04 +08:00
|
|
|
/* Pairs with the lockless read in hrtimer_is_queued() */
|
|
|
|
WRITE_ONCE(timer->state, HRTIMER_STATE_ENQUEUED);
|
2009-01-05 18:28:22 +08:00
|
|
|
|
2015-04-15 05:08:47 +08:00
|
|
|
return timerqueue_add(&base->active, &timer->node);
|
2006-01-12 18:25:54 +08:00
|
|
|
}
|
2006-01-10 12:52:32 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* __remove_hrtimer - internal function to remove a timer
|
|
|
|
*
|
|
|
|
* Caller must hold the base lock.
|
2007-02-16 17:28:11 +08:00
|
|
|
*
|
|
|
|
* High resolution timer mode reprograms the clock event device when the
|
|
|
|
* timer is the one which expires next. The caller can disable this by setting
|
|
|
|
* reprogram to zero. This is useful, when the context does a reprogramming
|
|
|
|
* anyway (e.g. timer interrupt)
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
2007-02-16 17:27:50 +08:00
|
|
|
static void __remove_hrtimer(struct hrtimer *timer,
|
2007-02-16 17:27:51 +08:00
|
|
|
struct hrtimer_clock_base *base,
|
2016-01-15 00:54:46 +08:00
|
|
|
u8 newstate, int reprogram)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2015-04-15 05:08:39 +08:00
|
|
|
struct hrtimer_cpu_base *cpu_base = base->cpu_base;
|
2016-01-15 00:54:46 +08:00
|
|
|
u8 state = timer->state;
|
2015-04-15 05:08:39 +08:00
|
|
|
|
2019-11-07 01:48:04 +08:00
|
|
|
/* Pairs with the lockless read in hrtimer_is_queued() */
|
|
|
|
WRITE_ONCE(timer->state, newstate);
|
2015-04-15 05:08:49 +08:00
|
|
|
if (!(state & HRTIMER_STATE_ENQUEUED))
|
|
|
|
return;
|
2009-09-02 11:03:33 +08:00
|
|
|
|
2015-04-15 05:08:47 +08:00
|
|
|
if (!timerqueue_del(&base->active, &timer->node))
|
2015-04-15 05:08:39 +08:00
|
|
|
cpu_base->active_bases &= ~(1 << base->index);
|
2009-09-02 11:03:33 +08:00
|
|
|
|
2015-04-15 05:08:49 +08:00
|
|
|
/*
|
|
|
|
* Note: If reprogram is false we do not update
|
|
|
|
* cpu_base->next_timer. This happens when we remove the first
|
|
|
|
* timer on a remote cpu. No harm as we never dereference
|
|
|
|
* cpu_base->next_timer. So the worst thing what can happen is
|
2021-03-23 05:39:03 +08:00
|
|
|
* an superfluous call to hrtimer_force_reprogram() on the
|
2015-04-15 05:08:49 +08:00
|
|
|
* remote cpu later on if the same timer gets enqueued again.
|
|
|
|
*/
|
|
|
|
if (reprogram && timer == cpu_base->next_timer)
|
|
|
|
hrtimer_force_reprogram(cpu_base, 1);
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* remove hrtimer, called with base lock held
|
|
|
|
*/
|
|
|
|
static inline int
|
2021-07-13 21:39:46 +08:00
|
|
|
remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
|
|
|
|
bool restart, bool keep_local)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2019-11-07 01:48:04 +08:00
|
|
|
u8 state = timer->state;
|
|
|
|
|
|
|
|
if (state & HRTIMER_STATE_ENQUEUED) {
|
2021-07-13 21:39:46 +08:00
|
|
|
bool reprogram;
|
2007-02-16 17:28:11 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove the timer and force reprogramming when high
|
|
|
|
* resolution mode is active and the timer is on the current
|
|
|
|
* CPU. If we remove a timer on another CPU, reprogramming is
|
|
|
|
* skipped. The interrupt event on this CPU is fired and
|
|
|
|
* reprogramming happens in the interrupt handler. This is a
|
|
|
|
* rare case and less expensive than a smp call.
|
|
|
|
*/
|
2009-08-10 10:51:23 +08:00
|
|
|
debug_deactivate(timer);
|
2014-08-18 01:30:26 +08:00
|
|
|
reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
|
2015-06-11 20:46:45 +08:00
|
|
|
|
2021-07-13 21:39:46 +08:00
|
|
|
/*
|
|
|
|
* If the timer is not restarted then reprogramming is
|
|
|
|
* required if the timer is local. If it is local and about
|
|
|
|
* to be restarted, avoid programming it twice (on removal
|
|
|
|
* and a moment later when it's requeued).
|
|
|
|
*/
|
2015-06-11 20:46:48 +08:00
|
|
|
if (!restart)
|
|
|
|
state = HRTIMER_STATE_INACTIVE;
|
2021-07-13 21:39:46 +08:00
|
|
|
else
|
|
|
|
reprogram &= !keep_local;
|
2015-06-11 20:46:48 +08:00
|
|
|
|
2010-10-12 22:25:19 +08:00
|
|
|
__remove_hrtimer(timer, base, state, reprogram);
|
2006-01-10 12:52:32 +08:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-01-15 00:54:46 +08:00
|
|
|
static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
|
|
|
|
const enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_TIME_LOW_RES
|
|
|
|
/*
|
|
|
|
* CONFIG_TIME_LOW_RES indicates that the system has no way to return
|
|
|
|
* granular time values. For relative timers we add hrtimer_resolution
|
|
|
|
* (i.e. one jiffie) to prevent short timeouts.
|
|
|
|
*/
|
|
|
|
timer->is_rel = mode & HRTIMER_MODE_REL;
|
|
|
|
if (timer->is_rel)
|
2016-12-25 19:30:41 +08:00
|
|
|
tim = ktime_add_safe(tim, hrtimer_resolution);
|
2016-01-15 00:54:46 +08:00
|
|
|
#endif
|
|
|
|
return tim;
|
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
static void
|
|
|
|
hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
|
|
|
|
{
|
|
|
|
ktime_t expires;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Find the next SOFT expiration.
|
|
|
|
*/
|
|
|
|
expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* reprogramming needs to be triggered, even if the next soft
|
|
|
|
* hrtimer expires at the same time than the next hard
|
|
|
|
* hrtimer. cpu_base->softirq_expires_next needs to be updated!
|
|
|
|
*/
|
|
|
|
if (expires == KTIME_MAX)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* cpu_base->*next_timer is recomputed by __hrtimer_get_next_event()
|
|
|
|
* cpu_base->*expires_next is only set by hrtimer_reprogram()
|
|
|
|
*/
|
|
|
|
hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
|
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:52 +08:00
|
|
|
static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
|
|
|
|
u64 delta_ns, const enum hrtimer_mode mode,
|
|
|
|
struct hrtimer_clock_base *base)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2017-12-21 18:41:52 +08:00
|
|
|
struct hrtimer_clock_base *new_base;
|
2021-07-13 21:39:46 +08:00
|
|
|
bool force_local, first;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2021-07-13 21:39:46 +08:00
|
|
|
/*
|
|
|
|
* If the timer is on the local cpu base and is the first expiring
|
|
|
|
* timer then this might end up reprogramming the hardware twice
|
|
|
|
* (on removal and on enqueue). To avoid that by prevent the
|
|
|
|
* reprogram on removal, keep the timer local to the current CPU
|
|
|
|
* and enforce reprogramming after it is queued no matter whether
|
|
|
|
* it is the new first expiring timer again or not.
|
|
|
|
*/
|
|
|
|
force_local = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
|
|
|
|
force_local &= base->cpu_base->next_timer == timer;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove an active timer from the queue. In case it is not queued
|
|
|
|
* on the current CPU, make sure that remove_hrtimer() updates the
|
|
|
|
* remote data correctly.
|
|
|
|
*
|
|
|
|
* If it's on the current CPU and the first expiring timer, then
|
|
|
|
* skip reprogramming, keep the timer local and enforce
|
|
|
|
* reprogramming later if it was the first expiring timer. This
|
|
|
|
* avoids programming the underlying clock event twice (once at
|
|
|
|
* removal and once after enqueue).
|
|
|
|
*/
|
|
|
|
remove_hrtimer(timer, base, true, force_local);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2016-01-15 00:54:46 +08:00
|
|
|
if (mode & HRTIMER_MODE_REL)
|
2014-05-12 16:12:29 +08:00
|
|
|
tim = ktime_add_safe(tim, base->get_time());
|
2016-01-15 00:54:46 +08:00
|
|
|
|
|
|
|
tim = hrtimer_update_lowres(timer, tim, mode);
|
2008-04-30 15:55:04 +08:00
|
|
|
|
2008-09-08 01:47:46 +08:00
|
|
|
hrtimer_set_expires_range_ns(timer, tim, delta_ns);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2014-05-12 16:12:29 +08:00
|
|
|
/* Switch the timer base, if necessary: */
|
2021-07-13 21:39:46 +08:00
|
|
|
if (!force_local) {
|
|
|
|
new_base = switch_hrtimer_base(timer, base,
|
|
|
|
mode & HRTIMER_MODE_PINNED);
|
|
|
|
} else {
|
|
|
|
new_base = base;
|
|
|
|
}
|
|
|
|
|
|
|
|
first = enqueue_hrtimer(timer, new_base, mode);
|
|
|
|
if (!force_local)
|
|
|
|
return first;
|
2014-05-12 16:12:29 +08:00
|
|
|
|
2021-07-13 21:39:46 +08:00
|
|
|
/*
|
|
|
|
* Timer was forced to stay on the current CPU to avoid
|
|
|
|
* reprogramming on removal and enqueue. Force reprogram the
|
|
|
|
* hardware by evaluating the new first expiring timer.
|
|
|
|
*/
|
|
|
|
hrtimer_force_reprogram(new_base->cpu_base, 1);
|
|
|
|
return 0;
|
2017-12-21 18:41:52 +08:00
|
|
|
}
|
2017-12-21 18:41:57 +08:00
|
|
|
|
2017-12-21 18:41:52 +08:00
|
|
|
/**
|
|
|
|
* hrtimer_start_range_ns - (re)start an hrtimer
|
|
|
|
* @timer: the timer to be added
|
|
|
|
* @tim: expiry time
|
|
|
|
* @delta_ns: "slack" range for the timer
|
|
|
|
* @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
|
2017-12-21 18:41:57 +08:00
|
|
|
* relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
|
|
|
|
* softirq based mode is considered for debug purpose only!
|
2017-12-21 18:41:52 +08:00
|
|
|
*/
|
|
|
|
void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
|
|
|
|
u64 delta_ns, const enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
struct hrtimer_clock_base *base;
|
|
|
|
unsigned long flags;
|
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
/*
|
|
|
|
* Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
|
2019-07-31 02:15:25 +08:00
|
|
|
* match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
|
|
|
|
* expiry mode because unmarked timers are moved to softirq expiry.
|
2017-12-21 18:41:57 +08:00
|
|
|
*/
|
2019-07-31 02:15:25 +08:00
|
|
|
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
|
|
|
|
WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
|
|
|
|
else
|
|
|
|
WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
|
2017-12-21 18:41:57 +08:00
|
|
|
|
2017-12-21 18:41:52 +08:00
|
|
|
base = lock_hrtimer_base(timer, &flags);
|
|
|
|
|
|
|
|
if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
|
2017-12-21 18:41:57 +08:00
|
|
|
hrtimer_reprogram(timer, true);
|
2014-06-23 16:09:37 +08:00
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
unlock_hrtimer_base(timer, &flags);
|
2009-03-13 19:21:27 +08:00
|
|
|
}
|
2008-09-08 01:47:46 +08:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
|
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
/**
|
|
|
|
* hrtimer_try_to_cancel - try to deactivate a timer
|
|
|
|
* @timer: hrtimer to stop
|
|
|
|
*
|
|
|
|
* Returns:
|
2019-06-24 18:33:26 +08:00
|
|
|
*
|
|
|
|
* * 0 when the timer was not active
|
|
|
|
* * 1 when the timer was active
|
|
|
|
* * -1 when the timer is currently executing the callback function and
|
2006-06-25 20:49:15 +08:00
|
|
|
* cannot be stopped
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
|
|
|
int hrtimer_try_to_cancel(struct hrtimer *timer)
|
|
|
|
{
|
2007-02-16 17:27:50 +08:00
|
|
|
struct hrtimer_clock_base *base;
|
2006-01-10 12:52:32 +08:00
|
|
|
unsigned long flags;
|
|
|
|
int ret = -1;
|
|
|
|
|
2015-04-15 05:09:25 +08:00
|
|
|
/*
|
|
|
|
* Check lockless first. If the timer is not active (neither
|
|
|
|
* enqueued nor running the callback, nothing to do here. The
|
|
|
|
* base lock does not serialize against a concurrent enqueue,
|
|
|
|
* so we can avoid taking it.
|
|
|
|
*/
|
|
|
|
if (!hrtimer_active(timer))
|
|
|
|
return 0;
|
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
base = lock_hrtimer_base(timer, &flags);
|
|
|
|
|
2007-02-16 17:27:51 +08:00
|
|
|
if (!hrtimer_callback_running(timer))
|
2021-07-13 21:39:46 +08:00
|
|
|
ret = remove_hrtimer(timer, base, false, false);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
|
|
|
unlock_hrtimer_base(timer, &flags);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
}
|
2006-05-31 12:26:09 +08:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2019-07-27 02:30:59 +08:00
|
|
|
#ifdef CONFIG_PREEMPT_RT
|
|
|
|
static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base)
|
|
|
|
{
|
|
|
|
spin_lock_init(&base->softirq_expiry_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base)
|
|
|
|
{
|
|
|
|
spin_lock(&base->softirq_expiry_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base)
|
|
|
|
{
|
|
|
|
spin_unlock(&base->softirq_expiry_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The counterpart to hrtimer_cancel_wait_running().
|
|
|
|
*
|
|
|
|
* If there is a waiter for cpu_base->expiry_lock, then it was waiting for
|
2021-03-23 05:39:03 +08:00
|
|
|
* the timer callback to finish. Drop expiry_lock and reacquire it. That
|
2019-07-27 02:30:59 +08:00
|
|
|
* allows the waiter to acquire the lock and make progress.
|
|
|
|
*/
|
|
|
|
static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,
|
|
|
|
unsigned long flags)
|
|
|
|
{
|
|
|
|
if (atomic_read(&cpu_base->timer_waiters)) {
|
|
|
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
|
|
|
spin_unlock(&cpu_base->softirq_expiry_lock);
|
|
|
|
spin_lock(&cpu_base->softirq_expiry_lock);
|
|
|
|
raw_spin_lock_irq(&cpu_base->lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This function is called on PREEMPT_RT kernels when the fast path
|
|
|
|
* deletion of a timer failed because the timer callback function was
|
|
|
|
* running.
|
|
|
|
*
|
2019-08-20 21:12:23 +08:00
|
|
|
* This prevents priority inversion: if the soft irq thread is preempted
|
|
|
|
* in the middle of a timer callback, then calling del_timer_sync() can
|
|
|
|
* lead to two issues:
|
|
|
|
*
|
|
|
|
* - If the caller is on a remote CPU then it has to spin wait for the timer
|
|
|
|
* handler to complete. This can result in unbound priority inversion.
|
|
|
|
*
|
|
|
|
* - If the caller originates from the task which preempted the timer
|
|
|
|
* handler on the same CPU, then spin waiting for the timer handler to
|
|
|
|
* complete is never going to end.
|
2019-07-27 02:30:59 +08:00
|
|
|
*/
|
|
|
|
void hrtimer_cancel_wait_running(const struct hrtimer *timer)
|
|
|
|
{
|
2019-08-21 17:24:07 +08:00
|
|
|
/* Lockless read. Prevent the compiler from reloading it below */
|
|
|
|
struct hrtimer_clock_base *base = READ_ONCE(timer->base);
|
2019-07-27 02:30:59 +08:00
|
|
|
|
2019-08-21 17:24:09 +08:00
|
|
|
/*
|
|
|
|
* Just relax if the timer expires in hard interrupt context or if
|
|
|
|
* it is currently on the migration base.
|
|
|
|
*/
|
2019-09-04 22:55:27 +08:00
|
|
|
if (!timer->is_soft || is_migration_base(base)) {
|
2019-07-27 02:30:59 +08:00
|
|
|
cpu_relax();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Mark the base as contended and grab the expiry lock, which is
|
|
|
|
* held by the softirq across the timer callback. Drop the lock
|
|
|
|
* immediately so the softirq can expire the next timer. In theory
|
|
|
|
* the timer could already be running again, but that's more than
|
|
|
|
* unlikely and just causes another wait loop.
|
|
|
|
*/
|
|
|
|
atomic_inc(&base->cpu_base->timer_waiters);
|
|
|
|
spin_lock_bh(&base->cpu_base->softirq_expiry_lock);
|
|
|
|
atomic_dec(&base->cpu_base->timer_waiters);
|
|
|
|
spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline void
|
|
|
|
hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
|
|
|
|
static inline void
|
|
|
|
hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
|
|
|
|
static inline void
|
|
|
|
hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
|
|
|
|
static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base,
|
|
|
|
unsigned long flags) { }
|
|
|
|
#endif
|
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
/**
|
|
|
|
* hrtimer_cancel - cancel a timer and wait for the handler to finish.
|
|
|
|
* @timer: the timer to be cancelled
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* 0 when the timer was not active
|
|
|
|
* 1 when the timer was active
|
|
|
|
*/
|
|
|
|
int hrtimer_cancel(struct hrtimer *timer)
|
|
|
|
{
|
2019-07-27 02:30:59 +08:00
|
|
|
int ret;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2019-07-27 02:30:59 +08:00
|
|
|
do {
|
|
|
|
ret = hrtimer_try_to_cancel(timer);
|
|
|
|
|
|
|
|
if (ret < 0)
|
|
|
|
hrtimer_cancel_wait_running(timer);
|
|
|
|
} while (ret < 0);
|
|
|
|
return ret;
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
2006-05-31 12:26:09 +08:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_cancel);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
|
|
|
/**
|
2020-11-16 18:18:14 +08:00
|
|
|
* __hrtimer_get_remaining - get remaining time for the timer
|
2006-01-10 12:52:32 +08:00
|
|
|
* @timer: the timer to read
|
2016-01-15 00:54:46 +08:00
|
|
|
* @adjust: adjust relative timers when CONFIG_TIME_LOW_RES=y
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
2016-01-15 00:54:46 +08:00
|
|
|
ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
ktime_t rem;
|
|
|
|
|
2010-08-11 05:17:51 +08:00
|
|
|
lock_hrtimer_base(timer, &flags);
|
2016-01-15 00:54:46 +08:00
|
|
|
if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust)
|
|
|
|
rem = hrtimer_expires_remaining_adjusted(timer);
|
|
|
|
else
|
|
|
|
rem = hrtimer_expires_remaining(timer);
|
2006-01-10 12:52:32 +08:00
|
|
|
unlock_hrtimer_base(timer, &flags);
|
|
|
|
|
|
|
|
return rem;
|
|
|
|
}
|
2016-01-15 00:54:46 +08:00
|
|
|
EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2011-08-11 05:21:01 +08:00
|
|
|
#ifdef CONFIG_NO_HZ_COMMON
|
2006-03-07 07:42:45 +08:00
|
|
|
/**
|
|
|
|
* hrtimer_get_next_event - get the time until next expiry event
|
|
|
|
*
|
2015-04-15 05:08:58 +08:00
|
|
|
* Returns the next expiry time or KTIME_MAX if no timer is pending.
|
2006-03-07 07:42:45 +08:00
|
|
|
*/
|
2015-04-15 05:08:58 +08:00
|
|
|
u64 hrtimer_get_next_event(void)
|
2006-03-07 07:42:45 +08:00
|
|
|
{
|
2014-08-18 01:30:26 +08:00
|
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
2015-04-15 05:08:58 +08:00
|
|
|
u64 expires = KTIME_MAX;
|
2006-03-07 07:42:45 +08:00
|
|
|
unsigned long flags;
|
|
|
|
|
2009-11-17 23:36:54 +08:00
|
|
|
raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
2007-02-16 17:27:50 +08:00
|
|
|
|
2015-04-15 05:08:39 +08:00
|
|
|
if (!__hrtimer_hres_active(cpu_base))
|
2017-12-21 18:41:57 +08:00
|
|
|
expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
|
2007-02-16 17:27:50 +08:00
|
|
|
|
2009-11-17 23:36:54 +08:00
|
|
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
2007-02-16 17:27:50 +08:00
|
|
|
|
2015-04-15 05:08:58 +08:00
|
|
|
return expires;
|
2006-03-07 07:42:45 +08:00
|
|
|
}
|
2018-04-04 05:17:00 +08:00
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_next_event_without - time until next expiry event w/o one timer
|
|
|
|
* @exclude: timer to exclude
|
|
|
|
*
|
|
|
|
* Returns the next expiry time over all timers except for the @exclude one or
|
|
|
|
* KTIME_MAX if none of them is pending.
|
|
|
|
*/
|
|
|
|
u64 hrtimer_next_event_without(const struct hrtimer *exclude)
|
|
|
|
{
|
|
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
|
|
|
u64 expires = KTIME_MAX;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
|
|
|
|
|
|
|
if (__hrtimer_hres_active(cpu_base)) {
|
|
|
|
unsigned int active;
|
|
|
|
|
|
|
|
if (!cpu_base->softirq_activated) {
|
|
|
|
active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
|
|
|
|
expires = __hrtimer_next_event_base(cpu_base, exclude,
|
|
|
|
active, KTIME_MAX);
|
|
|
|
}
|
|
|
|
active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
|
|
|
|
expires = __hrtimer_next_event_base(cpu_base, exclude, active,
|
|
|
|
expires);
|
|
|
|
}
|
|
|
|
|
|
|
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
|
|
|
|
|
|
|
return expires;
|
|
|
|
}
|
2006-03-07 07:42:45 +08:00
|
|
|
#endif
|
|
|
|
|
2016-01-16 01:41:09 +08:00
|
|
|
static inline int hrtimer_clockid_to_base(clockid_t clock_id)
|
|
|
|
{
|
|
|
|
if (likely(clock_id < MAX_CLOCKS)) {
|
|
|
|
int base = hrtimer_clock_to_base_table[clock_id];
|
|
|
|
|
|
|
|
if (likely(base != HRTIMER_MAX_CLOCK_BASES))
|
|
|
|
return base;
|
|
|
|
}
|
|
|
|
WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
|
|
|
|
return HRTIMER_BASE_MONOTONIC;
|
|
|
|
}
|
|
|
|
|
2008-04-30 15:55:04 +08:00
|
|
|
static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2017-12-21 18:41:58 +08:00
|
|
|
bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
|
2007-02-16 17:27:50 +08:00
|
|
|
struct hrtimer_cpu_base *cpu_base;
|
2019-07-27 02:30:57 +08:00
|
|
|
int base;
|
|
|
|
|
|
|
|
/*
|
2021-03-23 05:39:03 +08:00
|
|
|
* On PREEMPT_RT enabled kernels hrtimers which are not explicitly
|
2019-07-27 02:30:57 +08:00
|
|
|
* marked for hard interrupt expiry mode are moved into soft
|
|
|
|
* interrupt context for latency reasons and because the callbacks
|
|
|
|
* can invoke functions which might sleep on RT, e.g. spin_lock().
|
|
|
|
*/
|
|
|
|
if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD))
|
|
|
|
softtimer = true;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2006-02-01 19:05:11 +08:00
|
|
|
memset(timer, 0, sizeof(struct hrtimer));
|
|
|
|
|
2014-08-18 01:30:25 +08:00
|
|
|
cpu_base = raw_cpu_ptr(&hrtimer_bases);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2017-12-21 18:41:35 +08:00
|
|
|
/*
|
|
|
|
* POSIX magic: Relative CLOCK_REALTIME timers are not affected by
|
|
|
|
* clock modifications, so they needs to become CLOCK_MONOTONIC to
|
|
|
|
* ensure POSIX compliance.
|
|
|
|
*/
|
|
|
|
if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
|
2006-02-01 19:05:11 +08:00
|
|
|
clock_id = CLOCK_MONOTONIC;
|
|
|
|
|
2019-07-27 02:30:57 +08:00
|
|
|
base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
|
2017-12-21 18:41:58 +08:00
|
|
|
base += hrtimer_clockid_to_base(clock_id);
|
|
|
|
timer->is_soft = softtimer;
|
2020-03-21 19:26:02 +08:00
|
|
|
timer->is_hard = !!(mode & HRTIMER_MODE_HARD);
|
2010-12-15 11:37:07 +08:00
|
|
|
timer->base = &cpu_base->clock_base[base];
|
2010-09-21 10:19:17 +08:00
|
|
|
timerqueue_init(&timer->node);
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
2008-04-30 15:55:04 +08:00
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_init - initialize a timer to the given clock
|
|
|
|
* @timer: the timer to be initialized
|
|
|
|
* @clock_id: the clock to be used
|
2021-03-23 05:39:03 +08:00
|
|
|
* @mode: The modes which are relevant for initialization:
|
2017-12-21 18:41:58 +08:00
|
|
|
* HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
|
|
|
|
* HRTIMER_MODE_REL_SOFT
|
|
|
|
*
|
|
|
|
* The PINNED variants of the above can be handed in,
|
|
|
|
* but the PINNED bit is ignored as pinning happens
|
|
|
|
* when the hrtimer is started
|
2008-04-30 15:55:04 +08:00
|
|
|
*/
|
|
|
|
void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode)
|
|
|
|
{
|
2009-08-10 10:51:23 +08:00
|
|
|
debug_init(timer, clock_id, mode);
|
2008-04-30 15:55:04 +08:00
|
|
|
__hrtimer_init(timer, clock_id, mode);
|
|
|
|
}
|
2006-05-31 12:26:09 +08:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_init);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2015-06-11 20:46:48 +08:00
|
|
|
/*
|
|
|
|
* A timer is active, when it is enqueued into the rbtree or the
|
|
|
|
* callback function is running or it's in the state of being migrated
|
|
|
|
* to another cpu.
|
2006-01-10 12:52:32 +08:00
|
|
|
*
|
2015-06-11 20:46:48 +08:00
|
|
|
* It is important for this function to not return a false negative.
|
2006-01-10 12:52:32 +08:00
|
|
|
*/
|
2015-06-11 20:46:48 +08:00
|
|
|
bool hrtimer_active(const struct hrtimer *timer)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2017-12-21 18:41:40 +08:00
|
|
|
struct hrtimer_clock_base *base;
|
2015-06-11 20:46:48 +08:00
|
|
|
unsigned int seq;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2015-06-11 20:46:48 +08:00
|
|
|
do {
|
2017-12-21 18:41:40 +08:00
|
|
|
base = READ_ONCE(timer->base);
|
|
|
|
seq = raw_read_seqcount_begin(&base->seq);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2015-06-11 20:46:48 +08:00
|
|
|
if (timer->state != HRTIMER_STATE_INACTIVE ||
|
2017-12-21 18:41:40 +08:00
|
|
|
base->running == timer)
|
2015-06-11 20:46:48 +08:00
|
|
|
return true;
|
|
|
|
|
2017-12-21 18:41:40 +08:00
|
|
|
} while (read_seqcount_retry(&base->seq, seq) ||
|
|
|
|
base != READ_ONCE(timer->base));
|
2015-06-11 20:46:48 +08:00
|
|
|
|
|
|
|
return false;
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
2015-06-11 20:46:48 +08:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_active);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2015-06-11 20:46:48 +08:00
|
|
|
/*
|
|
|
|
* The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3
|
|
|
|
* distinct sections:
|
|
|
|
*
|
|
|
|
* - queued: the timer is queued
|
|
|
|
* - callback: the timer is being ran
|
|
|
|
* - post: the timer is inactive or (re)queued
|
|
|
|
*
|
|
|
|
* On the read side we ensure we observe timer->state and cpu_base->running
|
|
|
|
* from the same section, if anything changed while we looked at it, we retry.
|
|
|
|
* This includes timer->base changing because sequence numbers alone are
|
|
|
|
* insufficient for that.
|
|
|
|
*
|
|
|
|
* The sequence numbers are required because otherwise we could still observe
|
2021-03-23 05:39:03 +08:00
|
|
|
* a false negative if the read side got smeared over multiple consecutive
|
2015-06-11 20:46:48 +08:00
|
|
|
* __run_hrtimer() invocations.
|
|
|
|
*/
|
|
|
|
|
2015-04-15 05:08:35 +08:00
|
|
|
static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
|
|
|
|
struct hrtimer_clock_base *base,
|
2017-12-21 18:41:54 +08:00
|
|
|
struct hrtimer *timer, ktime_t *now,
|
2020-01-21 06:43:47 +08:00
|
|
|
unsigned long flags) __must_hold(&cpu_base->lock)
|
2008-01-26 04:08:31 +08:00
|
|
|
{
|
|
|
|
enum hrtimer_restart (*fn)(struct hrtimer *);
|
2020-04-01 04:18:49 +08:00
|
|
|
bool expires_in_hardirq;
|
2008-01-26 04:08:31 +08:00
|
|
|
int restart;
|
|
|
|
|
2015-06-11 20:46:48 +08:00
|
|
|
lockdep_assert_held(&cpu_base->lock);
|
2008-11-25 19:43:51 +08:00
|
|
|
|
2009-08-10 10:51:23 +08:00
|
|
|
debug_deactivate(timer);
|
2017-12-21 18:41:40 +08:00
|
|
|
base->running = timer;
|
2015-06-11 20:46:48 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Separate the ->running assignment from the ->state assignment.
|
|
|
|
*
|
|
|
|
* As with a regular write barrier, this ensures the read side in
|
2017-12-21 18:41:40 +08:00
|
|
|
* hrtimer_active() cannot observe base->running == NULL &&
|
2015-06-11 20:46:48 +08:00
|
|
|
* timer->state == INACTIVE.
|
|
|
|
*/
|
2017-12-21 18:41:40 +08:00
|
|
|
raw_write_seqcount_barrier(&base->seq);
|
2015-06-11 20:46:48 +08:00
|
|
|
|
|
|
|
__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
|
2008-01-26 04:08:31 +08:00
|
|
|
fn = timer->function;
|
2008-11-25 19:43:51 +08:00
|
|
|
|
2016-01-15 00:54:46 +08:00
|
|
|
/*
|
|
|
|
* Clear the 'is relative' flag for the TIME_LOW_RES case. If the
|
|
|
|
* timer is restarted with a period then it becomes an absolute
|
|
|
|
* timer. If its not restarted it does not matter.
|
|
|
|
*/
|
|
|
|
if (IS_ENABLED(CONFIG_TIME_LOW_RES))
|
|
|
|
timer->is_rel = false;
|
|
|
|
|
2008-11-25 19:43:51 +08:00
|
|
|
/*
|
2017-12-21 18:41:31 +08:00
|
|
|
* The timer is marked as running in the CPU base, so it is
|
|
|
|
* protected against migration to a different CPU even if the lock
|
|
|
|
* is dropped.
|
2008-11-25 19:43:51 +08:00
|
|
|
*/
|
2017-12-21 18:41:54 +08:00
|
|
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
2009-08-10 10:51:23 +08:00
|
|
|
trace_hrtimer_expire_entry(timer, now);
|
2020-04-01 04:18:49 +08:00
|
|
|
expires_in_hardirq = lockdep_hrtimer_enter(timer);
|
2020-03-21 19:26:02 +08:00
|
|
|
|
2008-11-25 19:43:51 +08:00
|
|
|
restart = fn(timer);
|
2020-03-21 19:26:02 +08:00
|
|
|
|
2020-04-01 04:18:49 +08:00
|
|
|
lockdep_hrtimer_exit(expires_in_hardirq);
|
2009-08-10 10:51:23 +08:00
|
|
|
trace_hrtimer_expire_exit(timer);
|
2017-12-21 18:41:54 +08:00
|
|
|
raw_spin_lock_irq(&cpu_base->lock);
|
2008-01-26 04:08:31 +08:00
|
|
|
|
|
|
|
/*
|
2015-06-11 20:46:48 +08:00
|
|
|
* Note: We clear the running state after enqueue_hrtimer and
|
2016-06-24 02:50:37 +08:00
|
|
|
* we do not reprogram the event hardware. Happens either in
|
2009-01-05 18:28:23 +08:00
|
|
|
* hrtimer_start_range_ns() or in hrtimer_interrupt()
|
2014-05-20 21:49:48 +08:00
|
|
|
*
|
|
|
|
* Note: Because we dropped the cpu_base->lock above,
|
|
|
|
* hrtimer_start_range_ns() can have popped in and enqueued the timer
|
|
|
|
* for us already.
|
2008-01-26 04:08:31 +08:00
|
|
|
*/
|
2014-05-20 21:49:48 +08:00
|
|
|
if (restart != HRTIMER_NORESTART &&
|
|
|
|
!(timer->state & HRTIMER_STATE_ENQUEUED))
|
2017-12-21 18:41:38 +08:00
|
|
|
enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);
|
2010-10-12 22:25:19 +08:00
|
|
|
|
2015-06-11 20:46:48 +08:00
|
|
|
/*
|
|
|
|
* Separate the ->running assignment from the ->state assignment.
|
|
|
|
*
|
|
|
|
* As with a regular write barrier, this ensures the read side in
|
2017-12-21 18:41:40 +08:00
|
|
|
* hrtimer_active() cannot observe base->running.timer == NULL &&
|
2015-06-11 20:46:48 +08:00
|
|
|
* timer->state == INACTIVE.
|
|
|
|
*/
|
2017-12-21 18:41:40 +08:00
|
|
|
raw_write_seqcount_barrier(&base->seq);
|
2010-10-12 22:25:19 +08:00
|
|
|
|
2017-12-21 18:41:40 +08:00
|
|
|
WARN_ON_ONCE(base->running != timer);
|
|
|
|
base->running = NULL;
|
2008-01-26 04:08:31 +08:00
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:54 +08:00
|
|
|
static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
|
2017-12-21 18:41:56 +08:00
|
|
|
unsigned long flags, unsigned int active_mask)
|
2007-02-16 17:28:11 +08:00
|
|
|
{
|
2017-12-21 18:41:39 +08:00
|
|
|
struct hrtimer_clock_base *base;
|
2017-12-21 18:41:56 +08:00
|
|
|
unsigned int active = cpu_base->active_bases & active_mask;
|
2009-07-10 20:57:05 +08:00
|
|
|
|
2017-12-21 18:41:39 +08:00
|
|
|
for_each_active_base(base, cpu_base, active) {
|
2010-09-21 10:19:17 +08:00
|
|
|
struct timerqueue_node *node;
|
2011-05-20 19:05:15 +08:00
|
|
|
ktime_t basenow;
|
|
|
|
|
2007-02-16 17:28:11 +08:00
|
|
|
basenow = ktime_add(now, base->offset);
|
|
|
|
|
2010-09-21 10:19:17 +08:00
|
|
|
while ((node = timerqueue_getnext(&base->active))) {
|
2007-02-16 17:28:11 +08:00
|
|
|
struct hrtimer *timer;
|
|
|
|
|
2010-09-21 10:19:17 +08:00
|
|
|
timer = container_of(node, struct hrtimer, node);
|
2007-02-16 17:28:11 +08:00
|
|
|
|
2008-09-02 06:47:08 +08:00
|
|
|
/*
|
|
|
|
* The immediate goal for using the softexpires is
|
|
|
|
* minimizing wakeups, not running timers at the
|
|
|
|
* earliest interrupt after their soft expiration.
|
|
|
|
* This allows us to avoid using a Priority Search
|
2021-03-23 05:39:03 +08:00
|
|
|
* Tree, which can answer a stabbing query for
|
2008-09-02 06:47:08 +08:00
|
|
|
* overlapping intervals and instead use the simple
|
|
|
|
* BST we already have.
|
|
|
|
* We don't add extra wakeups by delaying timers that
|
|
|
|
* are right-of a not yet expired timer, because that
|
|
|
|
* timer will have to trigger a wakeup anyway.
|
|
|
|
*/
|
2016-12-25 18:38:40 +08:00
|
|
|
if (basenow < hrtimer_get_softexpires_tv64(timer))
|
2007-02-16 17:28:11 +08:00
|
|
|
break;
|
|
|
|
|
2017-12-21 18:41:54 +08:00
|
|
|
__run_hrtimer(cpu_base, base, timer, &basenow, flags);
|
2019-07-27 02:30:59 +08:00
|
|
|
if (active_mask == HRTIMER_ACTIVE_SOFT)
|
|
|
|
hrtimer_sync_wait_running(cpu_base, flags);
|
2007-02-16 17:28:11 +08:00
|
|
|
}
|
|
|
|
}
|
2015-04-15 05:08:35 +08:00
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h)
|
|
|
|
{
|
|
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
|
|
|
unsigned long flags;
|
|
|
|
ktime_t now;
|
|
|
|
|
2019-07-27 02:30:59 +08:00
|
|
|
hrtimer_cpu_base_lock_expiry(cpu_base);
|
2017-12-21 18:41:57 +08:00
|
|
|
raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
|
|
|
|
|
|
|
now = hrtimer_update_base(cpu_base);
|
|
|
|
__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
|
|
|
|
|
|
|
|
cpu_base->softirq_activated = 0;
|
|
|
|
hrtimer_update_softirq_timer(cpu_base, true);
|
|
|
|
|
|
|
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
2019-07-27 02:30:59 +08:00
|
|
|
hrtimer_cpu_base_unlock_expiry(cpu_base);
|
2017-12-21 18:41:57 +08:00
|
|
|
}
|
|
|
|
|
2015-04-15 05:08:35 +08:00
|
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
|
|
|
|
|
|
|
/*
|
|
|
|
* High resolution timer interrupt
|
|
|
|
* Called with interrupts disabled
|
|
|
|
*/
|
|
|
|
void hrtimer_interrupt(struct clock_event_device *dev)
|
|
|
|
{
|
|
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
|
|
|
ktime_t expires_next, now, entry_time, delta;
|
2017-12-21 18:41:54 +08:00
|
|
|
unsigned long flags;
|
2015-04-15 05:08:35 +08:00
|
|
|
int retries = 0;
|
|
|
|
|
|
|
|
BUG_ON(!cpu_base->hres_active);
|
|
|
|
cpu_base->nr_events++;
|
2016-12-25 18:38:40 +08:00
|
|
|
dev->next_event = KTIME_MAX;
|
2015-04-15 05:08:35 +08:00
|
|
|
|
2017-12-21 18:41:54 +08:00
|
|
|
raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
2015-04-15 05:08:35 +08:00
|
|
|
entry_time = now = hrtimer_update_base(cpu_base);
|
|
|
|
retry:
|
|
|
|
cpu_base->in_hrtirq = 1;
|
|
|
|
/*
|
|
|
|
* We set expires_next to KTIME_MAX here with cpu_base->lock
|
|
|
|
* held to prevent that a timer is enqueued in our queue via
|
|
|
|
* the migration code. This does not affect enqueueing of
|
|
|
|
* timers which run their callback and need to be requeued on
|
|
|
|
* this CPU.
|
|
|
|
*/
|
2016-12-25 18:38:40 +08:00
|
|
|
cpu_base->expires_next = KTIME_MAX;
|
2015-04-15 05:08:35 +08:00
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
if (!ktime_before(now, cpu_base->softirq_expires_next)) {
|
|
|
|
cpu_base->softirq_expires_next = KTIME_MAX;
|
|
|
|
cpu_base->softirq_activated = 1;
|
|
|
|
raise_softirq_irqoff(HRTIMER_SOFTIRQ);
|
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:56 +08:00
|
|
|
__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
|
2015-04-15 05:08:35 +08:00
|
|
|
|
2021-02-24 00:02:40 +08:00
|
|
|
/* Reevaluate the clock bases for the [soft] next expiry */
|
|
|
|
expires_next = hrtimer_update_next_event(cpu_base);
|
2009-07-10 20:57:05 +08:00
|
|
|
/*
|
|
|
|
* Store the new expiry value so the migration code can verify
|
|
|
|
* against it.
|
|
|
|
*/
|
2007-02-16 17:28:11 +08:00
|
|
|
cpu_base->expires_next = expires_next;
|
2015-01-21 04:24:10 +08:00
|
|
|
cpu_base->in_hrtirq = 0;
|
2017-12-21 18:41:54 +08:00
|
|
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
2007-02-16 17:28:11 +08:00
|
|
|
|
|
|
|
/* Reprogramming necessary ? */
|
2015-04-03 11:34:05 +08:00
|
|
|
if (!tick_program_event(expires_next, 0)) {
|
2009-11-14 00:05:44 +08:00
|
|
|
cpu_base->hang_detected = 0;
|
|
|
|
return;
|
2007-02-16 17:28:11 +08:00
|
|
|
}
|
2009-11-14 00:05:44 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The next timer was already expired due to:
|
|
|
|
* - tracing
|
|
|
|
* - long lasting callbacks
|
|
|
|
* - being scheduled away when running in a VM
|
|
|
|
*
|
|
|
|
* We need to prevent that we loop forever in the hrtimer
|
|
|
|
* interrupt routine. We give it 3 attempts to avoid
|
|
|
|
* overreacting on some spurious event.
|
2012-07-11 06:43:25 +08:00
|
|
|
*
|
|
|
|
* Acquire base lock for updating the offsets and retrieving
|
|
|
|
* the current time.
|
2009-11-14 00:05:44 +08:00
|
|
|
*/
|
2017-12-21 18:41:54 +08:00
|
|
|
raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
2012-07-11 06:43:25 +08:00
|
|
|
now = hrtimer_update_base(cpu_base);
|
2009-11-14 00:05:44 +08:00
|
|
|
cpu_base->nr_retries++;
|
|
|
|
if (++retries < 3)
|
|
|
|
goto retry;
|
|
|
|
/*
|
|
|
|
* Give the system a chance to do something else than looping
|
|
|
|
* here. We stored the entry time, so we know exactly how long
|
|
|
|
* we spent here. We schedule the next event this amount of
|
|
|
|
* time away.
|
|
|
|
*/
|
|
|
|
cpu_base->nr_hangs++;
|
|
|
|
cpu_base->hang_detected = 1;
|
2017-12-21 18:41:54 +08:00
|
|
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
|
|
|
|
2009-11-14 00:05:44 +08:00
|
|
|
delta = ktime_sub(now, entry_time);
|
2016-12-25 18:38:40 +08:00
|
|
|
if ((unsigned int)delta > cpu_base->max_hang_time)
|
|
|
|
cpu_base->max_hang_time = (unsigned int) delta;
|
2009-11-14 00:05:44 +08:00
|
|
|
/*
|
|
|
|
* Limit it to a sensible value as we enforce a longer
|
|
|
|
* delay. Give the CPU at least 100ms to catch up.
|
|
|
|
*/
|
2016-12-25 18:38:40 +08:00
|
|
|
if (delta > 100 * NSEC_PER_MSEC)
|
2009-11-14 00:05:44 +08:00
|
|
|
expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC);
|
|
|
|
else
|
|
|
|
expires_next = ktime_add(now, delta);
|
|
|
|
tick_program_event(expires_next, 1);
|
2018-07-12 22:41:18 +08:00
|
|
|
pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta));
|
2007-02-16 17:28:11 +08:00
|
|
|
}
|
|
|
|
|
2017-03-17 09:08:13 +08:00
|
|
|
/* called with interrupts disabled */
|
2015-04-15 05:08:51 +08:00
|
|
|
static inline void __hrtimer_peek_ahead_timers(void)
|
2009-01-05 18:28:19 +08:00
|
|
|
{
|
|
|
|
struct tick_device *td;
|
|
|
|
|
|
|
|
if (!hrtimer_hres_active())
|
|
|
|
return;
|
|
|
|
|
2014-08-18 01:30:25 +08:00
|
|
|
td = this_cpu_ptr(&tick_cpu_device);
|
2009-01-05 18:28:19 +08:00
|
|
|
if (td && td->evtdev)
|
|
|
|
hrtimer_interrupt(td->evtdev);
|
|
|
|
}
|
|
|
|
|
2009-01-05 21:11:10 +08:00
|
|
|
#else /* CONFIG_HIGH_RES_TIMERS */
|
|
|
|
|
|
|
|
static inline void __hrtimer_peek_ahead_timers(void) { }
|
|
|
|
|
|
|
|
#endif /* !CONFIG_HIGH_RES_TIMERS */
|
[PATCH] Add debugging feature /proc/timer_stat
Add /proc/timer_stats support: debugging feature to profile timer expiration.
Both the starting site, process/PID and the expiration function is captured.
This allows the quick identification of timer event sources in a system.
Sample output:
# echo 1 > /proc/timer_stats
# cat /proc/timer_stats
Timer Stats Version: v0.1
Sample period: 4.010 s
24, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
11, 0 swapper sk_reset_timer (tcp_delack_timer)
6, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
17, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
4, 2050 pcscd do_nanosleep (hrtimer_wakeup)
5, 4179 sshd sk_reset_timer (tcp_write_timer)
4, 2248 yum-updatesd schedule_timeout (process_timeout)
18, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
3, 0 swapper sk_reset_timer (tcp_delack_timer)
1, 1 swapper neigh_table_init_no_netlink (neigh_periodic_timer)
2, 1 swapper e1000_up (e1000_watchdog)
1, 1 init schedule_timeout (process_timeout)
100 total events, 25.24 events/sec
[ cleanups and hrtimers support from Thomas Gleixner <tglx@linutronix.de> ]
[bunk@stusta.de: nr_entries can become static]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-16 17:28:13 +08:00
|
|
|
|
2008-01-26 04:08:31 +08:00
|
|
|
/*
|
2015-04-15 05:08:51 +08:00
|
|
|
* Called from run_local_timers in hardirq context every jiffy
|
2008-01-26 04:08:31 +08:00
|
|
|
*/
|
2008-04-19 04:39:00 +08:00
|
|
|
void hrtimer_run_queues(void)
|
2008-01-26 04:08:31 +08:00
|
|
|
{
|
2014-08-18 01:30:26 +08:00
|
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
2017-12-21 18:41:54 +08:00
|
|
|
unsigned long flags;
|
2015-04-15 05:08:35 +08:00
|
|
|
ktime_t now;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2015-04-15 05:08:39 +08:00
|
|
|
if (__hrtimer_hres_active(cpu_base))
|
2008-01-26 04:08:31 +08:00
|
|
|
return;
|
2007-02-16 17:28:11 +08:00
|
|
|
|
2008-01-26 04:08:31 +08:00
|
|
|
/*
|
2015-04-15 05:08:51 +08:00
|
|
|
* This _is_ ugly: We have to check periodically, whether we
|
|
|
|
* can switch to highres and / or nohz mode. The clocksource
|
|
|
|
* switch happens with xtime_lock held. Notification from
|
|
|
|
* there only sets the check bit in the tick_oneshot code,
|
|
|
|
* otherwise we might deadlock vs. xtime_lock.
|
2008-01-26 04:08:31 +08:00
|
|
|
*/
|
2015-04-15 05:08:51 +08:00
|
|
|
if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) {
|
2008-01-26 04:08:31 +08:00
|
|
|
hrtimer_switch_to_hres();
|
2006-03-31 18:31:20 +08:00
|
|
|
return;
|
2008-04-19 04:39:00 +08:00
|
|
|
}
|
2015-04-15 05:08:51 +08:00
|
|
|
|
2017-12-21 18:41:54 +08:00
|
|
|
raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
2015-04-15 05:08:35 +08:00
|
|
|
now = hrtimer_update_base(cpu_base);
|
2017-12-21 18:41:57 +08:00
|
|
|
|
|
|
|
if (!ktime_before(now, cpu_base->softirq_expires_next)) {
|
|
|
|
cpu_base->softirq_expires_next = KTIME_MAX;
|
|
|
|
cpu_base->softirq_activated = 1;
|
|
|
|
raise_softirq_irqoff(HRTIMER_SOFTIRQ);
|
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:56 +08:00
|
|
|
__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
|
2017-12-21 18:41:54 +08:00
|
|
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
|
|
|
|
2006-01-10 12:52:35 +08:00
|
|
|
/*
|
|
|
|
* Sleep related functions:
|
|
|
|
*/
|
2007-02-16 17:27:49 +08:00
|
|
|
static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
|
2006-03-31 18:31:17 +08:00
|
|
|
{
|
|
|
|
struct hrtimer_sleeper *t =
|
|
|
|
container_of(timer, struct hrtimer_sleeper, timer);
|
|
|
|
struct task_struct *task = t->task;
|
|
|
|
|
|
|
|
t->task = NULL;
|
|
|
|
if (task)
|
|
|
|
wake_up_process(task);
|
|
|
|
|
|
|
|
return HRTIMER_NORESTART;
|
|
|
|
}
|
|
|
|
|
2019-07-31 03:03:53 +08:00
|
|
|
/**
|
|
|
|
* hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
|
|
|
|
* @sl: sleeper to be started
|
|
|
|
* @mode: timer mode abs/rel
|
|
|
|
*
|
|
|
|
* Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers
|
|
|
|
* to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context)
|
|
|
|
*/
|
|
|
|
void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
|
|
|
|
enum hrtimer_mode mode)
|
|
|
|
{
|
2019-07-27 02:30:58 +08:00
|
|
|
/*
|
|
|
|
* Make the enqueue delivery mode check work on RT. If the sleeper
|
|
|
|
* was initialized for hard interrupt delivery, force the mode bit.
|
|
|
|
* This is a special case for hrtimer_sleepers because
|
|
|
|
* hrtimer_init_sleeper() determines the delivery mode on RT so the
|
|
|
|
* fiddling with this decision is avoided at the call sites.
|
|
|
|
*/
|
|
|
|
if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
|
|
|
|
mode |= HRTIMER_MODE_HARD;
|
|
|
|
|
2019-07-31 03:03:53 +08:00
|
|
|
hrtimer_start_expires(&sl->timer, mode);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
|
|
|
|
|
2019-07-27 02:30:50 +08:00
|
|
|
static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
|
|
|
|
clockid_t clock_id, enum hrtimer_mode mode)
|
2006-03-31 18:31:17 +08:00
|
|
|
{
|
2019-07-27 02:30:58 +08:00
|
|
|
/*
|
2021-03-23 05:39:03 +08:00
|
|
|
* On PREEMPT_RT enabled kernels hrtimers which are not explicitly
|
2019-07-27 02:30:58 +08:00
|
|
|
* marked for hard interrupt expiry mode are moved into soft
|
|
|
|
* interrupt context either for latency reasons or because the
|
|
|
|
* hrtimer callback takes regular spinlocks or invokes other
|
|
|
|
* functions which are not suitable for hard interrupt context on
|
|
|
|
* PREEMPT_RT.
|
|
|
|
*
|
|
|
|
* The hrtimer_sleeper callback is RT compatible in hard interrupt
|
|
|
|
* context, but there is a latency concern: Untrusted userspace can
|
|
|
|
* spawn many threads which arm timers for the same expiry time on
|
|
|
|
* the same CPU. That causes a latency spike due to the wakeup of
|
|
|
|
* a gazillion threads.
|
|
|
|
*
|
2021-03-23 05:39:03 +08:00
|
|
|
* OTOH, privileged real-time user space applications rely on the
|
2019-07-27 02:30:58 +08:00
|
|
|
* low latency of hard interrupt wakeups. If the current task is in
|
|
|
|
* a real-time scheduling class, mark the mode for hard interrupt
|
|
|
|
* expiry.
|
|
|
|
*/
|
|
|
|
if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
|
|
|
|
if (task_is_realtime(current) && !(mode & HRTIMER_MODE_SOFT))
|
|
|
|
mode |= HRTIMER_MODE_HARD;
|
|
|
|
}
|
|
|
|
|
2019-07-27 02:30:50 +08:00
|
|
|
__hrtimer_init(&sl->timer, clock_id, mode);
|
2006-03-31 18:31:17 +08:00
|
|
|
sl->timer.function = hrtimer_wakeup;
|
2019-07-27 02:30:49 +08:00
|
|
|
sl->task = current;
|
2006-03-31 18:31:17 +08:00
|
|
|
}
|
2019-07-27 02:30:50 +08:00
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_init_sleeper - initialize sleeper to the given clock
|
|
|
|
* @sl: sleeper to be initialized
|
|
|
|
* @clock_id: the clock to be used
|
|
|
|
* @mode: timer mode abs/rel
|
|
|
|
*/
|
|
|
|
void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
debug_init(&sl->timer, clock_id, mode);
|
|
|
|
__hrtimer_init_sleeper(sl, clock_id, mode);
|
|
|
|
|
|
|
|
}
|
2009-08-29 14:41:29 +08:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
|
2006-03-31 18:31:17 +08:00
|
|
|
|
2017-06-25 02:45:06 +08:00
|
|
|
int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
|
2017-06-07 16:42:32 +08:00
|
|
|
{
|
|
|
|
switch(restart->nanosleep.type) {
|
2018-06-18 22:07:59 +08:00
|
|
|
#ifdef CONFIG_COMPAT_32BIT_TIME
|
2017-06-07 16:42:32 +08:00
|
|
|
case TT_COMPAT:
|
y2038: globally rename compat_time to old_time32
Christoph Hellwig suggested a slightly different path for handling
backwards compatibility with the 32-bit time_t based system calls:
Rather than simply reusing the compat_sys_* entry points on 32-bit
architectures unchanged, we get rid of those entry points and the
compat_time types by renaming them to something that makes more sense
on 32-bit architectures (which don't have a compat mode otherwise),
and then share the entry points under the new name with the 64-bit
architectures that use them for implementing the compatibility.
The following types and interfaces are renamed here, and moved
from linux/compat_time.h to linux/time32.h:
old new
--- ---
compat_time_t old_time32_t
struct compat_timeval struct old_timeval32
struct compat_timespec struct old_timespec32
struct compat_itimerspec struct old_itimerspec32
ns_to_compat_timeval() ns_to_old_timeval32()
get_compat_itimerspec64() get_old_itimerspec32()
put_compat_itimerspec64() put_old_itimerspec32()
compat_get_timespec64() get_old_timespec32()
compat_put_timespec64() put_old_timespec32()
As we already have aliases in place, this patch addresses only the
instances that are relevant to the system call interface in particular,
not those that occur in device drivers and other modules. Those
will get handled separately, while providing the 64-bit version
of the respective interfaces.
I'm not renaming the timex, rusage and itimerval structures, as we are
still debating what the new interface will look like, and whether we
will need a replacement at all.
This also doesn't change the names of the syscall entry points, which can
be done more easily when we actually switch over the 32-bit architectures
to use them, at that point we need to change COMPAT_SYSCALL_DEFINEx to
SYSCALL_DEFINEx with a new name, e.g. with a _time32 suffix.
Suggested-by: Christoph Hellwig <hch@infradead.org>
Link: https://lore.kernel.org/lkml/20180705222110.GA5698@infradead.org/
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
2018-07-13 18:52:28 +08:00
|
|
|
if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp))
|
2017-06-07 16:42:32 +08:00
|
|
|
return -EFAULT;
|
|
|
|
break;
|
|
|
|
#endif
|
|
|
|
case TT_NATIVE:
|
2017-06-25 02:45:06 +08:00
|
|
|
if (put_timespec64(ts, restart->nanosleep.rmtp))
|
2017-06-07 16:42:32 +08:00
|
|
|
return -EFAULT;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
return -ERESTART_RESTARTBLOCK;
|
|
|
|
}
|
|
|
|
|
2006-03-31 18:31:19 +08:00
|
|
|
static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
|
2006-03-26 17:38:08 +08:00
|
|
|
{
|
2017-06-07 16:42:31 +08:00
|
|
|
struct restart_block *restart;
|
|
|
|
|
2006-03-26 17:38:08 +08:00
|
|
|
do {
|
freezer,sched: Rewrite core freezer logic
Rewrite the core freezer to behave better wrt thawing and be simpler
in general.
By replacing PF_FROZEN with TASK_FROZEN, a special block state, it is
ensured frozen tasks stay frozen until thawed and don't randomly wake
up early, as is currently possible.
As such, it does away with PF_FROZEN and PF_FREEZER_SKIP, freeing up
two PF_flags (yay!).
Specifically; the current scheme works a little like:
freezer_do_not_count();
schedule();
freezer_count();
And either the task is blocked, or it lands in try_to_freezer()
through freezer_count(). Now, when it is blocked, the freezer
considers it frozen and continues.
However, on thawing, once pm_freezing is cleared, freezer_count()
stops working, and any random/spurious wakeup will let a task run
before its time.
That is, thawing tries to thaw things in explicit order; kernel
threads and workqueues before doing bringing SMP back before userspace
etc.. However due to the above mentioned races it is entirely possible
for userspace tasks to thaw (by accident) before SMP is back.
This can be a fatal problem in asymmetric ISA architectures (eg ARMv9)
where the userspace task requires a special CPU to run.
As said; replace this with a special task state TASK_FROZEN and add
the following state transitions:
TASK_FREEZABLE -> TASK_FROZEN
__TASK_STOPPED -> TASK_FROZEN
__TASK_TRACED -> TASK_FROZEN
The new TASK_FREEZABLE can be set on any state part of TASK_NORMAL
(IOW. TASK_INTERRUPTIBLE and TASK_UNINTERRUPTIBLE) -- any such state
is already required to deal with spurious wakeups and the freezer
causes one such when thawing the task (since the original state is
lost).
The special __TASK_{STOPPED,TRACED} states *can* be restored since
their canonical state is in ->jobctl.
With this, frozen tasks need an explicit TASK_FROZEN wakeup and are
free of undue (early / spurious) wakeups.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Link: https://lore.kernel.org/r/20220822114649.055452969@infradead.org
2022-08-22 19:18:22 +08:00
|
|
|
set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
|
2019-07-31 03:03:53 +08:00
|
|
|
hrtimer_sleeper_start_expires(t, mode);
|
2006-03-26 17:38:08 +08:00
|
|
|
|
2007-02-16 17:28:11 +08:00
|
|
|
if (likely(t->task))
|
freezer,sched: Rewrite core freezer logic
Rewrite the core freezer to behave better wrt thawing and be simpler
in general.
By replacing PF_FROZEN with TASK_FROZEN, a special block state, it is
ensured frozen tasks stay frozen until thawed and don't randomly wake
up early, as is currently possible.
As such, it does away with PF_FROZEN and PF_FREEZER_SKIP, freeing up
two PF_flags (yay!).
Specifically; the current scheme works a little like:
freezer_do_not_count();
schedule();
freezer_count();
And either the task is blocked, or it lands in try_to_freezer()
through freezer_count(). Now, when it is blocked, the freezer
considers it frozen and continues.
However, on thawing, once pm_freezing is cleared, freezer_count()
stops working, and any random/spurious wakeup will let a task run
before its time.
That is, thawing tries to thaw things in explicit order; kernel
threads and workqueues before doing bringing SMP back before userspace
etc.. However due to the above mentioned races it is entirely possible
for userspace tasks to thaw (by accident) before SMP is back.
This can be a fatal problem in asymmetric ISA architectures (eg ARMv9)
where the userspace task requires a special CPU to run.
As said; replace this with a special task state TASK_FROZEN and add
the following state transitions:
TASK_FREEZABLE -> TASK_FROZEN
__TASK_STOPPED -> TASK_FROZEN
__TASK_TRACED -> TASK_FROZEN
The new TASK_FREEZABLE can be set on any state part of TASK_NORMAL
(IOW. TASK_INTERRUPTIBLE and TASK_UNINTERRUPTIBLE) -- any such state
is already required to deal with spurious wakeups and the freezer
causes one such when thawing the task (since the original state is
lost).
The special __TASK_{STOPPED,TRACED} states *can* be restored since
their canonical state is in ->jobctl.
With this, frozen tasks need an explicit TASK_FROZEN wakeup and are
free of undue (early / spurious) wakeups.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Link: https://lore.kernel.org/r/20220822114649.055452969@infradead.org
2022-08-22 19:18:22 +08:00
|
|
|
schedule();
|
2006-03-26 17:38:08 +08:00
|
|
|
|
2006-03-31 18:31:19 +08:00
|
|
|
hrtimer_cancel(&t->timer);
|
2007-02-16 17:27:49 +08:00
|
|
|
mode = HRTIMER_MODE_ABS;
|
2006-03-31 18:31:19 +08:00
|
|
|
|
|
|
|
} while (t->task && !signal_pending(current));
|
2006-03-26 17:38:08 +08:00
|
|
|
|
2008-02-02 00:45:13 +08:00
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
|
2017-06-07 16:42:29 +08:00
|
|
|
if (!t->task)
|
2008-02-01 22:29:05 +08:00
|
|
|
return 0;
|
|
|
|
|
2017-06-07 16:42:31 +08:00
|
|
|
restart = ¤t->restart_block;
|
|
|
|
if (restart->nanosleep.type != TT_NONE) {
|
2017-06-07 16:42:29 +08:00
|
|
|
ktime_t rem = hrtimer_expires_remaining(&t->timer);
|
2017-06-25 02:45:06 +08:00
|
|
|
struct timespec64 rmt;
|
2017-06-07 16:42:31 +08:00
|
|
|
|
2017-06-07 16:42:29 +08:00
|
|
|
if (rem <= 0)
|
|
|
|
return 0;
|
2017-06-25 02:45:06 +08:00
|
|
|
rmt = ktime_to_timespec64(rem);
|
2017-06-07 16:42:29 +08:00
|
|
|
|
2017-06-07 16:42:32 +08:00
|
|
|
return nanosleep_copyout(restart, &rmt);
|
2017-06-07 16:42:29 +08:00
|
|
|
}
|
|
|
|
return -ERESTART_RESTARTBLOCK;
|
2008-02-01 22:29:05 +08:00
|
|
|
}
|
|
|
|
|
2017-06-07 16:42:33 +08:00
|
|
|
static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
|
2006-01-10 12:52:35 +08:00
|
|
|
{
|
2006-03-31 18:31:19 +08:00
|
|
|
struct hrtimer_sleeper t;
|
2017-06-07 16:42:29 +08:00
|
|
|
int ret;
|
2006-01-10 12:52:35 +08:00
|
|
|
|
2019-07-27 02:30:50 +08:00
|
|
|
hrtimer_init_sleeper_on_stack(&t, restart->nanosleep.clockid,
|
|
|
|
HRTIMER_MODE_ABS);
|
2008-09-02 06:02:30 +08:00
|
|
|
hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
|
2017-06-07 16:42:29 +08:00
|
|
|
ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
|
2008-04-30 15:55:04 +08:00
|
|
|
destroy_hrtimer_on_stack(&t.timer);
|
|
|
|
return ret;
|
2006-01-10 12:52:35 +08:00
|
|
|
}
|
|
|
|
|
2019-11-12 09:27:05 +08:00
|
|
|
long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
|
|
|
|
const clockid_t clockid)
|
2006-01-10 12:52:35 +08:00
|
|
|
{
|
2017-06-07 16:42:29 +08:00
|
|
|
struct restart_block *restart;
|
2006-03-31 18:31:19 +08:00
|
|
|
struct hrtimer_sleeper t;
|
2008-04-30 15:55:04 +08:00
|
|
|
int ret = 0;
|
timer: convert timer_slack_ns from unsigned long to u64
This patchset introduces a /proc/<pid>/timerslack_ns interface which
would allow controlling processes to be able to set the timerslack value
on other processes in order to save power by avoiding wakeups (Something
Android currently does via out-of-tree patches).
The first patch tries to fix the internal timer_slack_ns usage which was
defined as a long, which limits the slack range to ~4 seconds on 32bit
systems. It converts it to a u64, which provides the same basically
unlimited slack (500 years) on both 32bit and 64bit machines.
The second patch introduces the /proc/<pid>/timerslack_ns interface
which allows the full 64bit slack range for a task to be read or set on
both 32bit and 64bit machines.
With these two patches, on a 32bit machine, after setting the slack on
bash to 10 seconds:
$ time sleep 1
real 0m10.747s
user 0m0.001s
sys 0m0.005s
The first patch is a little ugly, since I had to chase the slack delta
arguments through a number of functions converting them to u64s. Let me
know if it makes sense to break that up more or not.
Other than that things are fairly straightforward.
This patch (of 2):
The timer_slack_ns value in the task struct is currently a unsigned
long. This means that on 32bit applications, the maximum slack is just
over 4 seconds. However, on 64bit machines, its much much larger (~500
years).
This disparity could make application development a little (as well as
the default_slack) to a u64. This means both 32bit and 64bit systems
have the same effective internal slack range.
Now the existing ABI via PR_GET_TIMERSLACK and PR_SET_TIMERSLACK specify
the interface as a unsigned long, so we preserve that limitation on
32bit systems, where SET_TIMERSLACK can only set the slack to a unsigned
long value, and GET_TIMERSLACK will return ULONG_MAX if the slack is
actually larger then what can be stored by an unsigned long.
This patch also modifies hrtimer functions which specified the slack
delta as a unsigned long.
Signed-off-by: John Stultz <john.stultz@linaro.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Oren Laadan <orenl@cellrox.com>
Cc: Ruchi Kandoi <kandoiruchi@google.com>
Cc: Rom Lemarchand <romlem@android.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Android Kernel Team <kernel-team@android.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-18 05:20:51 +08:00
|
|
|
u64 slack;
|
2008-09-08 23:58:59 +08:00
|
|
|
|
|
|
|
slack = current->timer_slack_ns;
|
2023-01-24 01:32:05 +08:00
|
|
|
if (rt_task(current))
|
2008-09-08 23:58:59 +08:00
|
|
|
slack = 0;
|
2006-01-10 12:52:35 +08:00
|
|
|
|
2019-07-27 02:30:50 +08:00
|
|
|
hrtimer_init_sleeper_on_stack(&t, clockid, mode);
|
2019-11-12 09:27:05 +08:00
|
|
|
hrtimer_set_expires_range_ns(&t.timer, rqtp, slack);
|
2017-06-07 16:42:29 +08:00
|
|
|
ret = do_nanosleep(&t, mode);
|
|
|
|
if (ret != -ERESTART_RESTARTBLOCK)
|
2008-04-30 15:55:04 +08:00
|
|
|
goto out;
|
2006-01-10 12:52:35 +08:00
|
|
|
|
2006-02-01 19:05:11 +08:00
|
|
|
/* Absolute timers do not update the rmtp value and restart: */
|
2008-04-30 15:55:04 +08:00
|
|
|
if (mode == HRTIMER_MODE_ABS) {
|
|
|
|
ret = -ERESTARTNOHAND;
|
|
|
|
goto out;
|
|
|
|
}
|
2006-01-10 12:52:35 +08:00
|
|
|
|
2017-06-07 16:42:29 +08:00
|
|
|
restart = ¤t->restart_block;
|
2011-05-20 19:05:15 +08:00
|
|
|
restart->nanosleep.clockid = t.timer.base->clockid;
|
2008-09-02 06:02:30 +08:00
|
|
|
restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
|
2021-02-02 01:46:41 +08:00
|
|
|
set_restart_fn(restart, hrtimer_nanosleep_restart);
|
2008-04-30 15:55:04 +08:00
|
|
|
out:
|
|
|
|
destroy_hrtimer_on_stack(&t.timer);
|
|
|
|
return ret;
|
2006-01-10 12:52:35 +08:00
|
|
|
}
|
|
|
|
|
2019-04-23 23:43:50 +08:00
|
|
|
#ifdef CONFIG_64BIT
|
2018-03-14 12:03:33 +08:00
|
|
|
|
|
|
|
SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp,
|
|
|
|
struct __kernel_timespec __user *, rmtp)
|
2006-01-10 12:52:36 +08:00
|
|
|
{
|
2017-06-25 02:45:06 +08:00
|
|
|
struct timespec64 tu;
|
2006-01-10 12:52:36 +08:00
|
|
|
|
2017-06-25 02:45:06 +08:00
|
|
|
if (get_timespec64(&tu, rqtp))
|
2006-01-10 12:52:36 +08:00
|
|
|
return -EFAULT;
|
|
|
|
|
2017-06-25 02:45:06 +08:00
|
|
|
if (!timespec64_valid(&tu))
|
2006-01-10 12:52:36 +08:00
|
|
|
return -EINVAL;
|
|
|
|
|
timers: Prevent union confusion from unexpected restart_syscall()
The nanosleep syscalls use the restart_block mechanism, with a quirk:
The `type` and `rmtp`/`compat_rmtp` fields are set up unconditionally on
syscall entry, while the rest of the restart_block is only set up in the
unlikely case that the syscall is actually interrupted by a signal (or
pseudo-signal) that doesn't have a signal handler.
If the restart_block was set up by a previous syscall (futex(...,
FUTEX_WAIT, ...) or poll()) and hasn't been invalidated somehow since then,
this will clobber some of the union fields used by futex_wait_restart() and
do_restart_poll().
If userspace afterwards wrongly calls the restart_syscall syscall,
futex_wait_restart()/do_restart_poll() will read struct fields that have
been clobbered.
This doesn't actually lead to anything particularly interesting because
none of the union fields contain trusted kernel data, and
futex(..., FUTEX_WAIT, ...) and poll() aren't syscalls where it makes much
sense to apply seccomp filters to their arguments.
So the current consequences are just of the "if userspace does bad stuff,
it can damage itself, and that's not a problem" flavor.
But still, it seems like a hazard for future developers, so invalidate the
restart_block when partly setting it up in the nanosleep syscalls.
Signed-off-by: Jann Horn <jannh@google.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/r/20230105134403.754986-1-jannh@google.com
2023-01-05 21:44:03 +08:00
|
|
|
current->restart_block.fn = do_no_restart_syscall;
|
2017-06-07 16:42:31 +08:00
|
|
|
current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
|
2017-06-07 16:42:28 +08:00
|
|
|
current->restart_block.nanosleep.rmtp = rmtp;
|
2019-11-12 09:27:05 +08:00
|
|
|
return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL,
|
|
|
|
CLOCK_MONOTONIC);
|
2006-01-10 12:52:36 +08:00
|
|
|
}
|
|
|
|
|
2018-03-14 12:03:33 +08:00
|
|
|
#endif
|
|
|
|
|
2018-03-14 12:03:29 +08:00
|
|
|
#ifdef CONFIG_COMPAT_32BIT_TIME
|
2017-06-07 16:42:31 +08:00
|
|
|
|
2019-01-07 07:33:08 +08:00
|
|
|
SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp,
|
y2038: globally rename compat_time to old_time32
Christoph Hellwig suggested a slightly different path for handling
backwards compatibility with the 32-bit time_t based system calls:
Rather than simply reusing the compat_sys_* entry points on 32-bit
architectures unchanged, we get rid of those entry points and the
compat_time types by renaming them to something that makes more sense
on 32-bit architectures (which don't have a compat mode otherwise),
and then share the entry points under the new name with the 64-bit
architectures that use them for implementing the compatibility.
The following types and interfaces are renamed here, and moved
from linux/compat_time.h to linux/time32.h:
old new
--- ---
compat_time_t old_time32_t
struct compat_timeval struct old_timeval32
struct compat_timespec struct old_timespec32
struct compat_itimerspec struct old_itimerspec32
ns_to_compat_timeval() ns_to_old_timeval32()
get_compat_itimerspec64() get_old_itimerspec32()
put_compat_itimerspec64() put_old_itimerspec32()
compat_get_timespec64() get_old_timespec32()
compat_put_timespec64() put_old_timespec32()
As we already have aliases in place, this patch addresses only the
instances that are relevant to the system call interface in particular,
not those that occur in device drivers and other modules. Those
will get handled separately, while providing the 64-bit version
of the respective interfaces.
I'm not renaming the timex, rusage and itimerval structures, as we are
still debating what the new interface will look like, and whether we
will need a replacement at all.
This also doesn't change the names of the syscall entry points, which can
be done more easily when we actually switch over the 32-bit architectures
to use them, at that point we need to change COMPAT_SYSCALL_DEFINEx to
SYSCALL_DEFINEx with a new name, e.g. with a _time32 suffix.
Suggested-by: Christoph Hellwig <hch@infradead.org>
Link: https://lore.kernel.org/lkml/20180705222110.GA5698@infradead.org/
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
2018-07-13 18:52:28 +08:00
|
|
|
struct old_timespec32 __user *, rmtp)
|
2017-06-07 16:42:31 +08:00
|
|
|
{
|
2017-06-25 02:45:06 +08:00
|
|
|
struct timespec64 tu;
|
2017-06-07 16:42:31 +08:00
|
|
|
|
y2038: globally rename compat_time to old_time32
Christoph Hellwig suggested a slightly different path for handling
backwards compatibility with the 32-bit time_t based system calls:
Rather than simply reusing the compat_sys_* entry points on 32-bit
architectures unchanged, we get rid of those entry points and the
compat_time types by renaming them to something that makes more sense
on 32-bit architectures (which don't have a compat mode otherwise),
and then share the entry points under the new name with the 64-bit
architectures that use them for implementing the compatibility.
The following types and interfaces are renamed here, and moved
from linux/compat_time.h to linux/time32.h:
old new
--- ---
compat_time_t old_time32_t
struct compat_timeval struct old_timeval32
struct compat_timespec struct old_timespec32
struct compat_itimerspec struct old_itimerspec32
ns_to_compat_timeval() ns_to_old_timeval32()
get_compat_itimerspec64() get_old_itimerspec32()
put_compat_itimerspec64() put_old_itimerspec32()
compat_get_timespec64() get_old_timespec32()
compat_put_timespec64() put_old_timespec32()
As we already have aliases in place, this patch addresses only the
instances that are relevant to the system call interface in particular,
not those that occur in device drivers and other modules. Those
will get handled separately, while providing the 64-bit version
of the respective interfaces.
I'm not renaming the timex, rusage and itimerval structures, as we are
still debating what the new interface will look like, and whether we
will need a replacement at all.
This also doesn't change the names of the syscall entry points, which can
be done more easily when we actually switch over the 32-bit architectures
to use them, at that point we need to change COMPAT_SYSCALL_DEFINEx to
SYSCALL_DEFINEx with a new name, e.g. with a _time32 suffix.
Suggested-by: Christoph Hellwig <hch@infradead.org>
Link: https://lore.kernel.org/lkml/20180705222110.GA5698@infradead.org/
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
2018-07-13 18:52:28 +08:00
|
|
|
if (get_old_timespec32(&tu, rqtp))
|
2017-06-07 16:42:31 +08:00
|
|
|
return -EFAULT;
|
|
|
|
|
2017-06-25 02:45:06 +08:00
|
|
|
if (!timespec64_valid(&tu))
|
2017-06-07 16:42:31 +08:00
|
|
|
return -EINVAL;
|
|
|
|
|
timers: Prevent union confusion from unexpected restart_syscall()
The nanosleep syscalls use the restart_block mechanism, with a quirk:
The `type` and `rmtp`/`compat_rmtp` fields are set up unconditionally on
syscall entry, while the rest of the restart_block is only set up in the
unlikely case that the syscall is actually interrupted by a signal (or
pseudo-signal) that doesn't have a signal handler.
If the restart_block was set up by a previous syscall (futex(...,
FUTEX_WAIT, ...) or poll()) and hasn't been invalidated somehow since then,
this will clobber some of the union fields used by futex_wait_restart() and
do_restart_poll().
If userspace afterwards wrongly calls the restart_syscall syscall,
futex_wait_restart()/do_restart_poll() will read struct fields that have
been clobbered.
This doesn't actually lead to anything particularly interesting because
none of the union fields contain trusted kernel data, and
futex(..., FUTEX_WAIT, ...) and poll() aren't syscalls where it makes much
sense to apply seccomp filters to their arguments.
So the current consequences are just of the "if userspace does bad stuff,
it can damage itself, and that's not a problem" flavor.
But still, it seems like a hazard for future developers, so invalidate the
restart_block when partly setting it up in the nanosleep syscalls.
Signed-off-by: Jann Horn <jannh@google.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/r/20230105134403.754986-1-jannh@google.com
2023-01-05 21:44:03 +08:00
|
|
|
current->restart_block.fn = do_no_restart_syscall;
|
2017-06-07 16:42:31 +08:00
|
|
|
current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
|
|
|
|
current->restart_block.nanosleep.compat_rmtp = rmtp;
|
2019-11-12 09:27:05 +08:00
|
|
|
return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL,
|
|
|
|
CLOCK_MONOTONIC);
|
2017-06-07 16:42:31 +08:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
/*
|
|
|
|
* Functions related to boot-time initialization:
|
|
|
|
*/
|
2016-07-15 16:41:04 +08:00
|
|
|
int hrtimers_prepare_cpu(unsigned int cpu)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2007-02-16 17:27:50 +08:00
|
|
|
struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
|
2006-01-10 12:52:32 +08:00
|
|
|
int i;
|
|
|
|
|
2010-09-21 10:19:17 +08:00
|
|
|
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
|
2020-07-20 23:55:30 +08:00
|
|
|
struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];
|
|
|
|
|
|
|
|
clock_b->cpu_base = cpu_base;
|
|
|
|
seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);
|
|
|
|
timerqueue_init_head(&clock_b->active);
|
2010-09-21 10:19:17 +08:00
|
|
|
}
|
2007-02-16 17:27:50 +08:00
|
|
|
|
2014-06-22 07:29:15 +08:00
|
|
|
cpu_base->cpu = cpu;
|
2018-01-27 22:35:29 +08:00
|
|
|
cpu_base->active_bases = 0;
|
2017-12-21 18:41:42 +08:00
|
|
|
cpu_base->hres_active = 0;
|
2018-01-27 22:35:29 +08:00
|
|
|
cpu_base->hang_detected = 0;
|
|
|
|
cpu_base->next_timer = NULL;
|
|
|
|
cpu_base->softirq_next_timer = NULL;
|
2017-12-21 18:41:44 +08:00
|
|
|
cpu_base->expires_next = KTIME_MAX;
|
2017-12-21 18:41:57 +08:00
|
|
|
cpu_base->softirq_expires_next = KTIME_MAX;
|
2019-07-27 02:30:59 +08:00
|
|
|
hrtimer_cpu_base_init_expiry_lock(cpu_base);
|
2016-07-15 16:41:04 +08:00
|
|
|
return 0;
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
|
2008-11-25 19:43:51 +08:00
|
|
|
static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
|
2008-12-04 18:17:10 +08:00
|
|
|
struct hrtimer_clock_base *new_base)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
|
|
|
struct hrtimer *timer;
|
2010-09-21 10:19:17 +08:00
|
|
|
struct timerqueue_node *node;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2010-09-21 10:19:17 +08:00
|
|
|
while ((node = timerqueue_getnext(&old_base->active))) {
|
|
|
|
timer = container_of(node, struct hrtimer, node);
|
2007-02-16 17:28:11 +08:00
|
|
|
BUG_ON(hrtimer_callback_running(timer));
|
2009-08-10 10:51:23 +08:00
|
|
|
debug_deactivate(timer);
|
2008-09-29 21:44:46 +08:00
|
|
|
|
|
|
|
/*
|
2015-06-11 20:46:44 +08:00
|
|
|
* Mark it as ENQUEUED not INACTIVE otherwise the
|
2008-09-29 21:44:46 +08:00
|
|
|
* timer could be seen as !active and just vanish away
|
|
|
|
* under us on another CPU
|
|
|
|
*/
|
2015-06-11 20:46:44 +08:00
|
|
|
__remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);
|
2006-01-10 12:52:32 +08:00
|
|
|
timer->base = new_base;
|
2007-02-16 17:28:11 +08:00
|
|
|
/*
|
2009-01-05 18:28:23 +08:00
|
|
|
* Enqueue the timers on the new cpu. This does not
|
|
|
|
* reprogram the event device in case the timer
|
|
|
|
* expires before the earliest on this CPU, but we run
|
|
|
|
* hrtimer_interrupt after we migrated everything to
|
|
|
|
* sort out already expired timers and reprogram the
|
|
|
|
* event device.
|
2007-02-16 17:28:11 +08:00
|
|
|
*/
|
2017-12-21 18:41:38 +08:00
|
|
|
enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-07-15 16:41:04 +08:00
|
|
|
int hrtimers_dead_cpu(unsigned int scpu)
|
2006-01-10 12:52:32 +08:00
|
|
|
{
|
2007-02-16 17:27:50 +08:00
|
|
|
struct hrtimer_cpu_base *old_base, *new_base;
|
2009-01-05 18:28:21 +08:00
|
|
|
int i;
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2008-12-04 18:17:10 +08:00
|
|
|
BUG_ON(cpu_online(scpu));
|
|
|
|
tick_cancel_sched_timer(scpu);
|
2009-01-05 18:28:21 +08:00
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
/*
|
|
|
|
* this BH disable ensures that raise_softirq_irqoff() does
|
|
|
|
* not wakeup ksoftirqd (and acquire the pi-lock) while
|
|
|
|
* holding the cpu_base lock
|
|
|
|
*/
|
|
|
|
local_bh_disable();
|
2009-01-05 18:28:21 +08:00
|
|
|
local_irq_disable();
|
|
|
|
old_base = &per_cpu(hrtimer_bases, scpu);
|
2014-08-18 01:30:26 +08:00
|
|
|
new_base = this_cpu_ptr(&hrtimer_bases);
|
2008-08-21 07:46:04 +08:00
|
|
|
/*
|
|
|
|
* The caller is globally serialized and nobody else
|
|
|
|
* takes two locks at once, deadlock is not possible.
|
|
|
|
*/
|
2009-11-17 23:36:54 +08:00
|
|
|
raw_spin_lock(&new_base->lock);
|
|
|
|
raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
|
2006-01-10 12:52:32 +08:00
|
|
|
|
2007-02-16 17:27:50 +08:00
|
|
|
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
|
2008-11-25 19:43:51 +08:00
|
|
|
migrate_hrtimer_list(&old_base->clock_base[i],
|
2008-12-04 18:17:10 +08:00
|
|
|
&new_base->clock_base[i]);
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
|
|
|
|
2017-12-21 18:41:57 +08:00
|
|
|
/*
|
|
|
|
* The migration might have changed the first expiring softirq
|
|
|
|
* timer on this CPU. Update it.
|
|
|
|
*/
|
|
|
|
hrtimer_update_softirq_timer(new_base, false);
|
|
|
|
|
2009-11-17 23:36:54 +08:00
|
|
|
raw_spin_unlock(&old_base->lock);
|
|
|
|
raw_spin_unlock(&new_base->lock);
|
2008-12-04 18:17:10 +08:00
|
|
|
|
2009-01-05 18:28:21 +08:00
|
|
|
/* Check, if we got expired work to do */
|
|
|
|
__hrtimer_peek_ahead_timers();
|
|
|
|
local_irq_enable();
|
2017-12-21 18:41:57 +08:00
|
|
|
local_bh_enable();
|
2016-07-15 16:41:04 +08:00
|
|
|
return 0;
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
2008-12-04 18:17:10 +08:00
|
|
|
|
2006-01-10 12:52:32 +08:00
|
|
|
#endif /* CONFIG_HOTPLUG_CPU */
|
|
|
|
|
|
|
|
void __init hrtimers_init(void)
|
|
|
|
{
|
2016-07-15 16:41:04 +08:00
|
|
|
hrtimers_prepare_cpu(smp_processor_id());
|
2017-12-21 18:41:57 +08:00
|
|
|
open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
|
2006-01-10 12:52:32 +08:00
|
|
|
}
|
|
|
|
|
2008-08-31 23:05:58 +08:00
|
|
|
/**
|
2010-04-03 04:40:19 +08:00
|
|
|
* schedule_hrtimeout_range_clock - sleep until timeout
|
2008-08-31 23:05:58 +08:00
|
|
|
* @expires: timeout value (ktime_t)
|
2023-01-24 01:32:06 +08:00
|
|
|
* @delta: slack in expires timeout (ktime_t) for SCHED_OTHER tasks
|
2017-12-21 18:41:33 +08:00
|
|
|
* @mode: timer mode
|
|
|
|
* @clock_id: timer clock to be used
|
2008-08-31 23:05:58 +08:00
|
|
|
*/
|
2010-04-03 04:40:19 +08:00
|
|
|
int __sched
|
timer: convert timer_slack_ns from unsigned long to u64
This patchset introduces a /proc/<pid>/timerslack_ns interface which
would allow controlling processes to be able to set the timerslack value
on other processes in order to save power by avoiding wakeups (Something
Android currently does via out-of-tree patches).
The first patch tries to fix the internal timer_slack_ns usage which was
defined as a long, which limits the slack range to ~4 seconds on 32bit
systems. It converts it to a u64, which provides the same basically
unlimited slack (500 years) on both 32bit and 64bit machines.
The second patch introduces the /proc/<pid>/timerslack_ns interface
which allows the full 64bit slack range for a task to be read or set on
both 32bit and 64bit machines.
With these two patches, on a 32bit machine, after setting the slack on
bash to 10 seconds:
$ time sleep 1
real 0m10.747s
user 0m0.001s
sys 0m0.005s
The first patch is a little ugly, since I had to chase the slack delta
arguments through a number of functions converting them to u64s. Let me
know if it makes sense to break that up more or not.
Other than that things are fairly straightforward.
This patch (of 2):
The timer_slack_ns value in the task struct is currently a unsigned
long. This means that on 32bit applications, the maximum slack is just
over 4 seconds. However, on 64bit machines, its much much larger (~500
years).
This disparity could make application development a little (as well as
the default_slack) to a u64. This means both 32bit and 64bit systems
have the same effective internal slack range.
Now the existing ABI via PR_GET_TIMERSLACK and PR_SET_TIMERSLACK specify
the interface as a unsigned long, so we preserve that limitation on
32bit systems, where SET_TIMERSLACK can only set the slack to a unsigned
long value, and GET_TIMERSLACK will return ULONG_MAX if the slack is
actually larger then what can be stored by an unsigned long.
This patch also modifies hrtimer functions which specified the slack
delta as a unsigned long.
Signed-off-by: John Stultz <john.stultz@linaro.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Oren Laadan <orenl@cellrox.com>
Cc: Ruchi Kandoi <kandoiruchi@google.com>
Cc: Rom Lemarchand <romlem@android.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Android Kernel Team <kernel-team@android.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-18 05:20:51 +08:00
|
|
|
schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
|
2017-12-21 18:41:33 +08:00
|
|
|
const enum hrtimer_mode mode, clockid_t clock_id)
|
2008-08-31 23:05:58 +08:00
|
|
|
{
|
|
|
|
struct hrtimer_sleeper t;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Optimize when a zero timeout value is given. It does not
|
|
|
|
* matter whether this is an absolute or a relative time.
|
|
|
|
*/
|
2016-12-25 18:38:40 +08:00
|
|
|
if (expires && *expires == 0) {
|
2008-08-31 23:05:58 +08:00
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2010-12-23 02:01:47 +08:00
|
|
|
* A NULL parameter means "infinite"
|
2008-08-31 23:05:58 +08:00
|
|
|
*/
|
|
|
|
if (!expires) {
|
|
|
|
schedule();
|
|
|
|
return -EINTR;
|
|
|
|
}
|
|
|
|
|
2023-01-24 01:32:06 +08:00
|
|
|
/*
|
|
|
|
* Override any slack passed by the user if under
|
|
|
|
* rt contraints.
|
|
|
|
*/
|
|
|
|
if (rt_task(current))
|
|
|
|
delta = 0;
|
|
|
|
|
2019-07-27 02:30:50 +08:00
|
|
|
hrtimer_init_sleeper_on_stack(&t, clock_id, mode);
|
2008-09-02 06:47:08 +08:00
|
|
|
hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
|
2019-07-31 03:03:53 +08:00
|
|
|
hrtimer_sleeper_start_expires(&t, mode);
|
2008-08-31 23:05:58 +08:00
|
|
|
|
|
|
|
if (likely(t.task))
|
|
|
|
schedule();
|
|
|
|
|
|
|
|
hrtimer_cancel(&t.timer);
|
|
|
|
destroy_hrtimer_on_stack(&t.timer);
|
|
|
|
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
|
|
|
|
return !t.task ? 0 : -EINTR;
|
|
|
|
}
|
wireguard: ratelimiter: use hrtimer in selftest
Using msleep() is problematic because it's compared against
ratelimiter.c's ktime_get_coarse_boottime_ns(), which means on systems
with slow jiffies (such as UML's forced HZ=100), the result is
inaccurate. So switch to using schedule_hrtimeout().
However, hrtimer gives us access only to the traditional posix timers,
and none of the _COARSE variants. So now, rather than being too
imprecise like jiffies, it's too precise.
One solution would be to give it a large "range" value, but this will
still fire early on a loaded system. A better solution is to align the
timeout to the actual coarse timer, and then round up to the nearest
tick, plus change.
So add the timeout to the current coarse time, and then
schedule_hrtimer() until the absolute computed time.
This should hopefully reduce flakes in CI as well. Note that we keep the
retry loop in case the entire function is running behind, because the
test could still be scheduled out, by either the kernel or by the
hypervisor's kernel, in which case restarting the test and hoping to not
be scheduled out still helps.
Fixes: e7096c131e51 ("net: WireGuard secure network tunnel")
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2022-08-02 20:56:10 +08:00
|
|
|
EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock);
|
2010-04-03 04:40:19 +08:00
|
|
|
|
|
|
|
/**
|
|
|
|
* schedule_hrtimeout_range - sleep until timeout
|
|
|
|
* @expires: timeout value (ktime_t)
|
2023-01-24 01:32:06 +08:00
|
|
|
* @delta: slack in expires timeout (ktime_t) for SCHED_OTHER tasks
|
2017-12-21 18:41:33 +08:00
|
|
|
* @mode: timer mode
|
2010-04-03 04:40:19 +08:00
|
|
|
*
|
|
|
|
* Make the current task sleep until the given expiry time has
|
|
|
|
* elapsed. The routine will return immediately unless
|
|
|
|
* the current task state has been set (see set_current_state()).
|
|
|
|
*
|
|
|
|
* The @delta argument gives the kernel the freedom to schedule the
|
2023-01-24 01:32:06 +08:00
|
|
|
* actual wakeup to a time that is both power and performance friendly
|
|
|
|
* for regular (non RT/DL) tasks.
|
2010-04-03 04:40:19 +08:00
|
|
|
* The kernel give the normal best effort behavior for "@expires+@delta",
|
|
|
|
* but may decide to fire the timer earlier, but no earlier than @expires.
|
|
|
|
*
|
|
|
|
* You can set the task state as follows -
|
|
|
|
*
|
|
|
|
* %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
|
2016-10-21 23:58:51 +08:00
|
|
|
* pass before the routine returns unless the current task is explicitly
|
|
|
|
* woken up, (e.g. by wake_up_process()).
|
2010-04-03 04:40:19 +08:00
|
|
|
*
|
|
|
|
* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
|
2016-10-21 23:58:51 +08:00
|
|
|
* delivered to the current task or the current task is explicitly woken
|
|
|
|
* up.
|
2010-04-03 04:40:19 +08:00
|
|
|
*
|
|
|
|
* The current task state is guaranteed to be TASK_RUNNING when this
|
|
|
|
* routine returns.
|
|
|
|
*
|
2016-10-21 23:58:51 +08:00
|
|
|
* Returns 0 when the timer has expired. If the task was woken before the
|
|
|
|
* timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
|
|
|
|
* by an explicit wakeup, it returns -EINTR.
|
2010-04-03 04:40:19 +08:00
|
|
|
*/
|
timer: convert timer_slack_ns from unsigned long to u64
This patchset introduces a /proc/<pid>/timerslack_ns interface which
would allow controlling processes to be able to set the timerslack value
on other processes in order to save power by avoiding wakeups (Something
Android currently does via out-of-tree patches).
The first patch tries to fix the internal timer_slack_ns usage which was
defined as a long, which limits the slack range to ~4 seconds on 32bit
systems. It converts it to a u64, which provides the same basically
unlimited slack (500 years) on both 32bit and 64bit machines.
The second patch introduces the /proc/<pid>/timerslack_ns interface
which allows the full 64bit slack range for a task to be read or set on
both 32bit and 64bit machines.
With these two patches, on a 32bit machine, after setting the slack on
bash to 10 seconds:
$ time sleep 1
real 0m10.747s
user 0m0.001s
sys 0m0.005s
The first patch is a little ugly, since I had to chase the slack delta
arguments through a number of functions converting them to u64s. Let me
know if it makes sense to break that up more or not.
Other than that things are fairly straightforward.
This patch (of 2):
The timer_slack_ns value in the task struct is currently a unsigned
long. This means that on 32bit applications, the maximum slack is just
over 4 seconds. However, on 64bit machines, its much much larger (~500
years).
This disparity could make application development a little (as well as
the default_slack) to a u64. This means both 32bit and 64bit systems
have the same effective internal slack range.
Now the existing ABI via PR_GET_TIMERSLACK and PR_SET_TIMERSLACK specify
the interface as a unsigned long, so we preserve that limitation on
32bit systems, where SET_TIMERSLACK can only set the slack to a unsigned
long value, and GET_TIMERSLACK will return ULONG_MAX if the slack is
actually larger then what can be stored by an unsigned long.
This patch also modifies hrtimer functions which specified the slack
delta as a unsigned long.
Signed-off-by: John Stultz <john.stultz@linaro.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Oren Laadan <orenl@cellrox.com>
Cc: Ruchi Kandoi <kandoiruchi@google.com>
Cc: Rom Lemarchand <romlem@android.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Android Kernel Team <kernel-team@android.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-18 05:20:51 +08:00
|
|
|
int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
|
2010-04-03 04:40:19 +08:00
|
|
|
const enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
return schedule_hrtimeout_range_clock(expires, delta, mode,
|
|
|
|
CLOCK_MONOTONIC);
|
|
|
|
}
|
2008-09-02 06:47:08 +08:00
|
|
|
EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* schedule_hrtimeout - sleep until timeout
|
|
|
|
* @expires: timeout value (ktime_t)
|
2017-12-21 18:41:33 +08:00
|
|
|
* @mode: timer mode
|
2008-09-02 06:47:08 +08:00
|
|
|
*
|
|
|
|
* Make the current task sleep until the given expiry time has
|
|
|
|
* elapsed. The routine will return immediately unless
|
|
|
|
* the current task state has been set (see set_current_state()).
|
|
|
|
*
|
|
|
|
* You can set the task state as follows -
|
|
|
|
*
|
|
|
|
* %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
|
2016-10-21 23:58:51 +08:00
|
|
|
* pass before the routine returns unless the current task is explicitly
|
|
|
|
* woken up, (e.g. by wake_up_process()).
|
2008-09-02 06:47:08 +08:00
|
|
|
*
|
|
|
|
* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
|
2016-10-21 23:58:51 +08:00
|
|
|
* delivered to the current task or the current task is explicitly woken
|
|
|
|
* up.
|
2008-09-02 06:47:08 +08:00
|
|
|
*
|
|
|
|
* The current task state is guaranteed to be TASK_RUNNING when this
|
|
|
|
* routine returns.
|
|
|
|
*
|
2016-10-21 23:58:51 +08:00
|
|
|
* Returns 0 when the timer has expired. If the task was woken before the
|
|
|
|
* timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
|
|
|
|
* by an explicit wakeup, it returns -EINTR.
|
2008-09-02 06:47:08 +08:00
|
|
|
*/
|
|
|
|
int __sched schedule_hrtimeout(ktime_t *expires,
|
|
|
|
const enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
return schedule_hrtimeout_range(expires, 0, mode);
|
|
|
|
}
|
2008-08-31 23:05:58 +08:00
|
|
|
EXPORT_SYMBOL_GPL(schedule_hrtimeout);
|