2019-01-18 02:05:33 +08:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0+ */
|
2011-06-18 06:53:19 +08:00
|
|
|
/*
|
|
|
|
|
* Read-Copy Update definitions shared among RCU implementations.
|
|
|
|
|
*
|
|
|
|
|
* Copyright IBM Corporation, 2011
|
|
|
|
|
*
|
2019-01-18 02:05:33 +08:00
|
|
|
* Author: Paul E. McKenney <paulmck@linux.ibm.com>
|
2011-06-18 06:53:19 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef __LINUX_RCU_H
|
|
|
|
|
#define __LINUX_RCU_H
|
|
|
|
|
|
2014-02-20 03:33:27 +08:00
|
|
|
#include <trace/events/rcu.h>
|
2011-06-21 15:13:44 +08:00
|
|
|
|
2017-02-21 06:57:17 +08:00
|
|
|
/*
|
|
|
|
|
* Grace-period counter management.
|
|
|
|
|
*/
|
|
|
|
|
|
2017-03-22 01:35:57 +08:00
|
|
|
#define RCU_SEQ_CTR_SHIFT 2
|
2017-03-21 22:28:14 +08:00
|
|
|
#define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1)
|
|
|
|
|
|
2022-04-14 06:17:25 +08:00
|
|
|
/* Low-order bit definition for polled grace-period APIs. */
|
|
|
|
|
#define RCU_GET_STATE_COMPLETED 0x1
|
|
|
|
|
|
2022-02-15 19:45:59 +08:00
|
|
|
extern int sysctl_sched_rt_runtime;
|
|
|
|
|
|
2017-03-21 22:28:14 +08:00
|
|
|
/*
|
|
|
|
|
* Return the counter portion of a sequence number previously returned
|
|
|
|
|
* by rcu_seq_snap() or rcu_seq_current().
|
|
|
|
|
*/
|
|
|
|
|
static inline unsigned long rcu_seq_ctr(unsigned long s)
|
|
|
|
|
{
|
|
|
|
|
return s >> RCU_SEQ_CTR_SHIFT;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Return the state portion of a sequence number previously returned
|
|
|
|
|
* by rcu_seq_snap() or rcu_seq_current().
|
|
|
|
|
*/
|
|
|
|
|
static inline int rcu_seq_state(unsigned long s)
|
|
|
|
|
{
|
|
|
|
|
return s & RCU_SEQ_STATE_MASK;
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-23 06:26:18 +08:00
|
|
|
/*
|
|
|
|
|
* Set the state portion of the pointed-to sequence number.
|
|
|
|
|
* The caller is responsible for preventing conflicting updates.
|
|
|
|
|
*/
|
|
|
|
|
static inline void rcu_seq_set_state(unsigned long *sp, int newstate)
|
|
|
|
|
{
|
|
|
|
|
WARN_ON_ONCE(newstate & ~RCU_SEQ_STATE_MASK);
|
|
|
|
|
WRITE_ONCE(*sp, (*sp & ~RCU_SEQ_STATE_MASK) + newstate);
|
|
|
|
|
}
|
|
|
|
|
|
2017-02-21 06:57:17 +08:00
|
|
|
/* Adjust sequence number for start of update-side operation. */
|
|
|
|
|
static inline void rcu_seq_start(unsigned long *sp)
|
|
|
|
|
{
|
|
|
|
|
WRITE_ONCE(*sp, *sp + 1);
|
|
|
|
|
smp_mb(); /* Ensure update-side operation after counter increment. */
|
2017-03-21 22:28:14 +08:00
|
|
|
WARN_ON_ONCE(rcu_seq_state(*sp) != 1);
|
2017-02-21 06:57:17 +08:00
|
|
|
}
|
|
|
|
|
|
2018-02-01 11:23:24 +08:00
|
|
|
/* Compute the end-of-grace-period value for the specified sequence number. */
|
|
|
|
|
static inline unsigned long rcu_seq_endval(unsigned long *sp)
|
|
|
|
|
{
|
|
|
|
|
return (*sp | RCU_SEQ_STATE_MASK) + 1;
|
|
|
|
|
}
|
|
|
|
|
|
2017-02-21 06:57:17 +08:00
|
|
|
/* Adjust sequence number for end of update-side operation. */
|
|
|
|
|
static inline void rcu_seq_end(unsigned long *sp)
|
|
|
|
|
{
|
|
|
|
|
smp_mb(); /* Ensure update-side operation before counter increment. */
|
2017-03-21 22:28:14 +08:00
|
|
|
WARN_ON_ONCE(!rcu_seq_state(*sp));
|
2018-02-01 11:23:24 +08:00
|
|
|
WRITE_ONCE(*sp, rcu_seq_endval(sp));
|
2017-02-21 06:57:17 +08:00
|
|
|
}
|
|
|
|
|
|
2018-05-23 14:38:13 +08:00
|
|
|
/*
|
|
|
|
|
* rcu_seq_snap - Take a snapshot of the update side's sequence number.
|
|
|
|
|
*
|
|
|
|
|
* This function returns the earliest value of the grace-period sequence number
|
|
|
|
|
* that will indicate that a full grace period has elapsed since the current
|
|
|
|
|
* time. Once the grace-period sequence number has reached this value, it will
|
|
|
|
|
* be safe to invoke all callbacks that have been registered prior to the
|
|
|
|
|
* current time. This value is the current grace-period number plus two to the
|
|
|
|
|
* power of the number of low-order bits reserved for state, then rounded up to
|
|
|
|
|
* the next value in which the state bits are all zero.
|
|
|
|
|
*/
|
2017-02-21 06:57:17 +08:00
|
|
|
static inline unsigned long rcu_seq_snap(unsigned long *sp)
|
|
|
|
|
{
|
|
|
|
|
unsigned long s;
|
|
|
|
|
|
2017-03-21 22:28:14 +08:00
|
|
|
s = (READ_ONCE(*sp) + 2 * RCU_SEQ_STATE_MASK + 1) & ~RCU_SEQ_STATE_MASK;
|
2017-02-21 06:57:17 +08:00
|
|
|
smp_mb(); /* Above access must not bleed into critical section. */
|
|
|
|
|
return s;
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-14 07:48:18 +08:00
|
|
|
/* Return the current value the update side's sequence number, no ordering. */
|
|
|
|
|
static inline unsigned long rcu_seq_current(unsigned long *sp)
|
|
|
|
|
{
|
|
|
|
|
return READ_ONCE(*sp);
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-16 02:53:41 +08:00
|
|
|
/*
|
|
|
|
|
* Given a snapshot from rcu_seq_snap(), determine whether or not the
|
|
|
|
|
* corresponding update-side operation has started.
|
|
|
|
|
*/
|
|
|
|
|
static inline bool rcu_seq_started(unsigned long *sp, unsigned long s)
|
|
|
|
|
{
|
|
|
|
|
return ULONG_CMP_LT((s - 1) & ~RCU_SEQ_STATE_MASK, READ_ONCE(*sp));
|
|
|
|
|
}
|
|
|
|
|
|
2017-02-21 06:57:17 +08:00
|
|
|
/*
|
|
|
|
|
* Given a snapshot from rcu_seq_snap(), determine whether or not a
|
|
|
|
|
* full update-side operation has occurred.
|
|
|
|
|
*/
|
|
|
|
|
static inline bool rcu_seq_done(unsigned long *sp, unsigned long s)
|
|
|
|
|
{
|
|
|
|
|
return ULONG_CMP_GE(READ_ONCE(*sp), s);
|
|
|
|
|
}
|
|
|
|
|
|
2022-03-22 09:41:46 +08:00
|
|
|
/*
|
|
|
|
|
* Given a snapshot from rcu_seq_snap(), determine whether or not a
|
|
|
|
|
* full update-side operation has occurred, but do not allow the
|
|
|
|
|
* (ULONG_MAX / 2) safety-factor/guard-band.
|
|
|
|
|
*/
|
|
|
|
|
static inline bool rcu_seq_done_exact(unsigned long *sp, unsigned long s)
|
|
|
|
|
{
|
|
|
|
|
unsigned long cur_s = READ_ONCE(*sp);
|
|
|
|
|
|
|
|
|
|
return ULONG_CMP_GE(cur_s, s) || ULONG_CMP_LT(cur_s, s - (2 * RCU_SEQ_STATE_MASK + 1));
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-28 07:01:46 +08:00
|
|
|
/*
|
|
|
|
|
* Has a grace period completed since the time the old gp_seq was collected?
|
|
|
|
|
*/
|
|
|
|
|
static inline bool rcu_seq_completed_gp(unsigned long old, unsigned long new)
|
|
|
|
|
{
|
|
|
|
|
return ULONG_CMP_LT(old, new & ~RCU_SEQ_STATE_MASK);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Has a grace period started since the time the old gp_seq was collected?
|
|
|
|
|
*/
|
|
|
|
|
static inline bool rcu_seq_new_gp(unsigned long old, unsigned long new)
|
|
|
|
|
{
|
|
|
|
|
return ULONG_CMP_LT((old + RCU_SEQ_STATE_MASK) & ~RCU_SEQ_STATE_MASK,
|
|
|
|
|
new);
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-16 06:24:41 +08:00
|
|
|
/*
|
|
|
|
|
* Roughly how many full grace periods have elapsed between the collection
|
|
|
|
|
* of the two specified grace periods?
|
|
|
|
|
*/
|
|
|
|
|
static inline unsigned long rcu_seq_diff(unsigned long new, unsigned long old)
|
|
|
|
|
{
|
2018-06-09 16:22:20 +08:00
|
|
|
unsigned long rnd_diff;
|
|
|
|
|
|
|
|
|
|
if (old == new)
|
|
|
|
|
return 0;
|
|
|
|
|
/*
|
|
|
|
|
* Compute the number of grace periods (still shifted up), plus
|
|
|
|
|
* one if either of new and old is not an exact grace period.
|
|
|
|
|
*/
|
|
|
|
|
rnd_diff = (new & ~RCU_SEQ_STATE_MASK) -
|
|
|
|
|
((old + RCU_SEQ_STATE_MASK) & ~RCU_SEQ_STATE_MASK) +
|
|
|
|
|
((new & RCU_SEQ_STATE_MASK) || (old & RCU_SEQ_STATE_MASK));
|
|
|
|
|
if (ULONG_CMP_GE(RCU_SEQ_STATE_MASK, rnd_diff))
|
|
|
|
|
return 1; /* Definitely no grace period has elapsed. */
|
|
|
|
|
return ((rnd_diff - RCU_SEQ_STATE_MASK - 1) >> RCU_SEQ_CTR_SHIFT) + 2;
|
2018-05-16 06:24:41 +08:00
|
|
|
}
|
|
|
|
|
|
2011-06-18 06:53:19 +08:00
|
|
|
/*
|
|
|
|
|
* debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally
|
2018-07-08 09:12:26 +08:00
|
|
|
* by call_rcu() and rcu callback execution, and are therefore not part
|
|
|
|
|
* of the RCU API. These are in rcupdate.h because they are used by all
|
|
|
|
|
* RCU implementations.
|
2011-06-18 06:53:19 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
|
|
|
|
|
# define STATE_RCU_HEAD_READY 0
|
|
|
|
|
# define STATE_RCU_HEAD_QUEUED 1
|
|
|
|
|
|
2020-08-15 08:40:27 +08:00
|
|
|
extern const struct debug_obj_descr rcuhead_debug_descr;
|
2011-06-18 06:53:19 +08:00
|
|
|
|
2013-04-24 04:20:57 +08:00
|
|
|
static inline int debug_rcu_head_queue(struct rcu_head *head)
|
2011-06-18 06:53:19 +08:00
|
|
|
{
|
2013-04-24 04:20:57 +08:00
|
|
|
int r1;
|
|
|
|
|
|
|
|
|
|
r1 = debug_object_activate(head, &rcuhead_debug_descr);
|
2011-06-18 06:53:19 +08:00
|
|
|
debug_object_active_state(head, &rcuhead_debug_descr,
|
|
|
|
|
STATE_RCU_HEAD_READY,
|
|
|
|
|
STATE_RCU_HEAD_QUEUED);
|
2013-04-24 04:20:57 +08:00
|
|
|
return r1;
|
2011-06-18 06:53:19 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void debug_rcu_head_unqueue(struct rcu_head *head)
|
|
|
|
|
{
|
|
|
|
|
debug_object_active_state(head, &rcuhead_debug_descr,
|
|
|
|
|
STATE_RCU_HEAD_QUEUED,
|
|
|
|
|
STATE_RCU_HEAD_READY);
|
|
|
|
|
debug_object_deactivate(head, &rcuhead_debug_descr);
|
|
|
|
|
}
|
|
|
|
|
#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
|
2013-04-24 04:20:57 +08:00
|
|
|
static inline int debug_rcu_head_queue(struct rcu_head *head)
|
2011-06-18 06:53:19 +08:00
|
|
|
{
|
2013-04-24 04:20:57 +08:00
|
|
|
return 0;
|
2011-06-18 06:53:19 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void debug_rcu_head_unqueue(struct rcu_head *head)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
|
|
|
|
|
|
2019-12-06 03:29:01 +08:00
|
|
|
extern int rcu_cpu_stall_suppress_at_boot;
|
|
|
|
|
|
|
|
|
|
static inline bool rcu_stall_is_suppressed_at_boot(void)
|
|
|
|
|
{
|
|
|
|
|
return rcu_cpu_stall_suppress_at_boot && !rcu_inkernel_boot_has_ended();
|
|
|
|
|
}
|
|
|
|
|
|
2012-10-20 03:49:17 +08:00
|
|
|
#ifdef CONFIG_RCU_STALL_COMMON
|
|
|
|
|
|
2019-06-14 06:30:49 +08:00
|
|
|
extern int rcu_cpu_stall_ftrace_dump;
|
2012-10-20 03:49:17 +08:00
|
|
|
extern int rcu_cpu_stall_suppress;
|
2019-01-12 08:10:57 +08:00
|
|
|
extern int rcu_cpu_stall_timeout;
|
2022-02-16 21:52:09 +08:00
|
|
|
extern int rcu_exp_cpu_stall_timeout;
|
2022-11-19 17:25:06 +08:00
|
|
|
extern int rcu_cpu_stall_cputime;
|
2022-12-20 10:02:20 +08:00
|
|
|
extern bool rcu_exp_stall_task_details __read_mostly;
|
2012-10-20 03:49:17 +08:00
|
|
|
int rcu_jiffies_till_stall_check(void);
|
2022-02-16 21:52:09 +08:00
|
|
|
int rcu_exp_jiffies_till_stall_check(void);
|
2012-10-20 03:49:17 +08:00
|
|
|
|
2019-12-06 03:29:01 +08:00
|
|
|
static inline bool rcu_stall_is_suppressed(void)
|
|
|
|
|
{
|
|
|
|
|
return rcu_stall_is_suppressed_at_boot() || rcu_cpu_stall_suppress;
|
|
|
|
|
}
|
|
|
|
|
|
2017-09-02 05:40:54 +08:00
|
|
|
#define rcu_ftrace_dump_stall_suppress() \
|
|
|
|
|
do { \
|
|
|
|
|
if (!rcu_cpu_stall_suppress) \
|
|
|
|
|
rcu_cpu_stall_suppress = 3; \
|
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
|
|
#define rcu_ftrace_dump_stall_unsuppress() \
|
|
|
|
|
do { \
|
|
|
|
|
if (rcu_cpu_stall_suppress == 3) \
|
|
|
|
|
rcu_cpu_stall_suppress = 0; \
|
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
|
|
#else /* #endif #ifdef CONFIG_RCU_STALL_COMMON */
|
2019-12-06 03:29:01 +08:00
|
|
|
|
|
|
|
|
static inline bool rcu_stall_is_suppressed(void)
|
|
|
|
|
{
|
|
|
|
|
return rcu_stall_is_suppressed_at_boot();
|
|
|
|
|
}
|
2017-09-02 05:40:54 +08:00
|
|
|
#define rcu_ftrace_dump_stall_suppress()
|
|
|
|
|
#define rcu_ftrace_dump_stall_unsuppress()
|
2012-10-20 03:49:17 +08:00
|
|
|
#endif /* #ifdef CONFIG_RCU_STALL_COMMON */
|
|
|
|
|
|
2013-08-18 09:08:37 +08:00
|
|
|
/*
|
|
|
|
|
* Strings used in tracepoints need to be exported via the
|
|
|
|
|
* tracing system such that tools like perf and trace-cmd can
|
|
|
|
|
* translate the string address pointers to actual text.
|
|
|
|
|
*/
|
|
|
|
|
#define TPS(x) tracepoint_string(x)
|
|
|
|
|
|
2017-05-04 03:28:59 +08:00
|
|
|
/*
|
|
|
|
|
* Dump the ftrace buffer, but only one time per callsite per boot.
|
|
|
|
|
*/
|
|
|
|
|
#define rcu_ftrace_dump(oops_dump_mode) \
|
|
|
|
|
do { \
|
|
|
|
|
static atomic_t ___rfd_beenhere = ATOMIC_INIT(0); \
|
|
|
|
|
\
|
|
|
|
|
if (!atomic_read(&___rfd_beenhere) && \
|
2017-09-01 07:47:08 +08:00
|
|
|
!atomic_xchg(&___rfd_beenhere, 1)) { \
|
|
|
|
|
tracing_off(); \
|
2017-09-02 05:40:54 +08:00
|
|
|
rcu_ftrace_dump_stall_suppress(); \
|
2017-05-04 03:28:59 +08:00
|
|
|
ftrace_dump(oops_dump_mode); \
|
2017-09-02 05:40:54 +08:00
|
|
|
rcu_ftrace_dump_stall_unsuppress(); \
|
2017-09-01 07:47:08 +08:00
|
|
|
} \
|
2017-05-04 03:28:59 +08:00
|
|
|
} while (0)
|
|
|
|
|
|
2014-09-19 23:32:29 +08:00
|
|
|
void rcu_early_boot_tests(void);
|
rcu: Narrow early boot window of illegal synchronous grace periods
The current preemptible RCU implementation goes through three phases
during bootup. In the first phase, there is only one CPU that is running
with preemption disabled, so that a no-op is a synchronous grace period.
In the second mid-boot phase, the scheduler is running, but RCU has
not yet gotten its kthreads spawned (and, for expedited grace periods,
workqueues are not yet running. During this time, any attempt to do
a synchronous grace period will hang the system (or complain bitterly,
depending). In the third and final phase, RCU is fully operational and
everything works normally.
This has been OK for some time, but there has recently been some
synchronous grace periods showing up during the second mid-boot phase.
This code worked "by accident" for awhile, but started failing as soon
as expedited RCU grace periods switched over to workqueues in commit
8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue").
Note that the code was buggy even before this commit, as it was subject
to failure on real-time systems that forced all expedited grace periods
to run as normal grace periods (for example, using the rcu_normal ksysfs
parameter). The callchain from the failure case is as follows:
early_amd_iommu_init()
|-> acpi_put_table(ivrs_base);
|-> acpi_tb_put_table(table_desc);
|-> acpi_tb_invalidate_table(table_desc);
|-> acpi_tb_release_table(...)
|-> acpi_os_unmap_memory
|-> acpi_os_unmap_iomem
|-> acpi_os_map_cleanup
|-> synchronize_rcu_expedited
The kernel showing this callchain was built with CONFIG_PREEMPT_RCU=y,
which caused the code to try using workqueues before they were
initialized, which did not go well.
This commit therefore reworks RCU to permit synchronous grace periods
to proceed during this mid-boot phase. This commit is therefore a
fix to a regression introduced in v4.9, and is therefore being put
forward post-merge-window in v4.10.
This commit sets a flag from the existing rcu_scheduler_starting()
function which causes all synchronous grace periods to take the expedited
path. The expedited path now checks this flag, using the requesting task
to drive the expedited grace period forward during the mid-boot phase.
Finally, this flag is updated by a core_initcall() function named
rcu_exp_runtime_mode(), which causes the runtime codepaths to be used.
Note that this arrangement assumes that tasks are not sent POSIX signals
(or anything similar) from the time that the first task is spawned
through core_initcall() time.
Fixes: 8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue")
Reported-by: "Zheng, Lv" <lv.zheng@intel.com>
Reported-by: Borislav Petkov <bp@alien8.de>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: Stan Kain <stan.kain@gmail.com>
Tested-by: Ivan <waffolz@hotmail.com>
Tested-by: Emanuel Castelo <emanuel.castelo@gmail.com>
Tested-by: Bruno Pesavento <bpesavento@infinito.it>
Tested-by: Borislav Petkov <bp@suse.de>
Tested-by: Frederic Bezies <fredbezies@gmail.com>
Cc: <stable@vger.kernel.org> # 4.9.0-
2017-01-10 18:28:26 +08:00
|
|
|
void rcu_test_sync_prims(void);
|
2014-09-19 23:32:29 +08:00
|
|
|
|
2014-12-09 17:53:34 +08:00
|
|
|
/*
|
|
|
|
|
* This function really isn't for public consumption, but RCU is special in
|
|
|
|
|
* that context switches can allow the state machine to make progress.
|
|
|
|
|
*/
|
|
|
|
|
extern void resched_cpu(int cpu);
|
|
|
|
|
|
2022-11-23 05:53:57 +08:00
|
|
|
#if !defined(CONFIG_TINY_RCU)
|
2017-03-15 05:29:53 +08:00
|
|
|
|
|
|
|
|
#include <linux/rcu_node_tree.h>
|
|
|
|
|
|
|
|
|
|
extern int rcu_num_lvls;
|
2017-03-16 04:11:11 +08:00
|
|
|
extern int num_rcu_lvl[];
|
2017-03-15 05:29:53 +08:00
|
|
|
extern int rcu_num_nodes;
|
|
|
|
|
static bool rcu_fanout_exact;
|
|
|
|
|
static int rcu_fanout_leaf;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Compute the per-level fanout, either using the exact fanout specified
|
|
|
|
|
* or balancing the tree, depending on the rcu_fanout_exact boot parameter.
|
|
|
|
|
*/
|
|
|
|
|
static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt)
|
|
|
|
|
{
|
|
|
|
|
int i;
|
|
|
|
|
|
2019-09-19 01:10:31 +08:00
|
|
|
for (i = 0; i < RCU_NUM_LVLS; i++)
|
|
|
|
|
levelspread[i] = INT_MIN;
|
2017-03-15 05:29:53 +08:00
|
|
|
if (rcu_fanout_exact) {
|
|
|
|
|
levelspread[rcu_num_lvls - 1] = rcu_fanout_leaf;
|
|
|
|
|
for (i = rcu_num_lvls - 2; i >= 0; i--)
|
|
|
|
|
levelspread[i] = RCU_FANOUT;
|
|
|
|
|
} else {
|
|
|
|
|
int ccur;
|
|
|
|
|
int cprv;
|
|
|
|
|
|
|
|
|
|
cprv = nr_cpu_ids;
|
|
|
|
|
for (i = rcu_num_lvls - 1; i >= 0; i--) {
|
|
|
|
|
ccur = levelcnt[i];
|
|
|
|
|
levelspread[i] = (cprv + ccur - 1) / ccur;
|
|
|
|
|
cprv = ccur;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
srcu: Fix broken node geometry after early ssp init
An srcu_struct structure that is initialized before rcu_init_geometry()
will have its srcu_node hierarchy based on CONFIG_NR_CPUS. Once
rcu_init_geometry() is called, this hierarchy is compressed as needed
for the actual maximum number of CPUs for this system.
Later on, that srcu_struct structure is confused, sometimes referring
to its initial CONFIG_NR_CPUS-based hierarchy, and sometimes instead
to the new num_possible_cpus() hierarchy. For example, each of its
->mynode fields continues to reference the original leaf rcu_node
structures, some of which might no longer exist. On the other hand,
srcu_for_each_node_breadth_first() traverses to the new node hierarchy.
There are at least two bad possible outcomes to this:
1) a) A callback enqueued early on an srcu_data structure (call it
*sdp) is recorded pending on sdp->mynode->srcu_data_have_cbs in
srcu_funnel_gp_start() with sdp->mynode pointing to a deep leaf
(say 3 levels).
b) The grace period ends after rcu_init_geometry() shrinks the
nodes level to a single one. srcu_gp_end() walks through the new
srcu_node hierarchy without ever reaching the old leaves so the
callback is never executed.
This is easily reproduced on an 8 CPUs machine with CONFIG_NR_CPUS >= 32
and "rcupdate.rcu_self_test=1". The srcu_barrier() after early tests
verification never completes and the boot hangs:
[ 5413.141029] INFO: task swapper/0:1 blocked for more than 4915 seconds.
[ 5413.147564] Not tainted 5.12.0-rc4+ #28
[ 5413.151927] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 5413.159753] task:swapper/0 state:D stack: 0 pid: 1 ppid: 0 flags:0x00004000
[ 5413.168099] Call Trace:
[ 5413.170555] __schedule+0x36c/0x930
[ 5413.174057] ? wait_for_completion+0x88/0x110
[ 5413.178423] schedule+0x46/0xf0
[ 5413.181575] schedule_timeout+0x284/0x380
[ 5413.185591] ? wait_for_completion+0x88/0x110
[ 5413.189957] ? mark_held_locks+0x61/0x80
[ 5413.193882] ? mark_held_locks+0x61/0x80
[ 5413.197809] ? _raw_spin_unlock_irq+0x24/0x50
[ 5413.202173] ? wait_for_completion+0x88/0x110
[ 5413.206535] wait_for_completion+0xb4/0x110
[ 5413.210724] ? srcu_torture_stats_print+0x110/0x110
[ 5413.215610] srcu_barrier+0x187/0x200
[ 5413.219277] ? rcu_tasks_verify_self_tests+0x50/0x50
[ 5413.224244] ? rdinit_setup+0x2b/0x2b
[ 5413.227907] rcu_verify_early_boot_tests+0x2d/0x40
[ 5413.232700] do_one_initcall+0x63/0x310
[ 5413.236541] ? rdinit_setup+0x2b/0x2b
[ 5413.240207] ? rcu_read_lock_sched_held+0x52/0x80
[ 5413.244912] kernel_init_freeable+0x253/0x28f
[ 5413.249273] ? rest_init+0x250/0x250
[ 5413.252846] kernel_init+0xa/0x110
[ 5413.256257] ret_from_fork+0x22/0x30
2) An srcu_struct structure that is initialized before rcu_init_geometry()
and used afterward will always have stale rdp->mynode references,
resulting in callbacks to be missed in srcu_gp_end(), just like in
the previous scenario.
This commit therefore causes init_srcu_struct_nodes to initialize the
geometry, if needed. This ensures that the srcu_node hierarchy is
properly built and distributed from the get-go.
Suggested-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Lai Jiangshan <jiangshanlai@gmail.com>
Cc: Neeraj Upadhyay <neeraju@codeaurora.org>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Joel Fernandes <joel@joelfernandes.org>
Cc: Uladzislau Rezki <urezki@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-04-17 21:16:49 +08:00
|
|
|
extern void rcu_init_geometry(void);
|
|
|
|
|
|
2018-07-08 09:12:26 +08:00
|
|
|
/* Returns a pointer to the first leaf rcu_node structure. */
|
2018-07-05 05:33:59 +08:00
|
|
|
#define rcu_first_leaf_node() (rcu_state.level[rcu_num_lvls - 1])
|
2018-04-14 08:11:44 +08:00
|
|
|
|
|
|
|
|
/* Is this rcu_node a leaf? */
|
|
|
|
|
#define rcu_is_leaf_node(rnp) ((rnp)->level == rcu_num_lvls - 1)
|
|
|
|
|
|
2018-04-25 02:03:39 +08:00
|
|
|
/* Is this rcu_node the last leaf? */
|
2018-07-05 05:33:59 +08:00
|
|
|
#define rcu_is_last_leaf_node(rnp) ((rnp) == &rcu_state.node[rcu_num_nodes - 1])
|
2018-04-25 02:03:39 +08:00
|
|
|
|
2017-03-16 04:07:53 +08:00
|
|
|
/*
|
2018-07-05 05:33:59 +08:00
|
|
|
* Do a full breadth-first scan of the {s,}rcu_node structures for the
|
2018-07-08 09:12:26 +08:00
|
|
|
* specified state structure (for SRCU) or the only rcu_state structure
|
|
|
|
|
* (for RCU).
|
2017-03-16 04:07:53 +08:00
|
|
|
*/
|
2018-07-05 05:33:59 +08:00
|
|
|
#define srcu_for_each_node_breadth_first(sp, rnp) \
|
|
|
|
|
for ((rnp) = &(sp)->node[0]; \
|
|
|
|
|
(rnp) < &(sp)->node[rcu_num_nodes]; (rnp)++)
|
|
|
|
|
#define rcu_for_each_node_breadth_first(rnp) \
|
|
|
|
|
srcu_for_each_node_breadth_first(&rcu_state, rnp)
|
2017-03-16 04:07:53 +08:00
|
|
|
|
|
|
|
|
/*
|
2018-07-08 09:12:26 +08:00
|
|
|
* Scan the leaves of the rcu_node hierarchy for the rcu_state structure.
|
|
|
|
|
* Note that if there is a singleton rcu_node tree with but one rcu_node
|
|
|
|
|
* structure, this loop -will- visit the rcu_node structure. It is still
|
|
|
|
|
* a leaf node, even if it is also the root node.
|
2017-03-16 04:07:53 +08:00
|
|
|
*/
|
2018-07-05 05:33:59 +08:00
|
|
|
#define rcu_for_each_leaf_node(rnp) \
|
|
|
|
|
for ((rnp) = rcu_first_leaf_node(); \
|
|
|
|
|
(rnp) < &rcu_state.node[rcu_num_nodes]; (rnp)++)
|
2017-03-16 04:07:53 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Iterate over all possible CPUs in a leaf RCU node.
|
|
|
|
|
*/
|
|
|
|
|
#define for_each_leaf_node_possible_cpu(rnp, cpu) \
|
2019-12-16 03:38:57 +08:00
|
|
|
for (WARN_ON_ONCE(!rcu_is_leaf_node(rnp)), \
|
|
|
|
|
(cpu) = cpumask_next((rnp)->grplo - 1, cpu_possible_mask); \
|
2018-02-01 12:24:15 +08:00
|
|
|
(cpu) <= rnp->grphi; \
|
|
|
|
|
(cpu) = cpumask_next((cpu), cpu_possible_mask))
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Iterate over all CPUs in a leaf RCU node's specified mask.
|
|
|
|
|
*/
|
|
|
|
|
#define rcu_find_next_bit(rnp, cpu, mask) \
|
|
|
|
|
((rnp)->grplo + find_next_bit(&(mask), BITS_PER_LONG, (cpu)))
|
|
|
|
|
#define for_each_leaf_node_cpu_mask(rnp, cpu, mask) \
|
2019-12-16 03:38:57 +08:00
|
|
|
for (WARN_ON_ONCE(!rcu_is_leaf_node(rnp)), \
|
|
|
|
|
(cpu) = rcu_find_next_bit((rnp), 0, (mask)); \
|
2018-02-01 12:24:15 +08:00
|
|
|
(cpu) <= rnp->grphi; \
|
|
|
|
|
(cpu) = rcu_find_next_bit((rnp), (cpu) + 1 - (rnp->grplo), (mask)))
|
2017-03-16 04:07:53 +08:00
|
|
|
|
2022-11-23 05:53:57 +08:00
|
|
|
#endif /* !defined(CONFIG_TINY_RCU) */
|
|
|
|
|
|
|
|
|
|
#if !defined(CONFIG_TINY_RCU) || defined(CONFIG_TASKS_RCU_GENERIC)
|
|
|
|
|
|
2017-05-10 04:28:51 +08:00
|
|
|
/*
|
|
|
|
|
* Wrappers for the rcu_node::lock acquire and release.
|
|
|
|
|
*
|
|
|
|
|
* Because the rcu_nodes form a tree, the tree traversal locking will observe
|
|
|
|
|
* different lock values, this in turn means that an UNLOCK of one level
|
|
|
|
|
* followed by a LOCK of another level does not imply a full memory barrier;
|
|
|
|
|
* and most importantly transitivity is lost.
|
|
|
|
|
*
|
|
|
|
|
* In order to restore full ordering between tree levels, augment the regular
|
|
|
|
|
* lock acquire functions with smp_mb__after_unlock_lock().
|
|
|
|
|
*
|
|
|
|
|
* As ->lock of struct rcu_node is a __private field, therefore one should use
|
|
|
|
|
* these wrappers rather than directly call raw_spin_{lock,unlock}* on ->lock.
|
|
|
|
|
*/
|
|
|
|
|
#define raw_spin_lock_rcu_node(p) \
|
|
|
|
|
do { \
|
|
|
|
|
raw_spin_lock(&ACCESS_PRIVATE(p, lock)); \
|
|
|
|
|
smp_mb__after_unlock_lock(); \
|
|
|
|
|
} while (0)
|
|
|
|
|
|
2020-11-20 05:30:33 +08:00
|
|
|
#define raw_spin_unlock_rcu_node(p) \
|
|
|
|
|
do { \
|
|
|
|
|
lockdep_assert_irqs_disabled(); \
|
|
|
|
|
raw_spin_unlock(&ACCESS_PRIVATE(p, lock)); \
|
|
|
|
|
} while (0)
|
2017-05-10 04:28:51 +08:00
|
|
|
|
|
|
|
|
#define raw_spin_lock_irq_rcu_node(p) \
|
|
|
|
|
do { \
|
|
|
|
|
raw_spin_lock_irq(&ACCESS_PRIVATE(p, lock)); \
|
|
|
|
|
smp_mb__after_unlock_lock(); \
|
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
|
|
#define raw_spin_unlock_irq_rcu_node(p) \
|
2020-11-20 05:30:33 +08:00
|
|
|
do { \
|
|
|
|
|
lockdep_assert_irqs_disabled(); \
|
|
|
|
|
raw_spin_unlock_irq(&ACCESS_PRIVATE(p, lock)); \
|
|
|
|
|
} while (0)
|
2017-05-10 04:28:51 +08:00
|
|
|
|
2017-05-12 06:33:23 +08:00
|
|
|
#define raw_spin_lock_irqsave_rcu_node(p, flags) \
|
2017-05-10 04:28:51 +08:00
|
|
|
do { \
|
2017-05-12 06:33:23 +08:00
|
|
|
raw_spin_lock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \
|
2017-05-10 04:28:51 +08:00
|
|
|
smp_mb__after_unlock_lock(); \
|
|
|
|
|
} while (0)
|
|
|
|
|
|
2017-05-12 06:33:23 +08:00
|
|
|
#define raw_spin_unlock_irqrestore_rcu_node(p, flags) \
|
2020-11-20 05:30:33 +08:00
|
|
|
do { \
|
|
|
|
|
lockdep_assert_irqs_disabled(); \
|
|
|
|
|
raw_spin_unlock_irqrestore(&ACCESS_PRIVATE(p, lock), flags); \
|
|
|
|
|
} while (0)
|
2017-05-10 04:28:51 +08:00
|
|
|
|
|
|
|
|
#define raw_spin_trylock_rcu_node(p) \
|
|
|
|
|
({ \
|
|
|
|
|
bool ___locked = raw_spin_trylock(&ACCESS_PRIVATE(p, lock)); \
|
|
|
|
|
\
|
|
|
|
|
if (___locked) \
|
|
|
|
|
smp_mb__after_unlock_lock(); \
|
|
|
|
|
___locked; \
|
|
|
|
|
})
|
|
|
|
|
|
2018-01-17 22:24:30 +08:00
|
|
|
#define raw_lockdep_assert_held_rcu_node(p) \
|
|
|
|
|
lockdep_assert_held(&ACCESS_PRIVATE(p, lock))
|
|
|
|
|
|
2022-11-23 05:53:57 +08:00
|
|
|
#endif // #if !defined(CONFIG_TINY_RCU) || defined(CONFIG_TASKS_RCU_GENERIC)
|
2017-03-15 05:29:53 +08:00
|
|
|
|
2017-05-04 00:51:55 +08:00
|
|
|
#ifdef CONFIG_TINY_RCU
|
|
|
|
|
/* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */
|
2017-06-13 07:44:19 +08:00
|
|
|
static inline bool rcu_gp_is_normal(void) { return true; }
|
|
|
|
|
static inline bool rcu_gp_is_expedited(void) { return false; }
|
2023-01-12 08:52:22 +08:00
|
|
|
static inline bool rcu_async_should_hurry(void) { return false; }
|
2017-06-13 07:44:19 +08:00
|
|
|
static inline void rcu_expedite_gp(void) { }
|
|
|
|
|
static inline void rcu_unexpedite_gp(void) { }
|
2023-01-12 08:52:22 +08:00
|
|
|
static inline void rcu_async_hurry(void) { }
|
|
|
|
|
static inline void rcu_async_relax(void) { }
|
2018-01-12 04:58:53 +08:00
|
|
|
static inline void rcu_request_urgent_qs_task(struct task_struct *t) { }
|
2017-05-04 00:51:55 +08:00
|
|
|
#else /* #ifdef CONFIG_TINY_RCU */
|
|
|
|
|
bool rcu_gp_is_normal(void); /* Internal RCU use. */
|
|
|
|
|
bool rcu_gp_is_expedited(void); /* Internal RCU use. */
|
2023-01-12 08:52:22 +08:00
|
|
|
bool rcu_async_should_hurry(void); /* Internal RCU use. */
|
2017-05-04 00:51:55 +08:00
|
|
|
void rcu_expedite_gp(void);
|
|
|
|
|
void rcu_unexpedite_gp(void);
|
2023-01-12 08:52:22 +08:00
|
|
|
void rcu_async_hurry(void);
|
|
|
|
|
void rcu_async_relax(void);
|
2017-05-04 00:51:55 +08:00
|
|
|
void rcupdate_announce_bootup_oddness(void);
|
2021-04-21 01:58:07 +08:00
|
|
|
#ifdef CONFIG_TASKS_RCU_GENERIC
|
2020-03-17 02:01:55 +08:00
|
|
|
void show_rcu_tasks_gp_kthreads(void);
|
2021-04-21 01:58:07 +08:00
|
|
|
#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
|
|
|
|
|
static inline void show_rcu_tasks_gp_kthreads(void) {}
|
|
|
|
|
#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
|
2018-01-12 04:58:53 +08:00
|
|
|
void rcu_request_urgent_qs_task(struct task_struct *t);
|
2017-05-04 00:51:55 +08:00
|
|
|
#endif /* #else #ifdef CONFIG_TINY_RCU */
|
|
|
|
|
|
2017-05-04 02:13:24 +08:00
|
|
|
#define RCU_SCHEDULER_INACTIVE 0
|
|
|
|
|
#define RCU_SCHEDULER_INIT 1
|
|
|
|
|
#define RCU_SCHEDULER_RUNNING 2
|
|
|
|
|
|
2017-05-04 01:22:57 +08:00
|
|
|
enum rcutorture_type {
|
|
|
|
|
RCU_FLAVOR,
|
|
|
|
|
RCU_TASKS_FLAVOR,
|
2020-03-04 07:02:50 +08:00
|
|
|
RCU_TASKS_RUDE_FLAVOR,
|
2020-03-11 01:32:30 +08:00
|
|
|
RCU_TASKS_TRACING_FLAVOR,
|
rcutorture: Add trivial RCU implementation
I have been showing off a trivial RCU implementation for non-preemptive
environments for some time now:
#define rcu_read_lock()
#define rcu_read_unlock()
#define rcu_dereference(p) READ_ONCE(p)
#define rcu_assign_pointer(p, v) smp_store_release(&(p), (v))
void synchronize_rcu(void)
{
int cpu;
for_each_online_cpu(cpu)
sched_setaffinity(current->pid, cpumask_of(cpu));
}
Trivial or not, as the old saying goes, "if it ain't tested, it don't
work!". This commit therefore adds a "trivial" flavor to rcutorture
and a corresponding TRIVIAL test scenario. This variant does not handle
CPU hotplug, which is unconditionally enabled on x86 for post-v5.1-rc3
kernels, which is why the TRIVIAL.boot says "rcutorture.onoff_interval=0".
This commit actually does handle CONFIG_PREEMPT=y kernels, but only
because it turns back the Linux-kernel clock in order to provide these
alternative definitions (or the moral equivalent thereof):
#define rcu_read_lock() preempt_disable()
#define rcu_read_unlock() preempt_enable()
In CONFIG_PREEMPT=n kernels without debugging, these are equivalent to
empty macros give or take a compiler barrier. However, the have been
successfully tested with actual empty macros as well.
Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
[ paulmck: Fix symbol issue reported by kbuild test robot <lkp@intel.com>. ]
[ paulmck: Work around sched_setaffinity() issue noted by Andrea Parri. ]
[ paulmck: Add rcutorture.shuffle_interval=0 to TRIVIAL.boot to fix
interaction with shuffler task noted by Peter Zijlstra. ]
Tested-by: Andrea Parri <andrea.parri@amarulasolutions.com>
2019-04-19 22:38:27 +08:00
|
|
|
RCU_TRIVIAL_FLAVOR,
|
2017-05-04 01:22:57 +08:00
|
|
|
SRCU_FLAVOR,
|
|
|
|
|
INVALID_RCU_FLAVOR
|
|
|
|
|
};
|
|
|
|
|
|
2022-10-17 00:22:54 +08:00
|
|
|
#if defined(CONFIG_RCU_LAZY)
|
|
|
|
|
unsigned long rcu_lazy_get_jiffies_till_flush(void);
|
|
|
|
|
void rcu_lazy_set_jiffies_till_flush(unsigned long j);
|
|
|
|
|
#else
|
|
|
|
|
static inline unsigned long rcu_lazy_get_jiffies_till_flush(void) { return 0; }
|
|
|
|
|
static inline void rcu_lazy_set_jiffies_till_flush(unsigned long j) { }
|
|
|
|
|
#endif
|
|
|
|
|
|
2019-10-15 10:55:57 +08:00
|
|
|
#if defined(CONFIG_TREE_RCU)
|
2017-05-04 01:22:57 +08:00
|
|
|
void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
|
2018-05-01 21:42:51 +08:00
|
|
|
unsigned long *gp_seq);
|
2017-05-04 01:22:57 +08:00
|
|
|
void do_trace_rcu_torture_read(const char *rcutorturename,
|
|
|
|
|
struct rcu_head *rhp,
|
|
|
|
|
unsigned long secs,
|
|
|
|
|
unsigned long c_old,
|
|
|
|
|
unsigned long c);
|
2020-04-02 10:57:52 +08:00
|
|
|
void rcu_gp_set_torture_wait(int duration);
|
2017-05-04 01:22:57 +08:00
|
|
|
#else
|
|
|
|
|
static inline void rcutorture_get_gp_data(enum rcutorture_type test_type,
|
2018-05-01 21:42:51 +08:00
|
|
|
int *flags, unsigned long *gp_seq)
|
2017-05-04 01:22:57 +08:00
|
|
|
{
|
|
|
|
|
*flags = 0;
|
2018-05-01 21:42:51 +08:00
|
|
|
*gp_seq = 0;
|
2017-05-04 01:22:57 +08:00
|
|
|
}
|
|
|
|
|
#ifdef CONFIG_RCU_TRACE
|
|
|
|
|
void do_trace_rcu_torture_read(const char *rcutorturename,
|
|
|
|
|
struct rcu_head *rhp,
|
|
|
|
|
unsigned long secs,
|
|
|
|
|
unsigned long c_old,
|
|
|
|
|
unsigned long c);
|
|
|
|
|
#else
|
|
|
|
|
#define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
|
|
|
|
|
do { } while (0)
|
|
|
|
|
#endif
|
2020-04-02 10:57:52 +08:00
|
|
|
static inline void rcu_gp_set_torture_wait(int duration) { }
|
2017-05-04 01:22:57 +08:00
|
|
|
#endif
|
|
|
|
|
|
rcutorture: Add trivial RCU implementation
I have been showing off a trivial RCU implementation for non-preemptive
environments for some time now:
#define rcu_read_lock()
#define rcu_read_unlock()
#define rcu_dereference(p) READ_ONCE(p)
#define rcu_assign_pointer(p, v) smp_store_release(&(p), (v))
void synchronize_rcu(void)
{
int cpu;
for_each_online_cpu(cpu)
sched_setaffinity(current->pid, cpumask_of(cpu));
}
Trivial or not, as the old saying goes, "if it ain't tested, it don't
work!". This commit therefore adds a "trivial" flavor to rcutorture
and a corresponding TRIVIAL test scenario. This variant does not handle
CPU hotplug, which is unconditionally enabled on x86 for post-v5.1-rc3
kernels, which is why the TRIVIAL.boot says "rcutorture.onoff_interval=0".
This commit actually does handle CONFIG_PREEMPT=y kernels, but only
because it turns back the Linux-kernel clock in order to provide these
alternative definitions (or the moral equivalent thereof):
#define rcu_read_lock() preempt_disable()
#define rcu_read_unlock() preempt_enable()
In CONFIG_PREEMPT=n kernels without debugging, these are equivalent to
empty macros give or take a compiler barrier. However, the have been
successfully tested with actual empty macros as well.
Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
[ paulmck: Fix symbol issue reported by kbuild test robot <lkp@intel.com>. ]
[ paulmck: Work around sched_setaffinity() issue noted by Andrea Parri. ]
[ paulmck: Add rcutorture.shuffle_interval=0 to TRIVIAL.boot to fix
interaction with shuffler task noted by Peter Zijlstra. ]
Tested-by: Andrea Parri <andrea.parri@amarulasolutions.com>
2019-04-19 22:38:27 +08:00
|
|
|
#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST)
|
|
|
|
|
long rcutorture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask);
|
|
|
|
|
#endif
|
|
|
|
|
|
2017-05-04 01:22:57 +08:00
|
|
|
#ifdef CONFIG_TINY_SRCU
|
|
|
|
|
|
|
|
|
|
static inline void srcutorture_get_gp_data(enum rcutorture_type test_type,
|
|
|
|
|
struct srcu_struct *sp, int *flags,
|
2018-05-01 21:42:51 +08:00
|
|
|
unsigned long *gp_seq)
|
2017-05-04 01:22:57 +08:00
|
|
|
{
|
|
|
|
|
if (test_type != SRCU_FLAVOR)
|
|
|
|
|
return;
|
|
|
|
|
*flags = 0;
|
2018-05-01 21:42:51 +08:00
|
|
|
*gp_seq = sp->srcu_idx;
|
2017-05-04 01:22:57 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#elif defined(CONFIG_TREE_SRCU)
|
|
|
|
|
|
|
|
|
|
void srcutorture_get_gp_data(enum rcutorture_type test_type,
|
|
|
|
|
struct srcu_struct *sp, int *flags,
|
2018-05-01 21:42:51 +08:00
|
|
|
unsigned long *gp_seq);
|
2017-05-04 01:22:57 +08:00
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
2017-05-04 04:37:16 +08:00
|
|
|
#ifdef CONFIG_TINY_RCU
|
rcu-tasks: Avoid IPIing userspace/idle tasks if kernel is so built
Systems running CPU-bound real-time task do not want IPIs sent to CPUs
executing nohz_full userspace tasks. Battery-powered systems don't
want IPIs sent to idle CPUs in low-power mode. Unfortunately, RCU tasks
trace can and will send such IPIs in some cases.
Both of these situations occur only when the target CPU is in RCU
dyntick-idle mode, in other words, when RCU is not watching the
target CPU. This suggests that CPUs in dyntick-idle mode should use
memory barriers in outermost invocations of rcu_read_lock_trace()
and rcu_read_unlock_trace(), which would allow the RCU tasks trace
grace period to directly read out the target CPU's read-side state.
One challenge is that RCU tasks trace is not targeting a specific
CPU, but rather a task. And that task could switch from one CPU to
another at any time.
This commit therefore uses try_invoke_on_locked_down_task()
and checks for task_curr() in trc_inspect_reader_notrunning().
When this condition holds, the target task is running and cannot move.
If CONFIG_TASKS_TRACE_RCU_READ_MB=y, the new rcu_dynticks_zero_in_eqs()
function can be used to check if the specified integer (in this case,
t->trc_reader_nesting) is zero while the target CPU remains in that same
dyntick-idle sojourn. If so, the target task is in a quiescent state.
If not, trc_read_check_handler() must indicate failure so that the
grace-period kthread can take appropriate action or retry after an
appropriate delay, as the case may be.
With this change, given CONFIG_TASKS_TRACE_RCU_READ_MB=y, if a given
CPU remains idle or a given task continues executing in nohz_full mode,
the RCU tasks trace grace-period kthread will detect this without the
need to send an IPI.
Suggested-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-20 06:33:12 +08:00
|
|
|
static inline bool rcu_dynticks_zero_in_eqs(int cpu, int *vp) { return false; }
|
2018-04-28 02:39:34 +08:00
|
|
|
static inline unsigned long rcu_get_gp_seq(void) { return 0; }
|
2017-06-13 07:44:19 +08:00
|
|
|
static inline unsigned long rcu_exp_batches_completed(void) { return 0; }
|
|
|
|
|
static inline unsigned long
|
|
|
|
|
srcu_batches_completed(struct srcu_struct *sp) { return 0; }
|
|
|
|
|
static inline void rcu_force_quiescent_state(void) { }
|
2021-04-09 04:01:14 +08:00
|
|
|
static inline bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) { return true; }
|
2017-06-13 07:44:19 +08:00
|
|
|
static inline void show_rcu_gp_kthreads(void) { }
|
2018-06-20 06:14:18 +08:00
|
|
|
static inline int rcu_get_gp_kthreads_prio(void) { return 0; }
|
2018-10-02 08:40:54 +08:00
|
|
|
static inline void rcu_fwd_progress_check(unsigned long j) { }
|
2022-02-05 04:45:18 +08:00
|
|
|
static inline void rcu_gp_slow_register(atomic_t *rgssp) { }
|
|
|
|
|
static inline void rcu_gp_slow_unregister(atomic_t *rgssp) { }
|
2017-05-04 04:37:16 +08:00
|
|
|
#else /* #ifdef CONFIG_TINY_RCU */
|
rcu-tasks: Avoid IPIing userspace/idle tasks if kernel is so built
Systems running CPU-bound real-time task do not want IPIs sent to CPUs
executing nohz_full userspace tasks. Battery-powered systems don't
want IPIs sent to idle CPUs in low-power mode. Unfortunately, RCU tasks
trace can and will send such IPIs in some cases.
Both of these situations occur only when the target CPU is in RCU
dyntick-idle mode, in other words, when RCU is not watching the
target CPU. This suggests that CPUs in dyntick-idle mode should use
memory barriers in outermost invocations of rcu_read_lock_trace()
and rcu_read_unlock_trace(), which would allow the RCU tasks trace
grace period to directly read out the target CPU's read-side state.
One challenge is that RCU tasks trace is not targeting a specific
CPU, but rather a task. And that task could switch from one CPU to
another at any time.
This commit therefore uses try_invoke_on_locked_down_task()
and checks for task_curr() in trc_inspect_reader_notrunning().
When this condition holds, the target task is running and cannot move.
If CONFIG_TASKS_TRACE_RCU_READ_MB=y, the new rcu_dynticks_zero_in_eqs()
function can be used to check if the specified integer (in this case,
t->trc_reader_nesting) is zero while the target CPU remains in that same
dyntick-idle sojourn. If so, the target task is in a quiescent state.
If not, trc_read_check_handler() must indicate failure so that the
grace-period kthread can take appropriate action or retry after an
appropriate delay, as the case may be.
With this change, given CONFIG_TASKS_TRACE_RCU_READ_MB=y, if a given
CPU remains idle or a given task continues executing in nohz_full mode,
the RCU tasks trace grace-period kthread will detect this without the
need to send an IPI.
Suggested-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-20 06:33:12 +08:00
|
|
|
bool rcu_dynticks_zero_in_eqs(int cpu, int *vp);
|
2018-04-28 02:39:34 +08:00
|
|
|
unsigned long rcu_get_gp_seq(void);
|
2017-05-04 04:37:16 +08:00
|
|
|
unsigned long rcu_exp_batches_completed(void);
|
2017-05-05 02:31:04 +08:00
|
|
|
unsigned long srcu_batches_completed(struct srcu_struct *sp);
|
2021-04-09 04:01:14 +08:00
|
|
|
bool rcu_check_boost_fail(unsigned long gp_state, int *cpup);
|
2017-05-04 04:37:16 +08:00
|
|
|
void show_rcu_gp_kthreads(void);
|
2018-06-20 06:14:18 +08:00
|
|
|
int rcu_get_gp_kthreads_prio(void);
|
2018-10-02 08:40:54 +08:00
|
|
|
void rcu_fwd_progress_check(unsigned long j);
|
2017-05-04 04:37:16 +08:00
|
|
|
void rcu_force_quiescent_state(void);
|
2018-01-09 06:35:52 +08:00
|
|
|
extern struct workqueue_struct *rcu_gp_wq;
|
rcu: Move expedited grace period (GP) work to RT kthread_worker
Enabling CONFIG_RCU_BOOST did not reduce RCU expedited grace-period
latency because its workqueues run at SCHED_OTHER, and thus can be
delayed by normal processes. This commit avoids these delays by moving
the expedited GP work items to a real-time-priority kthread_worker.
This option is controlled by CONFIG_RCU_EXP_KTHREAD and disabled by
default on PREEMPT_RT=y kernels which disable expedited grace periods
after boot by unconditionally setting rcupdate.rcu_normal_after_boot=1.
The results were evaluated on arm64 Android devices (6GB ram) running
5.10 kernel, and capturing trace data in critical user-level code.
The table below shows the resulting order-of-magnitude improvements
in synchronize_rcu_expedited() latency:
------------------------------------------------------------------------
| | workqueues | kthread_worker | Diff |
------------------------------------------------------------------------
| Count | 725 | 688 | |
------------------------------------------------------------------------
| Min Duration (ns) | 326 | 447 | 37.12% |
------------------------------------------------------------------------
| Q1 (ns) | 39,428 | 38,971 | -1.16% |
------------------------------------------------------------------------
| Q2 - Median (ns) | 98,225 | 69,743 | -29.00% |
------------------------------------------------------------------------
| Q3 (ns) | 342,122 | 126,638 | -62.98% |
------------------------------------------------------------------------
| Max Duration (ns) | 372,766,967 | 2,329,671 | -99.38% |
------------------------------------------------------------------------
| Avg Duration (ns) | 2,746,353 | 151,242 | -94.49% |
------------------------------------------------------------------------
| Standard Deviation (ns) | 19,327,765 | 294,408 | |
------------------------------------------------------------------------
The below table show the range of maximums/minimums for
synchronize_rcu_expedited() latency from all experiments:
------------------------------------------------------------------------
| | workqueues | kthread_worker | Diff |
------------------------------------------------------------------------
| Total No. of Experiments | 25 | 23 | |
------------------------------------------------------------------------
| Largest Maximum (ns) | 372,766,967 | 2,329,671 | -99.38% |
------------------------------------------------------------------------
| Smallest Maximum (ns) | 38,819 | 86,954 | 124.00% |
------------------------------------------------------------------------
| Range of Maximums (ns) | 372,728,148 | 2,242,717 | |
------------------------------------------------------------------------
| Largest Minimum (ns) | 88,623 | 27,588 | -68.87% |
------------------------------------------------------------------------
| Smallest Minimum (ns) | 326 | 447 | 37.12% |
------------------------------------------------------------------------
| Range of Minimums (ns) | 88,297 | 27,141 | |
------------------------------------------------------------------------
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Reported-by: Tim Murray <timmurray@google.com>
Reported-by: Wei Wang <wvw@google.com>
Tested-by: Kyle Lin <kylelin@google.com>
Tested-by: Chunwei Lu <chunweilu@google.com>
Tested-by: Lulu Wang <luluw@google.com>
Signed-off-by: Kalesh Singh <kaleshsingh@google.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-04-09 08:35:27 +08:00
|
|
|
#ifdef CONFIG_RCU_EXP_KTHREAD
|
|
|
|
|
extern struct kthread_worker *rcu_exp_gp_kworker;
|
|
|
|
|
extern struct kthread_worker *rcu_exp_par_gp_kworker;
|
|
|
|
|
#else /* !CONFIG_RCU_EXP_KTHREAD */
|
2018-02-02 14:05:38 +08:00
|
|
|
extern struct workqueue_struct *rcu_par_gp_wq;
|
rcu: Move expedited grace period (GP) work to RT kthread_worker
Enabling CONFIG_RCU_BOOST did not reduce RCU expedited grace-period
latency because its workqueues run at SCHED_OTHER, and thus can be
delayed by normal processes. This commit avoids these delays by moving
the expedited GP work items to a real-time-priority kthread_worker.
This option is controlled by CONFIG_RCU_EXP_KTHREAD and disabled by
default on PREEMPT_RT=y kernels which disable expedited grace periods
after boot by unconditionally setting rcupdate.rcu_normal_after_boot=1.
The results were evaluated on arm64 Android devices (6GB ram) running
5.10 kernel, and capturing trace data in critical user-level code.
The table below shows the resulting order-of-magnitude improvements
in synchronize_rcu_expedited() latency:
------------------------------------------------------------------------
| | workqueues | kthread_worker | Diff |
------------------------------------------------------------------------
| Count | 725 | 688 | |
------------------------------------------------------------------------
| Min Duration (ns) | 326 | 447 | 37.12% |
------------------------------------------------------------------------
| Q1 (ns) | 39,428 | 38,971 | -1.16% |
------------------------------------------------------------------------
| Q2 - Median (ns) | 98,225 | 69,743 | -29.00% |
------------------------------------------------------------------------
| Q3 (ns) | 342,122 | 126,638 | -62.98% |
------------------------------------------------------------------------
| Max Duration (ns) | 372,766,967 | 2,329,671 | -99.38% |
------------------------------------------------------------------------
| Avg Duration (ns) | 2,746,353 | 151,242 | -94.49% |
------------------------------------------------------------------------
| Standard Deviation (ns) | 19,327,765 | 294,408 | |
------------------------------------------------------------------------
The below table show the range of maximums/minimums for
synchronize_rcu_expedited() latency from all experiments:
------------------------------------------------------------------------
| | workqueues | kthread_worker | Diff |
------------------------------------------------------------------------
| Total No. of Experiments | 25 | 23 | |
------------------------------------------------------------------------
| Largest Maximum (ns) | 372,766,967 | 2,329,671 | -99.38% |
------------------------------------------------------------------------
| Smallest Maximum (ns) | 38,819 | 86,954 | 124.00% |
------------------------------------------------------------------------
| Range of Maximums (ns) | 372,728,148 | 2,242,717 | |
------------------------------------------------------------------------
| Largest Minimum (ns) | 88,623 | 27,588 | -68.87% |
------------------------------------------------------------------------
| Smallest Minimum (ns) | 326 | 447 | 37.12% |
------------------------------------------------------------------------
| Range of Minimums (ns) | 88,297 | 27,141 | |
------------------------------------------------------------------------
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Reported-by: Tim Murray <timmurray@google.com>
Reported-by: Wei Wang <wvw@google.com>
Tested-by: Kyle Lin <kylelin@google.com>
Tested-by: Chunwei Lu <chunweilu@google.com>
Tested-by: Lulu Wang <luluw@google.com>
Signed-off-by: Kalesh Singh <kaleshsingh@google.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-04-09 08:35:27 +08:00
|
|
|
#endif /* CONFIG_RCU_EXP_KTHREAD */
|
2022-02-05 04:45:18 +08:00
|
|
|
void rcu_gp_slow_register(atomic_t *rgssp);
|
|
|
|
|
void rcu_gp_slow_unregister(atomic_t *rgssp);
|
2017-05-04 04:37:16 +08:00
|
|
|
#endif /* #else #ifdef CONFIG_TINY_RCU */
|
|
|
|
|
|
2017-05-16 07:26:34 +08:00
|
|
|
#ifdef CONFIG_RCU_NOCB_CPU
|
2018-09-22 09:08:09 +08:00
|
|
|
void rcu_bind_current_to_nocb(void);
|
2017-05-04 03:25:50 +08:00
|
|
|
#else
|
2018-09-22 09:08:09 +08:00
|
|
|
static inline void rcu_bind_current_to_nocb(void) { }
|
2017-05-04 03:25:50 +08:00
|
|
|
#endif
|
|
|
|
|
|
2020-09-16 08:08:03 +08:00
|
|
|
#if !defined(CONFIG_TINY_RCU) && defined(CONFIG_TASKS_RCU)
|
|
|
|
|
void show_rcu_tasks_classic_gp_kthread(void);
|
|
|
|
|
#else
|
|
|
|
|
static inline void show_rcu_tasks_classic_gp_kthread(void) {}
|
|
|
|
|
#endif
|
|
|
|
|
#if !defined(CONFIG_TINY_RCU) && defined(CONFIG_TASKS_RUDE_RCU)
|
|
|
|
|
void show_rcu_tasks_rude_gp_kthread(void);
|
|
|
|
|
#else
|
|
|
|
|
static inline void show_rcu_tasks_rude_gp_kthread(void) {}
|
|
|
|
|
#endif
|
|
|
|
|
#if !defined(CONFIG_TINY_RCU) && defined(CONFIG_TASKS_TRACE_RCU)
|
|
|
|
|
void show_rcu_tasks_trace_gp_kthread(void);
|
|
|
|
|
#else
|
|
|
|
|
static inline void show_rcu_tasks_trace_gp_kthread(void) {}
|
|
|
|
|
#endif
|
|
|
|
|
|
2011-06-18 06:53:19 +08:00
|
|
|
#endif /* __LINUX_RCU_H */
|