UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

rcu: Fixup noinstr warnings 

A KCSAN build revealed we have explicit annoations through atomic_*()
usage, switch to arch_atomic_*() for the respective functions.

vmlinux.o: warning: objtool: rcu_nmi_exit()+0x4d: call to __kcsan_check_access() leaves .noinstr.text section
vmlinux.o: warning: objtool: rcu_dynticks_eqs_enter()+0x25: call to __kcsan_check_access() leaves .noinstr.text section
vmlinux.o: warning: objtool: rcu_nmi_enter()+0x4f: call to __kcsan_check_access() leaves .noinstr.text section
vmlinux.o: warning: objtool: rcu_dynticks_eqs_exit()+0x2a: call to __kcsan_check_access() leaves .noinstr.text section
vmlinux.o: warning: objtool: __rcu_is_watching()+0x25: call to __kcsan_check_access() leaves .noinstr.text section

Additionally, without the NOP in instrumentation_begin(), objtool would
not detect the lack of the 'else instrumentation_begin();' branch in
rcu_nmi_enter().

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
Peter Zijlstra 2020-06-15 18:24:27 +02:00 committed by Paul E. McKenney
parent 5faafd5685
commit b58e733fd7
2 changed files with 26 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/compiler.h
View File

@ -123,7 +123,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
#ifdef CONFIG_DEBUG_ENTRY #ifdef CONFIG_DEBUG_ENTRY
/* Begin/end of an instrumentation safe region */ /* Begin/end of an instrumentation safe region */
#define instrumentation_begin() ({ \ #define instrumentation_begin() ({ \
asm volatile("%c0:\n\t" \ asm volatile("%c0: nop\n\t" \
".pushsection .discard.instr_begin\n\t" \ ".pushsection .discard.instr_begin\n\t" \
".long %c0b - .\n\t" \ ".long %c0b - .\n\t" \
".popsection\n\t" : : "i" (__COUNTER__)); \ ".popsection\n\t" : : "i" (__COUNTER__)); \

32
kernel/rcu/tree.c
View File

@ -250,7 +250,7 @@ static noinstr void rcu_dynticks_eqs_enter(void)
* next idle sojourn. * next idle sojourn.
*/ */
rcu_dynticks_task_trace_enter(); // Before ->dynticks update! rcu_dynticks_task_trace_enter(); // Before ->dynticks update!
seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
// RCU is no longer watching. Better be in extended quiescent state! // RCU is no longer watching. Better be in extended quiescent state!
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
(seq & RCU_DYNTICK_CTRL_CTR)); (seq & RCU_DYNTICK_CTRL_CTR));
@ -274,13 +274,13 @@ static noinstr void rcu_dynticks_eqs_exit(void)
* and we also must force ordering with the next RCU read-side * and we also must force ordering with the next RCU read-side
* critical section. * critical section.
*/ */
seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
// RCU is now watching. Better not be in an extended quiescent state! // RCU is now watching. Better not be in an extended quiescent state!
rcu_dynticks_task_trace_exit(); // After ->dynticks update! rcu_dynticks_task_trace_exit(); // After ->dynticks update!
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
!(seq & RCU_DYNTICK_CTRL_CTR)); !(seq & RCU_DYNTICK_CTRL_CTR));
if (seq & RCU_DYNTICK_CTRL_MASK) { if (seq & RCU_DYNTICK_CTRL_MASK) {
atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks); arch_atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks);
smp_mb__after_atomic(); /* _exit after clearing mask. */ smp_mb__after_atomic(); /* _exit after clearing mask. */
} }
} }
@ -313,7 +313,7 @@ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
{ {
struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
return !(atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR); return !(arch_atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);
} }
/* /*
@ -633,6 +633,10 @@ static noinstr void rcu_eqs_enter(bool user)
do_nocb_deferred_wakeup(rdp); do_nocb_deferred_wakeup(rdp);
rcu_prepare_for_idle(); rcu_prepare_for_idle();
rcu_preempt_deferred_qs(current); rcu_preempt_deferred_qs(current);
// instrumentation for the noinstr rcu_dynticks_eqs_enter()
instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
instrumentation_end(); instrumentation_end();
WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
// RCU is watching here ... // RCU is watching here ...
@ -692,6 +696,7 @@ noinstr void rcu_nmi_exit(void)
{ {
struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
instrumentation_begin();
/* /*
* Check for ->dynticks_nmi_nesting underflow and bad ->dynticks. * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
* (We are exiting an NMI handler, so RCU better be paying attention * (We are exiting an NMI handler, so RCU better be paying attention
@ -705,7 +710,6 @@ noinstr void rcu_nmi_exit(void)
* leave it in non-RCU-idle state. * leave it in non-RCU-idle state.
*/ */
if (rdp->dynticks_nmi_nesting != 1) { if (rdp->dynticks_nmi_nesting != 1) {
instrumentation_begin();
trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2, trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2,
atomic_read(&rdp->dynticks)); atomic_read(&rdp->dynticks));
WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */ WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */
@ -714,13 +718,15 @@ noinstr void rcu_nmi_exit(void)
return; return;
} }
instrumentation_begin();
/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks)); trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
if (!in_nmi()) if (!in_nmi())
rcu_prepare_for_idle(); rcu_prepare_for_idle();
// instrumentation for the noinstr rcu_dynticks_eqs_enter()
instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
instrumentation_end(); instrumentation_end();
// RCU is watching here ... // RCU is watching here ...
@ -838,6 +844,10 @@ static void noinstr rcu_eqs_exit(bool user)
rcu_dynticks_eqs_exit(); rcu_dynticks_eqs_exit();
// ... but is watching here. // ... but is watching here.
instrumentation_begin(); instrumentation_begin();
// instrumentation for the noinstr rcu_dynticks_eqs_exit()
instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
rcu_cleanup_after_idle(); rcu_cleanup_after_idle();
trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks)); trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
@ -983,13 +993,21 @@ noinstr void rcu_nmi_enter(void)
if (!in_nmi()) if (!in_nmi())
rcu_cleanup_after_idle(); rcu_cleanup_after_idle();
instrumentation_begin();
// instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs()
instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks));
// instrumentation for the noinstr rcu_dynticks_eqs_exit()
instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
incby = 1; incby = 1;
} else if (!in_nmi()) { } else if (!in_nmi()) {
instrumentation_begin(); instrumentation_begin();
rcu_irq_enter_check_tick(); rcu_irq_enter_check_tick();
instrumentation_end(); instrumentation_end();
} else {
instrumentation_begin();
} }
instrumentation_begin();
trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting,
rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks)); rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks));