2019-05-19 20:08:55 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2008-06-26 17:21:34 +08:00
|
|
|
/*
|
|
|
|
* Generic helpers for smp ipi calls
|
|
|
|
*
|
|
|
|
* (C) Jens Axboe <jens.axboe@oracle.com> 2008
|
|
|
|
*/
|
2016-10-26 13:37:53 +08:00
|
|
|
|
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
|
2014-05-08 07:37:48 +08:00
|
|
|
#include <linux/irq_work.h>
|
2008-06-26 17:21:34 +08:00
|
|
|
#include <linux/rcupdate.h>
|
2008-07-16 05:02:33 +08:00
|
|
|
#include <linux/rculist.h>
|
2009-03-13 17:47:34 +08:00
|
|
|
#include <linux/kernel.h>
|
2011-05-24 02:51:41 +08:00
|
|
|
#include <linux/export.h>
|
2009-02-25 23:52:11 +08:00
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/init.h>
|
2021-01-24 04:10:25 +08:00
|
|
|
#include <linux/interrupt.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 16:04:11 +08:00
|
|
|
#include <linux/gfp.h>
|
2008-06-26 17:21:34 +08:00
|
|
|
#include <linux/smp.h>
|
2009-02-25 20:59:47 +08:00
|
|
|
#include <linux/cpu.h>
|
2014-09-04 15:17:54 +08:00
|
|
|
#include <linux/sched.h>
|
2017-02-01 23:36:40 +08:00
|
|
|
#include <linux/sched/idle.h>
|
2016-08-29 14:48:43 +08:00
|
|
|
#include <linux/hypervisor.h>
|
2020-07-01 04:22:54 +08:00
|
|
|
#include <linux/sched/clock.h>
|
|
|
|
#include <linux/nmi.h>
|
|
|
|
#include <linux/sched/debug.h>
|
2021-03-01 18:13:34 +08:00
|
|
|
#include <linux/jump_label.h>
|
2008-06-26 17:21:34 +08:00
|
|
|
|
2012-04-21 08:08:50 +08:00
|
|
|
#include "smpboot.h"
|
2020-05-28 17:01:34 +08:00
|
|
|
#include "sched/smp.h"
|
2012-04-21 08:08:50 +08:00
|
|
|
|
2020-06-15 17:29:31 +08:00
|
|
|
#define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK)
|
2008-06-26 17:21:34 +08:00
|
|
|
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
|
|
|
|
union cfd_seq_cnt {
|
|
|
|
u64 val;
|
|
|
|
struct {
|
|
|
|
u64 src:16;
|
|
|
|
u64 dst:16;
|
|
|
|
#define CFD_SEQ_NOCPU 0xffff
|
|
|
|
u64 type:4;
|
|
|
|
#define CFD_SEQ_QUEUE 0
|
|
|
|
#define CFD_SEQ_IPI 1
|
|
|
|
#define CFD_SEQ_NOIPI 2
|
|
|
|
#define CFD_SEQ_PING 3
|
|
|
|
#define CFD_SEQ_PINGED 4
|
|
|
|
#define CFD_SEQ_HANDLE 5
|
|
|
|
#define CFD_SEQ_DEQUEUE 6
|
|
|
|
#define CFD_SEQ_IDLE 7
|
|
|
|
#define CFD_SEQ_GOTIPI 8
|
|
|
|
#define CFD_SEQ_HDLEND 9
|
|
|
|
u64 cnt:28;
|
|
|
|
} u;
|
|
|
|
};
|
|
|
|
|
|
|
|
static char *seq_type[] = {
|
|
|
|
[CFD_SEQ_QUEUE] = "queue",
|
|
|
|
[CFD_SEQ_IPI] = "ipi",
|
|
|
|
[CFD_SEQ_NOIPI] = "noipi",
|
|
|
|
[CFD_SEQ_PING] = "ping",
|
|
|
|
[CFD_SEQ_PINGED] = "pinged",
|
|
|
|
[CFD_SEQ_HANDLE] = "handle",
|
|
|
|
[CFD_SEQ_DEQUEUE] = "dequeue (src CPU 0 == empty)",
|
|
|
|
[CFD_SEQ_IDLE] = "idle",
|
|
|
|
[CFD_SEQ_GOTIPI] = "gotipi",
|
|
|
|
[CFD_SEQ_HDLEND] = "hdlend (src CPU 0 == early)",
|
|
|
|
};
|
|
|
|
|
|
|
|
struct cfd_seq_local {
|
|
|
|
u64 ping;
|
|
|
|
u64 pinged;
|
|
|
|
u64 handle;
|
|
|
|
u64 dequeue;
|
|
|
|
u64 idle;
|
|
|
|
u64 gotipi;
|
|
|
|
u64 hdlend;
|
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
2021-03-01 18:13:35 +08:00
|
|
|
struct cfd_percpu {
|
|
|
|
call_single_data_t csd;
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
|
|
|
|
u64 seq_queue;
|
|
|
|
u64 seq_ipi;
|
|
|
|
u64 seq_noipi;
|
|
|
|
#endif
|
2021-03-01 18:13:35 +08:00
|
|
|
};
|
|
|
|
|
2008-06-26 17:21:34 +08:00
|
|
|
struct call_function_data {
|
2021-03-01 18:13:35 +08:00
|
|
|
struct cfd_percpu __percpu *pcpu;
|
2009-02-25 23:52:11 +08:00
|
|
|
cpumask_var_t cpumask;
|
2017-05-19 15:53:31 +08:00
|
|
|
cpumask_var_t cpumask_ipi;
|
2008-06-26 17:21:34 +08:00
|
|
|
};
|
|
|
|
|
2019-06-13 14:48:11 +08:00
|
|
|
static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data);
|
2010-01-18 10:00:51 +08:00
|
|
|
|
2014-01-31 07:45:47 +08:00
|
|
|
static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
|
2009-02-25 20:59:47 +08:00
|
|
|
|
2022-04-13 21:31:03 +08:00
|
|
|
static void __flush_smp_call_function_queue(bool warn_cpu_offline);
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-24 04:22:02 +08:00
|
|
|
|
2016-07-14 01:17:01 +08:00
|
|
|
int smpcfd_prepare_cpu(unsigned int cpu)
|
2009-02-25 20:59:47 +08:00
|
|
|
{
|
|
|
|
struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
|
|
|
|
|
2016-07-14 01:17:01 +08:00
|
|
|
if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
|
|
|
|
cpu_to_node(cpu)))
|
|
|
|
return -ENOMEM;
|
2017-05-19 15:53:31 +08:00
|
|
|
if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL,
|
|
|
|
cpu_to_node(cpu))) {
|
|
|
|
free_cpumask_var(cfd->cpumask);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
2021-03-01 18:13:35 +08:00
|
|
|
cfd->pcpu = alloc_percpu(struct cfd_percpu);
|
|
|
|
if (!cfd->pcpu) {
|
2009-02-25 20:59:47 +08:00
|
|
|
free_cpumask_var(cfd->cpumask);
|
2017-05-19 15:53:31 +08:00
|
|
|
free_cpumask_var(cfd->cpumask_ipi);
|
2016-07-14 01:17:01 +08:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
2009-02-25 20:59:47 +08:00
|
|
|
}
|
|
|
|
|
2016-07-14 01:17:01 +08:00
|
|
|
int smpcfd_dead_cpu(unsigned int cpu)
|
|
|
|
{
|
|
|
|
struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
|
|
|
|
|
|
|
|
free_cpumask_var(cfd->cpumask);
|
2017-05-19 15:53:31 +08:00
|
|
|
free_cpumask_var(cfd->cpumask_ipi);
|
2021-03-01 18:13:35 +08:00
|
|
|
free_percpu(cfd->pcpu);
|
2016-07-14 01:17:01 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int smpcfd_dying_cpu(unsigned int cpu)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The IPIs for the smp-call-function callbacks queued by other
|
|
|
|
* CPUs might arrive late, either due to hardware latencies or
|
|
|
|
* because this CPU disabled interrupts (inside stop-machine)
|
|
|
|
* before the IPIs were sent. So flush out any pending callbacks
|
|
|
|
* explicitly (without waiting for the IPIs to arrive), to
|
|
|
|
* ensure that the outgoing CPU doesn't go offline with work
|
|
|
|
* still pending.
|
|
|
|
*/
|
2022-04-13 21:31:03 +08:00
|
|
|
__flush_smp_call_function_queue(false);
|
2020-05-27 00:11:00 +08:00
|
|
|
irq_work_run();
|
2016-07-14 01:17:01 +08:00
|
|
|
return 0;
|
|
|
|
}
|
2009-02-25 20:59:47 +08:00
|
|
|
|
2011-03-30 00:35:04 +08:00
|
|
|
void __init call_function_init(void)
|
2008-06-26 17:21:34 +08:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2014-01-31 07:45:47 +08:00
|
|
|
for_each_possible_cpu(i)
|
|
|
|
init_llist_head(&per_cpu(call_single_queue, i));
|
2009-02-25 20:59:47 +08:00
|
|
|
|
2016-07-14 01:17:01 +08:00
|
|
|
smpcfd_prepare_cpu(smp_processor_id());
|
2008-06-26 17:21:34 +08:00
|
|
|
}
|
|
|
|
|
2020-07-01 04:22:54 +08:00
|
|
|
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
|
|
|
|
|
2021-03-01 18:13:34 +08:00
|
|
|
static DEFINE_STATIC_KEY_FALSE(csdlock_debug_enabled);
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
static DEFINE_STATIC_KEY_FALSE(csdlock_debug_extended);
|
2021-03-01 18:13:34 +08:00
|
|
|
|
|
|
|
static int __init csdlock_debug(char *str)
|
|
|
|
{
|
|
|
|
unsigned int val = 0;
|
|
|
|
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
if (str && !strcmp(str, "ext")) {
|
|
|
|
val = 1;
|
|
|
|
static_branch_enable(&csdlock_debug_extended);
|
|
|
|
} else
|
|
|
|
get_option(&str, &val);
|
|
|
|
|
2021-03-01 18:13:34 +08:00
|
|
|
if (val)
|
|
|
|
static_branch_enable(&csdlock_debug_enabled);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("csdlock_debug", csdlock_debug);
|
|
|
|
|
2020-07-01 04:22:54 +08:00
|
|
|
static DEFINE_PER_CPU(call_single_data_t *, cur_csd);
|
|
|
|
static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func);
|
|
|
|
static DEFINE_PER_CPU(void *, cur_csd_info);
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
static DEFINE_PER_CPU(struct cfd_seq_local, cfd_seq_local);
|
2020-07-01 04:22:54 +08:00
|
|
|
|
2022-03-01 10:08:33 +08:00
|
|
|
static ulong csd_lock_timeout = 5000; /* CSD lock timeout in milliseconds. */
|
|
|
|
module_param(csd_lock_timeout, ulong, 0444);
|
|
|
|
|
2020-07-06 21:49:41 +08:00
|
|
|
static atomic_t csd_bug_count = ATOMIC_INIT(0);
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
static u64 cfd_seq;
|
|
|
|
|
|
|
|
#define CFD_SEQ(s, d, t, c) \
|
|
|
|
(union cfd_seq_cnt){ .u.src = s, .u.dst = d, .u.type = t, .u.cnt = c }
|
|
|
|
|
|
|
|
static u64 cfd_seq_inc(unsigned int src, unsigned int dst, unsigned int type)
|
|
|
|
{
|
|
|
|
union cfd_seq_cnt new, old;
|
|
|
|
|
|
|
|
new = CFD_SEQ(src, dst, type, 0);
|
|
|
|
|
|
|
|
do {
|
|
|
|
old.val = READ_ONCE(cfd_seq);
|
|
|
|
new.u.cnt = old.u.cnt + 1;
|
|
|
|
} while (cmpxchg(&cfd_seq, old.val, new.val) != old.val);
|
|
|
|
|
|
|
|
return old.val;
|
|
|
|
}
|
|
|
|
|
|
|
|
#define cfd_seq_store(var, src, dst, type) \
|
|
|
|
do { \
|
|
|
|
if (static_branch_unlikely(&csdlock_debug_extended)) \
|
|
|
|
var = cfd_seq_inc(src, dst, type); \
|
|
|
|
} while (0)
|
2020-07-01 04:22:54 +08:00
|
|
|
|
|
|
|
/* Record current CSD work for current CPU, NULL to erase. */
|
2021-05-06 05:12:42 +08:00
|
|
|
static void __csd_lock_record(struct __call_single_data *csd)
|
2020-07-01 04:22:54 +08:00
|
|
|
{
|
|
|
|
if (!csd) {
|
|
|
|
smp_mb(); /* NULL cur_csd after unlock. */
|
|
|
|
__this_cpu_write(cur_csd, NULL);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
__this_cpu_write(cur_csd_func, csd->func);
|
|
|
|
__this_cpu_write(cur_csd_info, csd->info);
|
|
|
|
smp_wmb(); /* func and info before csd. */
|
|
|
|
__this_cpu_write(cur_csd, csd);
|
|
|
|
smp_mb(); /* Update cur_csd before function call. */
|
|
|
|
/* Or before unlock, as the case may be. */
|
|
|
|
}
|
|
|
|
|
2021-05-06 05:12:42 +08:00
|
|
|
static __always_inline void csd_lock_record(struct __call_single_data *csd)
|
2021-03-01 18:13:34 +08:00
|
|
|
{
|
|
|
|
if (static_branch_unlikely(&csdlock_debug_enabled))
|
|
|
|
__csd_lock_record(csd);
|
|
|
|
}
|
|
|
|
|
2021-05-06 05:12:42 +08:00
|
|
|
static int csd_lock_wait_getcpu(struct __call_single_data *csd)
|
2020-07-01 04:22:54 +08:00
|
|
|
{
|
|
|
|
unsigned int csd_type;
|
|
|
|
|
|
|
|
csd_type = CSD_TYPE(csd);
|
|
|
|
if (csd_type == CSD_TYPE_ASYNC || csd_type == CSD_TYPE_SYNC)
|
2020-11-27 18:09:57 +08:00
|
|
|
return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */
|
2020-07-01 04:22:54 +08:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
static void cfd_seq_data_add(u64 val, unsigned int src, unsigned int dst,
|
|
|
|
unsigned int type, union cfd_seq_cnt *data,
|
|
|
|
unsigned int *n_data, unsigned int now)
|
|
|
|
{
|
|
|
|
union cfd_seq_cnt new[2];
|
|
|
|
unsigned int i, j, k;
|
|
|
|
|
|
|
|
new[0].val = val;
|
|
|
|
new[1] = CFD_SEQ(src, dst, type, new[0].u.cnt + 1);
|
|
|
|
|
|
|
|
for (i = 0; i < 2; i++) {
|
|
|
|
if (new[i].u.cnt <= now)
|
|
|
|
new[i].u.cnt |= 0x80000000U;
|
|
|
|
for (j = 0; j < *n_data; j++) {
|
|
|
|
if (new[i].u.cnt == data[j].u.cnt) {
|
|
|
|
/* Direct read value trumps generated one. */
|
|
|
|
if (i == 0)
|
|
|
|
data[j].val = new[i].val;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (new[i].u.cnt < data[j].u.cnt) {
|
|
|
|
for (k = *n_data; k > j; k--)
|
|
|
|
data[k].val = data[k - 1].val;
|
|
|
|
data[j].val = new[i].val;
|
|
|
|
(*n_data)++;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (j == *n_data) {
|
|
|
|
data[j].val = new[i].val;
|
|
|
|
(*n_data)++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static const char *csd_lock_get_type(unsigned int type)
|
|
|
|
{
|
|
|
|
return (type >= ARRAY_SIZE(seq_type)) ? "?" : seq_type[type];
|
|
|
|
}
|
|
|
|
|
2021-05-06 05:12:42 +08:00
|
|
|
static void csd_lock_print_extended(struct __call_single_data *csd, int cpu)
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
{
|
|
|
|
struct cfd_seq_local *seq = &per_cpu(cfd_seq_local, cpu);
|
|
|
|
unsigned int srccpu = csd->node.src;
|
|
|
|
struct call_function_data *cfd = per_cpu_ptr(&cfd_data, srccpu);
|
|
|
|
struct cfd_percpu *pcpu = per_cpu_ptr(cfd->pcpu, cpu);
|
|
|
|
unsigned int now;
|
|
|
|
union cfd_seq_cnt data[2 * ARRAY_SIZE(seq_type)];
|
|
|
|
unsigned int n_data = 0, i;
|
|
|
|
|
|
|
|
data[0].val = READ_ONCE(cfd_seq);
|
|
|
|
now = data[0].u.cnt;
|
|
|
|
|
|
|
|
cfd_seq_data_add(pcpu->seq_queue, srccpu, cpu, CFD_SEQ_QUEUE, data, &n_data, now);
|
|
|
|
cfd_seq_data_add(pcpu->seq_ipi, srccpu, cpu, CFD_SEQ_IPI, data, &n_data, now);
|
|
|
|
cfd_seq_data_add(pcpu->seq_noipi, srccpu, cpu, CFD_SEQ_NOIPI, data, &n_data, now);
|
|
|
|
|
|
|
|
cfd_seq_data_add(per_cpu(cfd_seq_local.ping, srccpu), srccpu, CFD_SEQ_NOCPU, CFD_SEQ_PING, data, &n_data, now);
|
|
|
|
cfd_seq_data_add(per_cpu(cfd_seq_local.pinged, srccpu), srccpu, CFD_SEQ_NOCPU, CFD_SEQ_PINGED, data, &n_data, now);
|
|
|
|
|
|
|
|
cfd_seq_data_add(seq->idle, CFD_SEQ_NOCPU, cpu, CFD_SEQ_IDLE, data, &n_data, now);
|
|
|
|
cfd_seq_data_add(seq->gotipi, CFD_SEQ_NOCPU, cpu, CFD_SEQ_GOTIPI, data, &n_data, now);
|
|
|
|
cfd_seq_data_add(seq->handle, CFD_SEQ_NOCPU, cpu, CFD_SEQ_HANDLE, data, &n_data, now);
|
|
|
|
cfd_seq_data_add(seq->dequeue, CFD_SEQ_NOCPU, cpu, CFD_SEQ_DEQUEUE, data, &n_data, now);
|
|
|
|
cfd_seq_data_add(seq->hdlend, CFD_SEQ_NOCPU, cpu, CFD_SEQ_HDLEND, data, &n_data, now);
|
|
|
|
|
|
|
|
for (i = 0; i < n_data; i++) {
|
|
|
|
pr_alert("\tcsd: cnt(%07x): %04x->%04x %s\n",
|
|
|
|
data[i].u.cnt & ~0x80000000U, data[i].u.src,
|
|
|
|
data[i].u.dst, csd_lock_get_type(data[i].u.type));
|
|
|
|
}
|
|
|
|
pr_alert("\tcsd: cnt now: %07x\n", now);
|
|
|
|
}
|
|
|
|
|
2020-07-01 04:22:54 +08:00
|
|
|
/*
|
|
|
|
* Complain if too much time spent waiting. Note that only
|
|
|
|
* the CSD_TYPE_SYNC/ASYNC types provide the destination CPU,
|
|
|
|
* so waiting on other types gets much less information.
|
|
|
|
*/
|
2021-05-06 05:12:42 +08:00
|
|
|
static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *ts1, int *bug_id)
|
2020-07-01 04:22:54 +08:00
|
|
|
{
|
|
|
|
int cpu = -1;
|
|
|
|
int cpux;
|
|
|
|
bool firsttime;
|
|
|
|
u64 ts2, ts_delta;
|
|
|
|
call_single_data_t *cpu_cur_csd;
|
2020-06-15 17:29:31 +08:00
|
|
|
unsigned int flags = READ_ONCE(csd->node.u_flags);
|
2022-03-01 10:08:33 +08:00
|
|
|
unsigned long long csd_lock_timeout_ns = csd_lock_timeout * NSEC_PER_MSEC;
|
2020-07-01 04:22:54 +08:00
|
|
|
|
|
|
|
if (!(flags & CSD_FLAG_LOCK)) {
|
|
|
|
if (!unlikely(*bug_id))
|
|
|
|
return true;
|
|
|
|
cpu = csd_lock_wait_getcpu(csd);
|
|
|
|
pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock.\n",
|
|
|
|
*bug_id, raw_smp_processor_id(), cpu);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
ts2 = sched_clock();
|
|
|
|
ts_delta = ts2 - *ts1;
|
2022-03-01 10:08:33 +08:00
|
|
|
if (likely(ts_delta <= csd_lock_timeout_ns || csd_lock_timeout_ns == 0))
|
2020-07-01 04:22:54 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
firsttime = !*bug_id;
|
|
|
|
if (firsttime)
|
|
|
|
*bug_id = atomic_inc_return(&csd_bug_count);
|
|
|
|
cpu = csd_lock_wait_getcpu(csd);
|
|
|
|
if (WARN_ONCE(cpu < 0 || cpu >= nr_cpu_ids, "%s: cpu = %d\n", __func__, cpu))
|
|
|
|
cpux = 0;
|
|
|
|
else
|
|
|
|
cpux = cpu;
|
|
|
|
cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */
|
|
|
|
pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %llu ns for CPU#%02d %pS(%ps).\n",
|
|
|
|
firsttime ? "Detected" : "Continued", *bug_id, raw_smp_processor_id(), ts2 - ts0,
|
|
|
|
cpu, csd->func, csd->info);
|
|
|
|
if (cpu_cur_csd && csd != cpu_cur_csd) {
|
|
|
|
pr_alert("\tcsd: CSD lock (#%d) handling prior %pS(%ps) request.\n",
|
|
|
|
*bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)),
|
|
|
|
READ_ONCE(per_cpu(cur_csd_info, cpux)));
|
|
|
|
} else {
|
|
|
|
pr_alert("\tcsd: CSD lock (#%d) %s.\n",
|
|
|
|
*bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request");
|
|
|
|
}
|
|
|
|
if (cpu >= 0) {
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
if (static_branch_unlikely(&csdlock_debug_extended))
|
|
|
|
csd_lock_print_extended(csd, cpu);
|
2020-07-01 04:22:54 +08:00
|
|
|
if (!trigger_single_cpu_backtrace(cpu))
|
|
|
|
dump_cpu_task(cpu);
|
|
|
|
if (!cpu_cur_csd) {
|
|
|
|
pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu);
|
|
|
|
arch_send_call_function_single_ipi(cpu);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
dump_stack();
|
|
|
|
*ts1 = ts2;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2009-02-25 20:59:47 +08:00
|
|
|
/*
|
|
|
|
* csd_lock/csd_unlock used to serialize access to per-cpu csd resources
|
|
|
|
*
|
2009-02-25 23:52:11 +08:00
|
|
|
* For non-synchronous ipi calls the csd can still be in use by the
|
|
|
|
* previous function call. For multi-cpu calls its even more interesting
|
|
|
|
* as we'll have to ensure no other cpu is observing our csd.
|
2009-02-25 20:59:47 +08:00
|
|
|
*/
|
2021-05-06 05:12:42 +08:00
|
|
|
static void __csd_lock_wait(struct __call_single_data *csd)
|
2020-07-01 04:22:54 +08:00
|
|
|
{
|
|
|
|
int bug_id = 0;
|
|
|
|
u64 ts0, ts1;
|
|
|
|
|
|
|
|
ts1 = ts0 = sched_clock();
|
|
|
|
for (;;) {
|
|
|
|
if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id))
|
|
|
|
break;
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
smp_acquire__after_ctrl_dep();
|
|
|
|
}
|
|
|
|
|
2021-05-06 05:12:42 +08:00
|
|
|
static __always_inline void csd_lock_wait(struct __call_single_data *csd)
|
2021-03-01 18:13:34 +08:00
|
|
|
{
|
|
|
|
if (static_branch_unlikely(&csdlock_debug_enabled)) {
|
|
|
|
__csd_lock_wait(csd);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
|
|
|
|
}
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
|
|
|
|
static void __smp_call_single_queue_debug(int cpu, struct llist_node *node)
|
|
|
|
{
|
|
|
|
unsigned int this_cpu = smp_processor_id();
|
|
|
|
struct cfd_seq_local *seq = this_cpu_ptr(&cfd_seq_local);
|
|
|
|
struct call_function_data *cfd = this_cpu_ptr(&cfd_data);
|
|
|
|
struct cfd_percpu *pcpu = per_cpu_ptr(cfd->pcpu, cpu);
|
|
|
|
|
|
|
|
cfd_seq_store(pcpu->seq_queue, this_cpu, cpu, CFD_SEQ_QUEUE);
|
|
|
|
if (llist_add(node, &per_cpu(call_single_queue, cpu))) {
|
|
|
|
cfd_seq_store(pcpu->seq_ipi, this_cpu, cpu, CFD_SEQ_IPI);
|
|
|
|
cfd_seq_store(seq->ping, this_cpu, cpu, CFD_SEQ_PING);
|
|
|
|
send_call_function_single_ipi(cpu);
|
|
|
|
cfd_seq_store(seq->pinged, this_cpu, cpu, CFD_SEQ_PINGED);
|
|
|
|
} else {
|
|
|
|
cfd_seq_store(pcpu->seq_noipi, this_cpu, cpu, CFD_SEQ_NOIPI);
|
|
|
|
}
|
|
|
|
}
|
2020-07-01 04:22:54 +08:00
|
|
|
#else
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
#define cfd_seq_store(var, src, dst, type)
|
|
|
|
|
2021-05-06 05:12:42 +08:00
|
|
|
static void csd_lock_record(struct __call_single_data *csd)
|
2020-07-01 04:22:54 +08:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2021-05-06 05:12:42 +08:00
|
|
|
static __always_inline void csd_lock_wait(struct __call_single_data *csd)
|
2009-02-25 20:59:47 +08:00
|
|
|
{
|
2020-06-15 17:29:31 +08:00
|
|
|
smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
|
2009-02-25 20:59:48 +08:00
|
|
|
}
|
2020-07-01 04:22:54 +08:00
|
|
|
#endif
|
2009-02-25 20:59:48 +08:00
|
|
|
|
2021-05-06 05:12:42 +08:00
|
|
|
static __always_inline void csd_lock(struct __call_single_data *csd)
|
2009-02-25 20:59:48 +08:00
|
|
|
{
|
2013-05-01 06:27:28 +08:00
|
|
|
csd_lock_wait(csd);
|
2020-06-15 17:29:31 +08:00
|
|
|
csd->node.u_flags |= CSD_FLAG_LOCK;
|
2009-02-25 20:59:47 +08:00
|
|
|
|
|
|
|
/*
|
2009-02-25 23:52:11 +08:00
|
|
|
* prevent CPU from reordering the above assignment
|
|
|
|
* to ->flags with any subsequent assignments to other
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-08 12:30:00 +08:00
|
|
|
* fields of the specified call_single_data_t structure:
|
2009-02-25 20:59:47 +08:00
|
|
|
*/
|
2015-02-12 04:42:10 +08:00
|
|
|
smp_wmb();
|
2009-02-25 20:59:47 +08:00
|
|
|
}
|
|
|
|
|
2021-05-06 05:12:42 +08:00
|
|
|
static __always_inline void csd_unlock(struct __call_single_data *csd)
|
2009-02-25 20:59:47 +08:00
|
|
|
{
|
2020-06-15 17:29:31 +08:00
|
|
|
WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));
|
2009-02-25 23:52:11 +08:00
|
|
|
|
2009-02-25 20:59:47 +08:00
|
|
|
/*
|
2009-02-25 23:52:11 +08:00
|
|
|
* ensure we're all done before releasing data:
|
2009-02-25 20:59:47 +08:00
|
|
|
*/
|
2020-06-15 17:29:31 +08:00
|
|
|
smp_store_release(&csd->node.u_flags, 0);
|
2008-06-26 17:21:34 +08:00
|
|
|
}
|
|
|
|
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-08 12:30:00 +08:00
|
|
|
static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
|
2014-02-24 23:39:58 +08:00
|
|
|
|
2020-05-27 00:11:02 +08:00
|
|
|
void __smp_call_single_queue(int cpu, struct llist_node *node)
|
|
|
|
{
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
|
|
|
|
if (static_branch_unlikely(&csdlock_debug_extended)) {
|
|
|
|
unsigned int type;
|
|
|
|
|
|
|
|
type = CSD_TYPE(container_of(node, call_single_data_t,
|
|
|
|
node.llist));
|
|
|
|
if (type == CSD_TYPE_SYNC || type == CSD_TYPE_ASYNC) {
|
|
|
|
__smp_call_single_queue_debug(cpu, node);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2020-05-27 00:11:02 +08:00
|
|
|
/*
|
|
|
|
* The list addition should be visible before sending the IPI
|
|
|
|
* handler locks the list to pull the entry off it because of
|
|
|
|
* normal cache coherency rules implied by spinlocks.
|
|
|
|
*
|
|
|
|
* If IPIs can go out of order to the cache coherency protocol
|
|
|
|
* in an architecture, sufficient synchronisation should be added
|
|
|
|
* to arch code to make it appear to obey cache coherency WRT
|
|
|
|
* locking and barrier primitives. Generic code isn't really
|
|
|
|
* equipped to do the right thing...
|
|
|
|
*/
|
|
|
|
if (llist_add(node, &per_cpu(call_single_queue, cpu)))
|
|
|
|
send_call_function_single_ipi(cpu);
|
|
|
|
}
|
|
|
|
|
2008-06-26 17:21:34 +08:00
|
|
|
/*
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-08 12:30:00 +08:00
|
|
|
* Insert a previously allocated call_single_data_t element
|
2009-02-25 23:52:11 +08:00
|
|
|
* for execution on the given CPU. data must already have
|
|
|
|
* ->func, ->info, and ->flags set.
|
2008-06-26 17:21:34 +08:00
|
|
|
*/
|
2021-05-06 05:12:42 +08:00
|
|
|
static int generic_exec_single(int cpu, struct __call_single_data *csd)
|
2008-06-26 17:21:34 +08:00
|
|
|
{
|
2014-02-24 23:39:58 +08:00
|
|
|
if (cpu == smp_processor_id()) {
|
2020-05-27 00:11:02 +08:00
|
|
|
smp_call_func_t func = csd->func;
|
|
|
|
void *info = csd->info;
|
2015-02-12 04:42:10 +08:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We can unlock early even for the synchronous on-stack case,
|
|
|
|
* since we're doing this from the same CPU..
|
|
|
|
*/
|
2020-07-01 04:22:54 +08:00
|
|
|
csd_lock_record(csd);
|
2015-02-12 04:42:10 +08:00
|
|
|
csd_unlock(csd);
|
2014-02-24 23:39:58 +08:00
|
|
|
local_irq_save(flags);
|
|
|
|
func(info);
|
2020-07-01 04:22:54 +08:00
|
|
|
csd_lock_record(NULL);
|
2014-02-24 23:39:58 +08:00
|
|
|
local_irq_restore(flags);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-04-19 16:56:03 +08:00
|
|
|
if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu)) {
|
|
|
|
csd_unlock(csd);
|
2014-02-24 23:39:58 +08:00
|
|
|
return -ENXIO;
|
2015-04-19 16:56:03 +08:00
|
|
|
}
|
2014-02-24 23:39:58 +08:00
|
|
|
|
2020-06-15 17:29:31 +08:00
|
|
|
__smp_call_single_queue(cpu, &csd->node.llist);
|
2008-06-26 17:21:34 +08:00
|
|
|
|
2014-02-24 23:39:58 +08:00
|
|
|
return 0;
|
2008-06-26 17:21:34 +08:00
|
|
|
}
|
|
|
|
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-24 04:22:02 +08:00
|
|
|
/**
|
|
|
|
* generic_smp_call_function_single_interrupt - Execute SMP IPI callbacks
|
|
|
|
*
|
|
|
|
* Invoked by arch to handle an IPI for call function single.
|
|
|
|
* Must be called with interrupts disabled.
|
2008-06-26 17:21:34 +08:00
|
|
|
*/
|
|
|
|
void generic_smp_call_function_single_interrupt(void)
|
|
|
|
{
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->gotipi, CFD_SEQ_NOCPU,
|
|
|
|
smp_processor_id(), CFD_SEQ_GOTIPI);
|
2022-04-13 21:31:03 +08:00
|
|
|
__flush_smp_call_function_queue(true);
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-24 04:22:02 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2022-04-13 21:31:03 +08:00
|
|
|
* __flush_smp_call_function_queue - Flush pending smp-call-function callbacks
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-24 04:22:02 +08:00
|
|
|
*
|
|
|
|
* @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
|
|
|
|
* offline CPU. Skip this check if set to 'false'.
|
|
|
|
*
|
|
|
|
* Flush any pending smp-call-function callbacks queued on this CPU. This is
|
|
|
|
* invoked by the generic IPI handler, as well as by a CPU about to go offline,
|
|
|
|
* to ensure that all pending IPI callbacks are run before it goes completely
|
|
|
|
* offline.
|
|
|
|
*
|
|
|
|
* Loop through the call_single_queue and run all the queued callbacks.
|
|
|
|
* Must be called with interrupts disabled.
|
|
|
|
*/
|
2022-04-13 21:31:03 +08:00
|
|
|
static void __flush_smp_call_function_queue(bool warn_cpu_offline)
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-24 04:22:02 +08:00
|
|
|
{
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-08 12:30:00 +08:00
|
|
|
call_single_data_t *csd, *csd_next;
|
2020-05-27 00:10:59 +08:00
|
|
|
struct llist_node *entry, *prev;
|
|
|
|
struct llist_head *head;
|
smp: print more useful debug info upon receiving IPI on an offline CPU
There is a longstanding problem related to CPU hotplug which causes IPIs
to be delivered to offline CPUs, and the smp-call-function IPI handler
code prints out a warning whenever this is detected. Every once in a
while this (usually harmless) warning gets reported on LKML, but so far
it has not been completely fixed. Usually the solution involves finding
out the IPI sender and fixing it by adding appropriate synchronization
with CPU hotplug.
However, while going through one such internal bug reports, I found that
there is a significant bug in the receiver side itself (more
specifically, in stop-machine) that can lead to this problem even when
the sender code is perfectly fine. This patchset fixes that
synchronization problem in the CPU hotplug stop-machine code.
Patch 1 adds some additional debug code to the smp-call-function
framework, to help debug such issues easily.
Patch 2 modifies the stop-machine code to ensure that any IPIs that were
sent while the target CPU was online, would be noticed and handled by
that CPU without fail before it goes offline. Thus, this avoids
scenarios where IPIs are received on offline CPUs (as long as the sender
uses proper hotplug synchronization).
In fact, I debugged the problem by using Patch 1, and found that the
payload of the IPI was always the block layer's trigger_softirq()
function. But I was not able to find anything wrong with the block
layer code. That's when I started looking at the stop-machine code and
realized that there is a race-window which makes the IPI _receiver_ the
culprit, not the sender. Patch 2 fixes that race and hence this should
put an end to most of the hard-to-debug IPI-to-offline-CPU issues.
This patch (of 2):
Today the smp-call-function code just prints a warning if we get an IPI
on an offline CPU. This info is sufficient to let us know that
something went wrong, but often it is very hard to debug exactly who
sent the IPI and why, from this info alone.
In most cases, we get the warning about the IPI to an offline CPU,
immediately after the CPU going offline comes out of the stop-machine
phase and reenables interrupts. Since all online CPUs participate in
stop-machine, the information regarding the sender of the IPI is already
lost by the time we exit the stop-machine loop. So even if we dump the
stack on each CPU at this point, we won't find anything useful since all
of them will show the stack-trace of the stopper thread. So we need a
better way to figure out who sent the IPI and why.
To achieve this, when we detect an IPI targeted to an offline CPU, loop
through the call-single-data linked list and print out the payload
(i.e., the name of the function which was supposed to be executed by the
target CPU). This would give us an insight as to who might have sent
the IPI and help us debug this further.
[akpm@linux-foundation.org: correctly suppress warning output on second and later occurrences]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-07 05:37:05 +08:00
|
|
|
static bool warned;
|
|
|
|
|
2017-11-06 23:01:22 +08:00
|
|
|
lockdep_assert_irqs_disabled();
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-24 04:22:02 +08:00
|
|
|
|
2014-08-18 01:30:24 +08:00
|
|
|
head = this_cpu_ptr(&call_single_queue);
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->handle, CFD_SEQ_NOCPU,
|
|
|
|
smp_processor_id(), CFD_SEQ_HANDLE);
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-24 04:22:02 +08:00
|
|
|
entry = llist_del_all(head);
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->dequeue,
|
|
|
|
/* Special meaning of source cpu: 0 == queue empty */
|
|
|
|
entry ? CFD_SEQ_NOCPU : 0,
|
|
|
|
smp_processor_id(), CFD_SEQ_DEQUEUE);
|
smp: print more useful debug info upon receiving IPI on an offline CPU
There is a longstanding problem related to CPU hotplug which causes IPIs
to be delivered to offline CPUs, and the smp-call-function IPI handler
code prints out a warning whenever this is detected. Every once in a
while this (usually harmless) warning gets reported on LKML, but so far
it has not been completely fixed. Usually the solution involves finding
out the IPI sender and fixing it by adding appropriate synchronization
with CPU hotplug.
However, while going through one such internal bug reports, I found that
there is a significant bug in the receiver side itself (more
specifically, in stop-machine) that can lead to this problem even when
the sender code is perfectly fine. This patchset fixes that
synchronization problem in the CPU hotplug stop-machine code.
Patch 1 adds some additional debug code to the smp-call-function
framework, to help debug such issues easily.
Patch 2 modifies the stop-machine code to ensure that any IPIs that were
sent while the target CPU was online, would be noticed and handled by
that CPU without fail before it goes offline. Thus, this avoids
scenarios where IPIs are received on offline CPUs (as long as the sender
uses proper hotplug synchronization).
In fact, I debugged the problem by using Patch 1, and found that the
payload of the IPI was always the block layer's trigger_softirq()
function. But I was not able to find anything wrong with the block
layer code. That's when I started looking at the stop-machine code and
realized that there is a race-window which makes the IPI _receiver_ the
culprit, not the sender. Patch 2 fixes that race and hence this should
put an end to most of the hard-to-debug IPI-to-offline-CPU issues.
This patch (of 2):
Today the smp-call-function code just prints a warning if we get an IPI
on an offline CPU. This info is sufficient to let us know that
something went wrong, but often it is very hard to debug exactly who
sent the IPI and why, from this info alone.
In most cases, we get the warning about the IPI to an offline CPU,
immediately after the CPU going offline comes out of the stop-machine
phase and reenables interrupts. Since all online CPUs participate in
stop-machine, the information regarding the sender of the IPI is already
lost by the time we exit the stop-machine loop. So even if we dump the
stack on each CPU at this point, we won't find anything useful since all
of them will show the stack-trace of the stopper thread. So we need a
better way to figure out who sent the IPI and why.
To achieve this, when we detect an IPI targeted to an offline CPU, loop
through the call-single-data linked list and print out the payload
(i.e., the name of the function which was supposed to be executed by the
target CPU). This would give us an insight as to who might have sent
the IPI and help us debug this further.
[akpm@linux-foundation.org: correctly suppress warning output on second and later occurrences]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-07 05:37:05 +08:00
|
|
|
entry = llist_reverse_order(entry);
|
2008-06-26 17:21:34 +08:00
|
|
|
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-24 04:22:02 +08:00
|
|
|
/* There shouldn't be any pending callbacks on an offline CPU. */
|
|
|
|
if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
|
2022-03-19 15:20:15 +08:00
|
|
|
!warned && entry != NULL)) {
|
smp: print more useful debug info upon receiving IPI on an offline CPU
There is a longstanding problem related to CPU hotplug which causes IPIs
to be delivered to offline CPUs, and the smp-call-function IPI handler
code prints out a warning whenever this is detected. Every once in a
while this (usually harmless) warning gets reported on LKML, but so far
it has not been completely fixed. Usually the solution involves finding
out the IPI sender and fixing it by adding appropriate synchronization
with CPU hotplug.
However, while going through one such internal bug reports, I found that
there is a significant bug in the receiver side itself (more
specifically, in stop-machine) that can lead to this problem even when
the sender code is perfectly fine. This patchset fixes that
synchronization problem in the CPU hotplug stop-machine code.
Patch 1 adds some additional debug code to the smp-call-function
framework, to help debug such issues easily.
Patch 2 modifies the stop-machine code to ensure that any IPIs that were
sent while the target CPU was online, would be noticed and handled by
that CPU without fail before it goes offline. Thus, this avoids
scenarios where IPIs are received on offline CPUs (as long as the sender
uses proper hotplug synchronization).
In fact, I debugged the problem by using Patch 1, and found that the
payload of the IPI was always the block layer's trigger_softirq()
function. But I was not able to find anything wrong with the block
layer code. That's when I started looking at the stop-machine code and
realized that there is a race-window which makes the IPI _receiver_ the
culprit, not the sender. Patch 2 fixes that race and hence this should
put an end to most of the hard-to-debug IPI-to-offline-CPU issues.
This patch (of 2):
Today the smp-call-function code just prints a warning if we get an IPI
on an offline CPU. This info is sufficient to let us know that
something went wrong, but often it is very hard to debug exactly who
sent the IPI and why, from this info alone.
In most cases, we get the warning about the IPI to an offline CPU,
immediately after the CPU going offline comes out of the stop-machine
phase and reenables interrupts. Since all online CPUs participate in
stop-machine, the information regarding the sender of the IPI is already
lost by the time we exit the stop-machine loop. So even if we dump the
stack on each CPU at this point, we won't find anything useful since all
of them will show the stack-trace of the stopper thread. So we need a
better way to figure out who sent the IPI and why.
To achieve this, when we detect an IPI targeted to an offline CPU, loop
through the call-single-data linked list and print out the payload
(i.e., the name of the function which was supposed to be executed by the
target CPU). This would give us an insight as to who might have sent
the IPI and help us debug this further.
[akpm@linux-foundation.org: correctly suppress warning output on second and later occurrences]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-07 05:37:05 +08:00
|
|
|
warned = true;
|
|
|
|
WARN(1, "IPI on offline CPU %d\n", smp_processor_id());
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't have to use the _safe() variant here
|
|
|
|
* because we are not invoking the IPI handlers yet.
|
|
|
|
*/
|
2020-06-15 17:29:31 +08:00
|
|
|
llist_for_each_entry(csd, entry, node.llist) {
|
2020-05-27 00:11:02 +08:00
|
|
|
switch (CSD_TYPE(csd)) {
|
|
|
|
case CSD_TYPE_ASYNC:
|
|
|
|
case CSD_TYPE_SYNC:
|
|
|
|
case CSD_TYPE_IRQ_WORK:
|
|
|
|
pr_warn("IPI callback %pS sent to offline CPU\n",
|
|
|
|
csd->func);
|
|
|
|
break;
|
|
|
|
|
2020-05-27 00:11:04 +08:00
|
|
|
case CSD_TYPE_TTWU:
|
|
|
|
pr_warn("IPI task-wakeup sent to offline CPU\n");
|
|
|
|
break;
|
|
|
|
|
2020-05-27 00:11:02 +08:00
|
|
|
default:
|
|
|
|
pr_warn("IPI callback, unknown type %d, sent to offline CPU\n",
|
|
|
|
CSD_TYPE(csd));
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
smp: print more useful debug info upon receiving IPI on an offline CPU
There is a longstanding problem related to CPU hotplug which causes IPIs
to be delivered to offline CPUs, and the smp-call-function IPI handler
code prints out a warning whenever this is detected. Every once in a
while this (usually harmless) warning gets reported on LKML, but so far
it has not been completely fixed. Usually the solution involves finding
out the IPI sender and fixing it by adding appropriate synchronization
with CPU hotplug.
However, while going through one such internal bug reports, I found that
there is a significant bug in the receiver side itself (more
specifically, in stop-machine) that can lead to this problem even when
the sender code is perfectly fine. This patchset fixes that
synchronization problem in the CPU hotplug stop-machine code.
Patch 1 adds some additional debug code to the smp-call-function
framework, to help debug such issues easily.
Patch 2 modifies the stop-machine code to ensure that any IPIs that were
sent while the target CPU was online, would be noticed and handled by
that CPU without fail before it goes offline. Thus, this avoids
scenarios where IPIs are received on offline CPUs (as long as the sender
uses proper hotplug synchronization).
In fact, I debugged the problem by using Patch 1, and found that the
payload of the IPI was always the block layer's trigger_softirq()
function. But I was not able to find anything wrong with the block
layer code. That's when I started looking at the stop-machine code and
realized that there is a race-window which makes the IPI _receiver_ the
culprit, not the sender. Patch 2 fixes that race and hence this should
put an end to most of the hard-to-debug IPI-to-offline-CPU issues.
This patch (of 2):
Today the smp-call-function code just prints a warning if we get an IPI
on an offline CPU. This info is sufficient to let us know that
something went wrong, but often it is very hard to debug exactly who
sent the IPI and why, from this info alone.
In most cases, we get the warning about the IPI to an offline CPU,
immediately after the CPU going offline comes out of the stop-machine
phase and reenables interrupts. Since all online CPUs participate in
stop-machine, the information regarding the sender of the IPI is already
lost by the time we exit the stop-machine loop. So even if we dump the
stack on each CPU at this point, we won't find anything useful since all
of them will show the stack-trace of the stopper thread. So we need a
better way to figure out who sent the IPI and why.
To achieve this, when we detect an IPI targeted to an offline CPU, loop
through the call-single-data linked list and print out the payload
(i.e., the name of the function which was supposed to be executed by the
target CPU). This would give us an insight as to who might have sent
the IPI and help us debug this further.
[akpm@linux-foundation.org: correctly suppress warning output on second and later occurrences]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-07 05:37:05 +08:00
|
|
|
}
|
2008-06-26 17:21:34 +08:00
|
|
|
|
2020-05-27 00:10:59 +08:00
|
|
|
/*
|
|
|
|
* First; run all SYNC callbacks, people are waiting for us.
|
|
|
|
*/
|
|
|
|
prev = NULL;
|
2020-06-15 17:29:31 +08:00
|
|
|
llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
|
2015-02-12 04:42:10 +08:00
|
|
|
/* Do we wait until *after* callback? */
|
2020-05-27 00:11:02 +08:00
|
|
|
if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {
|
|
|
|
smp_call_func_t func = csd->func;
|
|
|
|
void *info = csd->info;
|
|
|
|
|
2020-05-27 00:10:59 +08:00
|
|
|
if (prev) {
|
2020-06-15 17:29:31 +08:00
|
|
|
prev->next = &csd_next->node.llist;
|
2020-05-27 00:10:59 +08:00
|
|
|
} else {
|
2020-06-15 17:29:31 +08:00
|
|
|
entry = &csd_next->node.llist;
|
2020-05-27 00:10:59 +08:00
|
|
|
}
|
2020-05-27 00:11:02 +08:00
|
|
|
|
2020-07-01 04:22:54 +08:00
|
|
|
csd_lock_record(csd);
|
2015-02-12 04:42:10 +08:00
|
|
|
func(info);
|
|
|
|
csd_unlock(csd);
|
2020-07-01 04:22:54 +08:00
|
|
|
csd_lock_record(NULL);
|
2015-02-12 04:42:10 +08:00
|
|
|
} else {
|
2020-06-15 17:29:31 +08:00
|
|
|
prev = &csd->node.llist;
|
2015-02-12 04:42:10 +08:00
|
|
|
}
|
2008-06-26 17:21:34 +08:00
|
|
|
}
|
2014-05-08 07:37:48 +08:00
|
|
|
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
if (!entry) {
|
|
|
|
cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->hdlend,
|
|
|
|
0, smp_processor_id(),
|
|
|
|
CFD_SEQ_HDLEND);
|
2020-05-27 00:11:04 +08:00
|
|
|
return;
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
}
|
2020-05-27 00:11:04 +08:00
|
|
|
|
2014-05-08 07:37:48 +08:00
|
|
|
/*
|
2020-05-27 00:10:59 +08:00
|
|
|
* Second; run all !SYNC callbacks.
|
2014-05-08 07:37:48 +08:00
|
|
|
*/
|
2020-05-27 00:11:04 +08:00
|
|
|
prev = NULL;
|
2020-06-15 17:29:31 +08:00
|
|
|
llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
|
2020-05-27 00:11:02 +08:00
|
|
|
int type = CSD_TYPE(csd);
|
2020-05-27 00:10:59 +08:00
|
|
|
|
2020-05-27 00:11:04 +08:00
|
|
|
if (type != CSD_TYPE_TTWU) {
|
|
|
|
if (prev) {
|
2020-06-15 17:29:31 +08:00
|
|
|
prev->next = &csd_next->node.llist;
|
2020-05-27 00:11:04 +08:00
|
|
|
} else {
|
2020-06-15 17:29:31 +08:00
|
|
|
entry = &csd_next->node.llist;
|
2020-05-27 00:11:04 +08:00
|
|
|
}
|
2020-05-27 00:11:02 +08:00
|
|
|
|
2020-05-27 00:11:04 +08:00
|
|
|
if (type == CSD_TYPE_ASYNC) {
|
|
|
|
smp_call_func_t func = csd->func;
|
|
|
|
void *info = csd->info;
|
|
|
|
|
2020-07-01 04:22:54 +08:00
|
|
|
csd_lock_record(csd);
|
2020-05-27 00:11:04 +08:00
|
|
|
csd_unlock(csd);
|
|
|
|
func(info);
|
2020-07-01 04:22:54 +08:00
|
|
|
csd_lock_record(NULL);
|
2020-05-27 00:11:04 +08:00
|
|
|
} else if (type == CSD_TYPE_IRQ_WORK) {
|
|
|
|
irq_work_single(csd);
|
|
|
|
}
|
|
|
|
|
|
|
|
} else {
|
2020-06-15 17:29:31 +08:00
|
|
|
prev = &csd->node.llist;
|
2020-05-27 00:11:02 +08:00
|
|
|
}
|
2020-05-27 00:10:59 +08:00
|
|
|
}
|
2020-05-27 00:11:04 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Third; only CSD_TYPE_TTWU is left, issue those.
|
|
|
|
*/
|
|
|
|
if (entry)
|
|
|
|
sched_ttwu_pending(entry);
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
|
|
|
|
cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->hdlend, CFD_SEQ_NOCPU,
|
|
|
|
smp_processor_id(), CFD_SEQ_HDLEND);
|
2008-06-26 17:21:34 +08:00
|
|
|
}
|
|
|
|
|
2022-04-13 21:31:03 +08:00
|
|
|
|
|
|
|
/**
|
|
|
|
* flush_smp_call_function_queue - Flush pending smp-call-function callbacks
|
|
|
|
* from task context (idle, migration thread)
|
|
|
|
*
|
|
|
|
* When TIF_POLLING_NRFLAG is supported and a CPU is in idle and has it
|
|
|
|
* set, then remote CPUs can avoid sending IPIs and wake the idle CPU by
|
|
|
|
* setting TIF_NEED_RESCHED. The idle task on the woken up CPU has to
|
|
|
|
* handle queued SMP function calls before scheduling.
|
|
|
|
*
|
|
|
|
* The migration thread has to ensure that an eventually pending wakeup has
|
|
|
|
* been handled before it migrates a task.
|
|
|
|
*/
|
|
|
|
void flush_smp_call_function_queue(void)
|
2020-05-27 00:11:01 +08:00
|
|
|
{
|
2022-04-13 21:31:05 +08:00
|
|
|
unsigned int was_pending;
|
2020-05-27 00:11:01 +08:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
if (llist_empty(this_cpu_ptr(&call_single_queue)))
|
|
|
|
return;
|
|
|
|
|
locking/csd_lock: Add more data to CSD lock debugging
In order to help identifying problems with IPI handling and remote
function execution add some more data to IPI debugging code.
There have been multiple reports of CPUs looping long times (many
seconds) in smp_call_function_many() waiting for another CPU executing
a function like tlb flushing. Most of these reports have been for
cases where the kernel was running as a guest on top of KVM or Xen
(there are rumours of that happening under VMWare, too, and even on
bare metal).
Finding the root cause hasn't been successful yet, even after more than
2 years of chasing this bug by different developers.
Commit:
35feb60474bf4f7 ("kernel/smp: Provide CSD lock timeout diagnostics")
tried to address this by adding some debug code and by issuing another
IPI when a hang was detected. This helped mitigating the problem
(the repeated IPI unlocks the hang), but the root cause is still unknown.
Current available data suggests that either an IPI wasn't sent when it
should have been, or that the IPI didn't result in the target CPU
executing the queued function (due to the IPI not reaching the CPU,
the IPI handler not being called, or the handler not seeing the queued
request).
Try to add more diagnostic data by introducing a global atomic counter
which is being incremented when doing critical operations (before and
after queueing a new request, when sending an IPI, and when dequeueing
a request). The counter value is stored in percpu variables which can
be printed out when a hang is detected.
The data of the last event (consisting of sequence counter, source
CPU, target CPU, and event type) is stored in a global variable. When
a new event is to be traced, the data of the last event is stored in
the event related percpu location and the global data is updated with
the new event's data. This allows to track two events in one data
location: one by the value of the event data (the event before the
current one), and one by the location itself (the current event).
A typical printout with a detected hang will look like this:
csd: Detected non-responsive CSD lock (#1) on CPU#1, waiting 5000000003 ns for CPU#06 scf_handler_1+0x0/0x50(0xffffa2a881bb1410).
csd: CSD lock (#1) handling prior scf_handler_1+0x0/0x50(0xffffa2a8813823c0) request.
csd: cnt(00008cc): ffff->0000 dequeue (src cpu 0 == empty)
csd: cnt(00008cd): ffff->0006 idle
csd: cnt(0003668): 0001->0006 queue
csd: cnt(0003669): 0001->0006 ipi
csd: cnt(0003e0f): 0007->000a queue
csd: cnt(0003e10): 0001->ffff ping
csd: cnt(0003e71): 0003->0000 ping
csd: cnt(0003e72): ffff->0006 gotipi
csd: cnt(0003e73): ffff->0006 handle
csd: cnt(0003e74): ffff->0006 dequeue (src cpu 0 == empty)
csd: cnt(0003e7f): 0004->0006 ping
csd: cnt(0003e80): 0001->ffff pinged
csd: cnt(0003eb2): 0005->0001 noipi
csd: cnt(0003eb3): 0001->0006 queue
csd: cnt(0003eb4): 0001->0006 noipi
csd: cnt now: 0003f00
The idea is to print only relevant entries. Those are all events which
are associated with the hang (so sender side events for the source CPU
of the hanging request, and receiver side events for the target CPU),
and the related events just before those (for adding data needed to
identify a possible race). Printing all available data would be
possible, but this would add large amounts of data printed on larger
configurations.
Signed-off-by: Juergen Gross <jgross@suse.com>
[ Minor readability edits. Breaks col80 but is far more readable. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20210301101336.7797-4-jgross@suse.com
2021-03-01 18:13:36 +08:00
|
|
|
cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->idle, CFD_SEQ_NOCPU,
|
|
|
|
smp_processor_id(), CFD_SEQ_IDLE);
|
2020-05-27 00:11:01 +08:00
|
|
|
local_irq_save(flags);
|
2022-04-13 21:31:05 +08:00
|
|
|
/* Get the already pending soft interrupts for RT enabled kernels */
|
|
|
|
was_pending = local_softirq_pending();
|
2022-04-13 21:31:03 +08:00
|
|
|
__flush_smp_call_function_queue(true);
|
2021-01-24 04:10:25 +08:00
|
|
|
if (local_softirq_pending())
|
2022-04-13 21:31:05 +08:00
|
|
|
do_softirq_post_smp_call_flush(was_pending);
|
2021-01-24 04:10:25 +08:00
|
|
|
|
2020-05-27 00:11:01 +08:00
|
|
|
local_irq_restore(flags);
|
2008-06-26 17:21:34 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* smp_call_function_single - Run a function on a specific CPU
|
|
|
|
* @func: The function to run. This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to the function.
|
|
|
|
* @wait: If true, wait until function has completed on other CPUs.
|
|
|
|
*
|
2009-10-22 19:19:34 +08:00
|
|
|
* Returns 0 on success, else a negative status code.
|
2008-06-26 17:21:34 +08:00
|
|
|
*/
|
2010-10-28 00:28:36 +08:00
|
|
|
int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
|
2008-06-06 17:18:06 +08:00
|
|
|
int wait)
|
2008-06-26 17:21:34 +08:00
|
|
|
{
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-08 12:30:00 +08:00
|
|
|
call_single_data_t *csd;
|
|
|
|
call_single_data_t csd_stack = {
|
2020-06-15 17:29:31 +08:00
|
|
|
.node = { .u_flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC, },
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-08 12:30:00 +08:00
|
|
|
};
|
2009-02-25 23:52:11 +08:00
|
|
|
int this_cpu;
|
2014-02-24 23:39:58 +08:00
|
|
|
int err;
|
2008-06-26 17:21:34 +08:00
|
|
|
|
2009-02-25 23:52:11 +08:00
|
|
|
/*
|
|
|
|
* prevent preemption and reschedule on another processor,
|
|
|
|
* as well as CPU removal
|
|
|
|
*/
|
|
|
|
this_cpu = get_cpu();
|
|
|
|
|
2009-08-20 09:05:35 +08:00
|
|
|
/*
|
|
|
|
* Can deadlock when called with interrupts disabled.
|
|
|
|
* We allow cpu's that are not yet online though, as no one else can
|
|
|
|
* send smp call function interrupt to this cpu and as such deadlocks
|
|
|
|
* can't happen.
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
|
|
|
|
&& !oops_in_progress);
|
2008-06-26 17:21:34 +08:00
|
|
|
|
2019-07-18 17:20:09 +08:00
|
|
|
/*
|
|
|
|
* When @wait we can deadlock when we interrupt between llist_add() and
|
|
|
|
* arch_send_call_function_ipi*(); when !@wait we can deadlock due to
|
|
|
|
* csd_lock() on because the interrupt context uses the same csd
|
|
|
|
* storage.
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(!in_task());
|
|
|
|
|
2015-02-12 04:42:10 +08:00
|
|
|
csd = &csd_stack;
|
|
|
|
if (!wait) {
|
|
|
|
csd = this_cpu_ptr(&csd_data);
|
|
|
|
csd_lock(csd);
|
|
|
|
}
|
|
|
|
|
2020-05-27 00:11:02 +08:00
|
|
|
csd->func = func;
|
|
|
|
csd->info = info;
|
2020-07-01 04:22:54 +08:00
|
|
|
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
|
2020-06-15 17:29:31 +08:00
|
|
|
csd->node.src = smp_processor_id();
|
|
|
|
csd->node.dst = cpu;
|
2020-06-30 08:21:32 +08:00
|
|
|
#endif
|
2020-05-27 00:11:02 +08:00
|
|
|
|
|
|
|
err = generic_exec_single(cpu, csd);
|
2015-02-12 04:42:10 +08:00
|
|
|
|
|
|
|
if (wait)
|
|
|
|
csd_lock_wait(csd);
|
2008-06-26 17:21:34 +08:00
|
|
|
|
|
|
|
put_cpu();
|
2009-02-25 23:52:11 +08:00
|
|
|
|
2008-08-26 08:07:14 +08:00
|
|
|
return err;
|
2008-06-26 17:21:34 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(smp_call_function_single);
|
|
|
|
|
2014-02-24 23:39:59 +08:00
|
|
|
/**
|
2021-08-11 06:50:51 +08:00
|
|
|
* smp_call_function_single_async() - Run an asynchronous function on a
|
2014-02-24 23:40:02 +08:00
|
|
|
* specific CPU.
|
2014-02-24 23:39:59 +08:00
|
|
|
* @cpu: The CPU to run on.
|
|
|
|
* @csd: Pre-allocated and setup data structure
|
|
|
|
*
|
2014-02-24 23:40:02 +08:00
|
|
|
* Like smp_call_function_single(), but the call is asynchonous and
|
|
|
|
* can thus be done from contexts with disabled interrupts.
|
|
|
|
*
|
|
|
|
* The caller passes his own pre-allocated data structure
|
|
|
|
* (ie: embedded in an object) and is responsible for synchronizing it
|
|
|
|
* such that the IPIs performed on the @csd are strictly serialized.
|
|
|
|
*
|
2019-12-17 05:31:23 +08:00
|
|
|
* If the function is called with one csd which has not yet been
|
|
|
|
* processed by previous call to smp_call_function_single_async(), the
|
|
|
|
* function will return immediately with -EBUSY showing that the csd
|
|
|
|
* object is still in progress.
|
|
|
|
*
|
2014-02-24 23:40:02 +08:00
|
|
|
* NOTE: Be careful, there is unfortunately no current debugging facility to
|
|
|
|
* validate the correctness of this serialization.
|
2021-08-11 06:50:51 +08:00
|
|
|
*
|
|
|
|
* Return: %0 on success or negative errno value on error
|
2014-02-24 23:39:59 +08:00
|
|
|
*/
|
2021-05-06 05:12:42 +08:00
|
|
|
int smp_call_function_single_async(int cpu, struct __call_single_data *csd)
|
2014-02-24 23:39:59 +08:00
|
|
|
{
|
|
|
|
int err = 0;
|
|
|
|
|
2014-02-24 23:40:01 +08:00
|
|
|
preempt_disable();
|
2015-02-12 04:42:10 +08:00
|
|
|
|
2020-06-15 17:29:31 +08:00
|
|
|
if (csd->node.u_flags & CSD_FLAG_LOCK) {
|
2019-12-17 05:31:23 +08:00
|
|
|
err = -EBUSY;
|
|
|
|
goto out;
|
|
|
|
}
|
2015-02-12 04:42:10 +08:00
|
|
|
|
2020-06-15 17:29:31 +08:00
|
|
|
csd->node.u_flags = CSD_FLAG_LOCK;
|
2015-02-12 04:42:10 +08:00
|
|
|
smp_wmb();
|
|
|
|
|
2020-05-27 00:11:02 +08:00
|
|
|
err = generic_exec_single(cpu, csd);
|
2019-12-17 05:31:23 +08:00
|
|
|
|
|
|
|
out:
|
2014-02-24 23:40:01 +08:00
|
|
|
preempt_enable();
|
2014-02-24 23:39:59 +08:00
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
2014-02-24 23:40:02 +08:00
|
|
|
EXPORT_SYMBOL_GPL(smp_call_function_single_async);
|
2014-02-24 23:39:59 +08:00
|
|
|
|
2009-11-18 06:27:27 +08:00
|
|
|
/*
|
|
|
|
* smp_call_function_any - Run a function on any of the given cpus
|
|
|
|
* @mask: The mask of cpus it can run on.
|
|
|
|
* @func: The function to run. This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to the function.
|
|
|
|
* @wait: If true, wait until function has completed.
|
|
|
|
*
|
|
|
|
* Returns 0 on success, else a negative status code (if no cpus were online).
|
|
|
|
*
|
|
|
|
* Selection preference:
|
|
|
|
* 1) current cpu if in @mask
|
|
|
|
* 2) any cpu of current node if in @mask
|
|
|
|
* 3) any other online cpu in @mask
|
|
|
|
*/
|
|
|
|
int smp_call_function_any(const struct cpumask *mask,
|
2010-10-28 00:28:36 +08:00
|
|
|
smp_call_func_t func, void *info, int wait)
|
2009-11-18 06:27:27 +08:00
|
|
|
{
|
|
|
|
unsigned int cpu;
|
|
|
|
const struct cpumask *nodemask;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
/* Try for same CPU (cheapest) */
|
|
|
|
cpu = get_cpu();
|
|
|
|
if (cpumask_test_cpu(cpu, mask))
|
|
|
|
goto call;
|
|
|
|
|
|
|
|
/* Try for same node. */
|
2010-01-16 09:01:23 +08:00
|
|
|
nodemask = cpumask_of_node(cpu_to_node(cpu));
|
2009-11-18 06:27:27 +08:00
|
|
|
for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids;
|
|
|
|
cpu = cpumask_next_and(cpu, nodemask, mask)) {
|
|
|
|
if (cpu_online(cpu))
|
|
|
|
goto call;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Any online will do: smp_call_function_single handles nr_cpu_ids. */
|
|
|
|
cpu = cpumask_any_and(mask, cpu_online_mask);
|
|
|
|
call:
|
|
|
|
ret = smp_call_function_single(cpu, func, info, wait);
|
|
|
|
put_cpu();
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(smp_call_function_any);
|
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
/*
|
|
|
|
* Flags to be used as scf_flags argument of smp_call_function_many_cond().
|
|
|
|
*
|
|
|
|
* %SCF_WAIT: Wait until function execution is completed
|
|
|
|
* %SCF_RUN_LOCAL: Run also locally if local cpu is set in cpumask
|
|
|
|
*/
|
|
|
|
#define SCF_WAIT (1U << 0)
|
|
|
|
#define SCF_RUN_LOCAL (1U << 1)
|
|
|
|
|
2020-01-17 17:01:36 +08:00
|
|
|
static void smp_call_function_many_cond(const struct cpumask *mask,
|
|
|
|
smp_call_func_t func, void *info,
|
2021-02-21 07:17:04 +08:00
|
|
|
unsigned int scf_flags,
|
|
|
|
smp_cond_func_t cond_func)
|
2008-06-26 17:21:34 +08:00
|
|
|
{
|
2021-02-21 07:17:04 +08:00
|
|
|
int cpu, last_cpu, this_cpu = smp_processor_id();
|
2013-05-01 06:27:28 +08:00
|
|
|
struct call_function_data *cfd;
|
2021-02-21 07:17:04 +08:00
|
|
|
bool wait = scf_flags & SCF_WAIT;
|
|
|
|
bool run_remote = false;
|
|
|
|
bool run_local = false;
|
|
|
|
int nr_cpus = 0;
|
|
|
|
|
|
|
|
lockdep_assert_preemption_disabled();
|
2008-06-26 17:21:34 +08:00
|
|
|
|
2009-08-20 09:05:35 +08:00
|
|
|
/*
|
|
|
|
* Can deadlock when called with interrupts disabled.
|
|
|
|
* We allow cpu's that are not yet online though, as no one else can
|
|
|
|
* send smp call function interrupt to this cpu and as such deadlocks
|
|
|
|
* can't happen.
|
|
|
|
*/
|
2021-02-21 07:17:04 +08:00
|
|
|
if (cpu_online(this_cpu) && !oops_in_progress &&
|
|
|
|
!early_boot_irqs_disabled)
|
|
|
|
lockdep_assert_irqs_enabled();
|
2008-06-26 17:21:34 +08:00
|
|
|
|
2019-07-18 17:20:09 +08:00
|
|
|
/*
|
|
|
|
* When @wait we can deadlock when we interrupt between llist_add() and
|
|
|
|
* arch_send_call_function_ipi*(); when !@wait we can deadlock due to
|
|
|
|
* csd_lock() on because the interrupt context uses the same csd
|
|
|
|
* storage.
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(!in_task());
|
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
/* Check if we need local execution. */
|
|
|
|
if ((scf_flags & SCF_RUN_LOCAL) && cpumask_test_cpu(this_cpu, mask))
|
|
|
|
run_local = true;
|
|
|
|
|
|
|
|
/* Check if we need remote execution, i.e., any CPU excluding this one. */
|
2008-12-30 06:35:16 +08:00
|
|
|
cpu = cpumask_first_and(mask, cpu_online_mask);
|
2009-02-25 23:52:11 +08:00
|
|
|
if (cpu == this_cpu)
|
2008-12-30 06:35:16 +08:00
|
|
|
cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
|
2021-02-21 07:17:04 +08:00
|
|
|
if (cpu < nr_cpu_ids)
|
|
|
|
run_remote = true;
|
2009-02-25 23:52:11 +08:00
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
if (run_remote) {
|
|
|
|
cfd = this_cpu_ptr(&cfd_data);
|
|
|
|
cpumask_and(cfd->cpumask, mask, cpu_online_mask);
|
|
|
|
__cpumask_clear_cpu(this_cpu, cfd->cpumask);
|
2011-03-16 03:27:16 +08:00
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
cpumask_clear(cfd->cpumask_ipi);
|
|
|
|
for_each_cpu(cpu, cfd->cpumask) {
|
2021-03-06 20:00:58 +08:00
|
|
|
struct cfd_percpu *pcpu = per_cpu_ptr(cfd->pcpu, cpu);
|
|
|
|
call_single_data_t *csd = &pcpu->csd;
|
smp: make smp_call_function_many() use logic similar to smp_call_function_single()
I'm testing swapout workload in a two-socket Xeon machine. The workload
has 10 threads, each thread sequentially accesses separate memory
region. TLB flush overhead is very big in the workload. For each page,
page reclaim need move it from active lru list and then unmap it. Both
need a TLB flush. And this is a multthread workload, TLB flush happens
in 10 CPUs. In X86, TLB flush uses generic smp_call)function. So this
workload stress smp_call_function_many heavily.
Without patch, perf shows:
+ 24.49% [k] generic_smp_call_function_interrupt
- 21.72% [k] _raw_spin_lock
- _raw_spin_lock
+ 79.80% __page_check_address
+ 6.42% generic_smp_call_function_interrupt
+ 3.31% get_swap_page
+ 2.37% free_pcppages_bulk
+ 1.75% handle_pte_fault
+ 1.54% put_super
+ 1.41% grab_super_passive
+ 1.36% __swap_duplicate
+ 0.68% blk_flush_plug_list
+ 0.62% swap_info_get
+ 6.55% [k] flush_tlb_func
+ 6.46% [k] smp_call_function_many
+ 5.09% [k] call_function_interrupt
+ 4.75% [k] default_send_IPI_mask_sequence_phys
+ 2.18% [k] find_next_bit
swapout throughput is around 1300M/s.
With the patch, perf shows:
- 27.23% [k] _raw_spin_lock
- _raw_spin_lock
+ 80.53% __page_check_address
+ 8.39% generic_smp_call_function_single_interrupt
+ 2.44% get_swap_page
+ 1.76% free_pcppages_bulk
+ 1.40% handle_pte_fault
+ 1.15% __swap_duplicate
+ 1.05% put_super
+ 0.98% grab_super_passive
+ 0.86% blk_flush_plug_list
+ 0.57% swap_info_get
+ 8.25% [k] default_send_IPI_mask_sequence_phys
+ 7.55% [k] call_function_interrupt
+ 7.47% [k] smp_call_function_many
+ 7.25% [k] flush_tlb_func
+ 3.81% [k] _raw_spin_lock_irqsave
+ 3.78% [k] generic_smp_call_function_single_interrupt
swapout throughput is around 1400M/s. So there is around a 7%
improvement, and total cpu utilization doesn't change.
Without the patch, cfd_data is shared by all CPUs.
generic_smp_call_function_interrupt does read/write cfd_data several times
which will create a lot of cache ping-pong. With the patch, the data
becomes per-cpu. The ping-pong is avoided. And from the perf data, this
doesn't make call_single_queue lock contend.
Next step is to remove generic_smp_call_function_interrupt() from arch
code.
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-22 08:43:03 +08:00
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
if (cond_func && !cond_func(cpu, info))
|
|
|
|
continue;
|
2020-01-17 17:01:36 +08:00
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
csd_lock(csd);
|
|
|
|
if (wait)
|
|
|
|
csd->node.u_flags |= CSD_TYPE_SYNC;
|
|
|
|
csd->func = func;
|
|
|
|
csd->info = info;
|
2020-07-01 04:22:54 +08:00
|
|
|
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
|
2021-02-21 07:17:04 +08:00
|
|
|
csd->node.src = smp_processor_id();
|
|
|
|
csd->node.dst = cpu;
|
2020-06-30 08:21:32 +08:00
|
|
|
#endif
|
2021-03-06 20:00:58 +08:00
|
|
|
cfd_seq_store(pcpu->seq_queue, this_cpu, cpu, CFD_SEQ_QUEUE);
|
2021-02-21 07:17:04 +08:00
|
|
|
if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) {
|
|
|
|
__cpumask_set_cpu(cpu, cfd->cpumask_ipi);
|
|
|
|
nr_cpus++;
|
|
|
|
last_cpu = cpu;
|
2011-03-16 03:27:17 +08:00
|
|
|
|
2021-03-06 20:00:58 +08:00
|
|
|
cfd_seq_store(pcpu->seq_ipi, this_cpu, cpu, CFD_SEQ_IPI);
|
|
|
|
} else {
|
|
|
|
cfd_seq_store(pcpu->seq_noipi, this_cpu, cpu, CFD_SEQ_NOIPI);
|
2021-02-21 07:17:04 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-06 20:00:58 +08:00
|
|
|
cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->ping, this_cpu, CFD_SEQ_NOCPU, CFD_SEQ_PING);
|
smp: make smp_call_function_many() use logic similar to smp_call_function_single()
I'm testing swapout workload in a two-socket Xeon machine. The workload
has 10 threads, each thread sequentially accesses separate memory
region. TLB flush overhead is very big in the workload. For each page,
page reclaim need move it from active lru list and then unmap it. Both
need a TLB flush. And this is a multthread workload, TLB flush happens
in 10 CPUs. In X86, TLB flush uses generic smp_call)function. So this
workload stress smp_call_function_many heavily.
Without patch, perf shows:
+ 24.49% [k] generic_smp_call_function_interrupt
- 21.72% [k] _raw_spin_lock
- _raw_spin_lock
+ 79.80% __page_check_address
+ 6.42% generic_smp_call_function_interrupt
+ 3.31% get_swap_page
+ 2.37% free_pcppages_bulk
+ 1.75% handle_pte_fault
+ 1.54% put_super
+ 1.41% grab_super_passive
+ 1.36% __swap_duplicate
+ 0.68% blk_flush_plug_list
+ 0.62% swap_info_get
+ 6.55% [k] flush_tlb_func
+ 6.46% [k] smp_call_function_many
+ 5.09% [k] call_function_interrupt
+ 4.75% [k] default_send_IPI_mask_sequence_phys
+ 2.18% [k] find_next_bit
swapout throughput is around 1300M/s.
With the patch, perf shows:
- 27.23% [k] _raw_spin_lock
- _raw_spin_lock
+ 80.53% __page_check_address
+ 8.39% generic_smp_call_function_single_interrupt
+ 2.44% get_swap_page
+ 1.76% free_pcppages_bulk
+ 1.40% handle_pte_fault
+ 1.15% __swap_duplicate
+ 1.05% put_super
+ 0.98% grab_super_passive
+ 0.86% blk_flush_plug_list
+ 0.57% swap_info_get
+ 8.25% [k] default_send_IPI_mask_sequence_phys
+ 7.55% [k] call_function_interrupt
+ 7.47% [k] smp_call_function_many
+ 7.25% [k] flush_tlb_func
+ 3.81% [k] _raw_spin_lock_irqsave
+ 3.78% [k] generic_smp_call_function_single_interrupt
swapout throughput is around 1400M/s. So there is around a 7%
improvement, and total cpu utilization doesn't change.
Without the patch, cfd_data is shared by all CPUs.
generic_smp_call_function_interrupt does read/write cfd_data several times
which will create a lot of cache ping-pong. With the patch, the data
becomes per-cpu. The ping-pong is avoided. And from the perf data, this
doesn't make call_single_queue lock contend.
Next step is to remove generic_smp_call_function_interrupt() from arch
code.
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-22 08:43:03 +08:00
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
/*
|
|
|
|
* Choose the most efficient way to send an IPI. Note that the
|
|
|
|
* number of CPUs might be zero due to concurrent changes to the
|
|
|
|
* provided mask.
|
|
|
|
*/
|
|
|
|
if (nr_cpus == 1)
|
2021-03-02 15:02:43 +08:00
|
|
|
send_call_function_single_ipi(last_cpu);
|
2021-02-21 07:17:04 +08:00
|
|
|
else if (likely(nr_cpus > 1))
|
|
|
|
arch_send_call_function_ipi_mask(cfd->cpumask_ipi);
|
2020-01-17 17:01:36 +08:00
|
|
|
|
2021-03-06 20:00:58 +08:00
|
|
|
cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->pinged, this_cpu, CFD_SEQ_NOCPU, CFD_SEQ_PINGED);
|
smp: make smp_call_function_many() use logic similar to smp_call_function_single()
I'm testing swapout workload in a two-socket Xeon machine. The workload
has 10 threads, each thread sequentially accesses separate memory
region. TLB flush overhead is very big in the workload. For each page,
page reclaim need move it from active lru list and then unmap it. Both
need a TLB flush. And this is a multthread workload, TLB flush happens
in 10 CPUs. In X86, TLB flush uses generic smp_call)function. So this
workload stress smp_call_function_many heavily.
Without patch, perf shows:
+ 24.49% [k] generic_smp_call_function_interrupt
- 21.72% [k] _raw_spin_lock
- _raw_spin_lock
+ 79.80% __page_check_address
+ 6.42% generic_smp_call_function_interrupt
+ 3.31% get_swap_page
+ 2.37% free_pcppages_bulk
+ 1.75% handle_pte_fault
+ 1.54% put_super
+ 1.41% grab_super_passive
+ 1.36% __swap_duplicate
+ 0.68% blk_flush_plug_list
+ 0.62% swap_info_get
+ 6.55% [k] flush_tlb_func
+ 6.46% [k] smp_call_function_many
+ 5.09% [k] call_function_interrupt
+ 4.75% [k] default_send_IPI_mask_sequence_phys
+ 2.18% [k] find_next_bit
swapout throughput is around 1300M/s.
With the patch, perf shows:
- 27.23% [k] _raw_spin_lock
- _raw_spin_lock
+ 80.53% __page_check_address
+ 8.39% generic_smp_call_function_single_interrupt
+ 2.44% get_swap_page
+ 1.76% free_pcppages_bulk
+ 1.40% handle_pte_fault
+ 1.15% __swap_duplicate
+ 1.05% put_super
+ 0.98% grab_super_passive
+ 0.86% blk_flush_plug_list
+ 0.57% swap_info_get
+ 8.25% [k] default_send_IPI_mask_sequence_phys
+ 7.55% [k] call_function_interrupt
+ 7.47% [k] smp_call_function_many
+ 7.25% [k] flush_tlb_func
+ 3.81% [k] _raw_spin_lock_irqsave
+ 3.78% [k] generic_smp_call_function_single_interrupt
swapout throughput is around 1400M/s. So there is around a 7%
improvement, and total cpu utilization doesn't change.
Without the patch, cfd_data is shared by all CPUs.
generic_smp_call_function_interrupt does read/write cfd_data several times
which will create a lot of cache ping-pong. With the patch, the data
becomes per-cpu. The ping-pong is avoided. And from the perf data, this
doesn't make call_single_queue lock contend.
Next step is to remove generic_smp_call_function_interrupt() from arch
code.
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-22 08:43:03 +08:00
|
|
|
}
|
2008-10-31 01:28:41 +08:00
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
if (run_local && (!cond_func || cond_func(this_cpu, info))) {
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
func(info);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
2008-06-26 17:21:34 +08:00
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
if (run_remote && wait) {
|
2013-05-01 06:27:28 +08:00
|
|
|
for_each_cpu(cpu, cfd->cpumask) {
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-08 12:30:00 +08:00
|
|
|
call_single_data_t *csd;
|
2013-05-01 06:27:28 +08:00
|
|
|
|
2021-03-01 18:13:35 +08:00
|
|
|
csd = &per_cpu_ptr(cfd->pcpu, cpu)->csd;
|
smp: make smp_call_function_many() use logic similar to smp_call_function_single()
I'm testing swapout workload in a two-socket Xeon machine. The workload
has 10 threads, each thread sequentially accesses separate memory
region. TLB flush overhead is very big in the workload. For each page,
page reclaim need move it from active lru list and then unmap it. Both
need a TLB flush. And this is a multthread workload, TLB flush happens
in 10 CPUs. In X86, TLB flush uses generic smp_call)function. So this
workload stress smp_call_function_many heavily.
Without patch, perf shows:
+ 24.49% [k] generic_smp_call_function_interrupt
- 21.72% [k] _raw_spin_lock
- _raw_spin_lock
+ 79.80% __page_check_address
+ 6.42% generic_smp_call_function_interrupt
+ 3.31% get_swap_page
+ 2.37% free_pcppages_bulk
+ 1.75% handle_pte_fault
+ 1.54% put_super
+ 1.41% grab_super_passive
+ 1.36% __swap_duplicate
+ 0.68% blk_flush_plug_list
+ 0.62% swap_info_get
+ 6.55% [k] flush_tlb_func
+ 6.46% [k] smp_call_function_many
+ 5.09% [k] call_function_interrupt
+ 4.75% [k] default_send_IPI_mask_sequence_phys
+ 2.18% [k] find_next_bit
swapout throughput is around 1300M/s.
With the patch, perf shows:
- 27.23% [k] _raw_spin_lock
- _raw_spin_lock
+ 80.53% __page_check_address
+ 8.39% generic_smp_call_function_single_interrupt
+ 2.44% get_swap_page
+ 1.76% free_pcppages_bulk
+ 1.40% handle_pte_fault
+ 1.15% __swap_duplicate
+ 1.05% put_super
+ 0.98% grab_super_passive
+ 0.86% blk_flush_plug_list
+ 0.57% swap_info_get
+ 8.25% [k] default_send_IPI_mask_sequence_phys
+ 7.55% [k] call_function_interrupt
+ 7.47% [k] smp_call_function_many
+ 7.25% [k] flush_tlb_func
+ 3.81% [k] _raw_spin_lock_irqsave
+ 3.78% [k] generic_smp_call_function_single_interrupt
swapout throughput is around 1400M/s. So there is around a 7%
improvement, and total cpu utilization doesn't change.
Without the patch, cfd_data is shared by all CPUs.
generic_smp_call_function_interrupt does read/write cfd_data several times
which will create a lot of cache ping-pong. With the patch, the data
becomes per-cpu. The ping-pong is avoided. And from the perf data, this
doesn't make call_single_queue lock contend.
Next step is to remove generic_smp_call_function_interrupt() from arch
code.
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-22 08:43:03 +08:00
|
|
|
csd_lock_wait(csd);
|
|
|
|
}
|
|
|
|
}
|
2008-06-26 17:21:34 +08:00
|
|
|
}
|
2020-01-17 17:01:36 +08:00
|
|
|
|
|
|
|
/**
|
2021-02-21 07:17:04 +08:00
|
|
|
* smp_call_function_many(): Run a function on a set of CPUs.
|
2020-01-17 17:01:36 +08:00
|
|
|
* @mask: The set of cpus to run on (only runs on online subset).
|
|
|
|
* @func: The function to run. This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to the function.
|
2021-08-11 06:50:51 +08:00
|
|
|
* @wait: Bitmask that controls the operation. If %SCF_WAIT is set, wait
|
2021-02-21 07:17:04 +08:00
|
|
|
* (atomically) until function has completed on other CPUs. If
|
|
|
|
* %SCF_RUN_LOCAL is set, the function will also be run locally
|
|
|
|
* if the local CPU is set in the @cpumask.
|
2020-01-17 17:01:36 +08:00
|
|
|
*
|
|
|
|
* If @wait is true, then returns once @func has returned.
|
|
|
|
*
|
|
|
|
* You must not call this function with disabled interrupts or from a
|
|
|
|
* hardware interrupt handler or from a bottom half handler. Preemption
|
|
|
|
* must be disabled when calling this function.
|
|
|
|
*/
|
|
|
|
void smp_call_function_many(const struct cpumask *mask,
|
|
|
|
smp_call_func_t func, void *info, bool wait)
|
|
|
|
{
|
2021-02-21 07:17:04 +08:00
|
|
|
smp_call_function_many_cond(mask, func, info, wait * SCF_WAIT, NULL);
|
2020-01-17 17:01:36 +08:00
|
|
|
}
|
2008-12-30 06:35:16 +08:00
|
|
|
EXPORT_SYMBOL(smp_call_function_many);
|
2008-06-26 17:21:34 +08:00
|
|
|
|
|
|
|
/**
|
|
|
|
* smp_call_function(): Run a function on all other CPUs.
|
|
|
|
* @func: The function to run. This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to the function.
|
2009-02-25 23:52:11 +08:00
|
|
|
* @wait: If true, wait (atomically) until function has completed
|
|
|
|
* on other CPUs.
|
2008-06-26 17:21:34 +08:00
|
|
|
*
|
2008-12-30 06:35:16 +08:00
|
|
|
* Returns 0.
|
2008-06-26 17:21:34 +08:00
|
|
|
*
|
|
|
|
* If @wait is true, then returns once @func has returned; otherwise
|
2009-10-22 19:19:34 +08:00
|
|
|
* it returns just before the target cpu calls @func.
|
2008-06-26 17:21:34 +08:00
|
|
|
*
|
|
|
|
* You must not call this function with disabled interrupts or from a
|
|
|
|
* hardware interrupt handler or from a bottom half handler.
|
|
|
|
*/
|
2019-06-13 14:48:05 +08:00
|
|
|
void smp_call_function(smp_call_func_t func, void *info, int wait)
|
2008-06-26 17:21:34 +08:00
|
|
|
{
|
|
|
|
preempt_disable();
|
2008-12-30 06:35:16 +08:00
|
|
|
smp_call_function_many(cpu_online_mask, func, info, wait);
|
2008-06-26 17:21:34 +08:00
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(smp_call_function);
|
2011-01-13 08:59:39 +08:00
|
|
|
|
2011-03-23 07:34:06 +08:00
|
|
|
/* Setup configured maximum number of CPUs to activate */
|
|
|
|
unsigned int setup_max_cpus = NR_CPUS;
|
|
|
|
EXPORT_SYMBOL(setup_max_cpus);
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Setup routine for controlling SMP activation
|
|
|
|
*
|
|
|
|
* Command-line option of "nosmp" or "maxcpus=0" will disable SMP
|
|
|
|
* activation entirely (the MPS table probe still happens, though).
|
|
|
|
*
|
|
|
|
* Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
|
|
|
|
* greater than 0, limits the maximum number of CPUs activated in
|
|
|
|
* SMP mode to <NUM>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
void __weak arch_disable_smp_support(void) { }
|
|
|
|
|
|
|
|
static int __init nosmp(char *str)
|
|
|
|
{
|
|
|
|
setup_max_cpus = 0;
|
|
|
|
arch_disable_smp_support();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
early_param("nosmp", nosmp);
|
|
|
|
|
|
|
|
/* this is hard limit */
|
|
|
|
static int __init nrcpus(char *str)
|
|
|
|
{
|
|
|
|
int nr_cpus;
|
|
|
|
|
2020-07-16 15:04:57 +08:00
|
|
|
if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids)
|
2011-03-23 07:34:06 +08:00
|
|
|
nr_cpu_ids = nr_cpus;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
early_param("nr_cpus", nrcpus);
|
|
|
|
|
|
|
|
static int __init maxcpus(char *str)
|
|
|
|
{
|
|
|
|
get_option(&str, &setup_max_cpus);
|
|
|
|
if (setup_max_cpus == 0)
|
|
|
|
arch_disable_smp_support();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
early_param("maxcpus", maxcpus);
|
|
|
|
|
|
|
|
/* Setup number of possible processor ids */
|
2017-09-09 07:14:18 +08:00
|
|
|
unsigned int nr_cpu_ids __read_mostly = NR_CPUS;
|
2011-03-23 07:34:06 +08:00
|
|
|
EXPORT_SYMBOL(nr_cpu_ids);
|
|
|
|
|
|
|
|
/* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
|
|
|
|
void __init setup_nr_cpu_ids(void)
|
|
|
|
{
|
|
|
|
nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Called by boot processor to activate the rest. */
|
|
|
|
void __init smp_init(void)
|
|
|
|
{
|
2016-10-26 13:37:54 +08:00
|
|
|
int num_nodes, num_cpus;
|
2011-03-23 07:34:06 +08:00
|
|
|
|
2012-04-21 08:08:50 +08:00
|
|
|
idle_threads_init();
|
2016-02-27 02:43:38 +08:00
|
|
|
cpuhp_threads_init();
|
2012-04-21 08:08:50 +08:00
|
|
|
|
2016-10-26 13:37:55 +08:00
|
|
|
pr_info("Bringing up secondary CPUs ...\n");
|
|
|
|
|
2020-03-23 21:51:09 +08:00
|
|
|
bringup_nonboot_cpus(setup_max_cpus);
|
2011-03-23 07:34:06 +08:00
|
|
|
|
2016-10-26 13:37:54 +08:00
|
|
|
num_nodes = num_online_nodes();
|
|
|
|
num_cpus = num_online_cpus();
|
|
|
|
pr_info("Brought up %d node%s, %d CPU%s\n",
|
|
|
|
num_nodes, (num_nodes > 1 ? "s" : ""),
|
|
|
|
num_cpus, (num_cpus > 1 ? "s" : ""));
|
|
|
|
|
2011-03-23 07:34:06 +08:00
|
|
|
/* Any cleanup work */
|
|
|
|
smp_cpus_done(setup_max_cpus);
|
|
|
|
}
|
|
|
|
|
2012-03-29 05:42:43 +08:00
|
|
|
/*
|
|
|
|
* on_each_cpu_cond(): Call a function on each processor for which
|
|
|
|
* the supplied function cond_func returns true, optionally waiting
|
|
|
|
* for all the required CPUs to finish. This may include the local
|
|
|
|
* processor.
|
|
|
|
* @cond_func: A callback function that is passed a cpu id and
|
2020-10-16 11:10:28 +08:00
|
|
|
* the info parameter. The function is called
|
2012-03-29 05:42:43 +08:00
|
|
|
* with preemption disabled. The function should
|
|
|
|
* return a blooean value indicating whether to IPI
|
|
|
|
* the specified CPU.
|
|
|
|
* @func: The function to run on all applicable CPUs.
|
|
|
|
* This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to both functions.
|
|
|
|
* @wait: If true, wait (atomically) until function has
|
|
|
|
* completed on other CPUs.
|
|
|
|
*
|
|
|
|
* Preemption is disabled to protect against CPUs going offline but not online.
|
|
|
|
* CPUs going online during the call will not be seen or sent an IPI.
|
|
|
|
*
|
|
|
|
* You must not call this function with disabled interrupts or
|
|
|
|
* from a hardware interrupt handler or from a bottom half handler.
|
|
|
|
*/
|
2020-01-17 17:01:35 +08:00
|
|
|
void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
|
2020-01-17 17:01:37 +08:00
|
|
|
void *info, bool wait, const struct cpumask *mask)
|
2012-03-29 05:42:43 +08:00
|
|
|
{
|
2021-02-21 07:17:04 +08:00
|
|
|
unsigned int scf_flags = SCF_RUN_LOCAL;
|
2020-01-17 17:01:36 +08:00
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
if (wait)
|
|
|
|
scf_flags |= SCF_WAIT;
|
2020-01-17 17:01:36 +08:00
|
|
|
|
2021-02-21 07:17:04 +08:00
|
|
|
preempt_disable();
|
|
|
|
smp_call_function_many_cond(mask, func, info, scf_flags, cond_func);
|
|
|
|
preempt_enable();
|
2012-03-29 05:42:43 +08:00
|
|
|
}
|
2018-09-26 11:58:41 +08:00
|
|
|
EXPORT_SYMBOL(on_each_cpu_cond_mask);
|
|
|
|
|
2012-05-08 01:59:48 +08:00
|
|
|
static void do_nothing(void *unused)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* kick_all_cpus_sync - Force all cpus out of idle
|
|
|
|
*
|
|
|
|
* Used to synchronize the update of pm_idle function pointer. It's
|
|
|
|
* called after the pointer is updated and returns after the dummy
|
|
|
|
* callback function has been executed on all cpus. The execution of
|
|
|
|
* the function can only happen on the remote cpus after they have
|
|
|
|
* left the idle function which had been called via pm_idle function
|
|
|
|
* pointer. So it's guaranteed that nothing uses the previous pointer
|
|
|
|
* anymore.
|
|
|
|
*/
|
|
|
|
void kick_all_cpus_sync(void)
|
|
|
|
{
|
|
|
|
/* Make sure the change is visible before we kick the cpus */
|
|
|
|
smp_mb();
|
|
|
|
smp_call_function(do_nothing, NULL, 1);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(kick_all_cpus_sync);
|
2014-09-04 15:17:54 +08:00
|
|
|
|
|
|
|
/**
|
|
|
|
* wake_up_all_idle_cpus - break all cpus out of idle
|
|
|
|
* wake_up_all_idle_cpus try to break all cpus which is in idle state even
|
|
|
|
* including idle polling cpus, for non-idle cpus, we will do nothing
|
|
|
|
* for them.
|
|
|
|
*/
|
|
|
|
void wake_up_all_idle_cpus(void)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
|
2021-10-18 22:41:05 +08:00
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
preempt_disable();
|
|
|
|
if (cpu != smp_processor_id() && cpu_online(cpu))
|
|
|
|
wake_up_if_idle(cpu);
|
|
|
|
preempt_enable();
|
2014-09-04 15:17:54 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(wake_up_all_idle_cpus);
|
2016-08-29 14:48:44 +08:00
|
|
|
|
|
|
|
/**
|
2021-08-11 06:50:51 +08:00
|
|
|
* struct smp_call_on_cpu_struct - Call a function on a specific CPU
|
|
|
|
* @work: &work_struct
|
|
|
|
* @done: &completion to signal
|
|
|
|
* @func: function to call
|
|
|
|
* @data: function's data argument
|
|
|
|
* @ret: return value from @func
|
|
|
|
* @cpu: target CPU (%-1 for any CPU)
|
2016-08-29 14:48:44 +08:00
|
|
|
*
|
|
|
|
* Used to call a function on a specific cpu and wait for it to return.
|
|
|
|
* Optionally make sure the call is done on a specified physical cpu via vcpu
|
|
|
|
* pinning in order to support virtualized environments.
|
|
|
|
*/
|
|
|
|
struct smp_call_on_cpu_struct {
|
|
|
|
struct work_struct work;
|
|
|
|
struct completion done;
|
|
|
|
int (*func)(void *);
|
|
|
|
void *data;
|
|
|
|
int ret;
|
|
|
|
int cpu;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void smp_call_on_cpu_callback(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct smp_call_on_cpu_struct *sscs;
|
|
|
|
|
|
|
|
sscs = container_of(work, struct smp_call_on_cpu_struct, work);
|
|
|
|
if (sscs->cpu >= 0)
|
|
|
|
hypervisor_pin_vcpu(sscs->cpu);
|
|
|
|
sscs->ret = sscs->func(sscs->data);
|
|
|
|
if (sscs->cpu >= 0)
|
|
|
|
hypervisor_pin_vcpu(-1);
|
|
|
|
|
|
|
|
complete(&sscs->done);
|
|
|
|
}
|
|
|
|
|
|
|
|
int smp_call_on_cpu(unsigned int cpu, int (*func)(void *), void *par, bool phys)
|
|
|
|
{
|
|
|
|
struct smp_call_on_cpu_struct sscs = {
|
|
|
|
.done = COMPLETION_INITIALIZER_ONSTACK(sscs.done),
|
|
|
|
.func = func,
|
|
|
|
.data = par,
|
|
|
|
.cpu = phys ? cpu : -1,
|
|
|
|
};
|
|
|
|
|
2016-09-11 16:36:26 +08:00
|
|
|
INIT_WORK_ONSTACK(&sscs.work, smp_call_on_cpu_callback);
|
|
|
|
|
2016-08-29 14:48:44 +08:00
|
|
|
if (cpu >= nr_cpu_ids || !cpu_online(cpu))
|
|
|
|
return -ENXIO;
|
|
|
|
|
|
|
|
queue_work_on(cpu, system_wq, &sscs.work);
|
|
|
|
wait_for_completion(&sscs.done);
|
|
|
|
|
|
|
|
return sscs.ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(smp_call_on_cpu);
|