2019-05-27 14:55:06 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2015-04-25 02:56:30 +08:00
|
|
|
/*
|
|
|
|
* Queued spinlock
|
|
|
|
*
|
|
|
|
* (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
|
2018-04-26 18:34:27 +08:00
|
|
|
* (C) Copyright 2013-2014,2018 Red Hat, Inc.
|
2015-04-25 02:56:30 +08:00
|
|
|
* (C) Copyright 2015 Intel Corp.
|
2015-11-10 08:09:21 +08:00
|
|
|
* (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
|
2015-04-25 02:56:30 +08:00
|
|
|
*
|
2018-04-26 18:34:27 +08:00
|
|
|
* Authors: Waiman Long <longman@redhat.com>
|
2015-04-25 02:56:30 +08:00
|
|
|
* Peter Zijlstra <peterz@infradead.org>
|
|
|
|
*/
|
2015-04-25 02:56:37 +08:00
|
|
|
|
|
|
|
#ifndef _GEN_PV_LOCK_SLOWPATH
|
|
|
|
|
2015-04-25 02:56:30 +08:00
|
|
|
#include <linux/smp.h>
|
|
|
|
#include <linux/bug.h>
|
|
|
|
#include <linux/cpumask.h>
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/hardirq.h>
|
|
|
|
#include <linux/mutex.h>
|
2017-07-08 03:56:58 +08:00
|
|
|
#include <linux/prefetch.h>
|
2015-04-25 02:56:34 +08:00
|
|
|
#include <asm/byteorder.h>
|
2015-04-25 02:56:30 +08:00
|
|
|
#include <asm/qspinlock.h>
|
2022-03-23 02:57:09 +08:00
|
|
|
#include <trace/events/lock.h>
|
2015-04-25 02:56:30 +08:00
|
|
|
|
2018-04-26 18:34:27 +08:00
|
|
|
/*
|
|
|
|
* Include queued spinlock statistics code
|
|
|
|
*/
|
|
|
|
#include "qspinlock_stat.h"
|
|
|
|
|
2015-04-25 02:56:30 +08:00
|
|
|
/*
|
|
|
|
* The basic principle of a queue-based spinlock can best be understood
|
|
|
|
* by studying a classic queue-based spinlock implementation called the
|
2020-01-08 01:49:14 +08:00
|
|
|
* MCS lock. A copy of the original MCS lock paper ("Algorithms for Scalable
|
|
|
|
* Synchronization on Shared-Memory Multiprocessors by Mellor-Crummey and
|
|
|
|
* Scott") is available at
|
2015-04-25 02:56:30 +08:00
|
|
|
*
|
2020-01-08 01:49:14 +08:00
|
|
|
* https://bugzilla.kernel.org/show_bug.cgi?id=206115
|
2015-04-25 02:56:30 +08:00
|
|
|
*
|
2020-01-08 01:49:14 +08:00
|
|
|
* This queued spinlock implementation is based on the MCS lock, however to
|
|
|
|
* make it fit the 4 bytes we assume spinlock_t to be, and preserve its
|
|
|
|
* existing API, we must modify it somehow.
|
2015-04-25 02:56:30 +08:00
|
|
|
*
|
|
|
|
* In particular; where the traditional MCS lock consists of a tail pointer
|
|
|
|
* (8 bytes) and needs the next pointer (another 8 bytes) of its own node to
|
|
|
|
* unlock the next pending (next->locked), we compress both these: {tail,
|
|
|
|
* next->locked} into a single u32 value.
|
|
|
|
*
|
|
|
|
* Since a spinlock disables recursion of its own context and there is a limit
|
|
|
|
* to the contexts that can nest; namely: task, softirq, hardirq, nmi. As there
|
|
|
|
* are at most 4 nesting levels, it can be encoded by a 2-bit number. Now
|
|
|
|
* we can encode the tail by combining the 2-bit nesting level with the cpu
|
|
|
|
* number. With one byte for the lock value and 3 bytes for the tail, only a
|
|
|
|
* 32-bit word is now needed. Even though we only need 1 bit for the lock,
|
|
|
|
* we extend it to a full byte to achieve better performance for architectures
|
|
|
|
* that support atomic byte write.
|
|
|
|
*
|
|
|
|
* We also change the first spinner to spin on the lock bit instead of its
|
|
|
|
* node; whereby avoiding the need to carry a node from lock to unlock, and
|
|
|
|
* preserving existing lock API. This also makes the unlock code simpler and
|
|
|
|
* faster.
|
2015-04-25 02:56:34 +08:00
|
|
|
*
|
|
|
|
* N.B. The current implementation only supports architectures that allow
|
|
|
|
* atomic operations on smaller 8-bit and 16-bit data types.
|
|
|
|
*
|
2015-04-25 02:56:30 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include "mcs_spinlock.h"
|
2018-10-16 21:45:07 +08:00
|
|
|
#define MAX_NODES 4
|
2015-04-25 02:56:30 +08:00
|
|
|
|
2018-10-16 21:45:07 +08:00
|
|
|
/*
|
|
|
|
* On 64-bit architectures, the mcs_spinlock structure will be 16 bytes in
|
|
|
|
* size and four of them will fit nicely in one 64-byte cacheline. For
|
|
|
|
* pvqspinlock, however, we need more space for extra data. To accommodate
|
|
|
|
* that, we insert two more long words to pad it up to 32 bytes. IOW, only
|
|
|
|
* two of them can fit in a cacheline in this case. That is OK as it is rare
|
|
|
|
* to have more than 2 levels of slowpath nesting in actual use. We don't
|
|
|
|
* want to penalize pvqspinlocks to optimize for a rare case in native
|
|
|
|
* qspinlocks.
|
|
|
|
*/
|
|
|
|
struct qnode {
|
|
|
|
struct mcs_spinlock mcs;
|
2015-04-25 02:56:37 +08:00
|
|
|
#ifdef CONFIG_PARAVIRT_SPINLOCKS
|
2018-10-16 21:45:07 +08:00
|
|
|
long reserved[2];
|
2015-04-25 02:56:37 +08:00
|
|
|
#endif
|
2018-10-16 21:45:07 +08:00
|
|
|
};
|
2015-04-25 02:56:37 +08:00
|
|
|
|
2018-04-26 18:34:17 +08:00
|
|
|
/*
|
|
|
|
* The pending bit spinning loop count.
|
|
|
|
* This heuristic is used to limit the number of lockword accesses
|
|
|
|
* made by atomic_cond_read_relaxed when waiting for the lock to
|
|
|
|
* transition out of the "== _Q_PENDING_VAL" state. We don't spin
|
|
|
|
* indefinitely because there's no guarantee that we'll make forward
|
|
|
|
* progress.
|
|
|
|
*/
|
|
|
|
#ifndef _Q_PENDING_LOOPS
|
|
|
|
#define _Q_PENDING_LOOPS 1
|
|
|
|
#endif
|
|
|
|
|
2015-04-25 02:56:30 +08:00
|
|
|
/*
|
|
|
|
* Per-CPU queue node structures; we can never have more than 4 nested
|
|
|
|
* contexts: task, softirq, hardirq, nmi.
|
|
|
|
*
|
|
|
|
* Exactly fits one 64-byte cacheline on a 64-bit architecture.
|
2015-04-25 02:56:37 +08:00
|
|
|
*
|
|
|
|
* PV doubles the storage and uses the second cacheline for PV state.
|
2015-04-25 02:56:30 +08:00
|
|
|
*/
|
2018-10-16 21:45:07 +08:00
|
|
|
static DEFINE_PER_CPU_ALIGNED(struct qnode, qnodes[MAX_NODES]);
|
2015-04-25 02:56:30 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We must be able to distinguish between no-tail and the tail at 0:0,
|
|
|
|
* therefore increment the cpu number by one.
|
|
|
|
*/
|
|
|
|
|
2016-06-08 15:12:30 +08:00
|
|
|
static inline __pure u32 encode_tail(int cpu, int idx)
|
2015-04-25 02:56:30 +08:00
|
|
|
{
|
|
|
|
u32 tail;
|
|
|
|
|
|
|
|
tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
|
|
|
|
tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
|
|
|
|
|
|
|
|
return tail;
|
|
|
|
}
|
|
|
|
|
2016-06-08 15:12:30 +08:00
|
|
|
static inline __pure struct mcs_spinlock *decode_tail(u32 tail)
|
2015-04-25 02:56:30 +08:00
|
|
|
{
|
|
|
|
int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
|
|
|
|
int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
|
|
|
|
|
2018-10-16 21:45:07 +08:00
|
|
|
return per_cpu_ptr(&qnodes[idx].mcs, cpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline __pure
|
|
|
|
struct mcs_spinlock *grab_mcs_node(struct mcs_spinlock *base, int idx)
|
|
|
|
{
|
|
|
|
return &((struct qnode *)base + idx)->mcs;
|
2015-04-25 02:56:30 +08:00
|
|
|
}
|
|
|
|
|
2015-04-25 02:56:32 +08:00
|
|
|
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
|
|
|
|
|
2015-04-25 02:56:35 +08:00
|
|
|
#if _Q_PENDING_BITS == 8
|
locking/qspinlock: Remove unbounded cmpxchg() loop from locking slowpath
The qspinlock locking slowpath utilises a "pending" bit as a simple form
of an embedded test-and-set lock that can avoid the overhead of explicit
queuing in cases where the lock is held but uncontended. This bit is
managed using a cmpxchg() loop which tries to transition the uncontended
lock word from (0,0,0) -> (0,0,1) or (0,0,1) -> (0,1,1).
Unfortunately, the cmpxchg() loop is unbounded and lockers can be starved
indefinitely if the lock word is seen to oscillate between unlocked
(0,0,0) and locked (0,0,1). This could happen if concurrent lockers are
able to take the lock in the cmpxchg() loop without queuing and pass it
around amongst themselves.
This patch fixes the problem by unconditionally setting _Q_PENDING_VAL
using atomic_fetch_or, and then inspecting the old value to see whether
we need to spin on the current lock owner, or whether we now effectively
hold the lock. The tricky scenario is when concurrent lockers end up
queuing on the lock and the lock becomes available, causing us to see
a lockword of (n,0,0). With pending now set, simply queuing could lead
to deadlock as the head of the queue may not have observed the pending
flag being cleared. Conversely, if the head of the queue did observe
pending being cleared, then it could transition the lock from (n,0,0) ->
(0,0,1) meaning that any attempt to "undo" our setting of the pending
bit could race with a concurrent locker trying to set it.
We handle this race by preserving the pending bit when taking the lock
after reaching the head of the queue and leaving the tail entry intact
if we saw pending set, because we know that the tail is going to be
updated shortly.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: boqun.feng@gmail.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: paulmck@linux.vnet.ibm.com
Link: http://lkml.kernel.org/r/1524738868-31318-6-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-04-26 18:34:19 +08:00
|
|
|
/**
|
|
|
|
* clear_pending - clear the pending bit.
|
|
|
|
* @lock: Pointer to queued spinlock structure
|
|
|
|
*
|
|
|
|
* *,1,* -> *,0,*
|
|
|
|
*/
|
|
|
|
static __always_inline void clear_pending(struct qspinlock *lock)
|
|
|
|
{
|
|
|
|
WRITE_ONCE(lock->pending, 0);
|
|
|
|
}
|
|
|
|
|
2015-04-25 02:56:34 +08:00
|
|
|
/**
|
|
|
|
* clear_pending_set_locked - take ownership and clear the pending bit.
|
|
|
|
* @lock: Pointer to queued spinlock structure
|
|
|
|
*
|
|
|
|
* *,1,0 -> *,0,1
|
|
|
|
*
|
|
|
|
* Lock stealing is not allowed if this function is used.
|
|
|
|
*/
|
|
|
|
static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
|
|
|
|
{
|
2018-04-26 18:34:16 +08:00
|
|
|
WRITE_ONCE(lock->locked_pending, _Q_LOCKED_VAL);
|
2015-04-25 02:56:34 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* xchg_tail - Put in the new queue tail code word & retrieve previous one
|
|
|
|
* @lock : Pointer to queued spinlock structure
|
|
|
|
* @tail : The new queue tail code word
|
|
|
|
* Return: The previous queue tail code word
|
|
|
|
*
|
2017-10-10 02:22:50 +08:00
|
|
|
* xchg(lock, tail), which heads an address dependency
|
2015-04-25 02:56:34 +08:00
|
|
|
*
|
|
|
|
* p,*,* -> n,*,* ; prev = xchg(lock, node)
|
|
|
|
*/
|
|
|
|
static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
|
|
|
|
{
|
2015-11-10 08:09:21 +08:00
|
|
|
/*
|
2018-04-26 18:34:25 +08:00
|
|
|
* We can use relaxed semantics since the caller ensures that the
|
|
|
|
* MCS node is properly initialized before updating the tail.
|
2015-11-10 08:09:21 +08:00
|
|
|
*/
|
2018-04-26 18:34:25 +08:00
|
|
|
return (u32)xchg_relaxed(&lock->tail,
|
2015-11-10 08:09:21 +08:00
|
|
|
tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
|
2015-04-25 02:56:34 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#else /* _Q_PENDING_BITS == 8 */
|
|
|
|
|
locking/qspinlock: Remove unbounded cmpxchg() loop from locking slowpath
The qspinlock locking slowpath utilises a "pending" bit as a simple form
of an embedded test-and-set lock that can avoid the overhead of explicit
queuing in cases where the lock is held but uncontended. This bit is
managed using a cmpxchg() loop which tries to transition the uncontended
lock word from (0,0,0) -> (0,0,1) or (0,0,1) -> (0,1,1).
Unfortunately, the cmpxchg() loop is unbounded and lockers can be starved
indefinitely if the lock word is seen to oscillate between unlocked
(0,0,0) and locked (0,0,1). This could happen if concurrent lockers are
able to take the lock in the cmpxchg() loop without queuing and pass it
around amongst themselves.
This patch fixes the problem by unconditionally setting _Q_PENDING_VAL
using atomic_fetch_or, and then inspecting the old value to see whether
we need to spin on the current lock owner, or whether we now effectively
hold the lock. The tricky scenario is when concurrent lockers end up
queuing on the lock and the lock becomes available, causing us to see
a lockword of (n,0,0). With pending now set, simply queuing could lead
to deadlock as the head of the queue may not have observed the pending
flag being cleared. Conversely, if the head of the queue did observe
pending being cleared, then it could transition the lock from (n,0,0) ->
(0,0,1) meaning that any attempt to "undo" our setting of the pending
bit could race with a concurrent locker trying to set it.
We handle this race by preserving the pending bit when taking the lock
after reaching the head of the queue and leaving the tail entry intact
if we saw pending set, because we know that the tail is going to be
updated shortly.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: boqun.feng@gmail.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: paulmck@linux.vnet.ibm.com
Link: http://lkml.kernel.org/r/1524738868-31318-6-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-04-26 18:34:19 +08:00
|
|
|
/**
|
|
|
|
* clear_pending - clear the pending bit.
|
|
|
|
* @lock: Pointer to queued spinlock structure
|
|
|
|
*
|
|
|
|
* *,1,* -> *,0,*
|
|
|
|
*/
|
|
|
|
static __always_inline void clear_pending(struct qspinlock *lock)
|
|
|
|
{
|
|
|
|
atomic_andnot(_Q_PENDING_VAL, &lock->val);
|
|
|
|
}
|
|
|
|
|
2015-04-25 02:56:33 +08:00
|
|
|
/**
|
|
|
|
* clear_pending_set_locked - take ownership and clear the pending bit.
|
|
|
|
* @lock: Pointer to queued spinlock structure
|
|
|
|
*
|
|
|
|
* *,1,0 -> *,0,1
|
|
|
|
*/
|
|
|
|
static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
|
|
|
|
{
|
|
|
|
atomic_add(-_Q_PENDING_VAL + _Q_LOCKED_VAL, &lock->val);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* xchg_tail - Put in the new queue tail code word & retrieve previous one
|
|
|
|
* @lock : Pointer to queued spinlock structure
|
|
|
|
* @tail : The new queue tail code word
|
|
|
|
* Return: The previous queue tail code word
|
|
|
|
*
|
|
|
|
* xchg(lock, tail)
|
|
|
|
*
|
|
|
|
* p,*,* -> n,*,* ; prev = xchg(lock, node)
|
|
|
|
*/
|
|
|
|
static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
|
|
|
|
{
|
|
|
|
u32 old, new, val = atomic_read(&lock->val);
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
new = (val & _Q_LOCKED_PENDING_MASK) | tail;
|
2015-11-10 08:09:21 +08:00
|
|
|
/*
|
2018-04-26 18:34:25 +08:00
|
|
|
* We can use relaxed semantics since the caller ensures that
|
|
|
|
* the MCS node is properly initialized before updating the
|
|
|
|
* tail.
|
2015-11-10 08:09:21 +08:00
|
|
|
*/
|
2018-04-26 18:34:25 +08:00
|
|
|
old = atomic_cmpxchg_relaxed(&lock->val, val, new);
|
2015-04-25 02:56:33 +08:00
|
|
|
if (old == val)
|
|
|
|
break;
|
|
|
|
|
|
|
|
val = old;
|
|
|
|
}
|
|
|
|
return old;
|
|
|
|
}
|
2015-04-25 02:56:34 +08:00
|
|
|
#endif /* _Q_PENDING_BITS == 8 */
|
2015-04-25 02:56:33 +08:00
|
|
|
|
locking/qspinlock, x86: Provide liveness guarantee
On x86 we cannot do fetch_or() with a single instruction and thus end up
using a cmpxchg loop, this reduces determinism. Replace the fetch_or()
with a composite operation: tas-pending + load.
Using two instructions of course opens a window we previously did not
have. Consider the scenario:
CPU0 CPU1 CPU2
1) lock
trylock -> (0,0,1)
2) lock
trylock /* fail */
3) unlock -> (0,0,0)
4) lock
trylock -> (0,0,1)
5) tas-pending -> (0,1,1)
load-val <- (0,1,0) from 3
6) clear-pending-set-locked -> (0,0,1)
FAIL: _2_ owners
where 5) is our new composite operation. When we consider each part of
the qspinlock state as a separate variable (as we can when
_Q_PENDING_BITS == 8) then the above is entirely possible, because
tas-pending will only RmW the pending byte, so the later load is able
to observe prior tail and lock state (but not earlier than its own
trylock, which operates on the whole word, due to coherence).
To avoid this we need 2 things:
- the load must come after the tas-pending (obviously, otherwise it
can trivially observe prior state).
- the tas-pending must be a full word RmW instruction, it cannot be an XCHGB for
example, such that we cannot observe other state prior to setting
pending.
On x86 we can realize this by using "LOCK BTS m32, r32" for
tas-pending followed by a regular load.
Note that observing later state is not a problem:
- if we fail to observe a later unlock, we'll simply spin-wait for
that store to become visible.
- if we observe a later xchg_tail(), there is no difference from that
xchg_tail() having taken place before the tas-pending.
Suggested-by: Will Deacon <will.deacon@arm.com>
Reported-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: andrea.parri@amarulasolutions.com
Cc: longman@redhat.com
Fixes: 59fb586b4a07 ("locking/qspinlock: Remove unbounded cmpxchg() loop from locking slowpath")
Link: https://lkml.kernel.org/r/20181003130957.183726335@infradead.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-09-26 19:01:20 +08:00
|
|
|
/**
|
|
|
|
* queued_fetch_set_pending_acquire - fetch the whole lock value and set pending
|
|
|
|
* @lock : Pointer to queued spinlock structure
|
|
|
|
* Return: The previous lock value
|
|
|
|
*
|
|
|
|
* *,*,* -> *,1,*
|
|
|
|
*/
|
|
|
|
#ifndef queued_fetch_set_pending_acquire
|
|
|
|
static __always_inline u32 queued_fetch_set_pending_acquire(struct qspinlock *lock)
|
|
|
|
{
|
|
|
|
return atomic_fetch_or_acquire(_Q_PENDING_VAL, &lock->val);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2015-04-25 02:56:35 +08:00
|
|
|
/**
|
|
|
|
* set_locked - Set the lock bit and own the lock
|
|
|
|
* @lock: Pointer to queued spinlock structure
|
|
|
|
*
|
|
|
|
* *,*,0 -> *,0,1
|
|
|
|
*/
|
|
|
|
static __always_inline void set_locked(struct qspinlock *lock)
|
|
|
|
{
|
2018-04-26 18:34:16 +08:00
|
|
|
WRITE_ONCE(lock->locked, _Q_LOCKED_VAL);
|
2015-04-25 02:56:35 +08:00
|
|
|
}
|
|
|
|
|
2015-04-25 02:56:37 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Generate the native code for queued_spin_unlock_slowpath(); provide NOPs for
|
|
|
|
* all the PV callbacks.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static __always_inline void __pv_init_node(struct mcs_spinlock *node) { }
|
2015-11-10 08:09:27 +08:00
|
|
|
static __always_inline void __pv_wait_node(struct mcs_spinlock *node,
|
|
|
|
struct mcs_spinlock *prev) { }
|
2015-07-12 04:36:52 +08:00
|
|
|
static __always_inline void __pv_kick_node(struct qspinlock *lock,
|
|
|
|
struct mcs_spinlock *node) { }
|
2015-11-11 05:18:56 +08:00
|
|
|
static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,
|
|
|
|
struct mcs_spinlock *node)
|
|
|
|
{ return 0; }
|
2015-04-25 02:56:37 +08:00
|
|
|
|
|
|
|
#define pv_enabled() false
|
|
|
|
|
|
|
|
#define pv_init_node __pv_init_node
|
|
|
|
#define pv_wait_node __pv_wait_node
|
|
|
|
#define pv_kick_node __pv_kick_node
|
2015-11-11 05:18:56 +08:00
|
|
|
#define pv_wait_head_or_lock __pv_wait_head_or_lock
|
2015-04-25 02:56:37 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_PARAVIRT_SPINLOCKS
|
|
|
|
#define queued_spin_lock_slowpath native_queued_spin_lock_slowpath
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#endif /* _GEN_PV_LOCK_SLOWPATH */
|
|
|
|
|
2015-04-25 02:56:30 +08:00
|
|
|
/**
|
|
|
|
* queued_spin_lock_slowpath - acquire the queued spinlock
|
|
|
|
* @lock: Pointer to queued spinlock structure
|
|
|
|
* @val: Current value of the queued spinlock 32-bit word
|
|
|
|
*
|
2015-04-25 02:56:32 +08:00
|
|
|
* (queue tail, pending bit, lock value)
|
2015-04-25 02:56:30 +08:00
|
|
|
*
|
2015-04-25 02:56:32 +08:00
|
|
|
* fast : slow : unlock
|
|
|
|
* : :
|
|
|
|
* uncontended (0,0,0) -:--> (0,0,1) ------------------------------:--> (*,*,0)
|
|
|
|
* : | ^--------.------. / :
|
|
|
|
* : v \ \ | :
|
|
|
|
* pending : (0,1,1) +--> (0,1,0) \ | :
|
|
|
|
* : | ^--' | | :
|
|
|
|
* : v | | :
|
|
|
|
* uncontended : (n,x,y) +--> (n,0,0) --' | :
|
|
|
|
* queue : | ^--' | :
|
|
|
|
* : v | :
|
|
|
|
* contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
|
|
|
|
* queue : ^--' :
|
2015-04-25 02:56:30 +08:00
|
|
|
*/
|
2022-08-11 06:03:46 +08:00
|
|
|
void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
|
2015-04-25 02:56:30 +08:00
|
|
|
{
|
|
|
|
struct mcs_spinlock *prev, *next, *node;
|
locking/qspinlock: Remove unbounded cmpxchg() loop from locking slowpath
The qspinlock locking slowpath utilises a "pending" bit as a simple form
of an embedded test-and-set lock that can avoid the overhead of explicit
queuing in cases where the lock is held but uncontended. This bit is
managed using a cmpxchg() loop which tries to transition the uncontended
lock word from (0,0,0) -> (0,0,1) or (0,0,1) -> (0,1,1).
Unfortunately, the cmpxchg() loop is unbounded and lockers can be starved
indefinitely if the lock word is seen to oscillate between unlocked
(0,0,0) and locked (0,0,1). This could happen if concurrent lockers are
able to take the lock in the cmpxchg() loop without queuing and pass it
around amongst themselves.
This patch fixes the problem by unconditionally setting _Q_PENDING_VAL
using atomic_fetch_or, and then inspecting the old value to see whether
we need to spin on the current lock owner, or whether we now effectively
hold the lock. The tricky scenario is when concurrent lockers end up
queuing on the lock and the lock becomes available, causing us to see
a lockword of (n,0,0). With pending now set, simply queuing could lead
to deadlock as the head of the queue may not have observed the pending
flag being cleared. Conversely, if the head of the queue did observe
pending being cleared, then it could transition the lock from (n,0,0) ->
(0,0,1) meaning that any attempt to "undo" our setting of the pending
bit could race with a concurrent locker trying to set it.
We handle this race by preserving the pending bit when taking the lock
after reaching the head of the queue and leaving the tail entry intact
if we saw pending set, because we know that the tail is going to be
updated shortly.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: boqun.feng@gmail.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: paulmck@linux.vnet.ibm.com
Link: http://lkml.kernel.org/r/1524738868-31318-6-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-04-26 18:34:19 +08:00
|
|
|
u32 old, tail;
|
2015-04-25 02:56:30 +08:00
|
|
|
int idx;
|
|
|
|
|
|
|
|
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
|
|
|
|
|
2015-04-25 02:56:37 +08:00
|
|
|
if (pv_enabled())
|
2018-04-26 18:34:27 +08:00
|
|
|
goto pv_queue;
|
2015-04-25 02:56:37 +08:00
|
|
|
|
2015-09-04 23:25:23 +08:00
|
|
|
if (virt_spin_lock(lock))
|
2015-04-25 02:56:36 +08:00
|
|
|
return;
|
|
|
|
|
2015-04-25 02:56:32 +08:00
|
|
|
/*
|
2018-04-26 18:34:17 +08:00
|
|
|
* Wait for in-progress pending->locked hand-overs with a bounded
|
|
|
|
* number of spins so that we guarantee forward progress.
|
2015-04-25 02:56:32 +08:00
|
|
|
*
|
|
|
|
* 0,1,0 -> 0,0,1
|
|
|
|
*/
|
|
|
|
if (val == _Q_PENDING_VAL) {
|
2018-04-26 18:34:17 +08:00
|
|
|
int cnt = _Q_PENDING_LOOPS;
|
|
|
|
val = atomic_cond_read_relaxed(&lock->val,
|
|
|
|
(VAL != _Q_PENDING_VAL) || !cnt--);
|
2015-04-25 02:56:32 +08:00
|
|
|
}
|
|
|
|
|
locking/qspinlock: Remove unbounded cmpxchg() loop from locking slowpath
The qspinlock locking slowpath utilises a "pending" bit as a simple form
of an embedded test-and-set lock that can avoid the overhead of explicit
queuing in cases where the lock is held but uncontended. This bit is
managed using a cmpxchg() loop which tries to transition the uncontended
lock word from (0,0,0) -> (0,0,1) or (0,0,1) -> (0,1,1).
Unfortunately, the cmpxchg() loop is unbounded and lockers can be starved
indefinitely if the lock word is seen to oscillate between unlocked
(0,0,0) and locked (0,0,1). This could happen if concurrent lockers are
able to take the lock in the cmpxchg() loop without queuing and pass it
around amongst themselves.
This patch fixes the problem by unconditionally setting _Q_PENDING_VAL
using atomic_fetch_or, and then inspecting the old value to see whether
we need to spin on the current lock owner, or whether we now effectively
hold the lock. The tricky scenario is when concurrent lockers end up
queuing on the lock and the lock becomes available, causing us to see
a lockword of (n,0,0). With pending now set, simply queuing could lead
to deadlock as the head of the queue may not have observed the pending
flag being cleared. Conversely, if the head of the queue did observe
pending being cleared, then it could transition the lock from (n,0,0) ->
(0,0,1) meaning that any attempt to "undo" our setting of the pending
bit could race with a concurrent locker trying to set it.
We handle this race by preserving the pending bit when taking the lock
after reaching the head of the queue and leaving the tail entry intact
if we saw pending set, because we know that the tail is going to be
updated shortly.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: boqun.feng@gmail.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: paulmck@linux.vnet.ibm.com
Link: http://lkml.kernel.org/r/1524738868-31318-6-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-04-26 18:34:19 +08:00
|
|
|
/*
|
|
|
|
* If we observe any contention; queue.
|
|
|
|
*/
|
|
|
|
if (val & ~_Q_LOCKED_MASK)
|
|
|
|
goto queue;
|
|
|
|
|
2015-04-25 02:56:32 +08:00
|
|
|
/*
|
|
|
|
* trylock || pending
|
|
|
|
*
|
2018-09-26 19:01:19 +08:00
|
|
|
* 0,0,* -> 0,1,* -> 0,0,1 pending, trylock
|
2015-04-25 02:56:32 +08:00
|
|
|
*/
|
locking/qspinlock, x86: Provide liveness guarantee
On x86 we cannot do fetch_or() with a single instruction and thus end up
using a cmpxchg loop, this reduces determinism. Replace the fetch_or()
with a composite operation: tas-pending + load.
Using two instructions of course opens a window we previously did not
have. Consider the scenario:
CPU0 CPU1 CPU2
1) lock
trylock -> (0,0,1)
2) lock
trylock /* fail */
3) unlock -> (0,0,0)
4) lock
trylock -> (0,0,1)
5) tas-pending -> (0,1,1)
load-val <- (0,1,0) from 3
6) clear-pending-set-locked -> (0,0,1)
FAIL: _2_ owners
where 5) is our new composite operation. When we consider each part of
the qspinlock state as a separate variable (as we can when
_Q_PENDING_BITS == 8) then the above is entirely possible, because
tas-pending will only RmW the pending byte, so the later load is able
to observe prior tail and lock state (but not earlier than its own
trylock, which operates on the whole word, due to coherence).
To avoid this we need 2 things:
- the load must come after the tas-pending (obviously, otherwise it
can trivially observe prior state).
- the tas-pending must be a full word RmW instruction, it cannot be an XCHGB for
example, such that we cannot observe other state prior to setting
pending.
On x86 we can realize this by using "LOCK BTS m32, r32" for
tas-pending followed by a regular load.
Note that observing later state is not a problem:
- if we fail to observe a later unlock, we'll simply spin-wait for
that store to become visible.
- if we observe a later xchg_tail(), there is no difference from that
xchg_tail() having taken place before the tas-pending.
Suggested-by: Will Deacon <will.deacon@arm.com>
Reported-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: andrea.parri@amarulasolutions.com
Cc: longman@redhat.com
Fixes: 59fb586b4a07 ("locking/qspinlock: Remove unbounded cmpxchg() loop from locking slowpath")
Link: https://lkml.kernel.org/r/20181003130957.183726335@infradead.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-09-26 19:01:20 +08:00
|
|
|
val = queued_fetch_set_pending_acquire(lock);
|
2018-09-26 19:01:19 +08:00
|
|
|
|
2018-09-26 19:01:18 +08:00
|
|
|
/*
|
2018-09-26 19:01:19 +08:00
|
|
|
* If we observe contention, there is a concurrent locker.
|
|
|
|
*
|
|
|
|
* Undo and queue; our setting of PENDING might have made the
|
|
|
|
* n,0,0 -> 0,0,0 transition fail and it will now be waiting
|
|
|
|
* on @next to become !NULL.
|
2018-09-26 19:01:18 +08:00
|
|
|
*/
|
|
|
|
if (unlikely(val & ~_Q_LOCKED_MASK)) {
|
2018-09-26 19:01:19 +08:00
|
|
|
|
|
|
|
/* Undo PENDING if we set it. */
|
2018-09-26 19:01:18 +08:00
|
|
|
if (!(val & _Q_PENDING_MASK))
|
|
|
|
clear_pending(lock);
|
2018-09-26 19:01:19 +08:00
|
|
|
|
2018-09-26 19:01:18 +08:00
|
|
|
goto queue;
|
locking/qspinlock: Remove unbounded cmpxchg() loop from locking slowpath
The qspinlock locking slowpath utilises a "pending" bit as a simple form
of an embedded test-and-set lock that can avoid the overhead of explicit
queuing in cases where the lock is held but uncontended. This bit is
managed using a cmpxchg() loop which tries to transition the uncontended
lock word from (0,0,0) -> (0,0,1) or (0,0,1) -> (0,1,1).
Unfortunately, the cmpxchg() loop is unbounded and lockers can be starved
indefinitely if the lock word is seen to oscillate between unlocked
(0,0,0) and locked (0,0,1). This could happen if concurrent lockers are
able to take the lock in the cmpxchg() loop without queuing and pass it
around amongst themselves.
This patch fixes the problem by unconditionally setting _Q_PENDING_VAL
using atomic_fetch_or, and then inspecting the old value to see whether
we need to spin on the current lock owner, or whether we now effectively
hold the lock. The tricky scenario is when concurrent lockers end up
queuing on the lock and the lock becomes available, causing us to see
a lockword of (n,0,0). With pending now set, simply queuing could lead
to deadlock as the head of the queue may not have observed the pending
flag being cleared. Conversely, if the head of the queue did observe
pending being cleared, then it could transition the lock from (n,0,0) ->
(0,0,1) meaning that any attempt to "undo" our setting of the pending
bit could race with a concurrent locker trying to set it.
We handle this race by preserving the pending bit when taking the lock
after reaching the head of the queue and leaving the tail entry intact
if we saw pending set, because we know that the tail is going to be
updated shortly.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: boqun.feng@gmail.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: paulmck@linux.vnet.ibm.com
Link: http://lkml.kernel.org/r/1524738868-31318-6-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-04-26 18:34:19 +08:00
|
|
|
}
|
2015-04-25 02:56:32 +08:00
|
|
|
|
|
|
|
/*
|
2018-09-26 19:01:18 +08:00
|
|
|
* We're pending, wait for the owner to go away.
|
|
|
|
*
|
2023-01-05 10:19:52 +08:00
|
|
|
* 0,1,1 -> *,1,0
|
2018-09-26 19:01:18 +08:00
|
|
|
*
|
|
|
|
* this wait loop must be a load-acquire such that we match the
|
|
|
|
* store-release that clears the locked bit and create lock
|
|
|
|
* sequentiality; this is because not all
|
|
|
|
* clear_pending_set_locked() implementations imply full
|
|
|
|
* barriers.
|
|
|
|
*/
|
|
|
|
if (val & _Q_LOCKED_MASK)
|
2023-01-05 10:19:52 +08:00
|
|
|
smp_cond_load_acquire(&lock->locked, !VAL);
|
2018-09-26 19:01:18 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* take ownership and clear the pending bit.
|
|
|
|
*
|
|
|
|
* 0,1,0 -> 0,0,1
|
2015-04-25 02:56:32 +08:00
|
|
|
*/
|
2018-09-26 19:01:18 +08:00
|
|
|
clear_pending_set_locked(lock);
|
2019-04-05 01:43:16 +08:00
|
|
|
lockevent_inc(lock_pending);
|
2018-09-26 19:01:18 +08:00
|
|
|
return;
|
2015-04-25 02:56:32 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* End of pending bit optimistic spinning and beginning of MCS
|
|
|
|
* queuing.
|
|
|
|
*/
|
|
|
|
queue:
|
2019-04-05 01:43:16 +08:00
|
|
|
lockevent_inc(lock_slowpath);
|
2018-04-26 18:34:27 +08:00
|
|
|
pv_queue:
|
2018-10-16 21:45:07 +08:00
|
|
|
node = this_cpu_ptr(&qnodes[0].mcs);
|
2015-04-25 02:56:30 +08:00
|
|
|
idx = node->count++;
|
|
|
|
tail = encode_tail(smp_processor_id(), idx);
|
|
|
|
|
2022-03-23 02:57:09 +08:00
|
|
|
trace_contention_begin(lock, LCB_F_SPIN);
|
|
|
|
|
2019-01-30 05:53:45 +08:00
|
|
|
/*
|
|
|
|
* 4 nodes are allocated based on the assumption that there will
|
|
|
|
* not be nested NMIs taking spinlocks. That may not be true in
|
|
|
|
* some architectures even though the chance of needing more than
|
|
|
|
* 4 nodes will still be extremely unlikely. When that happens,
|
|
|
|
* we fall back to spinning on the lock directly without using
|
|
|
|
* any MCS node. This is not the most elegant solution, but is
|
|
|
|
* simple enough.
|
|
|
|
*/
|
|
|
|
if (unlikely(idx >= MAX_NODES)) {
|
2019-04-05 01:43:16 +08:00
|
|
|
lockevent_inc(lock_no_node);
|
2019-01-30 05:53:45 +08:00
|
|
|
while (!queued_spin_trylock(lock))
|
|
|
|
cpu_relax();
|
|
|
|
goto release;
|
|
|
|
}
|
|
|
|
|
2018-10-16 21:45:07 +08:00
|
|
|
node = grab_mcs_node(node, idx);
|
2018-02-13 21:22:57 +08:00
|
|
|
|
2018-10-16 21:45:06 +08:00
|
|
|
/*
|
|
|
|
* Keep counts of non-zero index values:
|
|
|
|
*/
|
2019-04-05 01:43:16 +08:00
|
|
|
lockevent_cond_inc(lock_use_node2 + idx - 1, idx);
|
2018-10-16 21:45:06 +08:00
|
|
|
|
2018-02-13 21:22:57 +08:00
|
|
|
/*
|
|
|
|
* Ensure that we increment the head node->count before initialising
|
|
|
|
* the actual node. If the compiler is kind enough to reorder these
|
|
|
|
* stores, then an IRQ could overwrite our assignments.
|
|
|
|
*/
|
|
|
|
barrier();
|
|
|
|
|
2015-04-25 02:56:30 +08:00
|
|
|
node->locked = 0;
|
|
|
|
node->next = NULL;
|
2015-04-25 02:56:37 +08:00
|
|
|
pv_init_node(node);
|
2015-04-25 02:56:30 +08:00
|
|
|
|
|
|
|
/*
|
2015-04-25 02:56:33 +08:00
|
|
|
* We touched a (possibly) cold cacheline in the per-cpu queue node;
|
|
|
|
* attempt the trylock once more in the hope someone let go while we
|
|
|
|
* weren't watching.
|
2015-04-25 02:56:30 +08:00
|
|
|
*/
|
2015-04-25 02:56:33 +08:00
|
|
|
if (queued_spin_trylock(lock))
|
|
|
|
goto release;
|
2015-04-25 02:56:30 +08:00
|
|
|
|
|
|
|
/*
|
2018-04-26 18:34:25 +08:00
|
|
|
* Ensure that the initialisation of @node is complete before we
|
|
|
|
* publish the updated tail via xchg_tail() and potentially link
|
|
|
|
* @node into the waitqueue via WRITE_ONCE(prev->next, node) below.
|
|
|
|
*/
|
|
|
|
smp_wmb();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Publish the updated tail.
|
2015-04-25 02:56:33 +08:00
|
|
|
* We have already touched the queueing cacheline; don't bother with
|
|
|
|
* pending stuff.
|
|
|
|
*
|
|
|
|
* p,*,* -> n,*,*
|
2015-04-25 02:56:30 +08:00
|
|
|
*/
|
2015-04-25 02:56:33 +08:00
|
|
|
old = xchg_tail(lock, tail);
|
2015-11-10 08:09:23 +08:00
|
|
|
next = NULL;
|
2015-04-25 02:56:30 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* if there was a previous node; link it and wait until reaching the
|
|
|
|
* head of the waitqueue.
|
|
|
|
*/
|
2015-04-25 02:56:33 +08:00
|
|
|
if (old & _Q_TAIL_MASK) {
|
2015-04-25 02:56:30 +08:00
|
|
|
prev = decode_tail(old);
|
2018-02-13 21:22:56 +08:00
|
|
|
|
2018-04-26 18:34:25 +08:00
|
|
|
/* Link @node into the waitqueue. */
|
|
|
|
WRITE_ONCE(prev->next, node);
|
2015-04-25 02:56:30 +08:00
|
|
|
|
2015-11-10 08:09:27 +08:00
|
|
|
pv_wait_node(node, prev);
|
2015-04-25 02:56:30 +08:00
|
|
|
arch_mcs_spin_lock_contended(&node->locked);
|
2015-11-10 08:09:22 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* While waiting for the MCS lock, the next pointer may have
|
|
|
|
* been set by another lock waiter. We optimistically load
|
|
|
|
* the next pointer & prefetch the cacheline for writing
|
|
|
|
* to reduce latency in the upcoming MCS unlock operation.
|
|
|
|
*/
|
|
|
|
next = READ_ONCE(node->next);
|
|
|
|
if (next)
|
|
|
|
prefetchw(next);
|
2015-04-25 02:56:30 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2015-04-25 02:56:32 +08:00
|
|
|
* we're at the head of the waitqueue, wait for the owner & pending to
|
|
|
|
* go away.
|
2015-04-25 02:56:30 +08:00
|
|
|
*
|
2015-04-25 02:56:32 +08:00
|
|
|
* *,x,y -> *,0,0
|
2015-04-25 02:56:35 +08:00
|
|
|
*
|
|
|
|
* this wait loop must use a load-acquire such that we match the
|
|
|
|
* store-release that clears the locked bit and create lock
|
|
|
|
* sequentiality; this is because the set_locked() function below
|
|
|
|
* does not imply a full barrier.
|
|
|
|
*
|
2015-11-11 05:18:56 +08:00
|
|
|
* The PV pv_wait_head_or_lock function, if active, will acquire
|
|
|
|
* the lock and return a non-zero value. So we have to skip the
|
2018-04-26 18:34:21 +08:00
|
|
|
* atomic_cond_read_acquire() call. As the next PV queue head hasn't
|
|
|
|
* been designated yet, there is no way for the locked value to become
|
2015-11-11 05:18:56 +08:00
|
|
|
* _Q_SLOW_VAL. So both the set_locked() and the
|
|
|
|
* atomic_cmpxchg_relaxed() calls will be safe.
|
|
|
|
*
|
|
|
|
* If PV isn't active, 0 will be returned instead.
|
|
|
|
*
|
2015-04-25 02:56:30 +08:00
|
|
|
*/
|
2015-11-11 05:18:56 +08:00
|
|
|
if ((val = pv_wait_head_or_lock(lock, node)))
|
|
|
|
goto locked;
|
|
|
|
|
2018-04-26 18:34:21 +08:00
|
|
|
val = atomic_cond_read_acquire(&lock->val, !(VAL & _Q_LOCKED_PENDING_MASK));
|
2015-04-25 02:56:30 +08:00
|
|
|
|
2015-11-11 05:18:56 +08:00
|
|
|
locked:
|
2015-04-25 02:56:30 +08:00
|
|
|
/*
|
|
|
|
* claim the lock:
|
|
|
|
*
|
2015-04-25 02:56:32 +08:00
|
|
|
* n,0,0 -> 0,0,1 : lock, uncontended
|
locking/qspinlock: Remove unbounded cmpxchg() loop from locking slowpath
The qspinlock locking slowpath utilises a "pending" bit as a simple form
of an embedded test-and-set lock that can avoid the overhead of explicit
queuing in cases where the lock is held but uncontended. This bit is
managed using a cmpxchg() loop which tries to transition the uncontended
lock word from (0,0,0) -> (0,0,1) or (0,0,1) -> (0,1,1).
Unfortunately, the cmpxchg() loop is unbounded and lockers can be starved
indefinitely if the lock word is seen to oscillate between unlocked
(0,0,0) and locked (0,0,1). This could happen if concurrent lockers are
able to take the lock in the cmpxchg() loop without queuing and pass it
around amongst themselves.
This patch fixes the problem by unconditionally setting _Q_PENDING_VAL
using atomic_fetch_or, and then inspecting the old value to see whether
we need to spin on the current lock owner, or whether we now effectively
hold the lock. The tricky scenario is when concurrent lockers end up
queuing on the lock and the lock becomes available, causing us to see
a lockword of (n,0,0). With pending now set, simply queuing could lead
to deadlock as the head of the queue may not have observed the pending
flag being cleared. Conversely, if the head of the queue did observe
pending being cleared, then it could transition the lock from (n,0,0) ->
(0,0,1) meaning that any attempt to "undo" our setting of the pending
bit could race with a concurrent locker trying to set it.
We handle this race by preserving the pending bit when taking the lock
after reaching the head of the queue and leaving the tail entry intact
if we saw pending set, because we know that the tail is going to be
updated shortly.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: boqun.feng@gmail.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: paulmck@linux.vnet.ibm.com
Link: http://lkml.kernel.org/r/1524738868-31318-6-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-04-26 18:34:19 +08:00
|
|
|
* *,*,0 -> *,*,1 : lock, contended
|
2015-04-25 02:56:35 +08:00
|
|
|
*
|
locking/qspinlock: Remove unbounded cmpxchg() loop from locking slowpath
The qspinlock locking slowpath utilises a "pending" bit as a simple form
of an embedded test-and-set lock that can avoid the overhead of explicit
queuing in cases where the lock is held but uncontended. This bit is
managed using a cmpxchg() loop which tries to transition the uncontended
lock word from (0,0,0) -> (0,0,1) or (0,0,1) -> (0,1,1).
Unfortunately, the cmpxchg() loop is unbounded and lockers can be starved
indefinitely if the lock word is seen to oscillate between unlocked
(0,0,0) and locked (0,0,1). This could happen if concurrent lockers are
able to take the lock in the cmpxchg() loop without queuing and pass it
around amongst themselves.
This patch fixes the problem by unconditionally setting _Q_PENDING_VAL
using atomic_fetch_or, and then inspecting the old value to see whether
we need to spin on the current lock owner, or whether we now effectively
hold the lock. The tricky scenario is when concurrent lockers end up
queuing on the lock and the lock becomes available, causing us to see
a lockword of (n,0,0). With pending now set, simply queuing could lead
to deadlock as the head of the queue may not have observed the pending
flag being cleared. Conversely, if the head of the queue did observe
pending being cleared, then it could transition the lock from (n,0,0) ->
(0,0,1) meaning that any attempt to "undo" our setting of the pending
bit could race with a concurrent locker trying to set it.
We handle this race by preserving the pending bit when taking the lock
after reaching the head of the queue and leaving the tail entry intact
if we saw pending set, because we know that the tail is going to be
updated shortly.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: boqun.feng@gmail.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: paulmck@linux.vnet.ibm.com
Link: http://lkml.kernel.org/r/1524738868-31318-6-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-04-26 18:34:19 +08:00
|
|
|
* If the queue head is the only one in the queue (lock value == tail)
|
|
|
|
* and nobody is pending, clear the tail code and grab the lock.
|
|
|
|
* Otherwise, we only need to grab the lock.
|
2015-04-25 02:56:30 +08:00
|
|
|
*/
|
2018-04-26 18:34:20 +08:00
|
|
|
|
2018-04-26 18:34:26 +08:00
|
|
|
/*
|
2018-09-26 19:01:19 +08:00
|
|
|
* In the PV case we might already have _Q_LOCKED_VAL set, because
|
|
|
|
* of lock stealing; therefore we must also allow:
|
2018-04-26 18:34:26 +08:00
|
|
|
*
|
2018-09-26 19:01:19 +08:00
|
|
|
* n,0,1 -> 0,0,1
|
|
|
|
*
|
|
|
|
* Note: at this point: (val & _Q_PENDING_MASK) == 0, because of the
|
|
|
|
* above wait condition, therefore any concurrent setting of
|
|
|
|
* PENDING will make the uncontended transition fail.
|
2018-04-26 18:34:26 +08:00
|
|
|
*/
|
2018-09-26 19:01:19 +08:00
|
|
|
if ((val & _Q_TAIL_MASK) == tail) {
|
|
|
|
if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
|
|
|
|
goto release; /* No contention */
|
|
|
|
}
|
2015-04-25 02:56:30 +08:00
|
|
|
|
2018-09-26 19:01:19 +08:00
|
|
|
/*
|
|
|
|
* Either somebody is queued behind us or _Q_PENDING_VAL got set
|
|
|
|
* which will then detect the remaining tail and queue behind us
|
|
|
|
* ensuring we'll see a @next.
|
|
|
|
*/
|
2018-04-26 18:34:20 +08:00
|
|
|
set_locked(lock);
|
|
|
|
|
2015-04-25 02:56:30 +08:00
|
|
|
/*
|
2015-11-10 08:09:23 +08:00
|
|
|
* contended path; wait for next if not observed yet, release.
|
2015-04-25 02:56:30 +08:00
|
|
|
*/
|
2018-04-26 18:34:23 +08:00
|
|
|
if (!next)
|
|
|
|
next = smp_cond_load_relaxed(&node->next, (VAL));
|
2015-04-25 02:56:30 +08:00
|
|
|
|
2015-04-25 02:56:35 +08:00
|
|
|
arch_mcs_spin_unlock_contended(&next->locked);
|
2015-07-12 04:36:52 +08:00
|
|
|
pv_kick_node(lock, next);
|
2015-04-25 02:56:30 +08:00
|
|
|
|
|
|
|
release:
|
2022-03-23 02:57:09 +08:00
|
|
|
trace_contention_end(lock, 0);
|
|
|
|
|
2015-04-25 02:56:30 +08:00
|
|
|
/*
|
|
|
|
* release the node
|
|
|
|
*/
|
2018-10-16 21:45:07 +08:00
|
|
|
__this_cpu_dec(qnodes[0].mcs.count);
|
2015-04-25 02:56:30 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(queued_spin_lock_slowpath);
|
2015-04-25 02:56:37 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Generate the paravirt code for queued_spin_unlock_slowpath().
|
|
|
|
*/
|
|
|
|
#if !defined(_GEN_PV_LOCK_SLOWPATH) && defined(CONFIG_PARAVIRT_SPINLOCKS)
|
|
|
|
#define _GEN_PV_LOCK_SLOWPATH
|
|
|
|
|
|
|
|
#undef pv_enabled
|
|
|
|
#define pv_enabled() true
|
|
|
|
|
|
|
|
#undef pv_init_node
|
|
|
|
#undef pv_wait_node
|
|
|
|
#undef pv_kick_node
|
2015-11-11 05:18:56 +08:00
|
|
|
#undef pv_wait_head_or_lock
|
2015-04-25 02:56:37 +08:00
|
|
|
|
|
|
|
#undef queued_spin_lock_slowpath
|
|
|
|
#define queued_spin_lock_slowpath __pv_queued_spin_lock_slowpath
|
|
|
|
|
|
|
|
#include "qspinlock_paravirt.h"
|
|
|
|
#include "qspinlock.c"
|
|
|
|
|
2019-10-23 19:16:22 +08:00
|
|
|
bool nopvspin __initdata;
|
|
|
|
static __init int parse_nopvspin(char *arg)
|
|
|
|
{
|
|
|
|
nopvspin = true;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("nopvspin", parse_nopvspin);
|
2015-04-25 02:56:37 +08:00
|
|
|
#endif
|