Merge branch 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking fixes from Thomas Gleixner: "Two fixes to address shortcomings of the rwsem/percpu-rwsem lock debugging code which emits false positive warnings when the rwsem is anonymously locked and unlocked" * 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/percpu-rwsem: Annotate rwsem ownership transfer by setting RWSEM_OWNER_UNKNOWN locking/rwsem: Add a new RWSEM_ANONYMOUSLY_OWNED flag
This commit is contained in:
commit
4b65f455c7
|
@ -133,7 +133,7 @@ static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem,
|
||||||
lock_release(&sem->rw_sem.dep_map, 1, ip);
|
lock_release(&sem->rw_sem.dep_map, 1, ip);
|
||||||
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
|
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
|
||||||
if (!read)
|
if (!read)
|
||||||
sem->rw_sem.owner = NULL;
|
sem->rw_sem.owner = RWSEM_OWNER_UNKNOWN;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -141,6 +141,10 @@ static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem,
|
||||||
bool read, unsigned long ip)
|
bool read, unsigned long ip)
|
||||||
{
|
{
|
||||||
lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip);
|
lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip);
|
||||||
|
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
|
||||||
|
if (!read)
|
||||||
|
sem->rw_sem.owner = current;
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -44,6 +44,12 @@ struct rw_semaphore {
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Setting bit 0 of the owner field with other non-zero bits will indicate
|
||||||
|
* that the rwsem is writer-owned with an unknown owner.
|
||||||
|
*/
|
||||||
|
#define RWSEM_OWNER_UNKNOWN ((struct task_struct *)-1L)
|
||||||
|
|
||||||
extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
|
extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
|
||||||
extern struct rw_semaphore *rwsem_down_read_failed_killable(struct rw_semaphore *sem);
|
extern struct rw_semaphore *rwsem_down_read_failed_killable(struct rw_semaphore *sem);
|
||||||
extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
|
extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
|
||||||
|
|
|
@ -352,16 +352,15 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
|
||||||
struct task_struct *owner;
|
struct task_struct *owner;
|
||||||
bool ret = true;
|
bool ret = true;
|
||||||
|
|
||||||
|
BUILD_BUG_ON(!rwsem_has_anonymous_owner(RWSEM_OWNER_UNKNOWN));
|
||||||
|
|
||||||
if (need_resched())
|
if (need_resched())
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
rcu_read_lock();
|
rcu_read_lock();
|
||||||
owner = READ_ONCE(sem->owner);
|
owner = READ_ONCE(sem->owner);
|
||||||
if (!rwsem_owner_is_writer(owner)) {
|
if (!owner || !is_rwsem_owner_spinnable(owner)) {
|
||||||
/*
|
ret = !owner; /* !owner is spinnable */
|
||||||
* Don't spin if the rwsem is readers owned.
|
|
||||||
*/
|
|
||||||
ret = !rwsem_owner_is_reader(owner);
|
|
||||||
goto done;
|
goto done;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -382,11 +381,11 @@ static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem)
|
||||||
{
|
{
|
||||||
struct task_struct *owner = READ_ONCE(sem->owner);
|
struct task_struct *owner = READ_ONCE(sem->owner);
|
||||||
|
|
||||||
if (!rwsem_owner_is_writer(owner))
|
if (!is_rwsem_owner_spinnable(owner))
|
||||||
goto out;
|
return false;
|
||||||
|
|
||||||
rcu_read_lock();
|
rcu_read_lock();
|
||||||
while (sem->owner == owner) {
|
while (owner && (READ_ONCE(sem->owner) == owner)) {
|
||||||
/*
|
/*
|
||||||
* Ensure we emit the owner->on_cpu, dereference _after_
|
* Ensure we emit the owner->on_cpu, dereference _after_
|
||||||
* checking sem->owner still matches owner, if that fails,
|
* checking sem->owner still matches owner, if that fails,
|
||||||
|
@ -408,12 +407,12 @@ static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem)
|
||||||
cpu_relax();
|
cpu_relax();
|
||||||
}
|
}
|
||||||
rcu_read_unlock();
|
rcu_read_unlock();
|
||||||
out:
|
|
||||||
/*
|
/*
|
||||||
* If there is a new owner or the owner is not set, we continue
|
* If there is a new owner or the owner is not set, we continue
|
||||||
* spinning.
|
* spinning.
|
||||||
*/
|
*/
|
||||||
return !rwsem_owner_is_reader(READ_ONCE(sem->owner));
|
return is_rwsem_owner_spinnable(READ_ONCE(sem->owner));
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
|
static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
|
||||||
|
|
|
@ -221,5 +221,3 @@ void up_read_non_owner(struct rw_semaphore *sem)
|
||||||
EXPORT_SYMBOL(up_read_non_owner);
|
EXPORT_SYMBOL(up_read_non_owner);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,20 +1,24 @@
|
||||||
/* SPDX-License-Identifier: GPL-2.0 */
|
/* SPDX-License-Identifier: GPL-2.0 */
|
||||||
/*
|
/*
|
||||||
* The owner field of the rw_semaphore structure will be set to
|
* The owner field of the rw_semaphore structure will be set to
|
||||||
* RWSEM_READ_OWNED when a reader grabs the lock. A writer will clear
|
* RWSEM_READER_OWNED when a reader grabs the lock. A writer will clear
|
||||||
* the owner field when it unlocks. A reader, on the other hand, will
|
* the owner field when it unlocks. A reader, on the other hand, will
|
||||||
* not touch the owner field when it unlocks.
|
* not touch the owner field when it unlocks.
|
||||||
*
|
*
|
||||||
* In essence, the owner field now has the following 3 states:
|
* In essence, the owner field now has the following 4 states:
|
||||||
* 1) 0
|
* 1) 0
|
||||||
* - lock is free or the owner hasn't set the field yet
|
* - lock is free or the owner hasn't set the field yet
|
||||||
* 2) RWSEM_READER_OWNED
|
* 2) RWSEM_READER_OWNED
|
||||||
* - lock is currently or previously owned by readers (lock is free
|
* - lock is currently or previously owned by readers (lock is free
|
||||||
* or not set by owner yet)
|
* or not set by owner yet)
|
||||||
* 3) Other non-zero value
|
* 3) RWSEM_ANONYMOUSLY_OWNED bit set with some other bits set as well
|
||||||
* - a writer owns the lock
|
* - lock is owned by an anonymous writer, so spinning on the lock
|
||||||
|
* owner should be disabled.
|
||||||
|
* 4) Other non-zero value
|
||||||
|
* - a writer owns the lock and other writers can spin on the lock owner.
|
||||||
*/
|
*/
|
||||||
#define RWSEM_READER_OWNED ((struct task_struct *)1UL)
|
#define RWSEM_ANONYMOUSLY_OWNED (1UL << 0)
|
||||||
|
#define RWSEM_READER_OWNED ((struct task_struct *)RWSEM_ANONYMOUSLY_OWNED)
|
||||||
|
|
||||||
#ifdef CONFIG_DEBUG_RWSEMS
|
#ifdef CONFIG_DEBUG_RWSEMS
|
||||||
# define DEBUG_RWSEMS_WARN_ON(c) DEBUG_LOCKS_WARN_ON(c)
|
# define DEBUG_RWSEMS_WARN_ON(c) DEBUG_LOCKS_WARN_ON(c)
|
||||||
|
@ -51,14 +55,22 @@ static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
|
||||||
WRITE_ONCE(sem->owner, RWSEM_READER_OWNED);
|
WRITE_ONCE(sem->owner, RWSEM_READER_OWNED);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline bool rwsem_owner_is_writer(struct task_struct *owner)
|
/*
|
||||||
|
* Return true if the a rwsem waiter can spin on the rwsem's owner
|
||||||
|
* and steal the lock, i.e. the lock is not anonymously owned.
|
||||||
|
* N.B. !owner is considered spinnable.
|
||||||
|
*/
|
||||||
|
static inline bool is_rwsem_owner_spinnable(struct task_struct *owner)
|
||||||
{
|
{
|
||||||
return owner && owner != RWSEM_READER_OWNED;
|
return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline bool rwsem_owner_is_reader(struct task_struct *owner)
|
/*
|
||||||
|
* Return true if rwsem is owned by an anonymous writer or readers.
|
||||||
|
*/
|
||||||
|
static inline bool rwsem_has_anonymous_owner(struct task_struct *owner)
|
||||||
{
|
{
|
||||||
return owner == RWSEM_READER_OWNED;
|
return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED;
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
static inline void rwsem_set_owner(struct rw_semaphore *sem)
|
static inline void rwsem_set_owner(struct rw_semaphore *sem)
|
||||||
|
|
Loading…
Reference in New Issue