blockdev: turn a rw semaphore into a percpu rw semaphore
This avoids cache line bouncing when many processes lock the semaphore
for read.
New percpu lock implementation
The lock consists of an array of percpu unsigned integers, a boolean
variable and a mutex.
When we take the lock for read, we enter rcu read section, check for a
"locked" variable. If it is false, we increase a percpu counter on the
current cpu and exit the rcu section. If "locked" is true, we exit the
rcu section, take the mutex and drop it (this waits until a writer
finished) and retry.
Unlocking for read just decreases percpu variable. Note that we can
unlock on a difference cpu than where we locked, in this case the
counter underflows. The sum of all percpu counters represents the number
of processes that hold the lock for read.
When we need to lock for write, we take the mutex, set "locked" variable
to true and synchronize rcu. Since RCU has been synchronized, no
processes can create new read locks. We wait until the sum of percpu
counters is zero - when it is, there are no readers in the critical
section.
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-09-26 13:46:43 +08:00
|
|
|
#ifndef _LINUX_PERCPU_RWSEM_H
|
|
|
|
|
#define _LINUX_PERCPU_RWSEM_H
|
|
|
|
|
|
2012-12-18 08:01:36 +08:00
|
|
|
#include <linux/atomic.h>
|
2012-12-18 08:01:32 +08:00
|
|
|
#include <linux/rwsem.h>
|
blockdev: turn a rw semaphore into a percpu rw semaphore
This avoids cache line bouncing when many processes lock the semaphore
for read.
New percpu lock implementation
The lock consists of an array of percpu unsigned integers, a boolean
variable and a mutex.
When we take the lock for read, we enter rcu read section, check for a
"locked" variable. If it is false, we increase a percpu counter on the
current cpu and exit the rcu section. If "locked" is true, we exit the
rcu section, take the mutex and drop it (this waits until a writer
finished) and retry.
Unlocking for read just decreases percpu variable. Note that we can
unlock on a difference cpu than where we locked, in this case the
counter underflows. The sum of all percpu counters represents the number
of processes that hold the lock for read.
When we need to lock for write, we take the mutex, set "locked" variable
to true and synchronize rcu. Since RCU has been synchronized, no
processes can create new read locks. We wait until the sum of percpu
counters is zero - when it is, there are no readers in the critical
section.
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-09-26 13:46:43 +08:00
|
|
|
#include <linux/percpu.h>
|
2012-12-18 08:01:32 +08:00
|
|
|
#include <linux/wait.h>
|
2015-08-22 01:42:57 +08:00
|
|
|
#include <linux/rcu_sync.h>
|
2012-12-18 08:01:38 +08:00
|
|
|
#include <linux/lockdep.h>
|
blockdev: turn a rw semaphore into a percpu rw semaphore
This avoids cache line bouncing when many processes lock the semaphore
for read.
New percpu lock implementation
The lock consists of an array of percpu unsigned integers, a boolean
variable and a mutex.
When we take the lock for read, we enter rcu read section, check for a
"locked" variable. If it is false, we increase a percpu counter on the
current cpu and exit the rcu section. If "locked" is true, we exit the
rcu section, take the mutex and drop it (this waits until a writer
finished) and retry.
Unlocking for read just decreases percpu variable. Note that we can
unlock on a difference cpu than where we locked, in this case the
counter underflows. The sum of all percpu counters represents the number
of processes that hold the lock for read.
When we need to lock for write, we take the mutex, set "locked" variable
to true and synchronize rcu. Since RCU has been synchronized, no
processes can create new read locks. We wait until the sum of percpu
counters is zero - when it is, there are no readers in the critical
section.
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-09-26 13:46:43 +08:00
|
|
|
|
|
|
|
|
struct percpu_rw_semaphore {
|
2015-08-22 01:42:57 +08:00
|
|
|
struct rcu_sync rss;
|
2016-07-15 02:08:46 +08:00
|
|
|
unsigned int __percpu *read_count;
|
2012-12-18 08:01:32 +08:00
|
|
|
struct rw_semaphore rw_sem;
|
2016-07-15 02:08:46 +08:00
|
|
|
wait_queue_head_t writer;
|
|
|
|
|
int readers_block;
|
blockdev: turn a rw semaphore into a percpu rw semaphore
This avoids cache line bouncing when many processes lock the semaphore
for read.
New percpu lock implementation
The lock consists of an array of percpu unsigned integers, a boolean
variable and a mutex.
When we take the lock for read, we enter rcu read section, check for a
"locked" variable. If it is false, we increase a percpu counter on the
current cpu and exit the rcu section. If "locked" is true, we exit the
rcu section, take the mutex and drop it (this waits until a writer
finished) and retry.
Unlocking for read just decreases percpu variable. Note that we can
unlock on a difference cpu than where we locked, in this case the
counter underflows. The sum of all percpu counters represents the number
of processes that hold the lock for read.
When we need to lock for write, we take the mutex, set "locked" variable
to true and synchronize rcu. Since RCU has been synchronized, no
processes can create new read locks. We wait until the sum of percpu
counters is zero - when it is, there are no readers in the critical
section.
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-09-26 13:46:43 +08:00
|
|
|
};
|
|
|
|
|
|
2016-07-15 02:08:46 +08:00
|
|
|
extern int __percpu_down_read(struct percpu_rw_semaphore *, int);
|
|
|
|
|
extern void __percpu_up_read(struct percpu_rw_semaphore *);
|
|
|
|
|
|
|
|
|
|
static inline void percpu_down_read(struct percpu_rw_semaphore *sem)
|
|
|
|
|
{
|
|
|
|
|
might_sleep();
|
|
|
|
|
|
|
|
|
|
rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_);
|
|
|
|
|
|
|
|
|
|
preempt_disable();
|
|
|
|
|
/*
|
|
|
|
|
* We are in an RCU-sched read-side critical section, so the writer
|
|
|
|
|
* cannot both change sem->state from readers_fast and start checking
|
|
|
|
|
* counters while we are here. So if we see !sem->state, we know that
|
|
|
|
|
* the writer won't be checking until we're past the preempt_enable()
|
|
|
|
|
* and that one the synchronize_sched() is done, the writer will see
|
|
|
|
|
* anything we did within this RCU-sched read-size critical section.
|
|
|
|
|
*/
|
|
|
|
|
__this_cpu_inc(*sem->read_count);
|
|
|
|
|
if (unlikely(!rcu_sync_is_idle(&sem->rss)))
|
|
|
|
|
__percpu_down_read(sem, false); /* Unconditional memory barrier */
|
|
|
|
|
preempt_enable();
|
|
|
|
|
/*
|
|
|
|
|
* The barrier() from preempt_enable() prevents the compiler from
|
|
|
|
|
* bleeding the critical section out.
|
|
|
|
|
*/
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int percpu_down_read_trylock(struct percpu_rw_semaphore *sem)
|
|
|
|
|
{
|
|
|
|
|
int ret = 1;
|
|
|
|
|
|
|
|
|
|
preempt_disable();
|
|
|
|
|
/*
|
|
|
|
|
* Same as in percpu_down_read().
|
|
|
|
|
*/
|
|
|
|
|
__this_cpu_inc(*sem->read_count);
|
|
|
|
|
if (unlikely(!rcu_sync_is_idle(&sem->rss)))
|
|
|
|
|
ret = __percpu_down_read(sem, true); /* Unconditional memory barrier */
|
|
|
|
|
preempt_enable();
|
|
|
|
|
/*
|
|
|
|
|
* The barrier() from preempt_enable() prevents the compiler from
|
|
|
|
|
* bleeding the critical section out.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
if (ret)
|
|
|
|
|
rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 1, _RET_IP_);
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void percpu_up_read(struct percpu_rw_semaphore *sem)
|
|
|
|
|
{
|
|
|
|
|
/*
|
|
|
|
|
* The barrier() in preempt_disable() prevents the compiler from
|
|
|
|
|
* bleeding the critical section out.
|
|
|
|
|
*/
|
|
|
|
|
preempt_disable();
|
|
|
|
|
/*
|
|
|
|
|
* Same as in percpu_down_read().
|
|
|
|
|
*/
|
|
|
|
|
if (likely(rcu_sync_is_idle(&sem->rss)))
|
|
|
|
|
__this_cpu_dec(*sem->read_count);
|
|
|
|
|
else
|
|
|
|
|
__percpu_up_read(sem); /* Unconditional memory barrier */
|
|
|
|
|
preempt_enable();
|
|
|
|
|
|
|
|
|
|
rwsem_release(&sem->rw_sem.dep_map, 1, _RET_IP_);
|
|
|
|
|
}
|
2012-10-23 07:37:47 +08:00
|
|
|
|
2012-12-18 08:01:32 +08:00
|
|
|
extern void percpu_down_write(struct percpu_rw_semaphore *);
|
|
|
|
|
extern void percpu_up_write(struct percpu_rw_semaphore *);
|
blockdev: turn a rw semaphore into a percpu rw semaphore
This avoids cache line bouncing when many processes lock the semaphore
for read.
New percpu lock implementation
The lock consists of an array of percpu unsigned integers, a boolean
variable and a mutex.
When we take the lock for read, we enter rcu read section, check for a
"locked" variable. If it is false, we increase a percpu counter on the
current cpu and exit the rcu section. If "locked" is true, we exit the
rcu section, take the mutex and drop it (this waits until a writer
finished) and retry.
Unlocking for read just decreases percpu variable. Note that we can
unlock on a difference cpu than where we locked, in this case the
counter underflows. The sum of all percpu counters represents the number
of processes that hold the lock for read.
When we need to lock for write, we take the mutex, set "locked" variable
to true and synchronize rcu. Since RCU has been synchronized, no
processes can create new read locks. We wait until the sum of percpu
counters is zero - when it is, there are no readers in the critical
section.
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-09-26 13:46:43 +08:00
|
|
|
|
2012-12-18 08:01:38 +08:00
|
|
|
extern int __percpu_init_rwsem(struct percpu_rw_semaphore *,
|
|
|
|
|
const char *, struct lock_class_key *);
|
2016-07-15 02:08:46 +08:00
|
|
|
|
2012-12-18 08:01:32 +08:00
|
|
|
extern void percpu_free_rwsem(struct percpu_rw_semaphore *);
|
blockdev: turn a rw semaphore into a percpu rw semaphore
This avoids cache line bouncing when many processes lock the semaphore
for read.
New percpu lock implementation
The lock consists of an array of percpu unsigned integers, a boolean
variable and a mutex.
When we take the lock for read, we enter rcu read section, check for a
"locked" variable. If it is false, we increase a percpu counter on the
current cpu and exit the rcu section. If "locked" is true, we exit the
rcu section, take the mutex and drop it (this waits until a writer
finished) and retry.
Unlocking for read just decreases percpu variable. Note that we can
unlock on a difference cpu than where we locked, in this case the
counter underflows. The sum of all percpu counters represents the number
of processes that hold the lock for read.
When we need to lock for write, we take the mutex, set "locked" variable
to true and synchronize rcu. Since RCU has been synchronized, no
processes can create new read locks. We wait until the sum of percpu
counters is zero - when it is, there are no readers in the critical
section.
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-09-26 13:46:43 +08:00
|
|
|
|
2016-07-15 02:08:46 +08:00
|
|
|
#define percpu_init_rwsem(sem) \
|
2012-12-18 08:01:38 +08:00
|
|
|
({ \
|
|
|
|
|
static struct lock_class_key rwsem_key; \
|
2016-07-15 02:08:46 +08:00
|
|
|
__percpu_init_rwsem(sem, #sem, &rwsem_key); \
|
2012-12-18 08:01:38 +08:00
|
|
|
})
|
|
|
|
|
|
2015-07-22 02:26:44 +08:00
|
|
|
#define percpu_rwsem_is_held(sem) lockdep_is_held(&(sem)->rw_sem)
|
|
|
|
|
|
|
|
|
|
static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem,
|
|
|
|
|
bool read, unsigned long ip)
|
|
|
|
|
{
|
|
|
|
|
lock_release(&sem->rw_sem.dep_map, 1, ip);
|
|
|
|
|
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
|
|
|
|
|
if (!read)
|
|
|
|
|
sem->rw_sem.owner = NULL;
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem,
|
|
|
|
|
bool read, unsigned long ip)
|
|
|
|
|
{
|
|
|
|
|
lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip);
|
|
|
|
|
}
|
|
|
|
|
|
blockdev: turn a rw semaphore into a percpu rw semaphore
This avoids cache line bouncing when many processes lock the semaphore
for read.
New percpu lock implementation
The lock consists of an array of percpu unsigned integers, a boolean
variable and a mutex.
When we take the lock for read, we enter rcu read section, check for a
"locked" variable. If it is false, we increase a percpu counter on the
current cpu and exit the rcu section. If "locked" is true, we exit the
rcu section, take the mutex and drop it (this waits until a writer
finished) and retry.
Unlocking for read just decreases percpu variable. Note that we can
unlock on a difference cpu than where we locked, in this case the
counter underflows. The sum of all percpu counters represents the number
of processes that hold the lock for read.
When we need to lock for write, we take the mutex, set "locked" variable
to true and synchronize rcu. Since RCU has been synchronized, no
processes can create new read locks. We wait until the sum of percpu
counters is zero - when it is, there are no readers in the critical
section.
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-09-26 13:46:43 +08:00
|
|
|
#endif
|