201 lines
6.6 KiB
C
201 lines
6.6 KiB
C
/*
|
|
* Specialised local-global spinlock. Can only be declared as global variables
|
|
* to avoid overhead and keep things simple (and we don't want to start using
|
|
* these inside dynamically allocated structures).
|
|
*
|
|
* "local/global locks" (lglocks) can be used to:
|
|
*
|
|
* - Provide fast exclusive access to per-CPU data, with exclusive access to
|
|
* another CPU's data allowed but possibly subject to contention, and to
|
|
* provide very slow exclusive access to all per-CPU data.
|
|
* - Or to provide very fast and scalable read serialisation, and to provide
|
|
* very slow exclusive serialisation of data (not necessarily per-CPU data).
|
|
*
|
|
* Brlocks are also implemented as a short-hand notation for the latter use
|
|
* case.
|
|
*
|
|
* Copyright 2009, 2010, Nick Piggin, Novell Inc.
|
|
*/
|
|
#ifndef __LINUX_LGLOCK_H
|
|
#define __LINUX_LGLOCK_H
|
|
|
|
#include <linux/spinlock.h>
|
|
#include <linux/lockdep.h>
|
|
#include <linux/percpu.h>
|
|
#include <linux/cpu.h>
|
|
|
|
/* can make br locks by using local lock for read side, global lock for write */
|
|
#define br_lock_init(name) name##_lock_init()
|
|
#define br_read_lock(name) name##_local_lock()
|
|
#define br_read_unlock(name) name##_local_unlock()
|
|
#define br_write_lock(name) name##_global_lock_online()
|
|
#define br_write_unlock(name) name##_global_unlock_online()
|
|
|
|
#define DECLARE_BRLOCK(name) DECLARE_LGLOCK(name)
|
|
#define DEFINE_BRLOCK(name) DEFINE_LGLOCK(name)
|
|
|
|
|
|
#define lg_lock_init(name) name##_lock_init()
|
|
#define lg_local_lock(name) name##_local_lock()
|
|
#define lg_local_unlock(name) name##_local_unlock()
|
|
#define lg_local_lock_cpu(name, cpu) name##_local_lock_cpu(cpu)
|
|
#define lg_local_unlock_cpu(name, cpu) name##_local_unlock_cpu(cpu)
|
|
#define lg_global_lock(name) name##_global_lock()
|
|
#define lg_global_unlock(name) name##_global_unlock()
|
|
#define lg_global_lock_online(name) name##_global_lock_online()
|
|
#define lg_global_unlock_online(name) name##_global_unlock_online()
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
#define LOCKDEP_INIT_MAP lockdep_init_map
|
|
|
|
#define DEFINE_LGLOCK_LOCKDEP(name) \
|
|
struct lock_class_key name##_lock_key; \
|
|
struct lockdep_map name##_lock_dep_map; \
|
|
EXPORT_SYMBOL(name##_lock_dep_map)
|
|
|
|
#else
|
|
#define LOCKDEP_INIT_MAP(a, b, c, d)
|
|
|
|
#define DEFINE_LGLOCK_LOCKDEP(name)
|
|
#endif
|
|
|
|
|
|
#define DECLARE_LGLOCK(name) \
|
|
extern void name##_lock_init(void); \
|
|
extern void name##_local_lock(void); \
|
|
extern void name##_local_unlock(void); \
|
|
extern void name##_local_lock_cpu(int cpu); \
|
|
extern void name##_local_unlock_cpu(int cpu); \
|
|
extern void name##_global_lock(void); \
|
|
extern void name##_global_unlock(void); \
|
|
extern void name##_global_lock_online(void); \
|
|
extern void name##_global_unlock_online(void); \
|
|
|
|
#define DEFINE_LGLOCK(name) \
|
|
\
|
|
DEFINE_SPINLOCK(name##_cpu_lock); \
|
|
cpumask_t name##_cpus __read_mostly; \
|
|
DEFINE_PER_CPU(arch_spinlock_t, name##_lock); \
|
|
DEFINE_LGLOCK_LOCKDEP(name); \
|
|
\
|
|
static int \
|
|
name##_lg_cpu_callback(struct notifier_block *nb, \
|
|
unsigned long action, void *hcpu) \
|
|
{ \
|
|
switch (action & ~CPU_TASKS_FROZEN) { \
|
|
case CPU_UP_PREPARE: \
|
|
spin_lock(&name##_cpu_lock); \
|
|
cpu_set((unsigned long)hcpu, name##_cpus); \
|
|
spin_unlock(&name##_cpu_lock); \
|
|
break; \
|
|
case CPU_UP_CANCELED: case CPU_DEAD: \
|
|
spin_lock(&name##_cpu_lock); \
|
|
cpu_clear((unsigned long)hcpu, name##_cpus); \
|
|
spin_unlock(&name##_cpu_lock); \
|
|
} \
|
|
return NOTIFY_OK; \
|
|
} \
|
|
static struct notifier_block name##_lg_cpu_notifier = { \
|
|
.notifier_call = name##_lg_cpu_callback, \
|
|
}; \
|
|
void name##_lock_init(void) { \
|
|
int i; \
|
|
LOCKDEP_INIT_MAP(&name##_lock_dep_map, #name, &name##_lock_key, 0); \
|
|
for_each_possible_cpu(i) { \
|
|
arch_spinlock_t *lock; \
|
|
lock = &per_cpu(name##_lock, i); \
|
|
*lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; \
|
|
} \
|
|
register_hotcpu_notifier(&name##_lg_cpu_notifier); \
|
|
get_online_cpus(); \
|
|
for_each_online_cpu(i) \
|
|
cpu_set(i, name##_cpus); \
|
|
put_online_cpus(); \
|
|
} \
|
|
EXPORT_SYMBOL(name##_lock_init); \
|
|
\
|
|
void name##_local_lock(void) { \
|
|
arch_spinlock_t *lock; \
|
|
preempt_disable(); \
|
|
rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_); \
|
|
lock = &__get_cpu_var(name##_lock); \
|
|
arch_spin_lock(lock); \
|
|
} \
|
|
EXPORT_SYMBOL(name##_local_lock); \
|
|
\
|
|
void name##_local_unlock(void) { \
|
|
arch_spinlock_t *lock; \
|
|
rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_); \
|
|
lock = &__get_cpu_var(name##_lock); \
|
|
arch_spin_unlock(lock); \
|
|
preempt_enable(); \
|
|
} \
|
|
EXPORT_SYMBOL(name##_local_unlock); \
|
|
\
|
|
void name##_local_lock_cpu(int cpu) { \
|
|
arch_spinlock_t *lock; \
|
|
preempt_disable(); \
|
|
rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_); \
|
|
lock = &per_cpu(name##_lock, cpu); \
|
|
arch_spin_lock(lock); \
|
|
} \
|
|
EXPORT_SYMBOL(name##_local_lock_cpu); \
|
|
\
|
|
void name##_local_unlock_cpu(int cpu) { \
|
|
arch_spinlock_t *lock; \
|
|
rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_); \
|
|
lock = &per_cpu(name##_lock, cpu); \
|
|
arch_spin_unlock(lock); \
|
|
preempt_enable(); \
|
|
} \
|
|
EXPORT_SYMBOL(name##_local_unlock_cpu); \
|
|
\
|
|
void name##_global_lock_online(void) { \
|
|
int i; \
|
|
spin_lock(&name##_cpu_lock); \
|
|
rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_); \
|
|
for_each_cpu(i, &name##_cpus) { \
|
|
arch_spinlock_t *lock; \
|
|
lock = &per_cpu(name##_lock, i); \
|
|
arch_spin_lock(lock); \
|
|
} \
|
|
} \
|
|
EXPORT_SYMBOL(name##_global_lock_online); \
|
|
\
|
|
void name##_global_unlock_online(void) { \
|
|
int i; \
|
|
rwlock_release(&name##_lock_dep_map, 1, _RET_IP_); \
|
|
for_each_cpu(i, &name##_cpus) { \
|
|
arch_spinlock_t *lock; \
|
|
lock = &per_cpu(name##_lock, i); \
|
|
arch_spin_unlock(lock); \
|
|
} \
|
|
spin_unlock(&name##_cpu_lock); \
|
|
} \
|
|
EXPORT_SYMBOL(name##_global_unlock_online); \
|
|
\
|
|
void name##_global_lock(void) { \
|
|
int i; \
|
|
preempt_disable(); \
|
|
rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_); \
|
|
for_each_possible_cpu(i) { \
|
|
arch_spinlock_t *lock; \
|
|
lock = &per_cpu(name##_lock, i); \
|
|
arch_spin_lock(lock); \
|
|
} \
|
|
} \
|
|
EXPORT_SYMBOL(name##_global_lock); \
|
|
\
|
|
void name##_global_unlock(void) { \
|
|
int i; \
|
|
rwlock_release(&name##_lock_dep_map, 1, _RET_IP_); \
|
|
for_each_possible_cpu(i) { \
|
|
arch_spinlock_t *lock; \
|
|
lock = &per_cpu(name##_lock, i); \
|
|
arch_spin_unlock(lock); \
|
|
} \
|
|
preempt_enable(); \
|
|
} \
|
|
EXPORT_SYMBOL(name##_global_unlock);
|
|
#endif
|