[PATCH] mutex subsystem, add asm-generic/mutex-[dec|xchg|null].h implementations
Add three (generic) mutex fastpath implementations. The mutex-xchg.h implementation is atomic_xchg() based, and should work fine on every architecture. The mutex-dec.h implementation is atomic_dec_return() based - this one too should work on every architecture, but might not perform the most optimally on architectures that have no atomic-dec/inc instructions. The mutex-null.h implementation forces all calls into the slowpath. This is used for mutex debugging, but it can also be used on platforms that do not want (or need) a fastpath at all. Signed-off-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Arjan van de Ven <arjan@infradead.org>
This commit is contained in:
Ingo Molnar
Ingo Molnar
parent
711a660dc2
commit
620a6fd185
|
|
@ -0,0 +1,110 @@
|
|||
/*
|
||||
* asm-generic/mutex-dec.h
|
||||
*
|
||||
* Generic implementation of the mutex fastpath, based on atomic
|
||||
* decrement/increment.
|
||||
*/
|
||||
#ifndef _ASM_GENERIC_MUTEX_DEC_H
|
||||
#define _ASM_GENERIC_MUTEX_DEC_H
|
||||
|
||||
/**
|
||||
* __mutex_fastpath_lock - try to take the lock by moving the count
|
||||
* from 1 to a 0 value
|
||||
* @count: pointer of type atomic_t
|
||||
* @fail_fn: function to call if the original value was not 1
|
||||
*
|
||||
* Change the count from 1 to a value lower than 1, and call <fail_fn> if
|
||||
* it wasn't 1 originally. This function MUST leave the value lower than
|
||||
* 1 even when the "1" assertion wasn't true.
|
||||
*/
|
||||
#define __mutex_fastpath_lock(count, fail_fn) \
|
||||
do { \
|
||||
if (unlikely(atomic_dec_return(count) < 0)) \
|
||||
fail_fn(count); \
|
||||
else \
|
||||
smp_mb(); \
|
||||
} while (0)
|
||||
|
||||
/**
|
||||
* __mutex_fastpath_lock_retval - try to take the lock by moving the count
|
||||
* from 1 to a 0 value
|
||||
* @count: pointer of type atomic_t
|
||||
* @fail_fn: function to call if the original value was not 1
|
||||
*
|
||||
* Change the count from 1 to a value lower than 1, and call <fail_fn> if
|
||||
* it wasn't 1 originally. This function returns 0 if the fastpath succeeds,
|
||||
* or anything the slow path function returns.
|
||||
*/
|
||||
static inline int
|
||||
__mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
|
||||
{
|
||||
if (unlikely(atomic_dec_return(count) < 0))
|
||||
return fail_fn(count);
|
||||
else {
|
||||
smp_mb();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* __mutex_fastpath_unlock - try to promote the count from 0 to 1
|
||||
* @count: pointer of type atomic_t
|
||||
* @fail_fn: function to call if the original value was not 0
|
||||
*
|
||||
* Try to promote the count from 0 to 1. If it wasn't 0, call <fail_fn>.
|
||||
* In the failure case, this function is allowed to either set the value to
|
||||
* 1, or to set it to a value lower than 1.
|
||||
*
|
||||
* If the implementation sets it to a value of lower than 1, then the
|
||||
* __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs
|
||||
* to return 0 otherwise.
|
||||
*/
|
||||
#define __mutex_fastpath_unlock(count, fail_fn) \
|
||||
do { \
|
||||
smp_mb(); \
|
||||
if (unlikely(atomic_inc_return(count) <= 0)) \
|
||||
fail_fn(count); \
|
||||
} while (0)
|
||||
|
||||
#define __mutex_slowpath_needs_to_unlock() 1
|
||||
|
||||
/**
|
||||
* __mutex_fastpath_trylock - try to acquire the mutex, without waiting
|
||||
*
|
||||
* @count: pointer of type atomic_t
|
||||
* @fail_fn: fallback function
|
||||
*
|
||||
* Change the count from 1 to a value lower than 1, and return 0 (failure)
|
||||
* if it wasn't 1 originally, or return 1 (success) otherwise. This function
|
||||
* MUST leave the value lower than 1 even when the "1" assertion wasn't true.
|
||||
* Additionally, if the value was < 0 originally, this function must not leave
|
||||
* it to 0 on failure.
|
||||
*
|
||||
* If the architecture has no effective trylock variant, it should call the
|
||||
* <fail_fn> spinlock-based trylock variant unconditionally.
|
||||
*/
|
||||
static inline int
|
||||
__mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
|
||||
{
|
||||
/*
|
||||
* We have two variants here. The cmpxchg based one is the best one
|
||||
* because it never induce a false contention state. It is included
|
||||
* here because architectures using the inc/dec algorithms over the
|
||||
* xchg ones are much more likely to support cmpxchg natively.
|
||||
*
|
||||
* If not we fall back to the spinlock based variant - that is
|
||||
* just as efficient (and simpler) as a 'destructive' probing of
|
||||
* the mutex state would be.
|
||||
*/
|
||||
#ifdef __HAVE_ARCH_CMPXCHG
|
||||
if (likely(atomic_cmpxchg(count, 1, 0)) == 1) {
|
||||
smp_mb();
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
#else
|
||||
return fail_fn(count);
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/*
|
||||
* asm-generic/mutex-null.h
|
||||
*
|
||||
* Generic implementation of the mutex fastpath, based on NOP :-)
|
||||
*
|
||||
* This is used by the mutex-debugging infrastructure, but it can also
|
||||
* be used by architectures that (for whatever reason) want to use the
|
||||
* spinlock based slowpath.
|
||||
*/
|
||||
#ifndef _ASM_GENERIC_MUTEX_NULL_H
|
||||
#define _ASM_GENERIC_MUTEX_NULL_H
|
||||
|
||||
/* extra parameter only needed for mutex debugging: */
|
||||
#ifndef __IP__
|
||||
# define __IP__
|
||||
#endif
|
||||
|
||||
#define __mutex_fastpath_lock(count, fail_fn) fail_fn(count __RET_IP__)
|
||||
#define __mutex_fastpath_lock_retval(count, fail_fn) fail_fn(count __RET_IP__)
|
||||
#define __mutex_fastpath_unlock(count, fail_fn) fail_fn(count __RET_IP__)
|
||||
#define __mutex_fastpath_trylock(count, fail_fn) fail_fn(count)
|
||||
#define __mutex_slowpath_needs_to_unlock() 1
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
/*
|
||||
* asm-generic/mutex-xchg.h
|
||||
*
|
||||
* Generic implementation of the mutex fastpath, based on xchg().
|
||||
*
|
||||
* NOTE: An xchg based implementation is less optimal than an atomic
|
||||
* decrement/increment based implementation. If your architecture
|
||||
* has a reasonable atomic dec/inc then you should probably use
|
||||
* asm-generic/mutex-dec.h instead, or you could open-code an
|
||||
* optimized version in asm/mutex.h.
|
||||
*/
|
||||
#ifndef _ASM_GENERIC_MUTEX_XCHG_H
|
||||
#define _ASM_GENERIC_MUTEX_XCHG_H
|
||||
|
||||
/**
|
||||
* __mutex_fastpath_lock - try to take the lock by moving the count
|
||||
* from 1 to a 0 value
|
||||
* @count: pointer of type atomic_t
|
||||
* @fail_fn: function to call if the original value was not 1
|
||||
*
|
||||
* Change the count from 1 to a value lower than 1, and call <fail_fn> if it
|
||||
* wasn't 1 originally. This function MUST leave the value lower than 1
|
||||
* even when the "1" assertion wasn't true.
|
||||
*/
|
||||
#define __mutex_fastpath_lock(count, fail_fn) \
|
||||
do { \
|
||||
if (unlikely(atomic_xchg(count, 0) != 1)) \
|
||||
fail_fn(count); \
|
||||
else \
|
||||
smp_mb(); \
|
||||
} while (0)
|
||||
|
||||
|
||||
/**
|
||||
* __mutex_fastpath_lock_retval - try to take the lock by moving the count
|
||||
* from 1 to a 0 value
|
||||
* @count: pointer of type atomic_t
|
||||
* @fail_fn: function to call if the original value was not 1
|
||||
*
|
||||
* Change the count from 1 to a value lower than 1, and call <fail_fn> if it
|
||||
* wasn't 1 originally. This function returns 0 if the fastpath succeeds,
|
||||
* or anything the slow path function returns
|
||||
*/
|
||||
static inline int
|
||||
__mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
|
||||
{
|
||||
if (unlikely(atomic_xchg(count, 0) != 1))
|
||||
return fail_fn(count);
|
||||
else {
|
||||
smp_mb();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* __mutex_fastpath_unlock - try to promote the mutex from 0 to 1
|
||||
* @count: pointer of type atomic_t
|
||||
* @fail_fn: function to call if the original value was not 0
|
||||
*
|
||||
* try to promote the mutex from 0 to 1. if it wasn't 0, call <function>
|
||||
* In the failure case, this function is allowed to either set the value to
|
||||
* 1, or to set it to a value lower than one.
|
||||
* If the implementation sets it to a value of lower than one, the
|
||||
* __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs
|
||||
* to return 0 otherwise.
|
||||
*/
|
||||
#define __mutex_fastpath_unlock(count, fail_fn) \
|
||||
do { \
|
||||
smp_mb(); \
|
||||
if (unlikely(atomic_xchg(count, 1) != 0)) \
|
||||
fail_fn(count); \
|
||||
} while (0)
|
||||
|
||||
#define __mutex_slowpath_needs_to_unlock() 0
|
||||
|
||||
/**
|
||||
* __mutex_fastpath_trylock - try to acquire the mutex, without waiting
|
||||
*
|
||||
* @count: pointer of type atomic_t
|
||||
* @fail_fn: spinlock based trylock implementation
|
||||
*
|
||||
* Change the count from 1 to a value lower than 1, and return 0 (failure)
|
||||
* if it wasn't 1 originally, or return 1 (success) otherwise. This function
|
||||
* MUST leave the value lower than 1 even when the "1" assertion wasn't true.
|
||||
* Additionally, if the value was < 0 originally, this function must not leave
|
||||
* it to 0 on failure.
|
||||
*
|
||||
* If the architecture has no effective trylock variant, it should call the
|
||||
* <fail_fn> spinlock-based trylock variant unconditionally.
|
||||
*/
|
||||
static inline int
|
||||
__mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
|
||||
{
|
||||
int prev = atomic_xchg(count, 0);
|
||||
|
||||
if (unlikely(prev < 0)) {
|
||||
/*
|
||||
* The lock was marked contended so we must restore that
|
||||
* state. If while doing so we get back a prev value of 1
|
||||
* then we just own it.
|
||||
*
|
||||
* [ In the rare case of the mutex going to 1, to 0, to -1
|
||||
* and then back to 0 in this few-instructions window,
|
||||
* this has the potential to trigger the slowpath for the
|
||||
* owner's unlock path needlessly, but that's not a problem
|
||||
* in practice. ]
|
||||
*/
|
||||
prev = atomic_xchg(count, prev);
|
||||
if (prev < 0)
|
||||
prev = 0;
|
||||
}
|
||||
smp_mb();
|
||||
|
||||
return prev;
|
||||
}
|
||||
|
||||
#endif
|
||||
Loading…
Reference in New Issue