UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

s390/spinlock: add niai spinlock hints 

The z14 machine introduces new mode of the next-instruction-access-intent
NIAI instruction. With NIAI-8 it is possible to pin a cache-line on a
CPU for a small amount of time, NIAI-7 releases the cache-line again.
Finally NIAI-4 can be used to prevent the CPU to speculatively access
memory beyond the compare-and-swap instruction to get the lock.

Use these instruction in the spinlock code.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
This commit is contained in:
Martin Schwidefsky 2017-04-19 14:54:05 +02:00
parent 8351378f58
commit 7f7e6e28cd
2 changed files with 56 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
arch/s390/include/asm/spinlock.h
View File

@ -92,10 +92,11 @@ static inline void arch_spin_unlock(arch_spinlock_t *lp)
{ {
typecheck(int, lp->lock); typecheck(int, lp->lock);
asm volatile( asm volatile(
"st %1,%0\n" #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
: "+Q" (lp->lock) " .long 0xb2fa0070\n" /* NIAI 7 */
: "d" (0) #endif
: "cc", "memory"); " st %1,%0\n"
: "=Q" (lp->lock) : "d" (0) : "cc", "memory");
} }
static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)

87
arch/s390/lib/spinlock.c
View File

@ -32,42 +32,63 @@ static int __init spin_retry_setup(char *str)
} }
__setup("spin_retry=", spin_retry_setup); __setup("spin_retry=", spin_retry_setup);
static inline int arch_load_niai4(int *lock)
{
int owner;
asm volatile(
#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
" .long 0xb2fa0040\n" /* NIAI 4 */
#endif
" l %0,%1\n"
: "=d" (owner) : "Q" (*lock) : "memory");
return owner;
}
static inline int arch_cmpxchg_niai8(int *lock, int old, int new)
{
int expected = old;
asm volatile(
#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
" .long 0xb2fa0080\n" /* NIAI 8 */
#endif
" cs %0,%3,%1\n"
: "=d" (old), "=Q" (*lock)
: "0" (old), "d" (new), "Q" (*lock)
: "cc", "memory");
return expected == old;
}
void arch_spin_lock_wait(arch_spinlock_t *lp) void arch_spin_lock_wait(arch_spinlock_t *lp)
{ {
int cpu = SPINLOCK_LOCKVAL; int cpu = SPINLOCK_LOCKVAL;
int owner, count, first_diag; int owner, count;
first_diag = 1; /* Pass the virtual CPU to the lock holder if it is not running */
owner = arch_load_niai4(&lp->lock);
if (owner && arch_vcpu_is_preempted(~owner))
smp_yield_cpu(~owner);
count = spin_retry;
while (1) { while (1) {
owner = ACCESS_ONCE(lp->lock); owner = arch_load_niai4(&lp->lock);
/* Try to get the lock if it is free. */ /* Try to get the lock if it is free. */
if (!owner) { if (!owner) {
if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu)) if (arch_cmpxchg_niai8(&lp->lock, 0, cpu))
return; return;
continue; continue;
} }
/* First iteration: check if the lock owner is running. */ if (count-- >= 0)
if (first_diag && arch_vcpu_is_preempted(~owner)) {
smp_yield_cpu(~owner);
first_diag = 0;
continue; continue;
}
/* Loop for a while on the lock value. */
count = spin_retry; count = spin_retry;
do {
owner = ACCESS_ONCE(lp->lock);
} while (owner && count-- > 0);
if (!owner)
continue;
/* /*
* For multiple layers of hypervisors, e.g. z/VM + LPAR * For multiple layers of hypervisors, e.g. z/VM + LPAR
* yield the CPU unconditionally. For LPAR rely on the * yield the CPU unconditionally. For LPAR rely on the
* sense running status. * sense running status.
*/ */
if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner)) { if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner))
smp_yield_cpu(~owner); smp_yield_cpu(~owner);
first_diag = 0;
}
} }
} }
EXPORT_SYMBOL(arch_spin_lock_wait); EXPORT_SYMBOL(arch_spin_lock_wait);
@ -75,42 +96,36 @@ EXPORT_SYMBOL(arch_spin_lock_wait);
void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
{ {
int cpu = SPINLOCK_LOCKVAL; int cpu = SPINLOCK_LOCKVAL;
int owner, count, first_diag; int owner, count;
local_irq_restore(flags); local_irq_restore(flags);
first_diag = 1;
/* Pass the virtual CPU to the lock holder if it is not running */
owner = arch_load_niai4(&lp->lock);
if (owner && arch_vcpu_is_preempted(~owner))
smp_yield_cpu(~owner);
count = spin_retry;
while (1) { while (1) {
owner = ACCESS_ONCE(lp->lock); owner = arch_load_niai4(&lp->lock);
/* Try to get the lock if it is free. */ /* Try to get the lock if it is free. */
if (!owner) { if (!owner) {
local_irq_disable(); local_irq_disable();
if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu)) if (arch_cmpxchg_niai8(&lp->lock, 0, cpu))
return; return;
local_irq_restore(flags); local_irq_restore(flags);
continue; continue;
} }
/* Check if the lock owner is running. */ if (count-- >= 0)
if (first_diag && arch_vcpu_is_preempted(~owner)) {
smp_yield_cpu(~owner);
first_diag = 0;
continue; continue;
}
/* Loop for a while on the lock value. */
count = spin_retry; count = spin_retry;
do {
owner = ACCESS_ONCE(lp->lock);
} while (owner && count-- > 0);
if (!owner)
continue;
/* /*
* For multiple layers of hypervisors, e.g. z/VM + LPAR * For multiple layers of hypervisors, e.g. z/VM + LPAR
* yield the CPU unconditionally. For LPAR rely on the * yield the CPU unconditionally. For LPAR rely on the
* sense running status. * sense running status.
*/ */
if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner)) { if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner))
smp_yield_cpu(~owner); smp_yield_cpu(~owner);
first_diag = 0;
}
} }
} }
EXPORT_SYMBOL(arch_spin_lock_wait_flags); EXPORT_SYMBOL(arch_spin_lock_wait_flags);