UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

jump_label: move 'asm goto' support test to Kconfig 

Currently, CONFIG_JUMP_LABEL just means "I _want_ to use jump label".

The jump label is controlled by HAVE_JUMP_LABEL, which is defined
like this:

  #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
  # define HAVE_JUMP_LABEL
  #endif

We can improve this by testing 'asm goto' support in Kconfig, then
make JUMP_LABEL depend on CC_HAS_ASM_GOTO.

Ugly #ifdef HAVE_JUMP_LABEL will go away, and CONFIG_JUMP_LABEL will
match to the real kernel capability.

Signed-off-by: Masahiro Yamada <yamada.masahiro@socionext.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> (powerpc)
Tested-by: Sedat Dilek <sedat.dilek@gmail.com>
This commit is contained in:
Masahiro Yamada 2018-12-31 00:14:15 +09:00
parent 72d3ebb929
commit e9666d10a5
42 changed files with 65 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Makefile
View File

@ -514,13 +514,6 @@ RETPOLINE_VDSO_CFLAGS := $(call cc-option,$(RETPOLINE_VDSO_CFLAGS_GCC),$(call cc
export RETPOLINE_CFLAGS export RETPOLINE_CFLAGS
export RETPOLINE_VDSO_CFLAGS export RETPOLINE_VDSO_CFLAGS
# check for 'asm goto'
ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC) $(KBUILD_CFLAGS)), y)
CC_HAVE_ASM_GOTO := 1
KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO
KBUILD_AFLAGS += -DCC_HAVE_ASM_GOTO
endif
# The expansion should be delayed until arch/$(SRCARCH)/Makefile is included. # The expansion should be delayed until arch/$(SRCARCH)/Makefile is included.
# Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile. # Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile.
# CC_VERSION_TEXT is referenced from Kconfig (so it needs export), # CC_VERSION_TEXT is referenced from Kconfig (so it needs export),

1
arch/Kconfig
View File

@ -71,6 +71,7 @@ config KPROBES
config JUMP_LABEL config JUMP_LABEL
bool "Optimize very unlikely/likely branches" bool "Optimize very unlikely/likely branches"
depends on HAVE_ARCH_JUMP_LABEL depends on HAVE_ARCH_JUMP_LABEL
depends on CC_HAS_ASM_GOTO
help help
This option enables a transparent branch optimization that This option enables a transparent branch optimization that
makes certain almost-always-true or almost-always-false branch makes certain almost-always-true or almost-always-false branch

4
arch/arm/kernel/jump_label.c
View File

@ -4,8 +4,6 @@
#include <asm/patch.h> #include <asm/patch.h>
#include <asm/insn.h> #include <asm/insn.h>
#ifdef HAVE_JUMP_LABEL
static void __arch_jump_label_transform(struct jump_entry *entry, static void __arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type, enum jump_label_type type,
bool is_static) bool is_static)
@ -35,5 +33,3 @@ void arch_jump_label_transform_static(struct jump_entry *entry,
{ {
__arch_jump_label_transform(entry, type, true); __arch_jump_label_transform(entry, type, true);
} }
#endif

4
arch/arm64/kernel/jump_label.c
View File

@ -20,8 +20,6 @@
#include <linux/jump_label.h> #include <linux/jump_label.h>
#include <asm/insn.h> #include <asm/insn.h>
#ifdef HAVE_JUMP_LABEL
void arch_jump_label_transform(struct jump_entry *entry, void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type) enum jump_label_type type)
{ {
@ -49,5 +47,3 @@ void arch_jump_label_transform_static(struct jump_entry *entry,
* NOP needs to be replaced by a branch. * NOP needs to be replaced by a branch.
*/ */
} }
#endif /* HAVE_JUMP_LABEL */

4
arch/mips/kernel/jump_label.c
View File

@ -16,8 +16,6 @@
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/inst.h> #include <asm/inst.h>
#ifdef HAVE_JUMP_LABEL
/* /*
* Define parameters for the standard MIPS and the microMIPS jump * Define parameters for the standard MIPS and the microMIPS jump
* instruction encoding respectively: * instruction encoding respectively:
@ -70,5 +68,3 @@ void arch_jump_label_transform(struct jump_entry *e,
mutex_unlock(&text_mutex); mutex_unlock(&text_mutex);
} }
#endif /* HAVE_JUMP_LABEL */

2
arch/powerpc/include/asm/asm-prototypes.h
View File

@ -38,7 +38,7 @@ extern struct static_key hcall_tracepoint_key;
void __trace_hcall_entry(unsigned long opcode, unsigned long *args); void __trace_hcall_entry(unsigned long opcode, unsigned long *args);
void __trace_hcall_exit(long opcode, long retval, unsigned long *retbuf); void __trace_hcall_exit(long opcode, long retval, unsigned long *retbuf);
/* OPAL tracing */ /* OPAL tracing */
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
extern struct static_key opal_tracepoint_key; extern struct static_key opal_tracepoint_key;
#endif #endif

2
arch/powerpc/kernel/jump_label.c
View File

@ -11,7 +11,6 @@
#include <linux/jump_label.h> #include <linux/jump_label.h>
#include <asm/code-patching.h> #include <asm/code-patching.h>
#ifdef HAVE_JUMP_LABEL
void arch_jump_label_transform(struct jump_entry *entry, void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type) enum jump_label_type type)
{ {
@ -22,4 +21,3 @@ void arch_jump_label_transform(struct jump_entry *entry,
else else
patch_instruction(addr, PPC_INST_NOP); patch_instruction(addr, PPC_INST_NOP);
} }
#endif

2
arch/powerpc/platforms/powernv/opal-tracepoints.c
View File

@ -4,7 +4,7 @@
#include <asm/trace.h> #include <asm/trace.h>
#include <asm/asm-prototypes.h> #include <asm/asm-prototypes.h>
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
struct static_key opal_tracepoint_key = STATIC_KEY_INIT; struct static_key opal_tracepoint_key = STATIC_KEY_INIT;
int opal_tracepoint_regfunc(void) int opal_tracepoint_regfunc(void)

2
arch/powerpc/platforms/powernv/opal-wrappers.S
View File

@ -20,7 +20,7 @@
.section ".text" .section ".text"
#ifdef CONFIG_TRACEPOINTS #ifdef CONFIG_TRACEPOINTS
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
#define OPAL_BRANCH(LABEL) \ #define OPAL_BRANCH(LABEL) \
ARCH_STATIC_BRANCH(LABEL, opal_tracepoint_key) ARCH_STATIC_BRANCH(LABEL, opal_tracepoint_key)
#else #else

4
arch/powerpc/platforms/pseries/hvCall.S
View File

@ -19,7 +19,7 @@
#ifdef CONFIG_TRACEPOINTS #ifdef CONFIG_TRACEPOINTS
#ifndef HAVE_JUMP_LABEL #ifndef CONFIG_JUMP_LABEL
.section ".toc","aw" .section ".toc","aw"
.globl hcall_tracepoint_refcount .globl hcall_tracepoint_refcount
@ -79,7 +79,7 @@ hcall_tracepoint_refcount:
mr r5,BUFREG; \ mr r5,BUFREG; \
__HCALL_INST_POSTCALL __HCALL_INST_POSTCALL
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
#define HCALL_BRANCH(LABEL) \ #define HCALL_BRANCH(LABEL) \
ARCH_STATIC_BRANCH(LABEL, hcall_tracepoint_key) ARCH_STATIC_BRANCH(LABEL, hcall_tracepoint_key)
#else #else

2
arch/powerpc/platforms/pseries/lpar.c
View File

@ -1040,7 +1040,7 @@ EXPORT_SYMBOL(arch_free_page);
#endif /* CONFIG_PPC_BOOK3S_64 */ #endif /* CONFIG_PPC_BOOK3S_64 */
#ifdef CONFIG_TRACEPOINTS #ifdef CONFIG_TRACEPOINTS
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
struct static_key hcall_tracepoint_key = STATIC_KEY_INIT; struct static_key hcall_tracepoint_key = STATIC_KEY_INIT;
int hcall_tracepoint_regfunc(void) int hcall_tracepoint_regfunc(void)

3
arch/s390/kernel/Makefile
View File

@ -48,7 +48,7 @@ CFLAGS_ptrace.o += -DUTS_MACHINE='"$(UTS_MACHINE)"'
obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o
obj-y += processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o obj-y += processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o
obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o early_nobss.o obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o early_nobss.o
obj-y += sysinfo.o jump_label.o lgr.o os_info.o machine_kexec.o pgm_check.o obj-y += sysinfo.o lgr.o os_info.o machine_kexec.o pgm_check.o
obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o
obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o
obj-y += nospec-branch.o ipl_vmparm.o obj-y += nospec-branch.o ipl_vmparm.o
@ -72,6 +72,7 @@ obj-$(CONFIG_KPROBES) += kprobes.o
obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o
obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
obj-$(CONFIG_UPROBES) += uprobes.o obj-$(CONFIG_UPROBES) += uprobes.o
obj-$(CONFIG_JUMP_LABEL) += jump_label.o
obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o
obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o

4
arch/s390/kernel/jump_label.c
View File

@ -10,8 +10,6 @@
#include <linux/jump_label.h> #include <linux/jump_label.h>
#include <asm/ipl.h> #include <asm/ipl.h>
#ifdef HAVE_JUMP_LABEL
struct insn { struct insn {
u16 opcode; u16 opcode;
s32 offset; s32 offset;
@ -103,5 +101,3 @@ void arch_jump_label_transform_static(struct jump_entry *entry,
{ {
__jump_label_transform(entry, type, 1); __jump_label_transform(entry, type, 1);
} }
#endif

2
arch/sparc/kernel/Makefile
View File

@ -118,4 +118,4 @@ pc--$(CONFIG_PERF_EVENTS) := perf_event.o
obj-$(CONFIG_SPARC64) += $(pc--y) obj-$(CONFIG_SPARC64) += $(pc--y)
obj-$(CONFIG_UPROBES) += uprobes.o obj-$(CONFIG_UPROBES) += uprobes.o
obj-$(CONFIG_SPARC64) += jump_label.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o

4
arch/sparc/kernel/jump_label.c
View File

@ -9,8 +9,6 @@
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#ifdef HAVE_JUMP_LABEL
void arch_jump_label_transform(struct jump_entry *entry, void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type) enum jump_label_type type)
{ {
@ -47,5 +45,3 @@ void arch_jump_label_transform(struct jump_entry *entry,
flushi(insn); flushi(insn);
mutex_unlock(&text_mutex); mutex_unlock(&text_mutex);
} }
#endif

2
arch/x86/Makefile
View File

@ -289,7 +289,7 @@ vdso_install:
archprepare: checkbin archprepare: checkbin
checkbin: checkbin:
ifndef CC_HAVE_ASM_GOTO ifndef CONFIG_CC_HAS_ASM_GOTO
@echo Compiler lacks asm-goto support. @echo Compiler lacks asm-goto support.
@exit 1 @exit 1
endif endif

2
arch/x86/entry/calling.h
View File

@ -351,7 +351,7 @@ For 32-bit we have the following conventions - kernel is built with
*/ */
.macro CALL_enter_from_user_mode .macro CALL_enter_from_user_mode
#ifdef CONFIG_CONTEXT_TRACKING #ifdef CONFIG_CONTEXT_TRACKING
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0 STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0
#endif #endif
call enter_from_user_mode call enter_from_user_mode

2
arch/x86/include/asm/cpufeature.h
View File

@ -140,7 +140,7 @@ extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);
#define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit) #define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit)
#if defined(__clang__) && !defined(CC_HAVE_ASM_GOTO) #if defined(__clang__) && !defined(CONFIG_CC_HAS_ASM_GOTO)
/* /*
* Workaround for the sake of BPF compilation which utilizes kernel * Workaround for the sake of BPF compilation which utilizes kernel

13
arch/x86/include/asm/jump_label.h
View File

@ -2,19 +2,6 @@
#ifndef _ASM_X86_JUMP_LABEL_H #ifndef _ASM_X86_JUMP_LABEL_H
#define _ASM_X86_JUMP_LABEL_H #define _ASM_X86_JUMP_LABEL_H
#ifndef HAVE_JUMP_LABEL
/*
* For better or for worse, if jump labels (the gcc extension) are missing,
* then the entire static branch patching infrastructure is compiled out.
* If that happens, the code in here will malfunction. Raise a compiler
* error instead.
*
* In theory, jump labels and the static branch patching infrastructure
* could be decoupled to fix this.
*/
#error asm/jump_label.h included on a non-jump-label kernel
#endif
#define JUMP_LABEL_NOP_SIZE 5 #define JUMP_LABEL_NOP_SIZE 5
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64

6
arch/x86/include/asm/rmwcc.h
View File

@ -11,7 +11,7 @@
#define __CLOBBERS_MEM(clb...) "memory", ## clb #define __CLOBBERS_MEM(clb...) "memory", ## clb
#if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CC_HAVE_ASM_GOTO) #if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CONFIG_CC_HAS_ASM_GOTO)
/* Use asm goto */ /* Use asm goto */
@ -27,7 +27,7 @@ cc_label: c = true; \
c; \ c; \
}) })
#else /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */ #else /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CONFIG_CC_HAS_ASM_GOTO) */
/* Use flags output or a set instruction */ /* Use flags output or a set instruction */
@ -40,7 +40,7 @@ cc_label: c = true; \
c; \ c; \
}) })
#endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */ #endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CONFIG_CC_HAS_ASM_GOTO) */
#define GEN_UNARY_RMWcc_4(op, var, cc, arg0) \ #define GEN_UNARY_RMWcc_4(op, var, cc, arg0) \
__GEN_RMWcc(op " " arg0, var, cc, __CLOBBERS_MEM()) __GEN_RMWcc(op " " arg0, var, cc, __CLOBBERS_MEM())

3
arch/x86/kernel/Makefile
View File

@ -49,7 +49,8 @@ obj-$(CONFIG_COMPAT) += signal_compat.o
obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
obj-y += time.o ioport.o dumpstack.o nmi.o obj-y += time.o ioport.o dumpstack.o nmi.o
obj-$(CONFIG_MODIFY_LDT_SYSCALL) += ldt.o obj-$(CONFIG_MODIFY_LDT_SYSCALL) += ldt.o
obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o obj-y += setup.o x86_init.o i8259.o irqinit.o
obj-$(CONFIG_JUMP_LABEL) += jump_label.o
obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_IRQ_WORK) += irq_work.o
obj-y += probe_roms.o obj-y += probe_roms.o
obj-$(CONFIG_X86_64) += sys_x86_64.o obj-$(CONFIG_X86_64) += sys_x86_64.o

4
arch/x86/kernel/jump_label.c
View File

@ -16,8 +16,6 @@
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/text-patching.h> #include <asm/text-patching.h>
#ifdef HAVE_JUMP_LABEL
union jump_code_union { union jump_code_union {
char code[JUMP_LABEL_NOP_SIZE]; char code[JUMP_LABEL_NOP_SIZE];
struct { struct {
@ -130,5 +128,3 @@ __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
if (jlstate == JL_STATE_UPDATE) if (jlstate == JL_STATE_UPDATE)
__jump_label_transform(entry, type, text_poke_early, 1); __jump_label_transform(entry, type, text_poke_early, 1);
} }
#endif

2
arch/x86/kvm/emulate.c
View File

@ -456,7 +456,7 @@ FOP_END;
/* /*
* XXX: inoutclob user must know where the argument is being expanded. * XXX: inoutclob user must know where the argument is being expanded.
* Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault. * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
*/ */
#define asm_safe(insn, inoutclob...) \ #define asm_safe(insn, inoutclob...) \
({ \ ({ \

4
arch/xtensa/kernel/jump_label.c
View File

@ -10,8 +10,6 @@
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#ifdef HAVE_JUMP_LABEL
#define J_OFFSET_MASK 0x0003ffff #define J_OFFSET_MASK 0x0003ffff
#define J_SIGN_MASK (~(J_OFFSET_MASK >> 1)) #define J_SIGN_MASK (~(J_OFFSET_MASK >> 1))
@ -95,5 +93,3 @@ void arch_jump_label_transform(struct jump_entry *e,
patch_text(jump_entry_code(e), &insn, JUMP_LABEL_NOP_SIZE); patch_text(jump_entry_code(e), &insn, JUMP_LABEL_NOP_SIZE);
} }
#endif /* HAVE_JUMP_LABEL */

6
include/linux/dynamic_debug.h
View File

@ -2,7 +2,7 @@
#ifndef _DYNAMIC_DEBUG_H #ifndef _DYNAMIC_DEBUG_H
#define _DYNAMIC_DEBUG_H #define _DYNAMIC_DEBUG_H
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) #if defined(CONFIG_JUMP_LABEL)
#include <linux/jump_label.h> #include <linux/jump_label.h>
#endif #endif
@ -38,7 +38,7 @@ struct _ddebug {
#define _DPRINTK_FLAGS_DEFAULT 0 #define _DPRINTK_FLAGS_DEFAULT 0
#endif #endif
unsigned int flags:8; unsigned int flags:8;
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
union { union {
struct static_key_true dd_key_true; struct static_key_true dd_key_true;
struct static_key_false dd_key_false; struct static_key_false dd_key_false;
@ -83,7 +83,7 @@ void __dynamic_netdev_dbg(struct _ddebug *descriptor,
dd_key_init(key, init) \ dd_key_init(key, init) \
} }
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
#define dd_key_init(key, init) key = (init) #define dd_key_init(key, init) key = (init)

22
include/linux/jump_label.h
View File

@ -71,10 +71,6 @@
* Additional babbling in: Documentation/static-keys.txt * Additional babbling in: Documentation/static-keys.txt
*/ */
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
# define HAVE_JUMP_LABEL
#endif
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/types.h> #include <linux/types.h>
@ -86,7 +82,7 @@ extern bool static_key_initialized;
"%s(): static key '%pS' used before call to jump_label_init()", \ "%s(): static key '%pS' used before call to jump_label_init()", \
__func__, (key)) __func__, (key))
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
struct static_key { struct static_key {
atomic_t enabled; atomic_t enabled;
@ -114,10 +110,10 @@ struct static_key {
struct static_key { struct static_key {
atomic_t enabled; atomic_t enabled;
}; };
#endif /* HAVE_JUMP_LABEL */ #endif /* CONFIG_JUMP_LABEL */
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
#include <asm/jump_label.h> #include <asm/jump_label.h>
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
@ -192,7 +188,7 @@ enum jump_label_type {
struct module; struct module;
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
#define JUMP_TYPE_FALSE 0UL #define JUMP_TYPE_FALSE 0UL
#define JUMP_TYPE_TRUE 1UL #define JUMP_TYPE_TRUE 1UL
@ -245,7 +241,7 @@ extern void static_key_disable_cpuslocked(struct static_key *key);
{ .enabled = { 0 }, \ { .enabled = { 0 }, \
{ .entries = (void *)JUMP_TYPE_FALSE } } { .entries = (void *)JUMP_TYPE_FALSE } }
#else /* !HAVE_JUMP_LABEL */ #else /* !CONFIG_JUMP_LABEL */
#include <linux/atomic.h> #include <linux/atomic.h>
#include <linux/bug.h> #include <linux/bug.h>
@ -330,7 +326,7 @@ static inline void static_key_disable(struct static_key *key)
#define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) } #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
#define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) } #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
#endif /* HAVE_JUMP_LABEL */ #endif /* CONFIG_JUMP_LABEL */
#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
#define jump_label_enabled static_key_enabled #define jump_label_enabled static_key_enabled
@ -394,7 +390,7 @@ extern bool ____wrong_branch_error(void);
static_key_count((struct static_key *)x) > 0; \ static_key_count((struct static_key *)x) > 0; \
}) })
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
/* /*
* Combine the right initial value (type) with the right branch order * Combine the right initial value (type) with the right branch order
@ -476,12 +472,12 @@ extern bool ____wrong_branch_error(void);
unlikely(branch); \ unlikely(branch); \
}) })
#else /* !HAVE_JUMP_LABEL */ #else /* !CONFIG_JUMP_LABEL */
#define static_branch_likely(x) likely(static_key_enabled(&(x)->key)) #define static_branch_likely(x) likely(static_key_enabled(&(x)->key))
#define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key)) #define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key))
#endif /* HAVE_JUMP_LABEL */ #endif /* CONFIG_JUMP_LABEL */
/* /*
* Advanced usage; refcount, branch is enabled when: count != 0 * Advanced usage; refcount, branch is enabled when: count != 0

8
include/linux/jump_label_ratelimit.h
View File

@ -5,21 +5,19 @@
#include <linux/jump_label.h> #include <linux/jump_label.h>
#include <linux/workqueue.h> #include <linux/workqueue.h>
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) #if defined(CONFIG_JUMP_LABEL)
struct static_key_deferred { struct static_key_deferred {
struct static_key key; struct static_key key;
unsigned long timeout; unsigned long timeout;
struct delayed_work work; struct delayed_work work;
}; };
#endif
#ifdef HAVE_JUMP_LABEL
extern void static_key_slow_dec_deferred(struct static_key_deferred *key); extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
extern void static_key_deferred_flush(struct static_key_deferred *key); extern void static_key_deferred_flush(struct static_key_deferred *key);
extern void extern void
jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl); jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
#else /* !HAVE_JUMP_LABEL */ #else /* !CONFIG_JUMP_LABEL */
struct static_key_deferred { struct static_key_deferred {
struct static_key key; struct static_key key;
}; };
@ -38,5 +36,5 @@ jump_label_rate_limit(struct static_key_deferred *key,
{ {
STATIC_KEY_CHECK_USE(key); STATIC_KEY_CHECK_USE(key);
} }
#endif /* HAVE_JUMP_LABEL */ #endif /* CONFIG_JUMP_LABEL */
#endif /* _LINUX_JUMP_LABEL_RATELIMIT_H */ #endif /* _LINUX_JUMP_LABEL_RATELIMIT_H */

2
include/linux/module.h
View File

@ -436,7 +436,7 @@ struct module {
unsigned int num_bpf_raw_events; unsigned int num_bpf_raw_events;
struct bpf_raw_event_map *bpf_raw_events; struct bpf_raw_event_map *bpf_raw_events;
#endif #endif
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
struct jump_entry *jump_entries; struct jump_entry *jump_entries;
unsigned int num_jump_entries; unsigned int num_jump_entries;
#endif #endif

4
include/linux/netfilter.h
View File

@ -176,7 +176,7 @@ void nf_unregister_net_hooks(struct net *net, const struct nf_hook_ops *reg,
int nf_register_sockopt(struct nf_sockopt_ops *reg); int nf_register_sockopt(struct nf_sockopt_ops *reg);
void nf_unregister_sockopt(struct nf_sockopt_ops *reg); void nf_unregister_sockopt(struct nf_sockopt_ops *reg);
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
#endif #endif
@ -198,7 +198,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
struct nf_hook_entries *hook_head = NULL; struct nf_hook_entries *hook_head = NULL;
int ret = 1; int ret = 1;
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
if (__builtin_constant_p(pf) && if (__builtin_constant_p(pf) &&
__builtin_constant_p(hook) && __builtin_constant_p(hook) &&
!static_key_false(&nf_hooks_needed[pf][hook])) !static_key_false(&nf_hooks_needed[pf][hook]))

2
include/linux/netfilter_ingress.h
View File

@ -8,7 +8,7 @@
#ifdef CONFIG_NETFILTER_INGRESS #ifdef CONFIG_NETFILTER_INGRESS
static inline bool nf_hook_ingress_active(const struct sk_buff *skb) static inline bool nf_hook_ingress_active(const struct sk_buff *skb)
{ {
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
if (!static_key_false(&nf_hooks_needed[NFPROTO_NETDEV][NF_NETDEV_INGRESS])) if (!static_key_false(&nf_hooks_needed[NFPROTO_NETDEV][NF_NETDEV_INGRESS]))
return false; return false;
#endif #endif

3
init/Kconfig
View File

@ -23,6 +23,9 @@ config CLANG_VERSION
int int
default $(shell,$(srctree)/scripts/clang-version.sh $(CC)) default $(shell,$(srctree)/scripts/clang-version.sh $(CC))
config CC_HAS_ASM_GOTO
def_bool $(success,$(srctree)/scripts/gcc-goto.sh $(CC))
config CONSTRUCTORS config CONSTRUCTORS
bool bool
depends on !UML depends on !UML

10
kernel/jump_label.c
View File

@ -18,8 +18,6 @@
#include <linux/cpu.h> #include <linux/cpu.h>
#include <asm/sections.h> #include <asm/sections.h>
#ifdef HAVE_JUMP_LABEL
/* mutex to protect coming/going of the the jump_label table */ /* mutex to protect coming/going of the the jump_label table */
static DEFINE_MUTEX(jump_label_mutex); static DEFINE_MUTEX(jump_label_mutex);
@ -80,13 +78,13 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
static void jump_label_update(struct static_key *key); static void jump_label_update(struct static_key *key);
/* /*
* There are similar definitions for the !HAVE_JUMP_LABEL case in jump_label.h. * There are similar definitions for the !CONFIG_JUMP_LABEL case in jump_label.h.
* The use of 'atomic_read()' requires atomic.h and its problematic for some * The use of 'atomic_read()' requires atomic.h and its problematic for some
* kernel headers such as kernel.h and others. Since static_key_count() is not * kernel headers such as kernel.h and others. Since static_key_count() is not
* used in the branch statements as it is for the !HAVE_JUMP_LABEL case its ok * used in the branch statements as it is for the !CONFIG_JUMP_LABEL case its ok
* to have it be a function here. Similarly, for 'static_key_enable()' and * to have it be a function here. Similarly, for 'static_key_enable()' and
* 'static_key_disable()', which require bug.h. This should allow jump_label.h * 'static_key_disable()', which require bug.h. This should allow jump_label.h
* to be included from most/all places for HAVE_JUMP_LABEL. * to be included from most/all places for CONFIG_JUMP_LABEL.
*/ */
int static_key_count(struct static_key *key) int static_key_count(struct static_key *key)
{ {
@ -791,5 +789,3 @@ static __init int jump_label_test(void)
} }
early_initcall(jump_label_test); early_initcall(jump_label_test);
#endif /* STATIC_KEYS_SELFTEST */ #endif /* STATIC_KEYS_SELFTEST */
#endif /* HAVE_JUMP_LABEL */

2
kernel/module.c
View File

@ -3102,7 +3102,7 @@ static int find_module_sections(struct module *mod, struct load_info *info)
sizeof(*mod->bpf_raw_events), sizeof(*mod->bpf_raw_events),
&mod->num_bpf_raw_events); &mod->num_bpf_raw_events);
#endif #endif
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
mod->jump_entries = section_objs(info, "__jump_table", mod->jump_entries = section_objs(info, "__jump_table",
sizeof(*mod->jump_entries), sizeof(*mod->jump_entries),
&mod->num_jump_entries); &mod->num_jump_entries);

2
kernel/sched/core.c
View File

@ -24,7 +24,7 @@
DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL) #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_JUMP_LABEL)
/* /*
* Debugging: various feature bits * Debugging: various feature bits
* *

4
kernel/sched/debug.c
View File

@ -73,7 +73,7 @@ static int sched_feat_show(struct seq_file *m, void *v)
return 0; return 0;
} }
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
#define jump_label_key__true STATIC_KEY_INIT_TRUE #define jump_label_key__true STATIC_KEY_INIT_TRUE
#define jump_label_key__false STATIC_KEY_INIT_FALSE #define jump_label_key__false STATIC_KEY_INIT_FALSE
@ -99,7 +99,7 @@ static void sched_feat_enable(int i)
#else #else
static void sched_feat_disable(int i) { }; static void sched_feat_disable(int i) { };
static void sched_feat_enable(int i) { }; static void sched_feat_enable(int i) { };
#endif /* HAVE_JUMP_LABEL */ #endif /* CONFIG_JUMP_LABEL */
static int sched_feat_set(char *cmp) static int sched_feat_set(char *cmp)
{ {

6
kernel/sched/fair.c
View File

@ -4217,7 +4217,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
#ifdef CONFIG_CFS_BANDWIDTH #ifdef CONFIG_CFS_BANDWIDTH
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
static struct static_key __cfs_bandwidth_used; static struct static_key __cfs_bandwidth_used;
static inline bool cfs_bandwidth_used(void) static inline bool cfs_bandwidth_used(void)
@ -4234,7 +4234,7 @@ void cfs_bandwidth_usage_dec(void)
{ {
static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used);
} }
#else /* HAVE_JUMP_LABEL */ #else /* CONFIG_JUMP_LABEL */
static bool cfs_bandwidth_used(void) static bool cfs_bandwidth_used(void)
{ {
return true; return true;
@ -4242,7 +4242,7 @@ static bool cfs_bandwidth_used(void)
void cfs_bandwidth_usage_inc(void) {} void cfs_bandwidth_usage_inc(void) {}
void cfs_bandwidth_usage_dec(void) {} void cfs_bandwidth_usage_dec(void) {}
#endif /* HAVE_JUMP_LABEL */ #endif /* CONFIG_JUMP_LABEL */
/* /*
* default period for cfs group bandwidth. * default period for cfs group bandwidth.

6
kernel/sched/sched.h
View File

@ -1488,7 +1488,7 @@ enum {
#undef SCHED_FEAT #undef SCHED_FEAT
#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL) #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_JUMP_LABEL)
/* /*
* To support run-time toggling of sched features, all the translation units * To support run-time toggling of sched features, all the translation units
@ -1508,7 +1508,7 @@ static __always_inline bool static_branch_##name(struct static_key *key) \
extern struct static_key sched_feat_keys[__SCHED_FEAT_NR]; extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
#define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x])) #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
#else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */ #else /* !(SCHED_DEBUG && CONFIG_JUMP_LABEL) */
/* /*
* Each translation unit has its own copy of sysctl_sched_features to allow * Each translation unit has its own copy of sysctl_sched_features to allow
@ -1524,7 +1524,7 @@ static const_debug __maybe_unused unsigned int sysctl_sched_features =
#define sched_feat(x) !!(sysctl_sched_features & (1UL << __SCHED_FEAT_##x)) #define sched_feat(x) !!(sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
#endif /* SCHED_DEBUG && HAVE_JUMP_LABEL */ #endif /* SCHED_DEBUG && CONFIG_JUMP_LABEL */
extern struct static_key_false sched_numa_balancing; extern struct static_key_false sched_numa_balancing;
extern struct static_key_false sched_schedstats; extern struct static_key_false sched_schedstats;

2
lib/dynamic_debug.c
View File

@ -188,7 +188,7 @@ static int ddebug_change(const struct ddebug_query *query,
newflags = (dp->flags & mask) | flags; newflags = (dp->flags & mask) | flags;
if (newflags == dp->flags) if (newflags == dp->flags)
continue; continue;
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
if (dp->flags & _DPRINTK_FLAGS_PRINT) { if (dp->flags & _DPRINTK_FLAGS_PRINT) {
if (!(flags & _DPRINTK_FLAGS_PRINT)) if (!(flags & _DPRINTK_FLAGS_PRINT))
static_branch_disable(&dp->key.dd_key_true); static_branch_disable(&dp->key.dd_key_true);

6
net/core/dev.c
View File

@ -1821,7 +1821,7 @@ EXPORT_SYMBOL_GPL(net_dec_egress_queue);
#endif #endif
static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key); static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
static atomic_t netstamp_needed_deferred; static atomic_t netstamp_needed_deferred;
static atomic_t netstamp_wanted; static atomic_t netstamp_wanted;
static void netstamp_clear(struct work_struct *work) static void netstamp_clear(struct work_struct *work)
@ -1840,7 +1840,7 @@ static DECLARE_WORK(netstamp_work, netstamp_clear);
void net_enable_timestamp(void) void net_enable_timestamp(void)
{ {
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
int wanted; int wanted;
while (1) { while (1) {
@ -1860,7 +1860,7 @@ EXPORT_SYMBOL(net_enable_timestamp);
void net_disable_timestamp(void) void net_disable_timestamp(void)
{ {
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
int wanted; int wanted;
while (1) { while (1) {

6
net/netfilter/core.c
View File

@ -33,7 +33,7 @@ EXPORT_SYMBOL_GPL(nf_ipv6_ops);
DEFINE_PER_CPU(bool, nf_skb_duplicated); DEFINE_PER_CPU(bool, nf_skb_duplicated);
EXPORT_SYMBOL_GPL(nf_skb_duplicated); EXPORT_SYMBOL_GPL(nf_skb_duplicated);
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
EXPORT_SYMBOL(nf_hooks_needed); EXPORT_SYMBOL(nf_hooks_needed);
#endif #endif
@ -347,7 +347,7 @@ static int __nf_register_net_hook(struct net *net, int pf,
if (pf == NFPROTO_NETDEV && reg->hooknum == NF_NETDEV_INGRESS) if (pf == NFPROTO_NETDEV && reg->hooknum == NF_NETDEV_INGRESS)
net_inc_ingress_queue(); net_inc_ingress_queue();
#endif #endif
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
static_key_slow_inc(&nf_hooks_needed[pf][reg->hooknum]); static_key_slow_inc(&nf_hooks_needed[pf][reg->hooknum]);
#endif #endif
BUG_ON(p == new_hooks); BUG_ON(p == new_hooks);
@ -405,7 +405,7 @@ static void __nf_unregister_net_hook(struct net *net, int pf,
if (pf == NFPROTO_NETDEV && reg->hooknum == NF_NETDEV_INGRESS) if (pf == NFPROTO_NETDEV && reg->hooknum == NF_NETDEV_INGRESS)
net_dec_ingress_queue(); net_dec_ingress_queue();
#endif #endif
#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_JUMP_LABEL
static_key_slow_dec(&nf_hooks_needed[pf][reg->hooknum]); static_key_slow_dec(&nf_hooks_needed[pf][reg->hooknum]);
#endif #endif
} else { } else {

2
scripts/gcc-goto.sh
View File

@ -3,7 +3,7 @@
# Test for gcc 'asm goto' support # Test for gcc 'asm goto' support
# Copyright (C) 2010, Jason Baron <jbaron@redhat.com> # Copyright (C) 2010, Jason Baron <jbaron@redhat.com>
cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y" cat << "END" | $@ -x c - -fno-PIE -c -o /dev/null
int main(void) int main(void)
{ {
#if defined(__arm__) || defined(__aarch64__) #if defined(__arm__) || defined(__aarch64__)

6
tools/arch/x86/include/asm/rmwcc.h
View File

@ -2,7 +2,7 @@
#ifndef _TOOLS_LINUX_ASM_X86_RMWcc #ifndef _TOOLS_LINUX_ASM_X86_RMWcc
#define _TOOLS_LINUX_ASM_X86_RMWcc #define _TOOLS_LINUX_ASM_X86_RMWcc
#ifdef CC_HAVE_ASM_GOTO #ifdef CONFIG_CC_HAS_ASM_GOTO
#define __GEN_RMWcc(fullop, var, cc, ...) \ #define __GEN_RMWcc(fullop, var, cc, ...) \
do { \ do { \
@ -20,7 +20,7 @@ cc_label: \
#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ #define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
__GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val)) __GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val))
#else /* !CC_HAVE_ASM_GOTO */ #else /* !CONFIG_CC_HAS_ASM_GOTO */
#define __GEN_RMWcc(fullop, var, cc, ...) \ #define __GEN_RMWcc(fullop, var, cc, ...) \
do { \ do { \
@ -37,6 +37,6 @@ do { \
#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ #define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
__GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val)) __GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val))
#endif /* CC_HAVE_ASM_GOTO */ #endif /* CONFIG_CC_HAS_ASM_GOTO */
#endif /* _TOOLS_LINUX_ASM_X86_RMWcc */ #endif /* _TOOLS_LINUX_ASM_X86_RMWcc */