diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 7fd08755a1f9..933771b0b07a 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -187,6 +187,7 @@ config S390 select HAVE_KPROBES select HAVE_KPROBES_ON_FTRACE select HAVE_KRETPROBES + select HAVE_RETHOOK select HAVE_KVM select HAVE_LIVEPATCH select HAVE_MEMBLOCK_PHYS_MAP diff --git a/arch/s390/include/asm/kprobes.h b/arch/s390/include/asm/kprobes.h index 598095f4b924..6af75857156b 100644 --- a/arch/s390/include/asm/kprobes.h +++ b/arch/s390/include/asm/kprobes.h @@ -70,8 +70,7 @@ struct kprobe_ctlblk { }; void arch_remove_kprobe(struct kprobe *p); -void __kretprobe_trampoline(void); -void trampoline_probe_handler(struct pt_regs *regs); +unsigned long arch_rethook_trampoline_callback(struct pt_regs *regs); int kprobe_fault_handler(struct pt_regs *regs, int trapnr); int kprobe_exceptions_notify(struct notifier_block *self, diff --git a/arch/s390/include/asm/unwind.h b/arch/s390/include/asm/unwind.h index 02462e7100c1..b8ecf04e3468 100644 --- a/arch/s390/include/asm/unwind.h +++ b/arch/s390/include/asm/unwind.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include #include @@ -43,13 +43,15 @@ struct unwind_state { bool error; }; -/* Recover the return address modified by kretprobe and ftrace_graph. */ +/* Recover the return address modified by rethook and ftrace_graph. */ static inline unsigned long unwind_recover_ret_addr(struct unwind_state *state, unsigned long ip) { ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, ip, (void *)state->sp); - if (is_kretprobe_trampoline(ip)) - ip = kretprobe_find_ret_addr(state->task, (void *)state->sp, &state->kr_cur); +#ifdef CONFIG_RETHOOK + if (is_rethook_trampoline(ip)) + ip = rethook_find_ret_addr(state->task, state->sp, &state->kr_cur); +#endif return ip; } diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 5e6a23299790..6065fabb781a 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -58,6 +58,7 @@ obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_KPROBES) += kprobes_insn_page.o obj-$(CONFIG_KPROBES) += mcount.o +obj-$(CONFIG_RETHOOK) += rethook.o obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index 401f9c933ff9..5e713f318de3 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -281,16 +281,6 @@ static void pop_kprobe(struct kprobe_ctlblk *kcb) } NOKPROBE_SYMBOL(pop_kprobe); -void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) -{ - ri->ret_addr = (kprobe_opcode_t *)regs->gprs[14]; - ri->fp = (void *)regs->gprs[15]; - - /* Replace the return addr with trampoline addr */ - regs->gprs[14] = (unsigned long)&__kretprobe_trampoline; -} -NOKPROBE_SYMBOL(arch_prepare_kretprobe); - static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p) { switch (kcb->kprobe_status) { @@ -371,26 +361,6 @@ static int kprobe_handler(struct pt_regs *regs) } NOKPROBE_SYMBOL(kprobe_handler); -void arch_kretprobe_fixup_return(struct pt_regs *regs, - kprobe_opcode_t *correct_ret_addr) -{ - /* Replace fake return address with real one. */ - regs->gprs[14] = (unsigned long)correct_ret_addr; -} -NOKPROBE_SYMBOL(arch_kretprobe_fixup_return); - -/* - * Called from __kretprobe_trampoline - */ -void trampoline_probe_handler(struct pt_regs *regs) -{ - kretprobe_trampoline_handler(regs, (void *)regs->gprs[15]); -} -NOKPROBE_SYMBOL(trampoline_probe_handler); - -/* assembler function that handles the kretprobes must not be probed itself */ -NOKPROBE_SYMBOL(__kretprobe_trampoline); - /* * Called after single-stepping. p->addr is the address of the * instruction whose first byte has been replaced by the "breakpoint" diff --git a/arch/s390/kernel/mcount.S b/arch/s390/kernel/mcount.S index 4786bfe02144..43ff91073d2a 100644 --- a/arch/s390/kernel/mcount.S +++ b/arch/s390/kernel/mcount.S @@ -135,9 +135,9 @@ SYM_FUNC_END(return_to_handler) #endif #endif /* CONFIG_FUNCTION_TRACER */ -#ifdef CONFIG_KPROBES +#ifdef CONFIG_RETHOOK -SYM_FUNC_START(__kretprobe_trampoline) +SYM_FUNC_START(arch_rethook_trampoline) stg %r14,(__SF_GPRS+8*8)(%r15) lay %r15,-STACK_FRAME_SIZE(%r15) @@ -152,16 +152,16 @@ SYM_FUNC_START(__kretprobe_trampoline) epsw %r2,%r3 risbg %r3,%r2,0,31,32 stg %r3,STACK_PTREGS_PSW(%r15) - larl %r1,__kretprobe_trampoline + larl %r1,arch_rethook_trampoline stg %r1,STACK_PTREGS_PSW+8(%r15) lay %r2,STACK_PTREGS(%r15) - brasl %r14,trampoline_probe_handler + brasl %r14,arch_rethook_trampoline_callback mvc __SF_EMPTY(16,%r7),STACK_PTREGS_PSW(%r15) lmg %r0,%r15,STACK_PTREGS_GPRS(%r15) lpswe __SF_EMPTY(%r15) -SYM_FUNC_END(__kretprobe_trampoline) +SYM_FUNC_END(arch_rethook_trampoline) -#endif /* CONFIG_KPROBES */ +#endif /* CONFIG_RETHOOK */ diff --git a/arch/s390/kernel/rethook.c b/arch/s390/kernel/rethook.c new file mode 100644 index 000000000000..f2b6237bc35d --- /dev/null +++ b/arch/s390/kernel/rethook.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include +#include + +void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) +{ + rh->ret_addr = regs->gprs[14]; + rh->frame = regs->gprs[15]; + + /* Replace the return addr with trampoline addr */ + regs->gprs[14] = (unsigned long)&arch_rethook_trampoline; +} +NOKPROBE_SYMBOL(arch_rethook_prepare); + +void arch_rethook_fixup_return(struct pt_regs *regs, + unsigned long correct_ret_addr) +{ + /* Replace fake return address with real one. */ + regs->gprs[14] = correct_ret_addr; +} +NOKPROBE_SYMBOL(arch_rethook_fixup_return); + +/* + * Called from arch_rethook_trampoline + */ +unsigned long arch_rethook_trampoline_callback(struct pt_regs *regs) +{ + return rethook_trampoline_handler(regs, regs->gprs[15]); +} +NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); + +/* assembler function that handles the rethook must not be probed itself */ +NOKPROBE_SYMBOL(arch_rethook_trampoline); diff --git a/arch/s390/kernel/stacktrace.c b/arch/s390/kernel/stacktrace.c index 7ee455e8e3d5..0787010139f7 100644 --- a/arch/s390/kernel/stacktrace.c +++ b/arch/s390/kernel/stacktrace.c @@ -40,12 +40,12 @@ int arch_stack_walk_reliable(stack_trace_consume_fn consume_entry, if (!addr) return -EINVAL; -#ifdef CONFIG_KPROBES +#ifdef CONFIG_RETHOOK /* - * Mark stacktraces with kretprobed functions on them + * Mark stacktraces with krethook functions on them * as unreliable. */ - if (state.ip == (unsigned long)__kretprobe_trampoline) + if (state.ip == (unsigned long)arch_rethook_trampoline) return -EINVAL; #endif diff --git a/arch/s390/lib/test_unwind.c b/arch/s390/lib/test_unwind.c index 5a053b393d5c..7231bf97b93a 100644 --- a/arch/s390/lib/test_unwind.c +++ b/arch/s390/lib/test_unwind.c @@ -47,7 +47,7 @@ static void print_backtrace(char *bt) static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, unsigned long sp) { - int frame_count, prev_is_func2, seen_func2_func1, seen_kretprobe_trampoline; + int frame_count, prev_is_func2, seen_func2_func1, seen_arch_rethook_trampoline; const int max_frames = 128; struct unwind_state state; size_t bt_pos = 0; @@ -63,7 +63,7 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, frame_count = 0; prev_is_func2 = 0; seen_func2_func1 = 0; - seen_kretprobe_trampoline = 0; + seen_arch_rethook_trampoline = 0; unwind_for_each_frame(&state, task, regs, sp) { unsigned long addr = unwind_get_return_address(&state); char sym[KSYM_SYMBOL_LEN]; @@ -89,8 +89,8 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, if (prev_is_func2 && str_has_prefix(sym, "unwindme_func1")) seen_func2_func1 = 1; prev_is_func2 = str_has_prefix(sym, "unwindme_func2"); - if (str_has_prefix(sym, "__kretprobe_trampoline+0x0/")) - seen_kretprobe_trampoline = 1; + if (str_has_prefix(sym, "arch_rethook_trampoline+0x0/")) + seen_arch_rethook_trampoline = 1; } /* Check the results. */ @@ -106,8 +106,8 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, kunit_err(current_test, "Maximum number of frames exceeded\n"); ret = -EINVAL; } - if (seen_kretprobe_trampoline) { - kunit_err(current_test, "__kretprobe_trampoline+0x0 in unwinding results\n"); + if (seen_arch_rethook_trampoline) { + kunit_err(current_test, "arch_rethook_trampoline+0x0 in unwinding results\n"); ret = -EINVAL; } if (ret || force_bt)