From 28ac0a9e04d7dfb42220dc9d221164d93f20fb3a Mon Sep 17 00:00:00 2001 From: Qing Zhang Date: Sat, 10 Dec 2022 22:40:21 +0800 Subject: [PATCH] LoongArch: modules/ftrace: Initialize PLT at load time This patch implements ftrace trampolines through plt entry. Tested by forcing ftrace_make_call() to use the module PLT, and then loading up a module after setting up ftrace with: | echo ":mod:" > set_ftrace_filter; | echo function > current_tracer; | modprobe Since FTRACE_ADDR/FTRACE_REGS_ADDR is only defined when CONFIG_DYNAMIC_ FTRACE is selected, we wrap their usage in module_init_ftrace_plt() with ifdeffery rather than using IS_ENABLED(). Signed-off-by: Qing Zhang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/ftrace.h | 4 ++ arch/loongarch/include/asm/inst.h | 3 + arch/loongarch/include/asm/module.h | 5 +- arch/loongarch/include/asm/module.lds.h | 1 + arch/loongarch/kernel/ftrace_dyn.c | 93 +++++++++++++++++++++++++ arch/loongarch/kernel/inst.c | 11 +++ arch/loongarch/kernel/module-sections.c | 12 +++- arch/loongarch/kernel/module.c | 21 ++++++ 8 files changed, 148 insertions(+), 2 deletions(-) diff --git a/arch/loongarch/include/asm/ftrace.h b/arch/loongarch/include/asm/ftrace.h index 8c7d137e4871..90f9d3399b2a 100644 --- a/arch/loongarch/include/asm/ftrace.h +++ b/arch/loongarch/include/asm/ftrace.h @@ -6,6 +6,10 @@ #ifndef _ASM_LOONGARCH_FTRACE_H #define _ASM_LOONGARCH_FTRACE_H +#define FTRACE_PLT_IDX 0 +#define FTRACE_REGS_PLT_IDX 1 +#define NR_FTRACE_PLTS 2 + #define GRAPH_FAKE_OFFSET (sizeof(struct pt_regs) - offsetof(struct pt_regs, regs[1])) #ifdef CONFIG_FUNCTION_TRACER diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 88e1673524e1..c00e1512d4fa 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -13,10 +13,12 @@ #define ADDR_IMMMASK_LU52ID 0xFFF0000000000000 #define ADDR_IMMMASK_LU32ID 0x000FFFFF00000000 +#define ADDR_IMMMASK_LU12IW 0x00000000FFFFF000 #define ADDR_IMMMASK_ADDU16ID 0x00000000FFFF0000 #define ADDR_IMMSHIFT_LU52ID 52 #define ADDR_IMMSHIFT_LU32ID 32 +#define ADDR_IMMSHIFT_LU12IW 12 #define ADDR_IMMSHIFT_ADDU16ID 16 #define ADDR_IMM(addr, INSN) ((addr & ADDR_IMMMASK_##INSN) >> ADDR_IMMSHIFT_##INSN) @@ -360,6 +362,7 @@ u32 larch_insn_gen_bl(unsigned long pc, unsigned long dest); u32 larch_insn_gen_or(enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk); u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj); +u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm); u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm); u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm); u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, unsigned long pc, unsigned long dest); diff --git a/arch/loongarch/include/asm/module.h b/arch/loongarch/include/asm/module.h index 60dc62a1146e..12a0f1e66916 100644 --- a/arch/loongarch/include/asm/module.h +++ b/arch/loongarch/include/asm/module.h @@ -20,6 +20,9 @@ struct mod_arch_specific { struct mod_section got; struct mod_section plt; struct mod_section plt_idx; + + /* For CONFIG_DYNAMIC_FTRACE */ + struct plt_entry *ftrace_trampolines; }; struct got_entry { @@ -49,7 +52,7 @@ static inline struct plt_entry emit_plt_entry(unsigned long val) { u32 lu12iw, lu32id, lu52id, jirl; - lu12iw = (lu12iw_op << 25 | (((val >> 12) & 0xfffff) << 5) | LOONGARCH_GPR_T1); + lu12iw = larch_insn_gen_lu12iw(LOONGARCH_GPR_T1, ADDR_IMM(val, LU12IW)); lu32id = larch_insn_gen_lu32id(LOONGARCH_GPR_T1, ADDR_IMM(val, LU32ID)); lu52id = larch_insn_gen_lu52id(LOONGARCH_GPR_T1, LOONGARCH_GPR_T1, ADDR_IMM(val, LU52ID)); jirl = larch_insn_gen_jirl(0, LOONGARCH_GPR_T1, 0, (val & 0xfff)); diff --git a/arch/loongarch/include/asm/module.lds.h b/arch/loongarch/include/asm/module.lds.h index a3d1bc0fcc72..438f09d4ccf4 100644 --- a/arch/loongarch/include/asm/module.lds.h +++ b/arch/loongarch/include/asm/module.lds.h @@ -5,4 +5,5 @@ SECTIONS { .got : { BYTE(0) } .plt : { BYTE(0) } .plt.idx : { BYTE(0) } + .ftrace_trampoline : { BYTE(0) } } diff --git a/arch/loongarch/kernel/ftrace_dyn.c b/arch/loongarch/kernel/ftrace_dyn.c index e23c3be29baa..0f07591cab30 100644 --- a/arch/loongarch/kernel/ftrace_dyn.c +++ b/arch/loongarch/kernel/ftrace_dyn.c @@ -9,6 +9,7 @@ #include #include +#include static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, bool validate) { @@ -29,18 +30,78 @@ static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, bool validate) } #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + +#ifdef CONFIG_MODULES +static inline int __get_mod(struct module **mod, unsigned long addr) +{ + preempt_disable(); + *mod = __module_text_address(addr); + preempt_enable(); + + if (WARN_ON(!(*mod))) + return -EINVAL; + + return 0; +} + +static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr) +{ + struct plt_entry *plt = mod->arch.ftrace_trampolines; + + if (addr == FTRACE_ADDR) + return &plt[FTRACE_PLT_IDX]; + if (addr == FTRACE_REGS_ADDR && + IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) + return &plt[FTRACE_REGS_PLT_IDX]; + + return NULL; +} + +static unsigned long get_plt_addr(struct module *mod, unsigned long addr) +{ + struct plt_entry *plt; + + plt = get_ftrace_plt(mod, addr); + if (!plt) { + pr_err("ftrace: no module PLT for %ps\n", (void *)addr); + return -EINVAL; + } + + return (unsigned long)plt; +} +#endif + int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { u32 old, new; unsigned long pc; + long offset __maybe_unused; pc = rec->ip + LOONGARCH_INSN_SIZE; +#ifdef CONFIG_MODULES + offset = (long)pc - (long)addr; + + if (offset < -SZ_128M || offset >= SZ_128M) { + int ret; + struct module *mod; + + ret = __get_mod(&mod, pc); + if (ret) + return ret; + + addr = get_plt_addr(mod, addr); + + old_addr = get_plt_addr(mod, old_addr); + } +#endif + new = larch_insn_gen_bl(pc, addr); old = larch_insn_gen_bl(pc, old_addr); return ftrace_modify_code(pc, old, new, true); } + #endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ int ftrace_update_ftrace_func(ftrace_func_t func) @@ -91,9 +152,25 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { u32 old, new; unsigned long pc; + long offset __maybe_unused; pc = rec->ip + LOONGARCH_INSN_SIZE; +#ifdef CONFIG_MODULES + offset = (long)pc - (long)addr; + + if (offset < -SZ_128M || offset >= SZ_128M) { + int ret; + struct module *mod; + + ret = __get_mod(&mod, pc); + if (ret) + return ret; + + addr = get_plt_addr(mod, addr); + } +#endif + old = larch_insn_gen_nop(); new = larch_insn_gen_bl(pc, addr); @@ -104,9 +181,25 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long ad { u32 old, new; unsigned long pc; + long offset __maybe_unused; pc = rec->ip + LOONGARCH_INSN_SIZE; +#ifdef CONFIG_MODULES + offset = (long)pc - (long)addr; + + if (offset < -SZ_128M || offset >= SZ_128M) { + int ret; + struct module *mod; + + ret = __get_mod(&mod, pc); + if (ret) + return ret; + + addr = get_plt_addr(mod, addr); + } +#endif + new = larch_insn_gen_nop(); old = larch_insn_gen_bl(pc, addr); diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c index 39671e87e31c..512579d79b22 100644 --- a/arch/loongarch/kernel/inst.c +++ b/arch/loongarch/kernel/inst.c @@ -120,6 +120,17 @@ u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj) return larch_insn_gen_or(rd, rj, 0); } +u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm) +{ + union loongarch_instruction insn; + + insn.reg1i20_format.opcode = lu12iw_op; + insn.reg1i20_format.rd = rd; + insn.reg1i20_format.immediate = imm; + + return insn.word; +} + u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm) { union loongarch_instruction insn; diff --git a/arch/loongarch/kernel/module-sections.c b/arch/loongarch/kernel/module-sections.c index 13d9a427325a..d4dbcda1c4b0 100644 --- a/arch/loongarch/kernel/module-sections.c +++ b/arch/loongarch/kernel/module-sections.c @@ -6,6 +6,7 @@ #include #include #include +#include Elf_Addr module_emit_got_entry(struct module *mod, Elf_Shdr *sechdrs, Elf_Addr val) { @@ -103,7 +104,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, char *secstrings, struct module *mod) { unsigned int i, num_plts = 0, num_gots = 0; - Elf_Shdr *got_sec, *plt_sec, *plt_idx_sec; + Elf_Shdr *got_sec, *plt_sec, *plt_idx_sec, *tramp = NULL; /* * Find the empty .plt sections. @@ -115,6 +116,8 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, mod->arch.plt.shndx = i; else if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt.idx")) mod->arch.plt_idx.shndx = i; + else if (!strcmp(secstrings + sechdrs[i].sh_name, ".ftrace_trampoline")) + tramp = sechdrs + i; } if (!mod->arch.got.shndx) { @@ -170,5 +173,12 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, mod->arch.plt_idx.num_entries = 0; mod->arch.plt_idx.max_entries = num_plts; + if (tramp) { + tramp->sh_type = SHT_NOBITS; + tramp->sh_flags = SHF_EXECINSTR | SHF_ALLOC; + tramp->sh_addralign = __alignof__(struct plt_entry); + tramp->sh_size = NR_FTRACE_PLTS * sizeof(struct plt_entry); + } + return 0; } diff --git a/arch/loongarch/kernel/module.c b/arch/loongarch/kernel/module.c index 899dc677cec3..b8b86088b2dd 100644 --- a/arch/loongarch/kernel/module.c +++ b/arch/loongarch/kernel/module.c @@ -15,9 +15,11 @@ #include #include #include +#include #include #include #include +#include static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) { @@ -473,6 +475,23 @@ void *module_alloc(unsigned long size) GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); } +static void module_init_ftrace_plt(const Elf_Ehdr *hdr, + const Elf_Shdr *sechdrs, struct module *mod) +{ +#ifdef CONFIG_DYNAMIC_FTRACE + struct plt_entry *ftrace_plts; + + ftrace_plts = (void *)sechdrs->sh_addr; + + ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR); + + if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) + ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR); + + mod->arch.ftrace_trampolines = ftrace_plts; +#endif +} + int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod) { @@ -482,6 +501,8 @@ int module_finalize(const Elf_Ehdr *hdr, for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { if (!strcmp(".altinstructions", secstrs + s->sh_name)) apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size); + if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name)) + module_init_ftrace_plt(hdr, s, mod); } return 0;