UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'topic/kprobes' into next 

Although most of these kprobes patches are powerpc specific, there's a couple
that touch generic code (with Acks). At the moment there's one conflict with
acme's tree, but it's not too bad. Still just in case some other conflicts show
up, we've put these in a topic branch so another tree could merge some or all of
it if necessary.
This commit is contained in:
Michael Ellerman 2017-04-25 00:24:04 +10:00
parent 9a914aa682 24bd909e94
commit 9fc849144c
11 changed files with 316 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Documentation/features/debug/kprobes-on-ftrace/arch-support.txt
View File

@ -27,7 +27,7 @@
| nios2: | TODO |
| openrisc: | TODO |
| parisc: | TODO |
| powerpc: | TODO |
| powerpc: | ok |
| s390: | TODO |
| score: | TODO |
| sh: | TODO |

1
arch/powerpc/Kconfig
View File

@ -186,6 +186,7 @@ config PPC
select HAVE_IRQ_EXIT_ON_IRQ_STACK
select HAVE_KERNEL_GZIP
select HAVE_KPROBES
select HAVE_KPROBES_ON_FTRACE
select HAVE_KRETPROBES
select HAVE_LIVEPATCH if HAVE_DYNAMIC_FTRACE_WITH_REGS
select HAVE_MEMBLOCK

41
arch/powerpc/include/asm/code-patching.h
View File

@ -12,6 +12,8 @@
#include <asm/types.h>
#include <asm/ppc-opcode.h>
#include <linux/string.h>
#include <linux/kallsyms.h>
/* Flags for create_branch:
* "b" == create_branch(addr, target, 0);
@ -99,6 +101,45 @@ static inline unsigned long ppc_global_function_entry(void *func)
#endif
}
/*
* Wrapper around kallsyms_lookup() to return function entry address:
* - For ABIv1, we lookup the dot variant.
* - For ABIv2, we return the local entry point.
*/
static inline unsigned long ppc_kallsyms_lookup_name(const char *name)
{
unsigned long addr;
#ifdef PPC64_ELF_ABI_v1
/* check for dot variant */
char dot_name[1 + KSYM_NAME_LEN];
bool dot_appended = false;
if (strnlen(name, KSYM_NAME_LEN) >= KSYM_NAME_LEN)
return 0;
if (name[0] != '.') {
dot_name[0] = '.';
dot_name[1] = '\0';
strlcat(dot_name, name, sizeof(dot_name));
dot_appended = true;
} else {
dot_name[0] = '\0';
strlcat(dot_name, name, sizeof(dot_name));
}
addr = kallsyms_lookup_name(dot_name);
if (!addr && dot_appended)
/* Let's try the original non-dot symbol lookup */
addr = kallsyms_lookup_name(name);
#elif defined(PPC64_ELF_ABI_v2)
addr = kallsyms_lookup_name(name);
if (addr)
addr = ppc_function_entry((void *)addr);
#else
addr = kallsyms_lookup_name(name);
#endif
return addr;
}
#ifdef CONFIG_PPC64
/*
* Some instruction encodings commonly used in dynamic ftracing

63
arch/powerpc/include/asm/kprobes.h
View File

@ -61,59 +61,6 @@ extern kprobe_opcode_t optprobe_template_end[];
#define MAX_OPTINSN_SIZE (optprobe_template_end - optprobe_template_entry)
#define RELATIVEJUMP_SIZE sizeof(kprobe_opcode_t) /* 4 bytes */
#ifdef PPC64_ELF_ABI_v2
/* PPC64 ABIv2 needs local entry point */
#define kprobe_lookup_name(name, addr) \
{ \
addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); \
if (addr) \
addr = (kprobe_opcode_t *)ppc_function_entry(addr); \
}
#elif defined(PPC64_ELF_ABI_v1)
/*
* 64bit powerpc ABIv1 uses function descriptors:
* - Check for the dot variant of the symbol first.
* - If that fails, try looking up the symbol provided.
*
* This ensures we always get to the actual symbol and not the descriptor.
* Also handle <module:symbol> format.
*/
#define kprobe_lookup_name(name, addr) \
{ \
char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN]; \
const char *modsym; \
bool dot_appended = false; \
if ((modsym = strchr(name, ':')) != NULL) { \
modsym++; \
if (*modsym != '\0' && *modsym != '.') { \
/* Convert to <module:.symbol> */ \
strncpy(dot_name, name, modsym - name); \
dot_name[modsym - name] = '.'; \
dot_name[modsym - name + 1] = '\0'; \
strncat(dot_name, modsym, \
sizeof(dot_name) - (modsym - name) - 2);\
dot_appended = true; \
} else { \
dot_name[0] = '\0'; \
strncat(dot_name, name, sizeof(dot_name) - 1); \
} \
} else if (name[0] != '.') { \
dot_name[0] = '.'; \
dot_name[1] = '\0'; \
strncat(dot_name, name, KSYM_NAME_LEN - 2); \
dot_appended = true; \
} else { \
dot_name[0] = '\0'; \
strncat(dot_name, name, KSYM_NAME_LEN - 1); \
} \
addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name); \
if (!addr && dot_appended) { \
/* Let's try the original non-dot symbol lookup */ \
addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); \
} \
}
#endif
#define flush_insn_slot(p) do { } while (0)
#define kretprobe_blacklist_size 0
@ -156,6 +103,16 @@ extern int kprobe_exceptions_notify(struct notifier_block *self,
extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr);
extern int kprobe_handler(struct pt_regs *regs);
extern int kprobe_post_handler(struct pt_regs *regs);
#ifdef CONFIG_KPROBES_ON_FTRACE
extern int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb);
#else
static inline int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb)
{
return 0;
}
#endif
#else
static inline int kprobe_handler(struct pt_regs *regs) { return 0; }
static inline int kprobe_post_handler(struct pt_regs *regs) { return 0; }

3
arch/powerpc/kernel/Makefile
View File

@ -97,6 +97,7 @@ obj-$(CONFIG_BOOTX_TEXT) += btext.o
obj-$(CONFIG_SMP) += smp.o
obj-$(CONFIG_KPROBES) += kprobes.o
obj-$(CONFIG_OPTPROBES) += optprobes.o optprobes_head.o
obj-$(CONFIG_KPROBES_ON_FTRACE) += kprobes-ftrace.o
obj-$(CONFIG_UPROBES) += uprobes.o
obj-$(CONFIG_PPC_UDBG_16550) += legacy_serial.o udbg_16550.o
obj-$(CONFIG_STACKTRACE) += stacktrace.o
@ -150,6 +151,8 @@ GCOV_PROFILE_machine_kexec_32.o := n
UBSAN_SANITIZE_machine_kexec_32.o := n
GCOV_PROFILE_kprobes.o := n
UBSAN_SANITIZE_kprobes.o := n
GCOV_PROFILE_kprobes-ftrace.o := n
UBSAN_SANITIZE_kprobes-ftrace.o := n
UBSAN_SANITIZE_vdso.o := n
extra-$(CONFIG_PPC_FPU) += fpu.o

13
arch/powerpc/kernel/entry_64.S
View File

@ -1248,9 +1248,10 @@ _GLOBAL(ftrace_caller)
/* Get the _mcount() call site out of LR */
mflr r7
/* Save it as pt_regs->nip & pt_regs->link */
/* Save it as pt_regs->nip */
std r7, _NIP(r1)
std r7, _LINK(r1)
/* Save the read LR in pt_regs->link */
std r0, _LINK(r1)
/* Save callee's TOC in the ABI compliant location */
std r2, 24(r1)
@ -1297,16 +1298,16 @@ ftrace_call:
REST_8GPRS(16,r1)
REST_8GPRS(24,r1)
/* Restore possibly modified LR */
ld r0, _LINK(r1)
mtlr r0
/* Restore callee's TOC */
ld r2, 24(r1)
/* Pop our stack frame */
addi r1, r1, SWITCH_FRAME_SIZE
/* Restore original LR for return to B */
ld r0, LRSAVE(r1)
mtlr r0
#ifdef CONFIG_LIVEPATCH
/* Based on the cmpd above, if the NIP was altered handle livepatch */
bne- livepatch_handler

104
arch/powerpc/kernel/kprobes-ftrace.c Normal file
View File

@ -0,0 +1,104 @@
/*
* Dynamic Ftrace based Kprobes Optimization
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* Copyright (C) Hitachi Ltd., 2012
* Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
* IBM Corporation
*/
#include <linux/kprobes.h>
#include <linux/ptrace.h>
#include <linux/hardirq.h>
#include <linux/preempt.h>
#include <linux/ftrace.h>
static nokprobe_inline
int __skip_singlestep(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb, unsigned long orig_nip)
{
/*
* Emulate singlestep (and also recover regs->nip)
* as if there is a nop
*/
regs->nip = (unsigned long)p->addr + MCOUNT_INSN_SIZE;
if (unlikely(p->post_handler)) {
kcb->kprobe_status = KPROBE_HIT_SSDONE;
p->post_handler(p, regs, 0);
}
__this_cpu_write(current_kprobe, NULL);
if (orig_nip)
regs->nip = orig_nip;
return 1;
}
int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb)
{
if (kprobe_ftrace(p))
return __skip_singlestep(p, regs, kcb, 0);
else
return 0;
}
NOKPROBE_SYMBOL(skip_singlestep);
/* Ftrace callback handler for kprobes */
void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,
struct ftrace_ops *ops, struct pt_regs *regs)
{
struct kprobe *p;
struct kprobe_ctlblk *kcb;
unsigned long flags;
/* Disable irq for emulating a breakpoint and avoiding preempt */
local_irq_save(flags);
hard_irq_disable();
p = get_kprobe((kprobe_opcode_t *)nip);
if (unlikely(!p) || kprobe_disabled(p))
goto end;
kcb = get_kprobe_ctlblk();
if (kprobe_running()) {
kprobes_inc_nmissed_count(p);
} else {
unsigned long orig_nip = regs->nip;
/*
* On powerpc, NIP is *before* this instruction for the
* pre handler
*/
regs->nip -= MCOUNT_INSN_SIZE;
__this_cpu_write(current_kprobe, p);
kcb->kprobe_status = KPROBE_HIT_ACTIVE;
if (!p->pre_handler || !p->pre_handler(p, regs))
__skip_singlestep(p, regs, kcb, orig_nip);
/*
* If pre_handler returns !0, it sets regs->nip and
* resets current kprobe.
*/
}
end:
local_irq_restore(flags);
}
NOKPROBE_SYMBOL(kprobe_ftrace_handler);
int arch_prepare_kprobe_ftrace(struct kprobe *p)
{
p->ainsn.insn = NULL;
p->ainsn.boostable = -1;
return 0;
}

149
arch/powerpc/kernel/kprobes.c
View File

@ -51,6 +51,77 @@ bool arch_within_kprobe_blacklist(unsigned long addr)
addr < (unsigned long)__head_end);
}
kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset)
{
kprobe_opcode_t *addr;
#ifdef PPC64_ELF_ABI_v2
/* PPC64 ABIv2 needs local entry point */
addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
if (addr && !offset) {
#ifdef CONFIG_KPROBES_ON_FTRACE
unsigned long faddr;
/*
* Per livepatch.h, ftrace location is always within the first
* 16 bytes of a function on powerpc with -mprofile-kernel.
*/
faddr = ftrace_location_range((unsigned long)addr,
(unsigned long)addr + 16);
if (faddr)
addr = (kprobe_opcode_t *)faddr;
else
#endif
addr = (kprobe_opcode_t *)ppc_function_entry(addr);
}
#elif defined(PPC64_ELF_ABI_v1)
/*
* 64bit powerpc ABIv1 uses function descriptors:
* - Check for the dot variant of the symbol first.
* - If that fails, try looking up the symbol provided.
*
* This ensures we always get to the actual symbol and not
* the descriptor.
*
* Also handle <module:symbol> format.
*/
char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN];
const char *modsym;
bool dot_appended = false;
if ((modsym = strchr(name, ':')) != NULL) {
modsym++;
if (*modsym != '\0' && *modsym != '.') {
/* Convert to <module:.symbol> */
strncpy(dot_name, name, modsym - name);
dot_name[modsym - name] = '.';
dot_name[modsym - name + 1] = '\0';
strncat(dot_name, modsym,
sizeof(dot_name) - (modsym - name) - 2);
dot_appended = true;
} else {
dot_name[0] = '\0';
strncat(dot_name, name, sizeof(dot_name) - 1);
}
} else if (name[0] != '.') {
dot_name[0] = '.';
dot_name[1] = '\0';
strncat(dot_name, name, KSYM_NAME_LEN - 2);
dot_appended = true;
} else {
dot_name[0] = '\0';
strncat(dot_name, name, KSYM_NAME_LEN - 1);
}
addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name);
if (!addr && dot_appended) {
/* Let's try the original non-dot symbol lookup */
addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
}
#else
addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
#endif
return addr;
}
int arch_prepare_kprobe(struct kprobe *p)
{
int ret = 0;
@ -144,6 +215,19 @@ static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs
kcb->kprobe_saved_msr = regs->msr;
}
bool arch_function_offset_within_entry(unsigned long offset)
{
#ifdef PPC64_ELF_ABI_v2
#ifdef CONFIG_KPROBES_ON_FTRACE
return offset <= 16;
#else
return offset <= 8;
#endif
#else
return !offset;
#endif
}
void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *)regs->link;
@ -153,6 +237,36 @@ void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
}
NOKPROBE_SYMBOL(arch_prepare_kretprobe);
int try_to_emulate(struct kprobe *p, struct pt_regs *regs)
{
int ret;
unsigned int insn = *p->ainsn.insn;
/* regs->nip is also adjusted if emulate_step returns 1 */
ret = emulate_step(regs, insn);
if (ret > 0) {
/*
* Once this instruction has been boosted
* successfully, set the boostable flag
*/
if (unlikely(p->ainsn.boostable == 0))
p->ainsn.boostable = 1;
} else if (ret < 0) {
/*
* We don't allow kprobes on mtmsr(d)/rfi(d), etc.
* So, we should never get here... but, its still
* good to catch them, just in case...
*/
printk("Can't step on instruction %x\n", insn);
BUG();
} else if (ret == 0)
/* This instruction can't be boosted */
p->ainsn.boostable = -1;
return ret;
}
NOKPROBE_SYMBOL(try_to_emulate);
int kprobe_handler(struct pt_regs *regs)
{
struct kprobe *p;
@ -193,6 +307,14 @@ int kprobe_handler(struct pt_regs *regs)
kprobes_inc_nmissed_count(p);
prepare_singlestep(p, regs);
kcb->kprobe_status = KPROBE_REENTER;
if (p->ainsn.boostable >= 0) {
ret = try_to_emulate(p, regs);
if (ret > 0) {
restore_previous_kprobe(kcb);
return 1;
}
}
return 1;
} else {
if (*addr != BREAKPOINT_INSTRUCTION) {
@ -209,7 +331,9 @@ int kprobe_handler(struct pt_regs *regs)
}
p = __this_cpu_read(current_kprobe);
if (p->break_handler && p->break_handler(p, regs)) {
goto ss_probe;
if (!skip_singlestep(p, regs, kcb))
goto ss_probe;
ret = 1;
}
}
goto no_kprobe;
@ -247,18 +371,9 @@ int kprobe_handler(struct pt_regs *regs)
ss_probe:
if (p->ainsn.boostable >= 0) {
unsigned int insn = *p->ainsn.insn;
ret = try_to_emulate(p, regs);
/* regs->nip is also adjusted if emulate_step returns 1 */
ret = emulate_step(regs, insn);
if (ret > 0) {
/*
* Once this instruction has been boosted
* successfully, set the boostable flag
*/
if (unlikely(p->ainsn.boostable == 0))
p->ainsn.boostable = 1;
if (p->post_handler)
p->post_handler(p, regs, 0);
@ -266,17 +381,7 @@ ss_probe:
reset_current_kprobe();
preempt_enable_no_resched();
return 1;
} else if (ret < 0) {
/*
* We don't allow kprobes on mtmsr(d)/rfi(d), etc.
* So, we should never get here... but, its still
* good to catch them, just in case...
*/
printk("Can't step on instruction %x\n", insn);
BUG();
} else if (ret == 0)
/* This instruction can't be boosted */
p->ainsn.boostable = -1;
}
}
prepare_singlestep(p, regs);
kcb->kprobe_status = KPROBE_HIT_SS;

6
arch/powerpc/kernel/optprobes.c
View File

@ -243,10 +243,10 @@ int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
/*
* 2. branch to optimized_callback() and emulate_step()
*/
kprobe_lookup_name("optimized_callback", op_callback_addr);
kprobe_lookup_name("emulate_step", emulate_step_addr);
op_callback_addr = (kprobe_opcode_t *)ppc_kallsyms_lookup_name("optimized_callback");
emulate_step_addr = (kprobe_opcode_t *)ppc_kallsyms_lookup_name("emulate_step");
if (!op_callback_addr || !emulate_step_addr) {
WARN(1, "kprobe_lookup_name() failed\n");
WARN(1, "Unable to lookup optimized_callback()/emulate_step()\n");
goto error;
}

1
include/linux/kprobes.h
View File

@ -379,6 +379,7 @@ static inline struct kprobe_ctlblk *get_kprobe_ctlblk(void)
return this_cpu_ptr(&kprobe_ctlblk);
}
kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset);
int register_kprobe(struct kprobe *p);
void unregister_kprobe(struct kprobe *p);
int register_kprobes(struct kprobe **kps, int num);

32
kernel/kprobes.c
View File

@ -58,15 +58,6 @@
#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
/*
* Some oddball architectures like 64bit powerpc have function descriptors
* so this must be overridable.
*/
#ifndef kprobe_lookup_name
#define kprobe_lookup_name(name, addr) \
addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
#endif
static int kprobes_initialized;
static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
@ -81,6 +72,12 @@ static struct {
raw_spinlock_t lock ____cacheline_aligned_in_smp;
} kretprobe_table_locks[KPROBE_TABLE_SIZE];
kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
unsigned int __unused)
{
return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
}
static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
{
return &(kretprobe_table_locks[hash].lock);
@ -746,13 +743,20 @@ static void kill_optimized_kprobe(struct kprobe *p)
arch_remove_optimized_kprobe(op);
}
static inline
void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
{
if (!kprobe_ftrace(p))
arch_prepare_optimized_kprobe(op, p);
}
/* Try to prepare optimized instructions */
static void prepare_optimized_kprobe(struct kprobe *p)
{
struct optimized_kprobe *op;
op = container_of(p, struct optimized_kprobe, kp);
arch_prepare_optimized_kprobe(op, p);
__prepare_optimized_kprobe(op, p);
}
/* Allocate new optimized_kprobe and try to prepare optimized instructions */
@ -766,7 +770,7 @@ static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
INIT_LIST_HEAD(&op->list);
op->kp.addr = p->addr;
arch_prepare_optimized_kprobe(op, p);
__prepare_optimized_kprobe(op, p);
return &op->kp;
}
@ -1400,7 +1404,7 @@ static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
goto invalid;
if (p->symbol_name) {
kprobe_lookup_name(p->symbol_name, addr);
addr = kprobe_lookup_name(p->symbol_name, p->offset);
if (!addr)
return ERR_PTR(-ENOENT);
}
@ -2192,8 +2196,8 @@ static int __init init_kprobes(void)
if (kretprobe_blacklist_size) {
/* lookup the function address from its name */
for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
kprobe_lookup_name(kretprobe_blacklist[i].name,
kretprobe_blacklist[i].addr);
kretprobe_blacklist[i].addr =
kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
if (!kretprobe_blacklist[i].addr)
printk("kretprobe: lookup failed: %s\n",
kretprobe_blacklist[i].name);