hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

kprobes: Replace rp->free_instance with freelist 

Gets rid of rp->lock, and as a result kretprobes are now fully
lockless.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/159870623583.1229682.17472357584134058687.stgit@devnote2
This commit is contained in:
Peter Zijlstra 2020-08-29 22:03:56 +09:00 committed by Ingo Molnar
parent e563604a5f
commit 6e426e0fcd
2 changed files with 29 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/linux/kprobes.h
View File

@ -28,6 +28,7 @@
#include <linux/mutex.h> #include <linux/mutex.h>
#include <linux/ftrace.h> #include <linux/ftrace.h>
#include <linux/refcount.h> #include <linux/refcount.h>
#include <linux/freelist.h>
#include <asm/kprobes.h> #include <asm/kprobes.h>
#ifdef CONFIG_KPROBES #ifdef CONFIG_KPROBES
@ -157,17 +158,16 @@ struct kretprobe {
int maxactive; int maxactive;
int nmissed; int nmissed;
size_t data_size; size_t data_size;
struct hlist_head free_instances; struct freelist_head freelist;
struct kretprobe_holder *rph; struct kretprobe_holder *rph;
raw_spinlock_t lock;
}; };
struct kretprobe_instance { struct kretprobe_instance {
union { union {
struct llist_node llist; struct freelist_node freelist;
struct hlist_node hlist;
struct rcu_head rcu; struct rcu_head rcu;
}; };
struct llist_node llist;
struct kretprobe_holder *rph; struct kretprobe_holder *rph;
kprobe_opcode_t *ret_addr; kprobe_opcode_t *ret_addr;
void *fp; void *fp;

44
kernel/kprobes.c
View File

@ -1228,11 +1228,8 @@ static void recycle_rp_inst(struct kretprobe_instance *ri)
{ {
struct kretprobe *rp = get_kretprobe(ri); struct kretprobe *rp = get_kretprobe(ri);
INIT_HLIST_NODE(&ri->hlist);
if (likely(rp)) { if (likely(rp)) {
raw_spin_lock(&rp->lock); freelist_add(&ri->freelist, &rp->freelist);
hlist_add_head(&ri->hlist, &rp->free_instances);
raw_spin_unlock(&rp->lock);
} else } else
call_rcu(&ri->rcu, free_rp_inst_rcu); call_rcu(&ri->rcu, free_rp_inst_rcu);
} }
@ -1290,11 +1287,14 @@ NOKPROBE_SYMBOL(kprobe_flush_task);
static inline void free_rp_inst(struct kretprobe *rp) static inline void free_rp_inst(struct kretprobe *rp)
{ {
struct kretprobe_instance *ri; struct kretprobe_instance *ri;
struct hlist_node *next; struct freelist_node *node;
int count = 0; int count = 0;
hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) { node = rp->freelist.head;
hlist_del(&ri->hlist); while (node) {
ri = container_of(node, struct kretprobe_instance, freelist);
node = node->next;
kfree(ri); kfree(ri);
count++; count++;
} }
@ -1925,21 +1925,19 @@ NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
{ {
struct kretprobe *rp = container_of(p, struct kretprobe, kp); struct kretprobe *rp = container_of(p, struct kretprobe, kp);
unsigned long flags = 0;
struct kretprobe_instance *ri; struct kretprobe_instance *ri;
struct freelist_node *fn;
/* TODO: consider to only swap the RA after the last pre_handler fired */ fn = freelist_try_get(&rp->freelist);
raw_spin_lock_irqsave(&rp->lock, flags); if (!fn) {
if (!hlist_empty(&rp->free_instances)) { rp->nmissed++;
ri = hlist_entry(rp->free_instances.first, return 0;
struct kretprobe_instance, hlist); }
hlist_del(&ri->hlist);
raw_spin_unlock_irqrestore(&rp->lock, flags); ri = container_of(fn, struct kretprobe_instance, freelist);
if (rp->entry_handler && rp->entry_handler(ri, regs)) { if (rp->entry_handler && rp->entry_handler(ri, regs)) {
raw_spin_lock_irqsave(&rp->lock, flags); freelist_add(&ri->freelist, &rp->freelist);
hlist_add_head(&ri->hlist, &rp->free_instances);
raw_spin_unlock_irqrestore(&rp->lock, flags);
return 0; return 0;
} }
@ -1947,10 +1945,6 @@ static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
__llist_add(&ri->llist, &current->kretprobe_instances); __llist_add(&ri->llist, &current->kretprobe_instances);
} else {
rp->nmissed++;
raw_spin_unlock_irqrestore(&rp->lock, flags);
}
return 0; return 0;
} }
NOKPROBE_SYMBOL(pre_handler_kretprobe); NOKPROBE_SYMBOL(pre_handler_kretprobe);
@ -2007,8 +2001,7 @@ int register_kretprobe(struct kretprobe *rp)
rp->maxactive = num_possible_cpus(); rp->maxactive = num_possible_cpus();
#endif #endif
} }
raw_spin_lock_init(&rp->lock); rp->freelist.head = NULL;
INIT_HLIST_HEAD(&rp->free_instances);
rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL); rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL);
if (!rp->rph) if (!rp->rph)
return -ENOMEM; return -ENOMEM;
@ -2023,8 +2016,7 @@ int register_kretprobe(struct kretprobe *rp)
return -ENOMEM; return -ENOMEM;
} }
inst->rph = rp->rph; inst->rph = rp->rph;
INIT_HLIST_NODE(&inst->hlist); freelist_add(&inst->freelist, &rp->freelist);
hlist_add_head(&inst->hlist, &rp->free_instances);
} }
refcount_set(&rp->rph->ref, i); refcount_set(&rp->rph->ref, i);