hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

srcu: Make early-boot call_srcu() reuse workqueue lists 

Allocating a list_head structure that is almost never used, and, when
used, is used only during early boot (rcu_init() and earlier), is a bit
wasteful.  This commit therefore eliminates that list_head in favor of
the one in the work_struct structure.  This is safe because the work_struct
structure cannot be used until after rcu_init() returns.

Reported-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Lai Jiangshan <jiangshanlai@gmail.com>
Tested-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
This commit is contained in:
Paul E. McKenney 2018-08-14 14:41:49 -07:00
parent 55cda2290b
commit 4e6ea4ef56
4 changed files with 11 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/srcutiny.h
View File

@ -36,7 +36,6 @@ struct srcu_struct {
struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */
struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */ struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */
struct work_struct srcu_work; /* For driving grace periods. */ struct work_struct srcu_work; /* For driving grace periods. */
struct list_head srcu_boot_entry; /* Early-boot callbacks. */
#ifdef CONFIG_DEBUG_LOCK_ALLOC #ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map; struct lockdep_map dep_map;
#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
@ -49,7 +48,6 @@ void srcu_drive_gp(struct work_struct *wp);
.srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \ .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \
.srcu_cb_tail = &name.srcu_cb_head, \ .srcu_cb_tail = &name.srcu_cb_head, \
.srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \ .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \
.srcu_boot_entry = LIST_HEAD_INIT(name.srcu_boot_entry), \
__SRCU_DEP_MAP_INIT(name) \ __SRCU_DEP_MAP_INIT(name) \
} }

3
include/linux/srcutree.h
View File

@ -94,7 +94,6 @@ struct srcu_struct {
/* callback for the barrier */ /* callback for the barrier */
/* operation. */ /* operation. */
struct delayed_work work; struct delayed_work work;
struct list_head srcu_boot_entry; /* Early-boot callbacks. */
#ifdef CONFIG_DEBUG_LOCK_ALLOC #ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map; struct lockdep_map dep_map;
#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
@ -110,7 +109,7 @@ struct srcu_struct {
.sda = &pcpu_name, \ .sda = &pcpu_name, \
.lock = __SPIN_LOCK_UNLOCKED(name.lock), \ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
.srcu_gp_seq_needed = -1UL, \ .srcu_gp_seq_needed = -1UL, \
.srcu_boot_entry = LIST_HEAD_INIT(name.srcu_boot_entry), \ .work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \
__SRCU_DEP_MAP_INIT(name) \ __SRCU_DEP_MAP_INIT(name) \
} }

10
kernel/rcu/srcutiny.c
View File

@ -48,7 +48,7 @@ static int init_srcu_struct_fields(struct srcu_struct *sp)
sp->srcu_gp_waiting = false; sp->srcu_gp_waiting = false;
sp->srcu_idx = 0; sp->srcu_idx = 0;
INIT_WORK(&sp->srcu_work, srcu_drive_gp); INIT_WORK(&sp->srcu_work, srcu_drive_gp);
INIT_LIST_HEAD(&sp->srcu_boot_entry); INIT_LIST_HEAD(&sp->srcu_work.entry);
return 0; return 0;
} }
@ -185,8 +185,8 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,
if (!READ_ONCE(sp->srcu_gp_running)) { if (!READ_ONCE(sp->srcu_gp_running)) {
if (likely(srcu_init_done)) if (likely(srcu_init_done))
schedule_work(&sp->srcu_work); schedule_work(&sp->srcu_work);
else if (list_empty(&sp->srcu_boot_entry)) else if (list_empty(&sp->srcu_work.entry))
list_add(&sp->srcu_boot_entry, &srcu_boot_list); list_add(&sp->srcu_work.entry, &srcu_boot_list);
} }
} }
EXPORT_SYMBOL_GPL(call_srcu); EXPORT_SYMBOL_GPL(call_srcu);
@ -224,8 +224,8 @@ void __init srcu_init(void)
srcu_init_done = true; srcu_init_done = true;
while (!list_empty(&srcu_boot_list)) { while (!list_empty(&srcu_boot_list)) {
sp = list_first_entry(&srcu_boot_list, sp = list_first_entry(&srcu_boot_list,
struct srcu_struct, srcu_boot_entry); struct srcu_struct, srcu_work.entry);
list_del_init(&sp->srcu_boot_entry); list_del_init(&sp->srcu_work.entry);
schedule_work(&sp->srcu_work); schedule_work(&sp->srcu_work);
} }
} }

11
kernel/rcu/srcutree.c
View File

@ -186,7 +186,6 @@ static int init_srcu_struct_fields(struct srcu_struct *sp, bool is_static)
mutex_init(&sp->srcu_barrier_mutex); mutex_init(&sp->srcu_barrier_mutex);
atomic_set(&sp->srcu_barrier_cpu_cnt, 0); atomic_set(&sp->srcu_barrier_cpu_cnt, 0);
INIT_DELAYED_WORK(&sp->work, process_srcu); INIT_DELAYED_WORK(&sp->work, process_srcu);
INIT_LIST_HEAD(&sp->srcu_boot_entry);
if (!is_static) if (!is_static)
sp->sda = alloc_percpu(struct srcu_data); sp->sda = alloc_percpu(struct srcu_data);
init_srcu_struct_nodes(sp, is_static); init_srcu_struct_nodes(sp, is_static);
@ -708,8 +707,8 @@ static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp,
if (likely(srcu_init_done)) if (likely(srcu_init_done))
queue_delayed_work(rcu_gp_wq, &sp->work, queue_delayed_work(rcu_gp_wq, &sp->work,
srcu_get_delay(sp)); srcu_get_delay(sp));
else if (list_empty(&sp->srcu_boot_entry)) else if (list_empty(&sp->work.work.entry))
list_add(&sp->srcu_boot_entry, &srcu_boot_list); list_add(&sp->work.work.entry, &srcu_boot_list);
} }
spin_unlock_irqrestore_rcu_node(sp, flags); spin_unlock_irqrestore_rcu_node(sp, flags);
} }
@ -1323,10 +1322,10 @@ void __init srcu_init(void)
srcu_init_done = true; srcu_init_done = true;
while (!list_empty(&srcu_boot_list)) { while (!list_empty(&srcu_boot_list)) {
sp = list_first_entry(&srcu_boot_list, sp = list_first_entry(&srcu_boot_list, struct srcu_struct,
struct srcu_struct, srcu_boot_entry); work.work.entry);
check_init_srcu_struct(sp); check_init_srcu_struct(sp);
list_del_init(&sp->srcu_boot_entry); list_del_init(&sp->work.work.entry);
queue_work(rcu_gp_wq, &sp->work.work); queue_work(rcu_gp_wq, &sp->work.work);
} }
} }