rcu: Break up rcu_gp_kthread() into subfunctions

Then rcu_gp_kthread() function is too large and furthermore needs to
have the force_quiescent_state() code pulled in.  This commit therefore
breaks up rcu_gp_kthread() into rcu_gp_init() and rcu_gp_cleanup().

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

kernel/rcutree.c

@@ -1028,37 +1028,20 @@ rcu_start_gp_per_cpu(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_dat
 }
 
 /*
- * Body of kthread that handles grace periods.
+ * Initialize a new grace period.
  */
-static int __noreturn rcu_gp_kthread(void *arg)
+static int rcu_gp_init(struct rcu_state *rsp)
 {
-	unsigned long gp_duration;
 	struct rcu_data *rdp;
-	struct rcu_node *rnp;
+	struct rcu_node *rnp = rcu_get_root(rsp);
-	struct rcu_state *rsp = arg;
 
-	for (;;) {
-
-		/* Handle grace-period start. */
-		rnp = rcu_get_root(rsp);
-		for (;;) {
-			wait_event_interruptible(rsp->gp_wq, rsp->gp_flags);
-			if (rsp->gp_flags)
-				break;
-			flush_signals(current);
-		}
 	raw_spin_lock_irq(&rnp->lock);
 	rsp->gp_flags = 0;
-		rdp = this_cpu_ptr(rsp->rda);
 
 	if (rcu_gp_in_progress(rsp)) {
-			/*
+		/* Grace period already in progress, don't start another.  */
-			 * A grace period is already in progress, so
-			 * don't start another one.
-			 */
 		raw_spin_unlock_irq(&rnp->lock);
-			cond_resched();
+		return 0;
-			continue;
 	}
 
 	if (rsp->fqs_active) {
@@ -1068,8 +1051,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
 		 */
 		rsp->fqs_need_gp = 1;
 		raw_spin_unlock_irq(&rnp->lock);
-			cond_resched();
+		return 0;
-			continue;
 	}
 
 	/* Advance to a new grace period and initialize state. */
@@ -1086,17 +1068,20 @@ static int __noreturn rcu_gp_kthread(void *arg)
 
 	/*
 	 * Set the quiescent-state-needed bits in all the rcu_node
-		 * structures for all currently online CPUs in breadth-first
+	 * structures for all currently online CPUs in breadth-first order,
-		 * order, starting from the root rcu_node structure.
+	 * starting from the root rcu_node structure, relying on the layout
-		 * This operation relies on the layout of the hierarchy
+	 * of the tree within the rsp->node[] array.  Note that other CPUs
-		 * within the rsp->node[] array.  Note that other CPUs will
+	 * will access only the leaves of the hierarchy, thus seeing that no
-		 * access only the leaves of the hierarchy, which still
+	 * grace period is in progress, at least until the corresponding
-		 * indicate that no grace period is in progress, at least
+	 * leaf node has been initialized.  In addition, we have excluded
-		 * until the corresponding leaf node has been initialized.
+	 * CPU-hotplug operations.
-		 * In addition, we have excluded CPU-hotplug operations.
+	 *
+	 * The grace period cannot complete until the initialization
+	 * process finishes, because this kthread handles both.
 	 */
 	rcu_for_each_node_breadth_first(rsp, rnp) {
 		raw_spin_lock_irq(&rnp->lock);
+		rdp = this_cpu_ptr(rsp->rda);
 		rcu_preempt_check_blocked_tasks(rnp);
 		rnp->qsmask = rnp->qsmaskinit;
 		rnp->gpnum = rsp->gpnum;
@@ -1117,18 +1102,17 @@ static int __noreturn rcu_gp_kthread(void *arg)
 	rsp->fqs_state = RCU_SIGNAL_INIT;
 	raw_spin_unlock_irq(&rnp->lock);
 	put_online_cpus();
+	return 1;
+}
 
-		/* Handle grace-period end. */
+/*
-		rnp = rcu_get_root(rsp);
+ * Clean up after the old grace period.
-		for (;;) {
+ */
-			wait_event_interruptible(rsp->gp_wq,
+static int rcu_gp_cleanup(struct rcu_state *rsp)
-						 !ACCESS_ONCE(rnp->qsmask) &&
+{
-						 !rcu_preempt_blocked_readers_cgp(rnp));
+	unsigned long gp_duration;
-			if (!ACCESS_ONCE(rnp->qsmask) &&
+	struct rcu_data *rdp;
-			    !rcu_preempt_blocked_readers_cgp(rnp))
+	struct rcu_node *rnp = rcu_get_root(rsp);
-				break;
-			flush_signals(current);
-		}
 
 	raw_spin_lock_irq(&rnp->lock);
 	gp_duration = jiffies - rsp->gp_start;
@@ -1150,6 +1134,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
 	 * callbacks are waiting on the grace period that just now
 	 * completed.
 	 */
+	rdp = this_cpu_ptr(rsp->rda);
 	if (*rdp->nxttail[RCU_WAIT_TAIL] == NULL) {
 		raw_spin_unlock_irq(&rnp->lock);
 
@@ -1175,6 +1160,41 @@ static int __noreturn rcu_gp_kthread(void *arg)
 	if (cpu_needs_another_gp(rsp, rdp))
 		rsp->gp_flags = 1;
 	raw_spin_unlock_irq(&rnp->lock);
+	return 1;
+}
+
+/*
+ * Body of kthread that handles grace periods.
+ */
+static int __noreturn rcu_gp_kthread(void *arg)
+{
+	struct rcu_state *rsp = arg;
+	struct rcu_node *rnp = rcu_get_root(rsp);
+
+	for (;;) {
+
+		/* Handle grace-period start. */
+		for (;;) {
+			wait_event_interruptible(rsp->gp_wq, rsp->gp_flags);
+			if (rsp->gp_flags && rcu_gp_init(rsp))
+				break;
+			cond_resched();
+			flush_signals(current);
+		}
+
+		/* Handle grace-period end. */
+		rnp = rcu_get_root(rsp);
+		for (;;) {
+			wait_event_interruptible(rsp->gp_wq,
+						 !ACCESS_ONCE(rnp->qsmask) &&
+						 !rcu_preempt_blocked_readers_cgp(rnp));
+			if (!ACCESS_ONCE(rnp->qsmask) &&
+			    !rcu_preempt_blocked_readers_cgp(rnp) &&
+			    rcu_gp_cleanup(rsp))
+				break;
+			cond_resched();
+			flush_signals(current);
+		}
 	}
 }