rcu: Simplify and inline cpu_needs_another_gp()
Now that RCU no longer relies on failsafe checks, cpu_needs_another_gp() can be greatly simplified. This simplification eliminates the last call to rcu_future_needs_gp() and to rcu_segcblist_future_gp_needed(), both of which which can then be eliminated. And then, because cpu_needs_another_gp() is called only from __rcu_pending(), it can be inlined and eliminated. This commit carries out the simplification, inlining, and elimination called out above. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Tested-by: Nicholas Piggin <npiggin@gmail.com>
This commit is contained in:
parent
384f77f4cb
commit
c1935209df
|
@ -403,24 +403,6 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq)
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Scan the specified rcu_segcblist structure for callbacks that need
|
|
||||||
* a grace period later than the one specified by "seq". We don't look
|
|
||||||
* at the RCU_DONE_TAIL or RCU_NEXT_TAIL segments because they don't
|
|
||||||
* have a grace-period sequence number.
|
|
||||||
*/
|
|
||||||
bool rcu_segcblist_future_gp_needed(struct rcu_segcblist *rsclp,
|
|
||||||
unsigned long seq)
|
|
||||||
{
|
|
||||||
int i;
|
|
||||||
|
|
||||||
for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++)
|
|
||||||
if (rsclp->tails[i - 1] != rsclp->tails[i] &&
|
|
||||||
ULONG_CMP_LT(seq, rsclp->gp_seq[i]))
|
|
||||||
return true;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Merge the source rcu_segcblist structure into the destination
|
* Merge the source rcu_segcblist structure into the destination
|
||||||
* rcu_segcblist structure, then initialize the source. Any pending
|
* rcu_segcblist structure, then initialize the source. Any pending
|
||||||
|
|
|
@ -134,7 +134,5 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
|
||||||
struct rcu_cblist *rclp);
|
struct rcu_cblist *rclp);
|
||||||
void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
|
void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
|
||||||
bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
|
bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
|
||||||
bool rcu_segcblist_future_gp_needed(struct rcu_segcblist *rsclp,
|
|
||||||
unsigned long seq);
|
|
||||||
void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
|
void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
|
||||||
struct rcu_segcblist *src_rsclp);
|
struct rcu_segcblist *src_rsclp);
|
||||||
|
|
|
@ -708,42 +708,6 @@ static struct rcu_node *rcu_get_root(struct rcu_state *rsp)
|
||||||
return &rsp->node[0];
|
return &rsp->node[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Is there any need for future grace periods?
|
|
||||||
* Interrupts must be disabled. If the caller does not hold the root
|
|
||||||
* rnp_node structure's ->lock, the results are advisory only.
|
|
||||||
*/
|
|
||||||
static int rcu_future_needs_gp(struct rcu_state *rsp)
|
|
||||||
{
|
|
||||||
struct rcu_node *rnp = rcu_get_root(rsp);
|
|
||||||
|
|
||||||
lockdep_assert_irqs_disabled();
|
|
||||||
return need_any_future_gp(rnp);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Does the current CPU require a not-yet-started grace period?
|
|
||||||
* The caller must have disabled interrupts to prevent races with
|
|
||||||
* normal callback registry.
|
|
||||||
*/
|
|
||||||
static bool
|
|
||||||
cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp)
|
|
||||||
{
|
|
||||||
lockdep_assert_irqs_disabled();
|
|
||||||
if (rcu_gp_in_progress(rsp))
|
|
||||||
return false; /* No, a grace period is already in progress. */
|
|
||||||
if (rcu_future_needs_gp(rsp))
|
|
||||||
return true; /* Yes, a no-CBs CPU needs one. */
|
|
||||||
if (!rcu_segcblist_is_enabled(&rdp->cblist))
|
|
||||||
return false; /* No, this is a no-CBs (or offline) CPU. */
|
|
||||||
if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
|
|
||||||
return true; /* Yes, CPU has newly registered callbacks. */
|
|
||||||
if (rcu_segcblist_future_gp_needed(&rdp->cblist,
|
|
||||||
READ_ONCE(rsp->completed)))
|
|
||||||
return true; /* Yes, CBs for future grace period. */
|
|
||||||
return false; /* No grace period needed. */
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Enter an RCU extended quiescent state, which can be either the
|
* Enter an RCU extended quiescent state, which can be either the
|
||||||
* idle loop or adaptive-tickless usermode execution.
|
* idle loop or adaptive-tickless usermode execution.
|
||||||
|
@ -3298,7 +3262,9 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
|
||||||
return 1;
|
return 1;
|
||||||
|
|
||||||
/* Has RCU gone idle with this CPU needing another grace period? */
|
/* Has RCU gone idle with this CPU needing another grace period? */
|
||||||
if (cpu_needs_another_gp(rsp, rdp))
|
if (!rcu_gp_in_progress(rsp) &&
|
||||||
|
rcu_segcblist_is_enabled(&rdp->cblist) &&
|
||||||
|
!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
|
||||||
return 1;
|
return 1;
|
||||||
|
|
||||||
/* Has another RCU grace period completed? */
|
/* Has another RCU grace period completed? */
|
||||||
|
|
Loading…
Reference in New Issue