static int rcu_implicit_offline_qs(struct rcu_data *rdp) { if (cpu_is_offline(rdp->cpu) && ULONG_CMP_LT(rdp->rsp->gp_start + 2, jiffies)) { trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "ofl"); rdp->offline_fqs++; return 1; } return 0; }
/* * Scan the specified rcu_segcblist structure for callbacks that need * a grace period later than the one specified by "seq". We don't look * at the RCU_DONE_TAIL or RCU_NEXT_TAIL segments because they don't * have a grace-period sequence number. */ bool rcu_segcblist_future_gp_needed(struct rcu_segcblist *rsclp, unsigned long seq) { int i; for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) if (rsclp->tails[i - 1] != rsclp->tails[i] && ULONG_CMP_LT(seq, rsclp->gp_seq[i])) return true; return false; }
/* * "Accelerate" callbacks based on more-accurate grace-period information. * The reason for this is that RCU does not synchronize the beginnings and * ends of grace periods, and that callbacks are posted locally. This in * turn means that the callbacks must be labelled conservatively early * on, as getting exact information would degrade both performance and * scalability. When more accurate grace-period information becomes * available, previously posted callbacks can be "accelerated", marking * them to complete at the end of the earlier grace period. * * This function operates on an rcu_segcblist structure, and also the * grace-period sequence number seq at which new callbacks would become * ready to invoke. Returns true if there are callbacks that won't be * ready to invoke until seq, false otherwise. */ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) { int i; WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) return false; /* * Find the segment preceding the oldest segment of callbacks * whose ->gp_seq[] completion is at or after that passed in via * "seq", skipping any empty segments. This oldest segment, along * with any later segments, can be merged in with any newly arrived * callbacks in the RCU_NEXT_TAIL segment, and assigned "seq" * as their ->gp_seq[] grace-period completion sequence number. */ for (i = RCU_NEXT_READY_TAIL; i > RCU_DONE_TAIL; i--) if (rsclp->tails[i] != rsclp->tails[i - 1] && ULONG_CMP_LT(rsclp->gp_seq[i], seq)) break; /* * If all the segments contain callbacks that correspond to * earlier grace-period sequence numbers than "seq", leave. * Assuming that the rcu_segcblist structure has enough * segments in its arrays, this can only happen if some of * the non-done segments contain callbacks that really are * ready to invoke. This situation will get straightened * out by the next call to rcu_segcblist_advance(). * * Also advance to the oldest segment of callbacks whose * ->gp_seq[] completion is at or after that passed in via "seq", * skipping any empty segments. */ if (++i >= RCU_NEXT_TAIL) return false; /* * Merge all later callbacks, including newly arrived callbacks, * into the segment located by the for-loop above. Assign "seq" * as the ->gp_seq[] value in order to correctly handle the case * where there were no pending callbacks in the rcu_segcblist * structure other than in the RCU_NEXT_TAIL segment. */ for (; i < RCU_NEXT_TAIL; i++) { rsclp->tails[i] = rsclp->tails[RCU_NEXT_TAIL]; rsclp->gp_seq[i] = seq; } return true; }
/* * Advance the callbacks in the specified rcu_segcblist structure based * on the current value passed in for the grace-period counter. */ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) { int i, j; WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) return; /* * Find all callbacks whose ->gp_seq numbers indicate that they * are ready to invoke, and put them into the RCU_DONE_TAIL segment. */ for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { if (ULONG_CMP_LT(seq, rsclp->gp_seq[i])) break; rsclp->tails[RCU_DONE_TAIL] = rsclp->tails[i]; } /* If no callbacks moved, nothing more need be done. */ if (i == RCU_WAIT_TAIL) return; /* Clean up tail pointers that might have been misordered above. */ for (j = RCU_WAIT_TAIL; j < i; j++) rsclp->tails[j] = rsclp->tails[RCU_DONE_TAIL]; /* * Callbacks moved, so clean up the misordered ->tails[] pointers * that now point into the middle of the list of ready-to-invoke * callbacks. The overall effect is to copy down the later pointers * into the gap that was created by the now-ready segments. */ for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) { if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL]) break; /* No more callbacks. */ rsclp->tails[j] = rsclp->tails[i]; rsclp->gp_seq[j] = rsclp->gp_seq[i]; } }