Lines Matching +full:wait +full:- +full:state
1 /* SPDX-License-Identifier: GPL-2.0+ */
24 * Return the value that the expedited-grace-period counter will have
42 * Take a snapshot of the expedited-grace-period counter, which is the
67 * Reset the ->expmaskinit values in the rcu_node tree to reflect any
68 * recent CPU-online activity. Note that these masks are not cleared
71 * no-work-to-do fastpath.
94 if (rnp->expmaskinit == rnp->expmaskinitnext) { in sync_exp_reset_tree_hotplug()
100 oldmask = rnp->expmaskinit; in sync_exp_reset_tree_hotplug()
101 rnp->expmaskinit = rnp->expmaskinitnext; in sync_exp_reset_tree_hotplug()
109 mask = rnp->grpmask; in sync_exp_reset_tree_hotplug()
110 rnp_up = rnp->parent; in sync_exp_reset_tree_hotplug()
114 if (rnp_up->expmaskinit) in sync_exp_reset_tree_hotplug()
116 rnp_up->expmaskinit |= mask; in sync_exp_reset_tree_hotplug()
120 mask = rnp_up->grpmask; in sync_exp_reset_tree_hotplug()
121 rnp_up = rnp_up->parent; in sync_exp_reset_tree_hotplug()
127 * Reset the ->expmask values in the rcu_node tree in preparation for
138 WARN_ON_ONCE(rnp->expmask); in sync_exp_reset_tree()
139 WRITE_ONCE(rnp->expmask, rnp->expmaskinit); in sync_exp_reset_tree()
145 * Return non-zero if there is no RCU expedited grace period in progress
153 return READ_ONCE(rnp->exp_tasks) == NULL && in sync_rcu_exp_done()
154 READ_ONCE(rnp->expmask) == 0; in sync_rcu_exp_done()
159 * rcu_node's ->lock.
175 * Report the exit from RCU read-side critical section for the last task
176 * that queued itself during or before the current expedited preemptible-RCU
184 __releases(rnp->lock) in __rcu_report_exp_rnp()
191 if (!rnp->expmask) in __rcu_report_exp_rnp()
197 if (rnp->parent == NULL) { in __rcu_report_exp_rnp()
205 mask = rnp->grpmask; in __rcu_report_exp_rnp()
207 rnp = rnp->parent; in __rcu_report_exp_rnp()
209 WARN_ON_ONCE(!(rnp->expmask & mask)); in __rcu_report_exp_rnp()
210 WRITE_ONCE(rnp->expmask, rnp->expmask & ~mask); in __rcu_report_exp_rnp()
215 * Report expedited quiescent state for specified node. This is a
216 * lock-acquisition wrapper function for __rcu_report_exp_rnp().
227 * Report expedited quiescent state for multiple CPUs, all covered by the
238 if (!(rnp->expmask & mask)) { in rcu_report_exp_cpu_mult()
242 WRITE_ONCE(rnp->expmask, rnp->expmask & ~mask); in rcu_report_exp_cpu_mult()
245 if (!IS_ENABLED(CONFIG_NO_HZ_FULL) || !rdp->rcu_forced_tick_exp) in rcu_report_exp_cpu_mult()
247 rdp->rcu_forced_tick_exp = false; in rcu_report_exp_cpu_mult()
250 __rcu_report_exp_rnp(rnp, wake, flags); /* Releases rnp->lock. */ in rcu_report_exp_cpu_mult()
254 * Report expedited quiescent state for specified rcu_data (CPU).
258 WRITE_ONCE(rdp->exp_deferred_qs, false); in rcu_report_exp_rdp()
259 rcu_report_exp_cpu_mult(rdp->mynode, rdp->grpmask, true); in rcu_report_exp_rdp()
262 /* Common code for work-done checking. */
274 * Funnel-lock acquisition for expedited grace periods. Returns true
276 * can piggy-back on, and with no mutex held. Otherwise, returns false
283 struct rcu_node *rnp = rdp->mynode; in exp_funnel_lock()
286 /* Low-contention fastpath. */ in exp_funnel_lock()
287 if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s) && in exp_funnel_lock()
289 ULONG_CMP_LT(READ_ONCE(rnp_root->exp_seq_rq), s)) && in exp_funnel_lock()
296 * otherwise falls through to acquire ->exp_mutex. The mapping in exp_funnel_lock()
300 for (; rnp != NULL; rnp = rnp->parent) { in exp_funnel_lock()
304 /* Work not done, either wait here or go up. */ in exp_funnel_lock()
305 spin_lock(&rnp->exp_lock); in exp_funnel_lock()
306 if (ULONG_CMP_GE(rnp->exp_seq_rq, s)) { in exp_funnel_lock()
308 /* Someone else doing GP, so wait for them. */ in exp_funnel_lock()
309 spin_unlock(&rnp->exp_lock); in exp_funnel_lock()
310 trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level, in exp_funnel_lock()
311 rnp->grplo, rnp->grphi, in exp_funnel_lock()
312 TPS("wait")); in exp_funnel_lock()
313 wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3], in exp_funnel_lock()
317 WRITE_ONCE(rnp->exp_seq_rq, s); /* Followers can wait on us. */ in exp_funnel_lock()
318 spin_unlock(&rnp->exp_lock); in exp_funnel_lock()
319 trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level, in exp_funnel_lock()
320 rnp->grplo, rnp->grphi, TPS("nxtlvl")); in exp_funnel_lock()
335 * expedited grace period needs to wait for.
352 for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) { in sync_rcu_exp_select_node_cpus()
354 unsigned long mask = rdp->grpmask; in sync_rcu_exp_select_node_cpus()
358 !(rnp->qsmaskinitnext & mask)) { in sync_rcu_exp_select_node_cpus()
365 rdp->exp_dynticks_snap = snap; in sync_rcu_exp_select_node_cpus()
368 mask_ofl_ipi = rnp->expmask & ~mask_ofl_test; in sync_rcu_exp_select_node_cpus()
371 * Need to wait for any blocked tasks as well. Note that in sync_rcu_exp_select_node_cpus()
373 * until such time as the ->expmask bits are cleared. in sync_rcu_exp_select_node_cpus()
376 WRITE_ONCE(rnp->exp_tasks, rnp->blkd_tasks.next); in sync_rcu_exp_select_node_cpus()
379 /* IPI the remaining CPUs for expedited quiescent state. */ in sync_rcu_exp_select_node_cpus()
382 unsigned long mask = rdp->grpmask; in sync_rcu_exp_select_node_cpus()
385 if (rcu_dynticks_in_eqs_since(rdp, rdp->exp_dynticks_snap)) { in sync_rcu_exp_select_node_cpus()
401 if ((rnp->qsmaskinitnext & mask) && in sync_rcu_exp_select_node_cpus()
402 (rnp->expmask & mask)) { in sync_rcu_exp_select_node_cpus()
410 if (rnp->expmask & mask) in sync_rcu_exp_select_node_cpus()
421 * to wait for.
434 rnp->exp_need_flush = false; in sync_rcu_exp_select_cpus()
435 if (!READ_ONCE(rnp->expmask)) in sync_rcu_exp_select_cpus()
436 continue; /* Avoid early boot non-existent wq. */ in sync_rcu_exp_select_cpus()
441 sync_rcu_exp_select_node_cpus(&rnp->rew.rew_work); in sync_rcu_exp_select_cpus()
444 INIT_WORK(&rnp->rew.rew_work, sync_rcu_exp_select_node_cpus); in sync_rcu_exp_select_cpus()
445 cpu = find_next_bit(&rnp->ffmask, BITS_PER_LONG, -1); in sync_rcu_exp_select_cpus()
447 if (unlikely(cpu > rnp->grphi - rnp->grplo)) in sync_rcu_exp_select_cpus()
450 cpu += rnp->grplo; in sync_rcu_exp_select_cpus()
451 queue_work_on(cpu, rcu_par_gp_wq, &rnp->rew.rew_work); in sync_rcu_exp_select_cpus()
452 rnp->exp_need_flush = true; in sync_rcu_exp_select_cpus()
455 /* Wait for workqueue jobs (if any) to complete. */ in sync_rcu_exp_select_cpus()
457 if (rnp->exp_need_flush) in sync_rcu_exp_select_cpus()
458 flush_work(&rnp->rew.rew_work); in sync_rcu_exp_select_cpus()
462 * Wait for the expedited grace period to elapse, within time limit.
482 * Wait for the expedited grace period to elapse, issuing any needed
504 for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) { in synchronize_rcu_expedited_wait()
506 if (rdp->rcu_forced_tick_exp) in synchronize_rcu_expedited_wait()
508 rdp->rcu_forced_tick_exp = true; in synchronize_rcu_expedited_wait()
533 if (!(READ_ONCE(rnp->expmask) & mask)) in synchronize_rcu_expedited_wait()
537 pr_cont(" %d-%c%c%c", cpu, in synchronize_rcu_expedited_wait()
539 "o."[!!(rdp->grpmask & rnp->expmaskinit)], in synchronize_rcu_expedited_wait()
540 "N."[!!(rdp->grpmask & rnp->expmaskinitnext)]); in synchronize_rcu_expedited_wait()
544 jiffies - jiffies_start, rcu_state.expedited_sequence, in synchronize_rcu_expedited_wait()
545 data_race(rnp_root->expmask), in synchronize_rcu_expedited_wait()
546 ".T"[!!data_race(rnp_root->exp_tasks)]); in synchronize_rcu_expedited_wait()
554 pr_cont(" l=%u:%d-%d:%#lx/%c", in synchronize_rcu_expedited_wait()
555 rnp->level, rnp->grplo, rnp->grphi, in synchronize_rcu_expedited_wait()
556 data_race(rnp->expmask), in synchronize_rcu_expedited_wait()
557 ".T"[!!data_race(rnp->exp_tasks)]); in synchronize_rcu_expedited_wait()
564 if (!(READ_ONCE(rnp->expmask) & mask)) in synchronize_rcu_expedited_wait()
574 * Wait for the current expedited grace period to complete, and then
575 * wake up everyone who piggybacked on the just-completed expedited
576 * grace period. Also update all the ->exp_seq_rq counters as needed
577 * in order to avoid counter-wrap problems.
593 if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) { in rcu_exp_wait_wake()
594 spin_lock(&rnp->exp_lock); in rcu_exp_wait_wake()
596 if (ULONG_CMP_LT(rnp->exp_seq_rq, s)) in rcu_exp_wait_wake()
597 WRITE_ONCE(rnp->exp_seq_rq, s); in rcu_exp_wait_wake()
598 spin_unlock(&rnp->exp_lock); in rcu_exp_wait_wake()
601 wake_up_all(&rnp->exp_wq[rcu_seq_ctr(s) & 0x3]); in rcu_exp_wait_wake()
609 * workqueues and mid-boot-time tasks.
613 /* Initialize the rcu_node tree in preparation for the wait. */ in rcu_exp_sel_wait_wake()
616 /* Wait and clean up, including waking everyone. */ in rcu_exp_sel_wait_wake()
621 * Work-queue handler to drive an expedited grace period forward.
628 rcu_exp_sel_wait_wake(rewp->rew_s); in wait_rcu_exp_gp()
635 * RCU read-side critical section in effect, request that the
636 * next rcu_read_unlock() record the quiescent state up the
637 * ->expmask fields in the rcu_node tree. Otherwise, immediately
638 * report the quiescent state.
645 struct rcu_node *rnp = rdp->mynode; in rcu_exp_handler()
649 * First, the common case of not being in an RCU read-side in rcu_exp_handler()
651 * report the quiescent state, otherwise defer. in rcu_exp_handler()
658 rdp->exp_deferred_qs = true; in rcu_exp_handler()
666 * Second, the less-common case of being in an RCU read-side in rcu_exp_handler()
671 * grace period is still waiting on this CPU, set ->deferred_qs in rcu_exp_handler()
672 * so that the eventual quiescent state will be reported. in rcu_exp_handler()
674 * can have caused this quiescent state to already have been in rcu_exp_handler()
675 * reported, so we really do need to check ->expmask. in rcu_exp_handler()
679 if (rnp->expmask & rdp->grpmask) { in rcu_exp_handler()
680 rdp->exp_deferred_qs = true; in rcu_exp_handler()
681 t->rcu_read_unlock_special.b.exp_hint = true; in rcu_exp_handler()
697 * Scan the current list of tasks blocked within RCU read-side critical
707 if (!READ_ONCE(rnp->exp_tasks)) in rcu_print_task_exp_stall()
710 t = list_entry(rnp->exp_tasks->prev, in rcu_print_task_exp_stall()
712 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { in rcu_print_task_exp_stall()
713 pr_cont(" P%d", t->pid); in rcu_print_task_exp_stall()
722 /* Request an expedited quiescent state. */
732 /* Invoked on each online non-idle CPU for expedited quiescent state. */
736 struct rcu_node *rnp = rdp->mynode; in rcu_exp_handler()
738 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in rcu_exp_handler()
748 /* Send IPI for expedited cleanup if needed at end of CPU-hotplug operation. */
758 rnp = rdp->mynode; in sync_sched_exp_online_cleanup()
760 /* Quiescent state either not needed or already requested, leave. */ in sync_sched_exp_online_cleanup()
761 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in sync_sched_exp_online_cleanup()
766 /* Quiescent state needed on current CPU, so set it up locally. */ in sync_sched_exp_online_cleanup()
774 /* Quiescent state needed on some other CPU, send IPI. */ in sync_sched_exp_online_cleanup()
782 * tasks blocked within RCU read-side critical sections that are
793 * synchronize_rcu_expedited - Brute-force RCU grace period
795 * Wait for an RCU grace period, but expedite it. The basic idea is to
796 * IPI all non-idle non-nohz online CPUs. The IPI handler checks whether
798 * causes the outermost rcu_read_unlock() to report the quiescent state
799 * for RCU-preempt or asks the scheduler for help for RCU-sched. On the
800 * other hand, if the CPU is not in an RCU read-side critical section,
801 * the IPI handler reports the quiescent state immediately.
804 * implementations, it is still unfriendly to real-time workloads, so is
805 * thus not recommended for any sort of common-case code. In fact, if
822 "Illegal synchronize_rcu_expedited() in RCU read-side critical section"); in synchronize_rcu_expedited()
824 /* Is the state is such that the call is a grace period? */ in synchronize_rcu_expedited()
850 /* Wait for expedited grace period to complete. */ in synchronize_rcu_expedited()
852 wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3], in synchronize_rcu_expedited()