Lines Matching refs:rdp
86 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_get_gpwrap_count() local
88 return READ_ONCE(rdp->gpwrap_count); in rcu_get_gpwrap_count()
162 static void rcu_report_exp_rdp(struct rcu_data *rdp);
163 static void rcu_report_qs_rdp(struct rcu_data *rdp);
164 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
165 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp);
166 static bool rcu_rdp_cpu_online(struct rcu_data *rdp);
231 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_get_n_cbs_cpu() local
233 if (rcu_segcblist_is_enabled(&rdp->cblist)) in rcu_get_n_cbs_cpu()
234 return rcu_segcblist_n_cbs(&rdp->cblist); in rcu_get_n_cbs_cpu()
312 static bool rcu_watching_snap_stopped_since(struct rcu_data *rdp, int snap) in rcu_watching_snap_stopped_since() argument
325 return snap != ct_rcu_watching_cpu_acquire(rdp->cpu); in rcu_watching_snap_stopped_since()
515 static void force_qs_rnp(int (*f)(struct rcu_data *rdp));
601 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_irq_work_resched() local
610 if (do_nocb_deferred_wakeup(rdp) && need_resched()) { in rcu_irq_work_resched()
664 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in __rcu_irq_enter_check_tick() local
673 if (!tick_nohz_full_cpu(rdp->cpu) || in __rcu_irq_enter_check_tick()
674 !READ_ONCE(rdp->rcu_urgent_qs) || in __rcu_irq_enter_check_tick()
675 READ_ONCE(rdp->rcu_forced_tick)) { in __rcu_irq_enter_check_tick()
687 raw_spin_lock_rcu_node(rdp->mynode); in __rcu_irq_enter_check_tick()
688 if (READ_ONCE(rdp->rcu_urgent_qs) && !rdp->rcu_forced_tick) { in __rcu_irq_enter_check_tick()
691 WRITE_ONCE(rdp->rcu_forced_tick, true); in __rcu_irq_enter_check_tick()
692 tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU); in __rcu_irq_enter_check_tick()
694 raw_spin_unlock_rcu_node(rdp->mynode); in __rcu_irq_enter_check_tick()
721 static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp) in rcu_disable_urgency_upon_qs() argument
723 raw_lockdep_assert_held_rcu_node(rdp->mynode); in rcu_disable_urgency_upon_qs()
724 WRITE_ONCE(rdp->rcu_urgent_qs, false); in rcu_disable_urgency_upon_qs()
725 WRITE_ONCE(rdp->rcu_need_heavy_qs, false); in rcu_disable_urgency_upon_qs()
726 if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) { in rcu_disable_urgency_upon_qs()
727 tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU); in rcu_disable_urgency_upon_qs()
728 WRITE_ONCE(rdp->rcu_forced_tick, false); in rcu_disable_urgency_upon_qs()
802 static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_gpnum_ovf() argument
805 if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + seq_gpwrap_lag, in rcu_gpnum_ovf()
807 WRITE_ONCE(rdp->gpwrap, true); in rcu_gpnum_ovf()
808 WRITE_ONCE(rdp->gpwrap_count, READ_ONCE(rdp->gpwrap_count) + 1); in rcu_gpnum_ovf()
810 if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq)) in rcu_gpnum_ovf()
811 rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4; in rcu_gpnum_ovf()
819 static int rcu_watching_snap_save(struct rcu_data *rdp) in rcu_watching_snap_save() argument
832 rdp->watching_snap = ct_rcu_watching_cpu_acquire(rdp->cpu); in rcu_watching_snap_save()
833 if (rcu_watching_snap_in_eqs(rdp->watching_snap)) { in rcu_watching_snap_save()
834 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_watching_snap_save()
835 rcu_gpnum_ovf(rdp->mynode, rdp); in rcu_watching_snap_save()
855 static int rcu_watching_snap_recheck(struct rcu_data *rdp) in rcu_watching_snap_recheck() argument
859 struct rcu_node *rnp = rdp->mynode; in rcu_watching_snap_recheck()
869 if (rcu_watching_snap_stopped_since(rdp, rdp->watching_snap)) { in rcu_watching_snap_recheck()
870 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_watching_snap_recheck()
871 rcu_gpnum_ovf(rnp, rdp); in rcu_watching_snap_recheck()
893 if (WARN_ON_ONCE(!rcu_rdp_cpu_online(rdp))) { in rcu_watching_snap_recheck()
903 __func__, rdp->cpu, ".o"[rcu_rdp_cpu_online(rdp)], in rcu_watching_snap_recheck()
904 (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_state, in rcu_watching_snap_recheck()
905 (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_state); in rcu_watching_snap_recheck()
921 if (!READ_ONCE(rdp->rcu_need_heavy_qs) && in rcu_watching_snap_recheck()
925 WRITE_ONCE(rdp->rcu_need_heavy_qs, true); in rcu_watching_snap_recheck()
927 smp_store_release(&rdp->rcu_urgent_qs, true); in rcu_watching_snap_recheck()
929 WRITE_ONCE(rdp->rcu_urgent_qs, true); in rcu_watching_snap_recheck()
940 if (tick_nohz_full_cpu(rdp->cpu) && in rcu_watching_snap_recheck()
941 (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) || in rcu_watching_snap_recheck()
943 WRITE_ONCE(rdp->rcu_urgent_qs, true); in rcu_watching_snap_recheck()
944 WRITE_ONCE(rdp->last_fqs_resched, jiffies); in rcu_watching_snap_recheck()
957 READ_ONCE(rdp->last_fqs_resched) + jtsq)) { in rcu_watching_snap_recheck()
958 WRITE_ONCE(rdp->last_fqs_resched, jiffies); in rcu_watching_snap_recheck()
962 !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq && in rcu_watching_snap_recheck()
963 (rnp->ffmask & rdp->grpmask)) { in rcu_watching_snap_recheck()
964 rdp->rcu_iw_pending = true; in rcu_watching_snap_recheck()
965 rdp->rcu_iw_gp_seq = rnp->gp_seq; in rcu_watching_snap_recheck()
966 irq_work_queue_on(&rdp->rcu_iw, rdp->cpu); in rcu_watching_snap_recheck()
969 if (rcu_cpu_stall_cputime && rdp->snap_record.gp_seq != rdp->gp_seq) { in rcu_watching_snap_recheck()
970 int cpu = rdp->cpu; in rcu_watching_snap_recheck()
976 rsrp = &rdp->snap_record; in rcu_watching_snap_recheck()
984 rsrp->gp_seq = rdp->gp_seq; in rcu_watching_snap_recheck()
992 static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, in trace_rcu_this_gp() argument
1016 static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, in rcu_start_this_gp() argument
1032 trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, TPS("Startleaf")); in rcu_start_this_gp()
1040 trace_rcu_this_gp(rnp, rdp, gp_seq_req, in rcu_start_this_gp()
1052 trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, in rcu_start_this_gp()
1064 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot")); in rcu_start_this_gp()
1067 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot")); in rcu_start_this_gp()
1071 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread")); in rcu_start_this_gp()
1080 WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed); in rcu_start_this_gp()
1094 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_future_gp_cleanup() local
1099 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq, in rcu_future_gp_cleanup()
1143 static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_accelerate_cbs() argument
1148 rcu_lockdep_assert_cblist_protected(rdp); in rcu_accelerate_cbs()
1152 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_accelerate_cbs()
1155 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPreAcc")); in rcu_accelerate_cbs()
1168 if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) in rcu_accelerate_cbs()
1169 ret = rcu_start_this_gp(rnp, rdp, gp_seq_req); in rcu_accelerate_cbs()
1172 if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) in rcu_accelerate_cbs()
1177 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPostAcc")); in rcu_accelerate_cbs()
1190 struct rcu_data *rdp) in rcu_accelerate_cbs_unlocked() argument
1195 rcu_lockdep_assert_cblist_protected(rdp); in rcu_accelerate_cbs_unlocked()
1197 if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) { in rcu_accelerate_cbs_unlocked()
1199 (void)rcu_segcblist_accelerate(&rdp->cblist, c); in rcu_accelerate_cbs_unlocked()
1203 needwake = rcu_accelerate_cbs(rnp, rdp); in rcu_accelerate_cbs_unlocked()
1219 static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_advance_cbs() argument
1221 rcu_lockdep_assert_cblist_protected(rdp); in rcu_advance_cbs()
1225 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_advance_cbs()
1232 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
1235 return rcu_accelerate_cbs(rnp, rdp); in rcu_advance_cbs()
1243 struct rcu_data *rdp) in rcu_advance_cbs_nowake() argument
1245 rcu_lockdep_assert_cblist_protected(rdp); in rcu_advance_cbs_nowake()
1250 WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp)); in rcu_advance_cbs_nowake()
1273 static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) in __note_gp_changes() argument
1277 const bool offloaded = rcu_rdp_is_offloaded(rdp); in __note_gp_changes()
1281 if (rdp->gp_seq == rnp->gp_seq) in __note_gp_changes()
1285 if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) || in __note_gp_changes()
1286 unlikely(rdp->gpwrap)) { in __note_gp_changes()
1288 ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */ in __note_gp_changes()
1289 rdp->core_needs_qs = false; in __note_gp_changes()
1290 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend")); in __note_gp_changes()
1293 ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */ in __note_gp_changes()
1294 if (rdp->core_needs_qs) in __note_gp_changes()
1295 rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask); in __note_gp_changes()
1299 if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) || in __note_gp_changes()
1300 unlikely(rdp->gpwrap)) { in __note_gp_changes()
1307 need_qs = !!(rnp->qsmask & rdp->grpmask); in __note_gp_changes()
1308 rdp->cpu_no_qs.b.norm = need_qs; in __note_gp_changes()
1309 rdp->core_needs_qs = need_qs; in __note_gp_changes()
1310 zero_cpu_stall_ticks(rdp); in __note_gp_changes()
1312 rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */ in __note_gp_changes()
1313 if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap) in __note_gp_changes()
1314 WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed); in __note_gp_changes()
1315 if (IS_ENABLED(CONFIG_PROVE_RCU) && rdp->gpwrap) in __note_gp_changes()
1316 WRITE_ONCE(rdp->last_sched_clock, jiffies); in __note_gp_changes()
1317 WRITE_ONCE(rdp->gpwrap, false); in __note_gp_changes()
1318 rcu_gpnum_ovf(rnp, rdp); in __note_gp_changes()
1322 static void note_gp_changes(struct rcu_data *rdp) in note_gp_changes() argument
1329 rnp = rdp->mynode; in note_gp_changes()
1330 if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) && in note_gp_changes()
1331 !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */ in note_gp_changes()
1336 needwake = __note_gp_changes(rnp, rdp); in note_gp_changes()
1809 struct rcu_data *rdp; in rcu_gp_init() local
1957 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_init()
1961 if (rnp == rdp->mynode) in rcu_gp_init()
1962 (void)__note_gp_changes(rnp, rdp); in rcu_gp_init()
2157 struct rcu_data *rdp; in rcu_gp_cleanup() local
2198 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_cleanup()
2199 if (rnp == rdp->mynode) in rcu_gp_cleanup()
2200 needgp = __note_gp_changes(rnp, rdp) || needgp; in rcu_gp_cleanup()
2206 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_gp_cleanup()
2207 check_cb_ovld_locked(rdp, rnp); in rcu_gp_cleanup()
2225 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_cleanup()
2227 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed, in rcu_gp_cleanup()
2232 offloaded = rcu_rdp_is_offloaded(rdp); in rcu_gp_cleanup()
2233 if ((offloaded || !rcu_accelerate_cbs(rnp, rdp)) && needgp) { in rcu_gp_cleanup()
2443 rcu_report_qs_rdp(struct rcu_data *rdp) in rcu_report_qs_rdp() argument
2449 WARN_ON_ONCE(rdp->cpu != smp_processor_id()); in rcu_report_qs_rdp()
2450 rnp = rdp->mynode; in rcu_report_qs_rdp()
2452 if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq || in rcu_report_qs_rdp()
2453 rdp->gpwrap) { in rcu_report_qs_rdp()
2461 rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */ in rcu_report_qs_rdp()
2465 mask = rdp->grpmask; in rcu_report_qs_rdp()
2466 rdp->core_needs_qs = false; in rcu_report_qs_rdp()
2476 if (!rcu_rdp_is_offloaded(rdp)) { in rcu_report_qs_rdp()
2482 WARN_ON_ONCE(rcu_accelerate_cbs(rnp, rdp)); in rcu_report_qs_rdp()
2485 rcu_disable_urgency_upon_qs(rdp); in rcu_report_qs_rdp()
2498 rcu_check_quiescent_state(struct rcu_data *rdp) in rcu_check_quiescent_state() argument
2501 note_gp_changes(rdp); in rcu_check_quiescent_state()
2507 if (!rdp->core_needs_qs) in rcu_check_quiescent_state()
2514 if (rdp->cpu_no_qs.b.norm) in rcu_check_quiescent_state()
2521 rcu_report_qs_rdp(rdp); in rcu_check_quiescent_state()
2540 static void rcu_do_batch(struct rcu_data *rdp) in rcu_do_batch() argument
2555 if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { in rcu_do_batch()
2557 rcu_segcblist_n_cbs(&rdp->cblist), 0); in rcu_do_batch()
2559 !rcu_segcblist_empty(&rdp->cblist), in rcu_do_batch()
2561 rcu_is_callbacks_kthread(rdp)); in rcu_do_batch()
2576 rcu_nocb_lock_irqsave(rdp, flags); in rcu_do_batch()
2578 pending = rcu_segcblist_get_seglen(&rdp->cblist, RCU_DONE_TAIL); in rcu_do_batch()
2581 bl = max(rdp->blimit, pending >> div); in rcu_do_batch()
2582 if ((in_serving_softirq() || rdp->rcu_cpu_kthread_status == RCU_KTHREAD_RUNNING) && in rcu_do_batch()
2593 rcu_segcblist_n_cbs(&rdp->cblist), bl); in rcu_do_batch()
2594 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2595 if (rcu_rdp_is_offloaded(rdp)) in rcu_do_batch()
2596 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2598 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbDequeued")); in rcu_do_batch()
2599 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_do_batch()
2643 if (rdp->rcu_cpu_kthread_status == RCU_KTHREAD_RUNNING && in rcu_do_batch()
2645 rdp->rcu_cpu_has_work = 1; in rcu_do_batch()
2651 rcu_nocb_lock_irqsave(rdp, flags); in rcu_do_batch()
2652 rdp->n_cbs_invoked += count; in rcu_do_batch()
2654 is_idle_task(current), rcu_is_callbacks_kthread(rdp)); in rcu_do_batch()
2657 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2658 rcu_segcblist_add_len(&rdp->cblist, -count); in rcu_do_batch()
2661 count = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2662 if (rdp->blimit >= DEFAULT_MAX_RCU_BLIMIT && count <= qlowmark) in rcu_do_batch()
2663 rdp->blimit = blimit; in rcu_do_batch()
2666 if (count == 0 && rdp->qlen_last_fqs_check != 0) { in rcu_do_batch()
2667 rdp->qlen_last_fqs_check = 0; in rcu_do_batch()
2668 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcu_do_batch()
2669 } else if (count < rdp->qlen_last_fqs_check - qhimark) in rcu_do_batch()
2670 rdp->qlen_last_fqs_check = count; in rcu_do_batch()
2676 empty = rcu_segcblist_empty(&rdp->cblist); in rcu_do_batch()
2680 WARN_ON_ONCE(count == 0 && rcu_segcblist_n_segment_cbs(&rdp->cblist) != 0); in rcu_do_batch()
2681 WARN_ON_ONCE(!empty && rcu_segcblist_n_segment_cbs(&rdp->cblist) == 0); in rcu_do_batch()
2683 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_do_batch()
2732 static void force_qs_rnp(int (*f)(struct rcu_data *rdp)) in force_qs_rnp() argument
2762 struct rcu_data *rdp; in force_qs_rnp() local
2765 rdp = per_cpu_ptr(&rcu_data, cpu); in force_qs_rnp()
2766 ret = f(rdp); in force_qs_rnp()
2768 mask |= rdp->grpmask; in force_qs_rnp()
2769 rcu_disable_urgency_upon_qs(rdp); in force_qs_rnp()
2772 rsmask |= rdp->grpmask; in force_qs_rnp()
2837 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); in rcu_core() local
2838 struct rcu_node *rnp = rdp->mynode; in rcu_core()
2843 WARN_ON_ONCE(!rdp->beenonline); in rcu_core()
2854 rcu_check_quiescent_state(rdp); in rcu_core()
2858 rcu_segcblist_is_enabled(&rdp->cblist) && !rcu_rdp_is_offloaded(rdp)) { in rcu_core()
2860 if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_core()
2861 rcu_accelerate_cbs_unlocked(rnp, rdp); in rcu_core()
2864 rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); in rcu_core()
2867 if (!rcu_rdp_is_offloaded(rdp) && rcu_segcblist_ready_cbs(&rdp->cblist) && in rcu_core()
2869 rcu_do_batch(rdp); in rcu_core()
2871 if (rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_core()
2876 do_nocb_deferred_wakeup(rdp); in rcu_core()
2881 queue_work_on(rdp->cpu, rcu_gp_wq, &rdp->strict_work); in rcu_core()
2998 static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func) in rcutree_enqueue() argument
3000 rcu_segcblist_enqueue(&rdp->cblist, head); in rcutree_enqueue()
3002 rcu_segcblist_n_cbs(&rdp->cblist)); in rcutree_enqueue()
3003 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCBQueued")); in rcutree_enqueue()
3009 static void call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, in call_rcu_core() argument
3012 rcutree_enqueue(rdp, head, func); in call_rcu_core()
3031 if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) > in call_rcu_core()
3032 rdp->qlen_last_fqs_check + qhimark)) { in call_rcu_core()
3035 note_gp_changes(rdp); in call_rcu_core()
3039 rcu_accelerate_cbs_unlocked(rdp->mynode, rdp); in call_rcu_core()
3042 rdp->blimit = DEFAULT_MAX_RCU_BLIMIT; in call_rcu_core()
3043 if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap && in call_rcu_core()
3044 rcu_segcblist_first_pend_cb(&rdp->cblist) != head) in call_rcu_core()
3046 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in call_rcu_core()
3047 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in call_rcu_core()
3065 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp) in check_cb_ovld_locked() argument
3070 if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) in check_cb_ovld_locked()
3071 WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask | rdp->grpmask); in check_cb_ovld_locked()
3073 WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask); in check_cb_ovld_locked()
3088 static void check_cb_ovld(struct rcu_data *rdp) in check_cb_ovld() argument
3090 struct rcu_node *const rnp = rdp->mynode; in check_cb_ovld()
3093 ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) == in check_cb_ovld()
3094 !!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask))) in check_cb_ovld()
3097 check_cb_ovld_locked(rdp, rnp); in check_cb_ovld()
3107 struct rcu_data *rdp; in __call_rcu_common() local
3134 rdp = this_cpu_ptr(&rcu_data); in __call_rcu_common()
3135 RCU_LOCKDEP_WARN(!rcu_rdp_cpu_online(rdp), "Callback enqueued on offline CPU!"); in __call_rcu_common()
3140 if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { in __call_rcu_common()
3146 if (rcu_segcblist_empty(&rdp->cblist)) in __call_rcu_common()
3147 rcu_segcblist_init(&rdp->cblist); in __call_rcu_common()
3150 check_cb_ovld(rdp); in __call_rcu_common()
3152 if (unlikely(rcu_rdp_is_offloaded(rdp))) in __call_rcu_common()
3153 call_rcu_nocb(rdp, head, func, flags, lazy); in __call_rcu_common()
3155 call_rcu_core(rdp, head, func, flags); in __call_rcu_common()
3462 struct rcu_data *rdp; in start_poll_synchronize_rcu_common() local
3466 rdp = this_cpu_ptr(&rcu_data); in start_poll_synchronize_rcu_common()
3467 rnp = rdp->mynode; in start_poll_synchronize_rcu_common()
3475 needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq)); in start_poll_synchronize_rcu_common()
3673 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_pending() local
3674 struct rcu_node *rnp = rdp->mynode; in rcu_pending()
3679 check_cpu_stall(rdp); in rcu_pending()
3682 if (rcu_nocb_need_deferred_wakeup(rdp, RCU_NOCB_WAKE)) in rcu_pending()
3695 if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm && gp_in_progress) in rcu_pending()
3699 if (!rcu_rdp_is_offloaded(rdp) && in rcu_pending()
3700 rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_pending()
3704 if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && in rcu_pending()
3705 !rcu_rdp_is_offloaded(rdp) && in rcu_pending()
3706 !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_pending()
3710 if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq || in rcu_pending()
3711 unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */ in rcu_pending()
3754 static void rcu_barrier_entrain(struct rcu_data *rdp) in rcu_barrier_entrain() argument
3757 unsigned long lseq = READ_ONCE(rdp->barrier_seq_snap); in rcu_barrier_entrain()
3765 rdp->barrier_head.func = rcu_barrier_callback; in rcu_barrier_entrain()
3766 debug_rcu_head_queue(&rdp->barrier_head); in rcu_barrier_entrain()
3767 rcu_nocb_lock(rdp); in rcu_barrier_entrain()
3773 was_alldone = rcu_rdp_is_offloaded(rdp) && !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_barrier_entrain()
3774 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies, false)); in rcu_barrier_entrain()
3775 wake_nocb = was_alldone && rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_barrier_entrain()
3776 if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) { in rcu_barrier_entrain()
3779 debug_rcu_head_unqueue(&rdp->barrier_head); in rcu_barrier_entrain()
3782 rcu_nocb_unlock(rdp); in rcu_barrier_entrain()
3784 wake_nocb_gp(rdp); in rcu_barrier_entrain()
3785 smp_store_release(&rdp->barrier_seq_snap, gseq); in rcu_barrier_entrain()
3794 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier_handler() local
3797 WARN_ON_ONCE(cpu != rdp->cpu); in rcu_barrier_handler()
3800 rcu_barrier_entrain(rdp); in rcu_barrier_handler()
3822 struct rcu_data *rdp; in rcu_barrier() local
3861 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier()
3863 if (smp_load_acquire(&rdp->barrier_seq_snap) == gseq) in rcu_barrier()
3866 if (!rcu_segcblist_n_cbs(&rdp->cblist)) { in rcu_barrier()
3867 WRITE_ONCE(rdp->barrier_seq_snap, gseq); in rcu_barrier()
3872 if (!rcu_rdp_cpu_online(rdp)) { in rcu_barrier()
3873 rcu_barrier_entrain(rdp); in rcu_barrier()
3874 WARN_ON_ONCE(READ_ONCE(rdp->barrier_seq_snap) != gseq); in rcu_barrier()
3884 WARN_ON_ONCE(READ_ONCE(rdp->barrier_seq_snap) != gseq); in rcu_barrier()
3903 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier()
3905 WRITE_ONCE(rdp->barrier_seq_snap, gseq); in rcu_barrier()
4005 static bool rcu_rdp_cpu_online(struct rcu_data *rdp) in rcu_rdp_cpu_online() argument
4007 return !!(rdp->grpmask & rcu_rnp_online_cpus(rdp->mynode)); in rcu_rdp_cpu_online()
4012 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_online() local
4014 return rcu_rdp_cpu_online(rdp); in rcu_cpu_online()
4034 struct rcu_data *rdp; in rcu_lockdep_current_cpu_online() local
4040 rdp = this_cpu_ptr(&rcu_data); in rcu_lockdep_current_cpu_online()
4048 if (rcu_rdp_cpu_online(rdp) || arch_spin_is_locked(&rcu_state.ofl_lock)) in rcu_lockdep_current_cpu_online()
4144 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_boot_init_percpu_data() local
4147 rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu); in rcu_boot_init_percpu_data()
4148 INIT_WORK(&rdp->strict_work, strict_work_handler); in rcu_boot_init_percpu_data()
4151 rdp->barrier_seq_snap = rcu_state.barrier_sequence; in rcu_boot_init_percpu_data()
4152 rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4153 rdp->rcu_ofl_gp_state = RCU_GP_CLEANED; in rcu_boot_init_percpu_data()
4154 rdp->rcu_onl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4155 rdp->rcu_onl_gp_state = RCU_GP_CLEANED; in rcu_boot_init_percpu_data()
4156 rdp->last_sched_clock = jiffies; in rcu_boot_init_percpu_data()
4157 rdp->cpu = cpu; in rcu_boot_init_percpu_data()
4158 rcu_boot_init_nocb_percpu_data(rdp); in rcu_boot_init_percpu_data()
4244 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_prepare_cpu() local
4249 rdp->qlen_last_fqs_check = 0; in rcutree_prepare_cpu()
4250 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcutree_prepare_cpu()
4251 rdp->blimit = blimit; in rcutree_prepare_cpu()
4259 if (!rcu_segcblist_is_enabled(&rdp->cblist)) in rcutree_prepare_cpu()
4260 rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ in rcutree_prepare_cpu()
4267 rnp = rdp->mynode; in rcutree_prepare_cpu()
4269 rdp->gp_seq = READ_ONCE(rnp->gp_seq); in rcutree_prepare_cpu()
4270 rdp->gp_seq_needed = rdp->gp_seq; in rcutree_prepare_cpu()
4271 rdp->cpu_no_qs.b.norm = true; in rcutree_prepare_cpu()
4272 rdp->core_needs_qs = false; in rcutree_prepare_cpu()
4273 rdp->rcu_iw_pending = false; in rcutree_prepare_cpu()
4274 rdp->rcu_iw = IRQ_WORK_INIT_HARD(rcu_iw_handler); in rcutree_prepare_cpu()
4275 rdp->rcu_iw_gp_seq = rdp->gp_seq - 1; in rcutree_prepare_cpu()
4276 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl")); in rcutree_prepare_cpu()
4279 rcu_preempt_deferred_qs_init(rdp); in rcutree_prepare_cpu()
4293 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_beenfullyonline() local
4295 return smp_load_acquire(&rdp->beenonline); in rcu_cpu_beenfullyonline()
4305 struct rcu_data *rdp; in rcutree_online_cpu() local
4308 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_online_cpu()
4309 rnp = rdp->mynode; in rcutree_online_cpu()
4311 rnp->ffmask |= rdp->grpmask; in rcutree_online_cpu()
4338 struct rcu_data *rdp; in rcutree_report_cpu_starting() local
4343 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_report_cpu_starting()
4344 if (rdp->cpu_started) in rcutree_report_cpu_starting()
4346 rdp->cpu_started = true; in rcutree_report_cpu_starting()
4348 rnp = rdp->mynode; in rcutree_report_cpu_starting()
4349 mask = rdp->grpmask; in rcutree_report_cpu_starting()
4361 rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */ in rcutree_report_cpu_starting()
4362 rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcutree_report_cpu_starting()
4363 rdp->rcu_onl_gp_state = READ_ONCE(rcu_state.gp_state); in rcutree_report_cpu_starting()
4371 rcu_disable_urgency_upon_qs(rdp); in rcutree_report_cpu_starting()
4378 smp_store_release(&rdp->beenonline, true); in rcutree_report_cpu_starting()
4396 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcutree_report_cpu_dead() local
4397 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ in rcutree_report_cpu_dead()
4409 WARN_ON_ONCE(rdp->cpu_no_qs.b.exp); in rcutree_report_cpu_dead()
4411 do_nocb_deferred_wakeup(rdp); in rcutree_report_cpu_dead()
4416 mask = rdp->grpmask; in rcutree_report_cpu_dead()
4426 rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcutree_report_cpu_dead()
4427 rdp->rcu_ofl_gp_state = READ_ONCE(rcu_state.gp_state); in rcutree_report_cpu_dead()
4430 rcu_disable_urgency_upon_qs(rdp); in rcutree_report_cpu_dead()
4438 rdp->cpu_started = false; in rcutree_report_cpu_dead()
4452 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_migrate_callbacks() local
4455 if (rcu_rdp_is_offloaded(rdp)) in rcutree_migrate_callbacks()
4459 if (rcu_segcblist_empty(&rdp->cblist)) { in rcutree_migrate_callbacks()
4464 WARN_ON_ONCE(rcu_rdp_cpu_online(rdp)); in rcutree_migrate_callbacks()
4465 rcu_barrier_entrain(rdp); in rcutree_migrate_callbacks()
4472 needwake = rcu_advance_cbs(my_rnp, rdp) || in rcutree_migrate_callbacks()
4474 rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); in rcutree_migrate_callbacks()
4477 rcu_segcblist_disable(&rdp->cblist); in rcutree_migrate_callbacks()
4491 WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || in rcutree_migrate_callbacks()
4492 !rcu_segcblist_empty(&rdp->cblist), in rcutree_migrate_callbacks()
4494 cpu, rcu_segcblist_n_cbs(&rdp->cblist), in rcutree_migrate_callbacks()
4495 rcu_segcblist_first_cb(&rdp->cblist)); in rcutree_migrate_callbacks()
4520 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_dying_cpu() local
4521 struct rcu_node *rnp = rdp->mynode; in rcutree_dying_cpu()
4523 blkd = !!(READ_ONCE(rnp->qsmask) & rdp->grpmask); in rcutree_dying_cpu()
4536 struct rcu_data *rdp; in rcutree_offline_cpu() local
4539 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_offline_cpu()
4540 rnp = rdp->mynode; in rcutree_offline_cpu()
4542 rnp->ffmask &= ~rdp->grpmask; in rcutree_offline_cpu()
4584 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_spawn_gp_kthread() local
4609 rcu_spawn_rnp_kthreads(rdp->mynode); in rcu_spawn_gp_kthread()