Lines Matching refs:rdp
20 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp) in rcu_current_is_nocb_kthread() argument
23 if (!rdp->nocb_cb_kthread || !rdp->nocb_gp_kthread) in rcu_current_is_nocb_kthread()
26 if (current == rdp->nocb_cb_kthread || current == rdp->nocb_gp_kthread) in rcu_current_is_nocb_kthread()
92 static void rcu_nocb_bypass_lock(struct rcu_data *rdp) in rcu_nocb_bypass_lock() argument
93 __acquires(&rdp->nocb_bypass_lock) in rcu_nocb_bypass_lock()
96 if (raw_spin_trylock(&rdp->nocb_bypass_lock)) in rcu_nocb_bypass_lock()
102 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_bypass_lock()
103 raw_spin_lock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_lock()
110 static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp) in rcu_nocb_bypass_trylock() argument
113 return raw_spin_trylock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_trylock()
119 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp) in rcu_nocb_bypass_unlock() argument
120 __releases(&rdp->nocb_bypass_lock) in rcu_nocb_bypass_unlock()
123 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_unlock()
130 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock() argument
133 if (!rcu_rdp_is_offloaded(rdp)) in rcu_nocb_lock()
135 raw_spin_lock(&rdp->nocb_lock); in rcu_nocb_lock()
142 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock() argument
144 if (rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_unlock()
146 raw_spin_unlock(&rdp->nocb_lock); in rcu_nocb_unlock()
154 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore() argument
157 if (rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_unlock_irqrestore()
159 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_unlock_irqrestore()
166 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected() argument
169 if (rcu_rdp_is_offloaded(rdp)) in rcu_lockdep_assert_cblist_protected()
170 lockdep_assert_held(&rdp->nocb_lock); in rcu_lockdep_assert_cblist_protected()
203 struct rcu_data *rdp, in __wake_nocb_gp()
211 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __wake_nocb_gp()
224 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake")); in __wake_nocb_gp()
234 static bool wake_nocb_gp(struct rcu_data *rdp) in wake_nocb_gp() argument
237 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp()
240 return __wake_nocb_gp(rdp_gp, rdp, flags); in wake_nocb_gp()
272 static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, in wake_nocb_gp_defer() argument
276 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp_defer()
300 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); in wake_nocb_gp_defer()
314 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp_in, in rcu_nocb_do_flush_bypass() argument
320 WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp)); in rcu_nocb_do_flush_bypass()
321 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_do_flush_bypass()
322 lockdep_assert_held(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
323 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) { in rcu_nocb_do_flush_bypass()
324 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
329 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_do_flush_bypass()
338 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass()
341 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass()
342 WRITE_ONCE(rdp->lazy_len, 0); in rcu_nocb_do_flush_bypass()
344 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl); in rcu_nocb_do_flush_bypass()
345 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_do_flush_bypass()
346 rcu_nocb_bypass_unlock(rdp); in rcu_nocb_do_flush_bypass()
358 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
361 if (!rcu_rdp_is_offloaded(rdp)) in rcu_nocb_flush_bypass()
363 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_flush_bypass()
364 rcu_nocb_bypass_lock(rdp); in rcu_nocb_flush_bypass()
365 return rcu_nocb_do_flush_bypass(rdp, rhp, j, lazy); in rcu_nocb_flush_bypass()
372 static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j) in rcu_nocb_try_flush_bypass() argument
374 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_try_flush_bypass()
375 if (!rcu_rdp_is_offloaded(rdp) || in rcu_nocb_try_flush_bypass()
376 !rcu_nocb_bypass_trylock(rdp)) in rcu_nocb_try_flush_bypass()
378 WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j, false)); in rcu_nocb_try_flush_bypass()
386 static bool nocb_bypass_needs_flush(struct rcu_data *rdp, long bypass_ncbs, in nocb_bypass_needs_flush() argument
402 bypass_first = READ_ONCE(rdp->nocb_bypass_first); in nocb_bypass_needs_flush()
431 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass() argument
438 long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
439 long lazy_len = READ_ONCE(rdp->lazy_len); in rcu_nocb_try_bypass()
446 if (!rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_try_bypass()
447 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
453 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
454 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
455 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
461 if (j == rdp->nocb_nobypass_last) { in rcu_nocb_try_bypass()
462 c = rdp->nocb_nobypass_count + 1; in rcu_nocb_try_bypass()
464 WRITE_ONCE(rdp->nocb_nobypass_last, j); in rcu_nocb_try_bypass()
465 c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy; in rcu_nocb_try_bypass()
466 if (ULONG_CMP_LT(rdp->nocb_nobypass_count, in rcu_nocb_try_bypass()
472 WRITE_ONCE(rdp->nocb_nobypass_count, c); in rcu_nocb_try_bypass()
478 if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy && !lazy) { in rcu_nocb_try_bypass()
479 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
480 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
482 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
485 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j, false)); in rcu_nocb_try_bypass()
486 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
492 if (nocb_bypass_needs_flush(rdp, ncbs, lazy_len, j, true)) { in rcu_nocb_try_bypass()
493 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
494 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
496 if (!rcu_nocb_flush_bypass(rdp, rhp, j, lazy)) { in rcu_nocb_try_bypass()
498 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
500 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
503 if (j != rdp->nocb_gp_adv_time && in rcu_nocb_try_bypass()
504 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in rcu_nocb_try_bypass()
505 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in rcu_nocb_try_bypass()
506 rcu_advance_cbs_nowake(rdp->mynode, rdp); in rcu_nocb_try_bypass()
507 rdp->nocb_gp_adv_time = j; in rcu_nocb_try_bypass()
513 __call_rcu_nocb_wake(rdp, *was_alldone, flags); in rcu_nocb_try_bypass()
519 rcu_nocb_bypass_lock(rdp); in rcu_nocb_try_bypass()
520 ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
521 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_try_bypass()
522 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_try_bypass()
525 WRITE_ONCE(rdp->lazy_len, rdp->lazy_len + 1); in rcu_nocb_try_bypass()
528 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_try_bypass()
529 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ")); in rcu_nocb_try_bypass()
531 rcu_nocb_bypass_unlock(rdp); in rcu_nocb_try_bypass()
542 rcu_nocb_lock(rdp); // Rare during call_rcu() flood. in rcu_nocb_try_bypass()
543 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_nocb_try_bypass()
544 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
546 __call_rcu_nocb_wake(rdp, true, flags); in rcu_nocb_try_bypass()
548 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
550 rcu_nocb_unlock(rdp); in rcu_nocb_try_bypass()
560 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, in __call_rcu_nocb_wake() argument
562 __releases(rdp->nocb_lock) in __call_rcu_nocb_wake()
570 t = READ_ONCE(rdp->nocb_gp_kthread); in __call_rcu_nocb_wake()
572 rcu_nocb_unlock(rdp); in __call_rcu_nocb_wake()
573 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
578 len = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_nocb_wake()
579 bypass_len = rcu_cblist_n_cbs(&rdp->nocb_bypass); in __call_rcu_nocb_wake()
580 lazy_len = READ_ONCE(rdp->lazy_len); in __call_rcu_nocb_wake()
582 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
583 rcu_nocb_unlock(rdp); in __call_rcu_nocb_wake()
586 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_LAZY, in __call_rcu_nocb_wake()
590 wake_nocb_gp(rdp); in __call_rcu_nocb_wake()
591 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
594 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE, in __call_rcu_nocb_wake()
601 rcu_nocb_unlock(rdp); in __call_rcu_nocb_wake()
602 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
605 static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, in call_rcu_nocb() argument
610 if (!rcu_nocb_try_bypass(rdp, head, &was_alldone, flags, lazy)) { in call_rcu_nocb()
612 rcutree_enqueue(rdp, head, func); in call_rcu_nocb()
613 __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */ in call_rcu_nocb()
617 static void nocb_gp_toggle_rdp(struct rcu_data *rdp_gp, struct rcu_data *rdp) in nocb_gp_toggle_rdp() argument
619 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_gp_toggle_rdp()
627 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in nocb_gp_toggle_rdp()
633 list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp); in nocb_gp_toggle_rdp()
640 list_del(&rdp->nocb_entry_rdp); in nocb_gp_toggle_rdp()
643 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in nocb_gp_toggle_rdp()
670 struct rcu_data *rdp, *rdp_toggling = NULL; in nocb_gp_wait() local
695 list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { in nocb_gp_wait()
700 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); in nocb_gp_wait()
701 rcu_nocb_lock_irqsave(rdp, flags); in nocb_gp_wait()
702 lockdep_assert_held(&rdp->nocb_lock); in nocb_gp_wait()
703 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
704 lazy_ncbs = READ_ONCE(rdp->lazy_len); in nocb_gp_wait()
706 flush_bypass = nocb_bypass_needs_flush(rdp, bypass_ncbs, lazy_ncbs, j, false); in nocb_gp_wait()
707 if (!flush_bypass && !bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { in nocb_gp_wait()
708 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
714 (void)rcu_nocb_try_flush_bypass(rdp, j); in nocb_gp_wait()
715 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
716 lazy_ncbs = READ_ONCE(rdp->lazy_len); in nocb_gp_wait()
720 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
727 rnp = rdp->mynode; in nocb_gp_wait()
731 if (!rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
733 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in nocb_gp_wait()
736 needwake_gp = rcu_advance_cbs(rnp, rdp); in nocb_gp_wait()
737 wasempty = rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
743 !rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
745 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { in nocb_gp_wait()
750 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
753 if (rcu_segcblist_ready_cbs(&rdp->cblist)) { in nocb_gp_wait()
754 needwake = rdp->nocb_cb_sleep; in nocb_gp_wait()
755 WRITE_ONCE(rdp->nocb_cb_sleep, false); in nocb_gp_wait()
759 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
761 swake_up_one(&rdp->nocb_cb_wq); in nocb_gp_wait()
858 struct rcu_data *rdp = arg; in rcu_nocb_gp_kthread() local
861 WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1); in rcu_nocb_gp_kthread()
862 nocb_gp_wait(rdp); in rcu_nocb_gp_kthread()
868 static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) in nocb_cb_wait_cond() argument
870 return !READ_ONCE(rdp->nocb_cb_sleep) || kthread_should_park(); in nocb_cb_wait_cond()
877 static void nocb_cb_wait(struct rcu_data *rdp) in nocb_cb_wait() argument
879 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_cb_wait()
883 struct rcu_node *rnp = rdp->mynode; in nocb_cb_wait()
885 swait_event_interruptible_exclusive(rdp->nocb_cb_wq, in nocb_cb_wait()
886 nocb_cb_wait_cond(rdp)); in nocb_cb_wait()
894 if (rdp->nocb_cb_sleep) { in nocb_cb_wait()
895 rcu_nocb_lock_irqsave(rdp, flags); in nocb_cb_wait()
896 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist)); in nocb_cb_wait()
897 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_cb_wait()
900 } else if (READ_ONCE(rdp->nocb_cb_sleep)) { in nocb_cb_wait()
902 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); in nocb_cb_wait()
905 WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp)); in nocb_cb_wait()
917 rcu_do_batch(rdp); in nocb_cb_wait()
920 rcu_nocb_lock_irqsave(rdp, flags); in nocb_cb_wait()
924 needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); in nocb_cb_wait()
929 WRITE_ONCE(rdp->nocb_cb_sleep, true); in nocb_cb_wait()
930 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); in nocb_cb_wait()
932 WRITE_ONCE(rdp->nocb_cb_sleep, false); in nocb_cb_wait()
935 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_cb_wait()
946 struct rcu_data *rdp = arg; in rcu_nocb_cb_kthread() local
951 nocb_cb_wait(rdp); in rcu_nocb_cb_kthread()
958 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level) in rcu_nocb_need_deferred_wakeup() argument
960 return READ_ONCE(rdp->nocb_defer_wakeup) >= level; in rcu_nocb_need_deferred_wakeup()
965 struct rcu_data *rdp, int level, in do_nocb_deferred_wakeup_common()
976 ret = __wake_nocb_gp(rdp_gp, rdp, flags); in do_nocb_deferred_wakeup_common()
977 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); in do_nocb_deferred_wakeup_common()
986 struct rcu_data *rdp = timer_container_of(rdp, t, nocb_timer); in do_nocb_deferred_wakeup_timer() local
988 WARN_ON_ONCE(rdp->nocb_gp_rdp != rdp); in do_nocb_deferred_wakeup_timer()
989 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer")); in do_nocb_deferred_wakeup_timer()
991 raw_spin_lock_irqsave(&rdp->nocb_gp_lock, flags); in do_nocb_deferred_wakeup_timer()
992 do_nocb_deferred_wakeup_common(rdp, rdp, RCU_NOCB_WAKE_BYPASS, flags); in do_nocb_deferred_wakeup_timer()
1000 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
1003 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in do_nocb_deferred_wakeup()
1009 return do_nocb_deferred_wakeup_common(rdp_gp, rdp, RCU_NOCB_WAKE, flags); in do_nocb_deferred_wakeup()
1018 static int rcu_nocb_queue_toggle_rdp(struct rcu_data *rdp) in rcu_nocb_queue_toggle_rdp() argument
1020 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rcu_nocb_queue_toggle_rdp()
1026 WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp); in rcu_nocb_queue_toggle_rdp()
1036 static bool rcu_nocb_rdp_deoffload_wait_cond(struct rcu_data *rdp) in rcu_nocb_rdp_deoffload_wait_cond() argument
1046 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload_wait_cond()
1047 ret = !rcu_segcblist_test_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_nocb_rdp_deoffload_wait_cond()
1048 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload_wait_cond()
1053 static int rcu_nocb_rdp_deoffload(struct rcu_data *rdp) in rcu_nocb_rdp_deoffload() argument
1057 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rcu_nocb_rdp_deoffload()
1060 WARN_ON_ONCE(cpu_online(rdp->cpu) && rdp->cpu != raw_smp_processor_id()); in rcu_nocb_rdp_deoffload()
1062 pr_info("De-offloading %d\n", rdp->cpu); in rcu_nocb_rdp_deoffload()
1071 if (rdp->nocb_cb_kthread) in rcu_nocb_rdp_deoffload()
1072 kthread_park(rdp->nocb_cb_kthread); in rcu_nocb_rdp_deoffload()
1074 rcu_nocb_lock_irqsave(rdp, flags); in rcu_nocb_rdp_deoffload()
1075 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_rdp_deoffload()
1076 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist)); in rcu_nocb_rdp_deoffload()
1077 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_nocb_rdp_deoffload()
1079 wake_gp = rcu_nocb_queue_toggle_rdp(rdp); in rcu_nocb_rdp_deoffload()
1087 swait_event_exclusive(rdp->nocb_state_wq, in rcu_nocb_rdp_deoffload()
1088 rcu_nocb_rdp_deoffload_wait_cond(rdp)); in rcu_nocb_rdp_deoffload()
1095 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload()
1096 rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_nocb_rdp_deoffload()
1097 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload()
1099 list_del(&rdp->nocb_entry_rdp); in rcu_nocb_rdp_deoffload()
1107 static bool rcu_nocb_rdp_offload_wait_cond(struct rcu_data *rdp) in rcu_nocb_rdp_offload_wait_cond() argument
1112 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_rdp_offload_wait_cond()
1113 ret = rcu_segcblist_test_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_nocb_rdp_offload_wait_cond()
1114 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_rdp_offload_wait_cond()
1119 static int rcu_nocb_rdp_offload(struct rcu_data *rdp) in rcu_nocb_rdp_offload() argument
1123 WARN_ON_ONCE(cpu_online(rdp->cpu)); in rcu_nocb_rdp_offload()
1128 if (!rdp->nocb_gp_rdp) in rcu_nocb_rdp_offload()
1131 if (WARN_ON_ONCE(!rdp->nocb_gp_kthread)) in rcu_nocb_rdp_offload()
1134 pr_info("Offloading %d\n", rdp->cpu); in rcu_nocb_rdp_offload()
1136 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_rdp_offload()
1137 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist)); in rcu_nocb_rdp_offload()
1139 wake_gp = rcu_nocb_queue_toggle_rdp(rdp); in rcu_nocb_rdp_offload()
1141 wake_up_process(rdp->nocb_gp_kthread); in rcu_nocb_rdp_offload()
1143 swait_event_exclusive(rdp->nocb_state_wq, in rcu_nocb_rdp_offload()
1144 rcu_nocb_rdp_offload_wait_cond(rdp)); in rcu_nocb_rdp_offload()
1146 kthread_unpark(rdp->nocb_cb_kthread); in rcu_nocb_rdp_offload()
1154 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_toggle_offload() local
1161 if (rcu_rdp_is_offloaded(rdp) == offload) in rcu_nocb_cpu_toggle_offload()
1166 offload ? "" : "de", rdp->cpu); in rcu_nocb_cpu_toggle_offload()
1172 ret = rcu_nocb_rdp_offload(rdp); in rcu_nocb_cpu_toggle_offload()
1176 ret = rcu_nocb_rdp_deoffload(rdp); in rcu_nocb_cpu_toggle_offload()
1215 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in lazy_rcu_shrink_count() local
1217 count += READ_ONCE(rdp->lazy_len); in lazy_rcu_shrink_count()
1250 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in lazy_rcu_shrink_scan() local
1253 if (WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp))) in lazy_rcu_shrink_scan()
1256 if (!READ_ONCE(rdp->lazy_len)) in lazy_rcu_shrink_scan()
1259 rcu_nocb_lock_irqsave(rdp, flags); in lazy_rcu_shrink_scan()
1265 _count = READ_ONCE(rdp->lazy_len); in lazy_rcu_shrink_scan()
1267 rcu_nocb_unlock_irqrestore(rdp, flags); in lazy_rcu_shrink_scan()
1270 rcu_nocb_try_flush_bypass(rdp, jiffies); in lazy_rcu_shrink_scan()
1271 rcu_nocb_unlock_irqrestore(rdp, flags); in lazy_rcu_shrink_scan()
1272 wake_nocb_gp(rdp); in lazy_rcu_shrink_scan()
1288 struct rcu_data *rdp; in rcu_init_nohz() local
1342 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_init_nohz()
1343 if (rcu_segcblist_empty(&rdp->cblist)) in rcu_init_nohz()
1344 rcu_segcblist_init(&rdp->cblist); in rcu_init_nohz()
1345 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_init_nohz()
1351 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
1353 init_swait_queue_head(&rdp->nocb_cb_wq); in rcu_boot_init_nocb_percpu_data()
1354 init_swait_queue_head(&rdp->nocb_gp_wq); in rcu_boot_init_nocb_percpu_data()
1355 init_swait_queue_head(&rdp->nocb_state_wq); in rcu_boot_init_nocb_percpu_data()
1356 raw_spin_lock_init(&rdp->nocb_lock); in rcu_boot_init_nocb_percpu_data()
1357 raw_spin_lock_init(&rdp->nocb_bypass_lock); in rcu_boot_init_nocb_percpu_data()
1358 raw_spin_lock_init(&rdp->nocb_gp_lock); in rcu_boot_init_nocb_percpu_data()
1359 timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0); in rcu_boot_init_nocb_percpu_data()
1360 rcu_cblist_init(&rdp->nocb_bypass); in rcu_boot_init_nocb_percpu_data()
1361 WRITE_ONCE(rdp->lazy_len, 0); in rcu_boot_init_nocb_percpu_data()
1362 mutex_init(&rdp->nocb_gp_kthread_mutex); in rcu_boot_init_nocb_percpu_data()
1372 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_spawn_cpu_nocb_kthread() local
1381 if (rdp->nocb_cb_kthread) in rcu_spawn_cpu_nocb_kthread()
1386 rdp_gp = rdp->nocb_gp_rdp; in rcu_spawn_cpu_nocb_kthread()
1402 t = kthread_create(rcu_nocb_cb_kthread, rdp, in rcu_spawn_cpu_nocb_kthread()
1407 if (rcu_rdp_is_offloaded(rdp)) in rcu_spawn_cpu_nocb_kthread()
1415 WRITE_ONCE(rdp->nocb_cb_kthread, t); in rcu_spawn_cpu_nocb_kthread()
1416 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread); in rcu_spawn_cpu_nocb_kthread()
1426 WARN_ON_ONCE(system_state > SYSTEM_BOOTING && rcu_segcblist_n_cbs(&rdp->cblist)); in rcu_spawn_cpu_nocb_kthread()
1428 if (rcu_rdp_is_offloaded(rdp)) { in rcu_spawn_cpu_nocb_kthread()
1429 rcu_nocb_rdp_deoffload(rdp); in rcu_spawn_cpu_nocb_kthread()
1450 struct rcu_data *rdp; in rcu_organize_nocb_kthreads() local
1466 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_organize_nocb_kthreads()
1467 if (rdp->cpu >= nl) { in rcu_organize_nocb_kthreads()
1470 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; in rcu_organize_nocb_kthreads()
1471 rdp_gp = rdp; in rcu_organize_nocb_kthreads()
1472 INIT_LIST_HEAD(&rdp->nocb_head_rdp); in rcu_organize_nocb_kthreads()
1488 rdp->nocb_gp_rdp = rdp_gp; in rcu_organize_nocb_kthreads()
1490 list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp); in rcu_organize_nocb_kthreads()
1524 static void show_rcu_nocb_gp_state(struct rcu_data *rdp) in show_rcu_nocb_gp_state() argument
1526 struct rcu_node *rnp = rdp->mynode; in show_rcu_nocb_gp_state()
1529 rdp->cpu, in show_rcu_nocb_gp_state()
1530 "kK"[!!rdp->nocb_gp_kthread], in show_rcu_nocb_gp_state()
1531 "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)], in show_rcu_nocb_gp_state()
1532 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_gp_state()
1533 "tT"[timer_pending(&rdp->nocb_timer)], in show_rcu_nocb_gp_state()
1534 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_gp_state()
1535 ".W"[swait_active(&rdp->nocb_gp_wq)], in show_rcu_nocb_gp_state()
1538 ".B"[!!rdp->nocb_gp_bypass], in show_rcu_nocb_gp_state()
1539 ".G"[!!rdp->nocb_gp_gp], in show_rcu_nocb_gp_state()
1540 (long)rdp->nocb_gp_seq, in show_rcu_nocb_gp_state()
1541 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops), in show_rcu_nocb_gp_state()
1542 rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.', in show_rcu_nocb_gp_state()
1543 rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, in show_rcu_nocb_gp_state()
1544 show_rcu_should_be_on_cpu(rdp->nocb_gp_kthread)); in show_rcu_nocb_gp_state()
1548 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state() argument
1556 struct rcu_segcblist *rsclp = &rdp->cblist; in show_rcu_nocb_state()
1560 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
1561 show_rcu_nocb_gp_state(rdp); in show_rcu_nocb_state()
1563 if (!rcu_segcblist_is_offloaded(&rdp->cblist)) in show_rcu_nocb_state()
1566 nocb_next_rdp = list_next_or_null_rcu(&rdp->nocb_gp_rdp->nocb_head_rdp, in show_rcu_nocb_state()
1567 &rdp->nocb_entry_rdp, in show_rcu_nocb_state()
1568 typeof(*rdp), in show_rcu_nocb_state()
1576 sprintf(bufb, "%ld", rcu_cblist_n_cbs(&rdp->nocb_bypass)); in show_rcu_nocb_state()
1578 rdp->cpu, rdp->nocb_gp_rdp->cpu, in show_rcu_nocb_state()
1580 "kK"[!!rdp->nocb_cb_kthread], in show_rcu_nocb_state()
1581 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)], in show_rcu_nocb_state()
1582 "lL"[raw_spin_is_locked(&rdp->nocb_lock)], in show_rcu_nocb_state()
1583 "sS"[!!rdp->nocb_cb_sleep], in show_rcu_nocb_state()
1584 ".W"[swait_active(&rdp->nocb_cb_wq)], in show_rcu_nocb_state()
1585 jiffies - rdp->nocb_bypass_first, in show_rcu_nocb_state()
1586 jiffies - rdp->nocb_nobypass_last, in show_rcu_nocb_state()
1587 rdp->nocb_nobypass_count, in show_rcu_nocb_state()
1596 ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)], in show_rcu_nocb_state()
1597 !rcu_cblist_n_cbs(&rdp->nocb_bypass) ? "" : bufb, in show_rcu_nocb_state()
1598 rcu_segcblist_n_cbs(&rdp->cblist), in show_rcu_nocb_state()
1599 rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.', in show_rcu_nocb_state()
1600 rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1, in show_rcu_nocb_state()
1601 show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread)); in show_rcu_nocb_state()
1604 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
1607 waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock); in show_rcu_nocb_state()
1608 wassleep = swait_active(&rdp->nocb_gp_wq); in show_rcu_nocb_state()
1609 if (!rdp->nocb_gp_sleep && !waslocked && !wassleep) in show_rcu_nocb_state()
1614 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_state()
1615 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_state()
1622 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock() argument
1627 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock() argument
1632 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore() argument
1639 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected() argument
1657 static bool wake_nocb_gp(struct rcu_data *rdp) in wake_nocb_gp() argument
1662 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
1668 static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, in call_rcu_nocb() argument
1674 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, in __call_rcu_nocb_wake() argument
1680 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
1684 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level) in rcu_nocb_need_deferred_wakeup() argument
1689 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
1698 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state() argument