76 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay);
123 static void init_srcu_struct_data(struct srcu_struct *ssp)  in init_srcu_struct_data()  argument
133 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_data()
137 		sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
138 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
144 		sdp->ssp = ssp;  in init_srcu_struct_data()
164 static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)  in init_srcu_struct_nodes()  argument
176 	ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags);  in init_srcu_struct_nodes()
177 	if (!ssp->srcu_sup->node)  in init_srcu_struct_nodes()
181 	ssp->srcu_sup->level[0] = &ssp->srcu_sup->node[0];  in init_srcu_struct_nodes()
183 		ssp->srcu_sup->level[i] = ssp->srcu_sup->level[i - 1] + num_rcu_lvl[i - 1];  in init_srcu_struct_nodes()
187 	srcu_for_each_node_breadth_first(ssp, snp) {  in init_srcu_struct_nodes()
198 		if (snp == &ssp->srcu_sup->node[0]) {  in init_srcu_struct_nodes()
205 		if (snp == ssp->srcu_sup->level[level + 1])  in init_srcu_struct_nodes()
207 		snp->srcu_parent = ssp->srcu_sup->level[level - 1] +  in init_srcu_struct_nodes()
208 				   (snp - ssp->srcu_sup->level[level]) /  in init_srcu_struct_nodes()
217 	snp_first = ssp->srcu_sup->level[level];  in init_srcu_struct_nodes()
219 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_nodes()
228 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_WAIT_BARRIER);  in init_srcu_struct_nodes()
237 static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)  in init_srcu_struct_fields()  argument
240 		ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL);  in init_srcu_struct_fields()
241 	if (!ssp->srcu_sup)  in init_srcu_struct_fields()
244 		spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in init_srcu_struct_fields()
245 	ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;  in init_srcu_struct_fields()
246 	ssp->srcu_sup->node = NULL;  in init_srcu_struct_fields()
247 	mutex_init(&ssp->srcu_sup->srcu_cb_mutex);  in init_srcu_struct_fields()
248 	mutex_init(&ssp->srcu_sup->srcu_gp_mutex);  in init_srcu_struct_fields()
249 	ssp->srcu_sup->srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL;  in init_srcu_struct_fields()
250 	ssp->srcu_sup->srcu_barrier_seq = 0;  in init_srcu_struct_fields()
251 	mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);  in init_srcu_struct_fields()
252 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);  in init_srcu_struct_fields()
253 	INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);  in init_srcu_struct_fields()
254 	ssp->srcu_sup->sda_is_static = is_static;  in init_srcu_struct_fields()
256 		ssp->sda = alloc_percpu(struct srcu_data);  in init_srcu_struct_fields()
257 		ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0];  in init_srcu_struct_fields()
259 	if (!ssp->sda)  in init_srcu_struct_fields()
261 	init_srcu_struct_data(ssp);  in init_srcu_struct_fields()
262 	ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL;  in init_srcu_struct_fields()
263 	ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();  in init_srcu_struct_fields()
264 	if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {  in init_srcu_struct_fields()
265 		if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC))  in init_srcu_struct_fields()
267 		WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);  in init_srcu_struct_fields()
269 	ssp->srcu_sup->srcu_ssp = ssp;  in init_srcu_struct_fields()
270 	smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed,  in init_srcu_struct_fields()
276 		free_percpu(ssp->sda);  in init_srcu_struct_fields()
277 		ssp->sda = NULL;  in init_srcu_struct_fields()
281 		kfree(ssp->srcu_sup);  in init_srcu_struct_fields()
282 		ssp->srcu_sup = NULL;  in init_srcu_struct_fields()
289 int __init_srcu_struct(struct srcu_struct *ssp, const char *name,  in __init_srcu_struct()  argument
293 	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));  in __init_srcu_struct()
294 	lockdep_init_map(&ssp->dep_map, name, key, 0);  in __init_srcu_struct()
295 	return init_srcu_struct_fields(ssp, false);  in __init_srcu_struct()
303  * @ssp: structure to initialize.
309 int init_srcu_struct(struct srcu_struct *ssp)  in init_srcu_struct()  argument
311 	return init_srcu_struct_fields(ssp, false);  in init_srcu_struct()
320 static void __srcu_transition_to_big(struct srcu_struct *ssp)  in __srcu_transition_to_big()  argument
322 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in __srcu_transition_to_big()
323 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC);  in __srcu_transition_to_big()
329 static void srcu_transition_to_big(struct srcu_struct *ssp)  in srcu_transition_to_big()  argument
334 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL)  in srcu_transition_to_big()
336 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
337 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) {  in srcu_transition_to_big()
338 		spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
341 	__srcu_transition_to_big(ssp);  in srcu_transition_to_big()
342 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
349 static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp)  in spin_lock_irqsave_check_contention()  argument
353 	if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state)  in spin_lock_irqsave_check_contention()
356 	if (ssp->srcu_sup->srcu_size_jiffies != j) {  in spin_lock_irqsave_check_contention()
357 		ssp->srcu_sup->srcu_size_jiffies = j;  in spin_lock_irqsave_check_contention()
358 		ssp->srcu_sup->srcu_n_lock_retries = 0;  in spin_lock_irqsave_check_contention()
360 	if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim)  in spin_lock_irqsave_check_contention()
362 	__srcu_transition_to_big(ssp);  in spin_lock_irqsave_check_contention()
373 	struct srcu_struct *ssp = sdp->ssp;  in spin_lock_irqsave_sdp_contention()  local
377 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_sdp_contention()
378 	spin_lock_irqsave_check_contention(ssp);  in spin_lock_irqsave_sdp_contention()
379 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_sdp_contention()
389 static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags)  in spin_lock_irqsave_ssp_contention()  argument
391 	if (spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags))  in spin_lock_irqsave_ssp_contention()
393 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_ssp_contention()
394 	spin_lock_irqsave_check_contention(ssp);  in spin_lock_irqsave_ssp_contention()
401  * to each update-side SRCU primitive.  Use ssp->lock, which -is-
405 static void check_init_srcu_struct(struct srcu_struct *ssp)  in check_init_srcu_struct()  argument
410 	if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed))) /*^^^*/  in check_init_srcu_struct()
412 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
413 	if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq_needed)) {  in check_init_srcu_struct()
414 		spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
417 	init_srcu_struct_fields(ssp, true);  in check_init_srcu_struct()
418 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
424 static bool srcu_gp_is_expedited(struct srcu_struct *ssp)  in srcu_gp_is_expedited()  argument
426 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_is_expedited()
437 static bool srcu_readers_lock_idx(struct srcu_struct *ssp, int idx, bool gp, unsigned long unlocks)  in srcu_readers_lock_idx()  argument
444 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_lock_idx()
451 		  "Mixed reader flavors for srcu_struct at %ps.\n", ssp);  in srcu_readers_lock_idx()
461 static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx, unsigned long *rdm)  in srcu_readers_unlock_idx()  argument
468 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_unlock_idx()
474 		  "Mixed reader flavors for srcu_struct at %ps.\n", ssp);  in srcu_readers_unlock_idx()
483 static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx)  in srcu_readers_active_idx_check()  argument
489 	unlocks = srcu_readers_unlock_idx(ssp, idx, &rdm);  in srcu_readers_active_idx_check()
565 	return srcu_readers_lock_idx(ssp, idx, did_gp, unlocks);  in srcu_readers_active_idx_check()
571  * @ssp: which srcu_struct to count active readers (holding srcu_read_lock).
577 static bool srcu_readers_active(struct srcu_struct *ssp)  in srcu_readers_active()  argument
583 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_active()
644 static unsigned long srcu_get_delay(struct srcu_struct *ssp)  in srcu_get_delay()  argument
649 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_get_delay()
651 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in srcu_get_delay()
652 	if (srcu_gp_is_expedited(ssp))  in srcu_get_delay()
671  * @ssp: structure to clean up.
676 void cleanup_srcu_struct(struct srcu_struct *ssp)  in cleanup_srcu_struct()  argument
680 	struct srcu_usage *sup = ssp->srcu_sup;  in cleanup_srcu_struct()
682 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in cleanup_srcu_struct()
683 	delay = srcu_get_delay(ssp);  in cleanup_srcu_struct()
684 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in cleanup_srcu_struct()
687 	if (WARN_ON(srcu_readers_active(ssp)))  in cleanup_srcu_struct()
691 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in cleanup_srcu_struct()
700 	    WARN_ON(srcu_readers_active(ssp))) {  in cleanup_srcu_struct()
702 			__func__, ssp, rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)),  in cleanup_srcu_struct()
713 		free_percpu(ssp->sda);  in cleanup_srcu_struct()
714 		ssp->sda = NULL;  in cleanup_srcu_struct()
716 		ssp->srcu_sup = NULL;  in cleanup_srcu_struct()
724 void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)  in __srcu_check_read_flavor()  argument
733 	sdp = raw_cpu_ptr(ssp->sda);  in __srcu_check_read_flavor()
750 int __srcu_read_lock(struct srcu_struct *ssp)  in __srcu_read_lock()  argument
752 	struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp);  in __srcu_read_lock()
756 	return __srcu_ptr_to_ctr(ssp, scp);  in __srcu_read_lock()
765 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock()  argument
768 	this_cpu_inc(__srcu_ctr_to_ptr(ssp, idx)->srcu_unlocks.counter);  in __srcu_read_unlock()
779 int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)  in __srcu_read_lock_nmisafe()  argument
781 	struct srcu_ctr __percpu *scpp = READ_ONCE(ssp->srcu_ctrp);  in __srcu_read_lock_nmisafe()
786 	return __srcu_ptr_to_ctr(ssp, scpp);  in __srcu_read_lock_nmisafe()
795 void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock_nmisafe()  argument
798 	atomic_long_inc(&raw_cpu_ptr(__srcu_ctr_to_ptr(ssp, idx))->srcu_unlocks);  in __srcu_read_unlock_nmisafe()
807 static void srcu_gp_start(struct srcu_struct *ssp)  in srcu_gp_start()  argument
811 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in srcu_gp_start()
812 	WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed));  in srcu_gp_start()
813 	WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies);  in srcu_gp_start()
814 	WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0);  in srcu_gp_start()
816 	rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
817 	state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
855 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_schedule_cbs_snp()  argument
863 		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay);  in srcu_schedule_cbs_snp()
876 static void srcu_gp_end(struct srcu_struct *ssp)  in srcu_gp_end()  argument
889 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_end()
898 	if (srcu_gp_is_expedited(ssp))  in srcu_gp_end()
913 		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()),  in srcu_gp_end()
917 		srcu_for_each_node_breadth_first(ssp, snp) {  in srcu_gp_end()
935 				srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay);  in srcu_gp_end()
942 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_gp_end()
959 		srcu_gp_start(ssp);  in srcu_gp_end()
961 		srcu_reschedule(ssp, 0);  in srcu_gp_end()
969 			init_srcu_struct_nodes(ssp, GFP_KERNEL);  in srcu_gp_end()
982 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_funnel_exp_start()  argument
991 			if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, s)) ||  in srcu_funnel_exp_start()
1003 	spin_lock_irqsave_ssp_contention(ssp, &flags);  in srcu_funnel_exp_start()
1004 	if (ULONG_CMP_LT(ssp->srcu_sup->srcu_gp_seq_needed_exp, s))  in srcu_funnel_exp_start()
1005 		WRITE_ONCE(ssp->srcu_sup->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
1006 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_funnel_exp_start()
1022 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,  in srcu_funnel_gp_start()  argument
1031 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_funnel_gp_start()
1055 					srcu_funnel_exp_start(ssp, snp, s);  in srcu_funnel_gp_start()
1068 	spin_lock_irqsave_ssp_contention(ssp, &flags);  in srcu_funnel_gp_start()
1082 		srcu_gp_start(ssp);  in srcu_funnel_gp_start()
1091 					   !!srcu_get_delay(ssp));  in srcu_funnel_gp_start()
1103 static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)  in try_check_zero()  argument
1107 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in try_check_zero()
1108 	curdelay = !srcu_get_delay(ssp);  in try_check_zero()
1109 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in try_check_zero()
1112 		if (srcu_readers_active_idx_check(ssp, idx))  in try_check_zero()
1125 static void srcu_flip(struct srcu_struct *ssp)  in srcu_flip()  argument
1159 	WRITE_ONCE(ssp->srcu_ctrp,  in srcu_flip()
1160 		   &ssp->sda->srcu_ctrs[!(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0])]);  in srcu_flip()
1196 static bool srcu_should_expedite(struct srcu_struct *ssp)  in srcu_should_expedite()  argument
1204 	check_init_srcu_struct(ssp);  in srcu_should_expedite()
1206 	if (this_cpu_read(ssp->sda->srcu_reader_flavor) & SRCU_READ_FLAVOR_SLOWGP)  in srcu_should_expedite()
1209 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_should_expedite()
1225 	tlast = READ_ONCE(ssp->srcu_sup->srcu_last_gp_end);  in srcu_should_expedite()
1231 	curseq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcu_should_expedite()
1233 	if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_sup->srcu_gp_seq_needed)))  in srcu_should_expedite()
1236 	if (curseq != rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq))  in srcu_should_expedite()
1251 static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,  in srcu_gp_start_if_needed()  argument
1263 	check_init_srcu_struct(ssp);  in srcu_gp_start_if_needed()
1269 	idx = __srcu_read_lock_nmisafe(ssp);  in srcu_gp_start_if_needed()
1270 	ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state);  in srcu_gp_start_if_needed()
1272 		sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());  in srcu_gp_start_if_needed()
1274 		sdp = raw_cpu_ptr(ssp->sda);  in srcu_gp_start_if_needed()
1314 	s = rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start_if_needed()
1317 				      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_gp_start_if_needed()
1344 		srcu_funnel_gp_start(ssp, sdp, s, do_norm);  in srcu_gp_start_if_needed()
1346 		srcu_funnel_exp_start(ssp, sdp_mynode, s);  in srcu_gp_start_if_needed()
1347 	__srcu_read_unlock_nmisafe(ssp, idx);  in srcu_gp_start_if_needed()
1379 static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in __call_srcu()  argument
1389 	(void)srcu_gp_start_if_needed(ssp, rhp, do_norm);  in __call_srcu()
1394  * @ssp: srcu_struct in queue the callback
1413 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in call_srcu()  argument
1416 	__call_srcu(ssp, rhp, func, true);  in call_srcu()
1423 static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm)  in __synchronize_srcu()  argument
1427 	srcu_lock_sync(&ssp->dep_map);  in __synchronize_srcu()
1429 	RCU_LOCKDEP_WARN(lockdep_is_held(ssp) ||  in __synchronize_srcu()
1438 	check_init_srcu_struct(ssp);  in __synchronize_srcu()
1441 	__call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm);  in __synchronize_srcu()
1457  * @ssp: srcu_struct with which to synchronize.
1465 void synchronize_srcu_expedited(struct srcu_struct *ssp)  in synchronize_srcu_expedited()  argument
1467 	__synchronize_srcu(ssp, rcu_gp_is_normal());  in synchronize_srcu_expedited()
1473  * @ssp: srcu_struct with which to synchronize.
1477  * the index=!(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]) to drain to zero
1520 void synchronize_srcu(struct srcu_struct *ssp)  in synchronize_srcu()  argument
1522 	if (srcu_should_expedite(ssp) || rcu_gp_is_expedited())  in synchronize_srcu()
1523 		synchronize_srcu_expedited(ssp);  in synchronize_srcu()
1525 		__synchronize_srcu(ssp, true);  in synchronize_srcu()
1531  * @ssp: srcu_struct to provide cookie for.
1539 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)  in get_state_synchronize_srcu()  argument
1544 	return rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in get_state_synchronize_srcu()
1550  * @ssp: srcu_struct to provide cookie for.
1558 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)  in start_poll_synchronize_srcu()  argument
1560 	return srcu_gp_start_if_needed(ssp, NULL, true);  in start_poll_synchronize_srcu()
1566  * @ssp: srcu_struct to provide cookie for.
1589 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)  in poll_state_synchronize_srcu()  argument
1592 	    !rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, cookie))  in poll_state_synchronize_srcu()
1607 	struct srcu_struct *ssp;  in srcu_barrier_cb()  local
1611 	ssp = sdp->ssp;  in srcu_barrier_cb()
1612 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier_cb()
1613 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier_cb()
1624 static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp)  in srcu_barrier_one_cpu()  argument
1627 	atomic_inc(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1633 		atomic_dec(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1640  * @ssp: srcu_struct on which to wait for in-flight callbacks.
1642 void srcu_barrier(struct srcu_struct *ssp)  in srcu_barrier()  argument
1646 	unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1648 	check_init_srcu_struct(ssp);  in srcu_barrier()
1649 	mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1650 	if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) {  in srcu_barrier()
1652 		mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1655 	rcu_seq_start(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1656 	init_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1659 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 1);  in srcu_barrier()
1661 	idx = __srcu_read_lock_nmisafe(ssp);  in srcu_barrier()
1662 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_barrier()
1663 		srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda,	get_boot_cpu_id()));  in srcu_barrier()
1666 			srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, cpu));  in srcu_barrier()
1667 	__srcu_read_unlock_nmisafe(ssp, idx);  in srcu_barrier()
1670 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier()
1671 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1672 	wait_for_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1674 	rcu_seq_end(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1675 	mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1681  * @ssp: srcu_struct on which to report batch completion.
1686 unsigned long srcu_batches_completed(struct srcu_struct *ssp)  in srcu_batches_completed()  argument
1688 	return READ_ONCE(ssp->srcu_sup->srcu_gp_seq);  in srcu_batches_completed()
1697 static void srcu_advance_state(struct srcu_struct *ssp)  in srcu_advance_state()  argument
1701 	mutex_lock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1713 	idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */  in srcu_advance_state()
1715 		spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1716 		if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_advance_state()
1717 			WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1718 			spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1719 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1722 		idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1724 			srcu_gp_start(ssp);  in srcu_advance_state()
1725 		spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1727 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1732 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) {  in srcu_advance_state()
1733 		idx = !(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]);  in srcu_advance_state()
1734 		if (!try_check_zero(ssp, idx, 1)) {  in srcu_advance_state()
1735 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1738 		srcu_flip(ssp);  in srcu_advance_state()
1739 		spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1740 		rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);  in srcu_advance_state()
1741 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1742 		spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1745 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) {  in srcu_advance_state()
1751 		idx = !(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]);  in srcu_advance_state()
1752 		if (!try_check_zero(ssp, idx, 2)) {  in srcu_advance_state()
1753 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1756 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1757 		srcu_gp_end(ssp);  /* Releases ->srcu_gp_mutex. */  in srcu_advance_state()
1774 	struct srcu_struct *ssp;  in srcu_invoke_callbacks()  local
1778 	ssp = sdp->ssp;  in srcu_invoke_callbacks()
1783 			      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_invoke_callbacks()
1828 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay)  in srcu_reschedule()  argument
1832 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1833 	if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_reschedule()
1834 		if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq))) {  in srcu_reschedule()
1838 	} else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) {  in srcu_reschedule()
1840 		srcu_gp_start(ssp);  in srcu_reschedule()
1842 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1845 		queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay);  in srcu_reschedule()
1855 	struct srcu_struct *ssp;  in process_srcu()  local
1859 	ssp = sup->srcu_ssp;  in process_srcu()
1861 	srcu_advance_state(ssp);  in process_srcu()
1862 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in process_srcu()
1863 	curdelay = srcu_get_delay(ssp);  in process_srcu()
1864 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in process_srcu()
1879 	srcu_reschedule(ssp, curdelay);  in process_srcu()
1882 void srcutorture_get_gp_data(struct srcu_struct *ssp, int *flags,  in srcutorture_get_gp_data()  argument
1886 	*gp_seq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcutorture_get_gp_data()
1903 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)  in srcu_torture_stats_print()  argument
1908 	int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state);  in srcu_torture_stats_print()
1911 	idx = ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0];  in srcu_torture_stats_print()
1915 		 tt, tf, rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq), ss_state,  in srcu_torture_stats_print()
1917 	if (!ssp->sda) {  in srcu_torture_stats_print()
1928 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_torture_stats_print()
1952 		srcu_transition_to_big(ssp);  in srcu_torture_stats_print()
2008 	struct srcu_struct *ssp;  in srcu_module_coming()  local
2012 		ssp = *(sspp++);  in srcu_module_coming()
2013 		ssp->sda = alloc_percpu(struct srcu_data);  in srcu_module_coming()
2014 		if (WARN_ON_ONCE(!ssp->sda))  in srcu_module_coming()
2016 		ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0];  in srcu_module_coming()
2025 	struct srcu_struct *ssp;  in srcu_module_going()  local
2029 		ssp = *(sspp++);  in srcu_module_going()
2030 		if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) &&  in srcu_module_going()
2031 		    !WARN_ON_ONCE(!ssp->srcu_sup->sda_is_static))  in srcu_module_going()
2032 			cleanup_srcu_struct(ssp);  in srcu_module_going()
2033 		if (!WARN_ON(srcu_readers_active(ssp)))  in srcu_module_going()
2034 			free_percpu(ssp->sda);  in srcu_module_going()