76 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay);
122 static void init_srcu_struct_data(struct srcu_struct *ssp)  in init_srcu_struct_data()  argument
134 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_data()
138 		sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
139 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
144 		sdp->ssp = ssp;  in init_srcu_struct_data()
164 static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)  in init_srcu_struct_nodes()  argument
176 	ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags);  in init_srcu_struct_nodes()
177 	if (!ssp->srcu_sup->node)  in init_srcu_struct_nodes()
181 	ssp->srcu_sup->level[0] = &ssp->srcu_sup->node[0];  in init_srcu_struct_nodes()
183 		ssp->srcu_sup->level[i] = ssp->srcu_sup->level[i - 1] + num_rcu_lvl[i - 1];  in init_srcu_struct_nodes()
187 	srcu_for_each_node_breadth_first(ssp, snp) {  in init_srcu_struct_nodes()
198 		if (snp == &ssp->srcu_sup->node[0]) {  in init_srcu_struct_nodes()
205 		if (snp == ssp->srcu_sup->level[level + 1])  in init_srcu_struct_nodes()
207 		snp->srcu_parent = ssp->srcu_sup->level[level - 1] +  in init_srcu_struct_nodes()
208 				   (snp - ssp->srcu_sup->level[level]) /  in init_srcu_struct_nodes()
217 	snp_first = ssp->srcu_sup->level[level];  in init_srcu_struct_nodes()
219 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_nodes()
228 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_WAIT_BARRIER);  in init_srcu_struct_nodes()
237 static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)  in init_srcu_struct_fields()  argument
240 		ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL);  in init_srcu_struct_fields()
241 	if (!ssp->srcu_sup)  in init_srcu_struct_fields()
244 		spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in init_srcu_struct_fields()
245 	ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;  in init_srcu_struct_fields()
246 	ssp->srcu_sup->node = NULL;  in init_srcu_struct_fields()
247 	mutex_init(&ssp->srcu_sup->srcu_cb_mutex);  in init_srcu_struct_fields()
248 	mutex_init(&ssp->srcu_sup->srcu_gp_mutex);  in init_srcu_struct_fields()
249 	ssp->srcu_idx = 0;  in init_srcu_struct_fields()
250 	ssp->srcu_sup->srcu_gp_seq = 0;  in init_srcu_struct_fields()
251 	ssp->srcu_sup->srcu_barrier_seq = 0;  in init_srcu_struct_fields()
252 	mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);  in init_srcu_struct_fields()
253 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);  in init_srcu_struct_fields()
254 	INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);  in init_srcu_struct_fields()
255 	ssp->srcu_sup->sda_is_static = is_static;  in init_srcu_struct_fields()
257 		ssp->sda = alloc_percpu(struct srcu_data);  in init_srcu_struct_fields()
258 	if (!ssp->sda)  in init_srcu_struct_fields()
260 	init_srcu_struct_data(ssp);  in init_srcu_struct_fields()
261 	ssp->srcu_sup->srcu_gp_seq_needed_exp = 0;  in init_srcu_struct_fields()
262 	ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();  in init_srcu_struct_fields()
263 	if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {  in init_srcu_struct_fields()
264 		if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC))  in init_srcu_struct_fields()
266 		WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);  in init_srcu_struct_fields()
268 	ssp->srcu_sup->srcu_ssp = ssp;  in init_srcu_struct_fields()
269 	smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */  in init_srcu_struct_fields()
274 		free_percpu(ssp->sda);  in init_srcu_struct_fields()
275 		ssp->sda = NULL;  in init_srcu_struct_fields()
279 		kfree(ssp->srcu_sup);  in init_srcu_struct_fields()
280 		ssp->srcu_sup = NULL;  in init_srcu_struct_fields()
287 int __init_srcu_struct(struct srcu_struct *ssp, const char *name,  in __init_srcu_struct()  argument
291 	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));  in __init_srcu_struct()
292 	lockdep_init_map(&ssp->dep_map, name, key, 0);  in __init_srcu_struct()
293 	return init_srcu_struct_fields(ssp, false);  in __init_srcu_struct()
301  * @ssp: structure to initialize.
307 int init_srcu_struct(struct srcu_struct *ssp)  in init_srcu_struct()  argument
309 	return init_srcu_struct_fields(ssp, false);  in init_srcu_struct()
318 static void __srcu_transition_to_big(struct srcu_struct *ssp)  in __srcu_transition_to_big()  argument
320 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in __srcu_transition_to_big()
321 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC);  in __srcu_transition_to_big()
327 static void srcu_transition_to_big(struct srcu_struct *ssp)  in srcu_transition_to_big()  argument
332 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL)  in srcu_transition_to_big()
334 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
335 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) {  in srcu_transition_to_big()
336 		spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
339 	__srcu_transition_to_big(ssp);  in srcu_transition_to_big()
340 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
347 static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp)  in spin_lock_irqsave_check_contention()  argument
351 	if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state)  in spin_lock_irqsave_check_contention()
354 	if (ssp->srcu_sup->srcu_size_jiffies != j) {  in spin_lock_irqsave_check_contention()
355 		ssp->srcu_sup->srcu_size_jiffies = j;  in spin_lock_irqsave_check_contention()
356 		ssp->srcu_sup->srcu_n_lock_retries = 0;  in spin_lock_irqsave_check_contention()
358 	if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim)  in spin_lock_irqsave_check_contention()
360 	__srcu_transition_to_big(ssp);  in spin_lock_irqsave_check_contention()
371 	struct srcu_struct *ssp = sdp->ssp;  in spin_lock_irqsave_sdp_contention()  local
375 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_sdp_contention()
376 	spin_lock_irqsave_check_contention(ssp);  in spin_lock_irqsave_sdp_contention()
377 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_sdp_contention()
387 static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags)  in spin_lock_irqsave_ssp_contention()  argument
389 	if (spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags))  in spin_lock_irqsave_ssp_contention()
391 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_ssp_contention()
392 	spin_lock_irqsave_check_contention(ssp);  in spin_lock_irqsave_ssp_contention()
399  * to each update-side SRCU primitive.  Use ssp->lock, which -is-
403 static void check_init_srcu_struct(struct srcu_struct *ssp)  in check_init_srcu_struct()  argument
408 	if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed))) /*^^^*/  in check_init_srcu_struct()
410 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
411 	if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq_needed)) {  in check_init_srcu_struct()
412 		spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
415 	init_srcu_struct_fields(ssp, true);  in check_init_srcu_struct()
416 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
423 static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_lock_idx()  argument
429 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_lock_idx()
440 static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_unlock_idx()  argument
447 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_unlock_idx()
454 		  "Mixed NMI-safe readers for srcu_struct at %ps.\n", ssp);  in srcu_readers_unlock_idx()
462 static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx)  in srcu_readers_active_idx_check()  argument
466 	unlocks = srcu_readers_unlock_idx(ssp, idx);  in srcu_readers_active_idx_check()
537 	return srcu_readers_lock_idx(ssp, idx) == unlocks;  in srcu_readers_active_idx_check()
543  * @ssp: which srcu_struct to count active readers (holding srcu_read_lock).
549 static bool srcu_readers_active(struct srcu_struct *ssp)  in srcu_readers_active()  argument
555 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_active()
616 static unsigned long srcu_get_delay(struct srcu_struct *ssp)  in srcu_get_delay()  argument
621 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_get_delay()
641  * @ssp: structure to clean up.
646 void cleanup_srcu_struct(struct srcu_struct *ssp)  in cleanup_srcu_struct()  argument
649 	struct srcu_usage *sup = ssp->srcu_sup;  in cleanup_srcu_struct()
651 	if (WARN_ON(!srcu_get_delay(ssp)))  in cleanup_srcu_struct()
653 	if (WARN_ON(srcu_readers_active(ssp)))  in cleanup_srcu_struct()
657 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in cleanup_srcu_struct()
666 	    WARN_ON(srcu_readers_active(ssp))) {  in cleanup_srcu_struct()
668 			__func__, ssp, rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)),  in cleanup_srcu_struct()
676 		free_percpu(ssp->sda);  in cleanup_srcu_struct()
677 		ssp->sda = NULL;  in cleanup_srcu_struct()
679 		ssp->srcu_sup = NULL;  in cleanup_srcu_struct()
688 void srcu_check_nmi_safety(struct srcu_struct *ssp, bool nmi_safe)  in srcu_check_nmi_safety()  argument
696 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_check_nmi_safety()
712 int __srcu_read_lock(struct srcu_struct *ssp)  in __srcu_read_lock()  argument
716 	idx = READ_ONCE(ssp->srcu_idx) & 0x1;  in __srcu_read_lock()
717 	this_cpu_inc(ssp->sda->srcu_lock_count[idx].counter);  in __srcu_read_lock()
728 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock()  argument
731 	this_cpu_inc(ssp->sda->srcu_unlock_count[idx].counter);  in __srcu_read_unlock()
742 int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)  in __srcu_read_lock_nmisafe()  argument
745 	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);  in __srcu_read_lock_nmisafe()
747 	idx = READ_ONCE(ssp->srcu_idx) & 0x1;  in __srcu_read_lock_nmisafe()
759 void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock_nmisafe()  argument
761 	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);  in __srcu_read_unlock_nmisafe()
773 static void srcu_gp_start(struct srcu_struct *ssp)  in srcu_gp_start()  argument
777 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in srcu_gp_start()
778 	WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed));  in srcu_gp_start()
779 	WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies);  in srcu_gp_start()
780 	WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0);  in srcu_gp_start()
782 	rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
783 	state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
821 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_schedule_cbs_snp()  argument
829 		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay);  in srcu_schedule_cbs_snp()
842 static void srcu_gp_end(struct srcu_struct *ssp)  in srcu_gp_end()  argument
856 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_end()
880 		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()),  in srcu_gp_end()
884 		srcu_for_each_node_breadth_first(ssp, snp) {  in srcu_gp_end()
902 				srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay);  in srcu_gp_end()
909 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_gp_end()
926 		srcu_gp_start(ssp);  in srcu_gp_end()
928 		srcu_reschedule(ssp, 0);  in srcu_gp_end()
936 			init_srcu_struct_nodes(ssp, GFP_KERNEL);  in srcu_gp_end()
949 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_funnel_exp_start()  argument
958 			if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, s)) ||  in srcu_funnel_exp_start()
970 	spin_lock_irqsave_ssp_contention(ssp, &flags);  in srcu_funnel_exp_start()
971 	if (ULONG_CMP_LT(ssp->srcu_sup->srcu_gp_seq_needed_exp, s))  in srcu_funnel_exp_start()
972 		WRITE_ONCE(ssp->srcu_sup->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
973 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_funnel_exp_start()
989 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,  in srcu_funnel_gp_start()  argument
998 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_funnel_gp_start()
1022 					srcu_funnel_exp_start(ssp, snp, s);  in srcu_funnel_gp_start()
1035 	spin_lock_irqsave_ssp_contention(ssp, &flags);  in srcu_funnel_gp_start()
1050 		srcu_gp_start(ssp);  in srcu_funnel_gp_start()
1059 					   !!srcu_get_delay(ssp));  in srcu_funnel_gp_start()
1071 static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)  in try_check_zero()  argument
1075 	curdelay = !srcu_get_delay(ssp);  in try_check_zero()
1078 		if (srcu_readers_active_idx_check(ssp, idx))  in try_check_zero()
1091 static void srcu_flip(struct srcu_struct *ssp)  in srcu_flip()  argument
1123 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); // Flip the counter.  in srcu_flip()
1157 static bool srcu_might_be_idle(struct srcu_struct *ssp)  in srcu_might_be_idle()  argument
1165 	check_init_srcu_struct(ssp);  in srcu_might_be_idle()
1167 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_might_be_idle()
1183 	tlast = READ_ONCE(ssp->srcu_sup->srcu_last_gp_end);  in srcu_might_be_idle()
1189 	curseq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcu_might_be_idle()
1191 	if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_sup->srcu_gp_seq_needed)))  in srcu_might_be_idle()
1194 	if (curseq != rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq))  in srcu_might_be_idle()
1209 static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,  in srcu_gp_start_if_needed()  argument
1221 	check_init_srcu_struct(ssp);  in srcu_gp_start_if_needed()
1227 	idx = __srcu_read_lock_nmisafe(ssp);  in srcu_gp_start_if_needed()
1228 	ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state);  in srcu_gp_start_if_needed()
1230 		sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());  in srcu_gp_start_if_needed()
1232 		sdp = raw_cpu_ptr(ssp->sda);  in srcu_gp_start_if_needed()
1263 	s = rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start_if_needed()
1266 				      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_gp_start_if_needed()
1286 		srcu_funnel_gp_start(ssp, sdp, s, do_norm);  in srcu_gp_start_if_needed()
1288 		srcu_funnel_exp_start(ssp, sdp_mynode, s);  in srcu_gp_start_if_needed()
1289 	__srcu_read_unlock_nmisafe(ssp, idx);  in srcu_gp_start_if_needed()
1321 static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in __call_srcu()  argument
1331 	(void)srcu_gp_start_if_needed(ssp, rhp, do_norm);  in __call_srcu()
1336  * @ssp: srcu_struct in queue the callback
1351 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in call_srcu()  argument
1354 	__call_srcu(ssp, rhp, func, true);  in call_srcu()
1361 static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm)  in __synchronize_srcu()  argument
1365 	srcu_lock_sync(&ssp->dep_map);  in __synchronize_srcu()
1367 	RCU_LOCKDEP_WARN(lockdep_is_held(ssp) ||  in __synchronize_srcu()
1376 	check_init_srcu_struct(ssp);  in __synchronize_srcu()
1379 	__call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm);  in __synchronize_srcu()
1395  * @ssp: srcu_struct with which to synchronize.
1403 void synchronize_srcu_expedited(struct srcu_struct *ssp)  in synchronize_srcu_expedited()  argument
1405 	__synchronize_srcu(ssp, rcu_gp_is_normal());  in synchronize_srcu_expedited()
1411  * @ssp: srcu_struct with which to synchronize.
1456 void synchronize_srcu(struct srcu_struct *ssp)  in synchronize_srcu()  argument
1458 	if (srcu_might_be_idle(ssp) || rcu_gp_is_expedited())  in synchronize_srcu()
1459 		synchronize_srcu_expedited(ssp);  in synchronize_srcu()
1461 		__synchronize_srcu(ssp, true);  in synchronize_srcu()
1467  * @ssp: srcu_struct to provide cookie for.
1475 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)  in get_state_synchronize_srcu()  argument
1480 	return rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in get_state_synchronize_srcu()
1486  * @ssp: srcu_struct to provide cookie for.
1494 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)  in start_poll_synchronize_srcu()  argument
1496 	return srcu_gp_start_if_needed(ssp, NULL, true);  in start_poll_synchronize_srcu()
1502  * @ssp: srcu_struct to provide cookie for.
1525 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)  in poll_state_synchronize_srcu()  argument
1527 	if (!rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, cookie))  in poll_state_synchronize_srcu()
1542 	struct srcu_struct *ssp;  in srcu_barrier_cb()  local
1545 	ssp = sdp->ssp;  in srcu_barrier_cb()
1546 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier_cb()
1547 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier_cb()
1558 static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp)  in srcu_barrier_one_cpu()  argument
1561 	atomic_inc(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1567 		atomic_dec(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1574  * @ssp: srcu_struct on which to wait for in-flight callbacks.
1576 void srcu_barrier(struct srcu_struct *ssp)  in srcu_barrier()  argument
1580 	unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1582 	check_init_srcu_struct(ssp);  in srcu_barrier()
1583 	mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1584 	if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) {  in srcu_barrier()
1586 		mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1589 	rcu_seq_start(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1590 	init_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1593 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 1);  in srcu_barrier()
1595 	idx = __srcu_read_lock_nmisafe(ssp);  in srcu_barrier()
1596 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_barrier()
1597 		srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda,	get_boot_cpu_id()));  in srcu_barrier()
1600 			srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, cpu));  in srcu_barrier()
1601 	__srcu_read_unlock_nmisafe(ssp, idx);  in srcu_barrier()
1604 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier()
1605 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1606 	wait_for_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1608 	rcu_seq_end(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1609 	mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1615  * @ssp: srcu_struct on which to report batch completion.
1620 unsigned long srcu_batches_completed(struct srcu_struct *ssp)  in srcu_batches_completed()  argument
1622 	return READ_ONCE(ssp->srcu_idx);  in srcu_batches_completed()
1631 static void srcu_advance_state(struct srcu_struct *ssp)  in srcu_advance_state()  argument
1635 	mutex_lock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1647 	idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */  in srcu_advance_state()
1649 		spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1650 		if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_advance_state()
1651 			WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1652 			spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1653 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1656 		idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1658 			srcu_gp_start(ssp);  in srcu_advance_state()
1659 		spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1661 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1666 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) {  in srcu_advance_state()
1667 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1668 		if (!try_check_zero(ssp, idx, 1)) {  in srcu_advance_state()
1669 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1672 		srcu_flip(ssp);  in srcu_advance_state()
1673 		spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1674 		rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);  in srcu_advance_state()
1675 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1676 		spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1679 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) {  in srcu_advance_state()
1685 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1686 		if (!try_check_zero(ssp, idx, 2)) {  in srcu_advance_state()
1687 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1690 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1691 		srcu_gp_end(ssp);  /* Releases ->srcu_gp_mutex. */  in srcu_advance_state()
1708 	struct srcu_struct *ssp;  in srcu_invoke_callbacks()  local
1712 	ssp = sdp->ssp;  in srcu_invoke_callbacks()
1717 			      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_invoke_callbacks()
1762 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay)  in srcu_reschedule()  argument
1766 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1767 	if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_reschedule()
1768 		if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq))) {  in srcu_reschedule()
1772 	} else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) {  in srcu_reschedule()
1774 		srcu_gp_start(ssp);  in srcu_reschedule()
1776 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1779 		queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay);  in srcu_reschedule()
1789 	struct srcu_struct *ssp;  in process_srcu()  local
1793 	ssp = sup->srcu_ssp;  in process_srcu()
1795 	srcu_advance_state(ssp);  in process_srcu()
1796 	curdelay = srcu_get_delay(ssp);  in process_srcu()
1810 	srcu_reschedule(ssp, curdelay);  in process_srcu()
1814 			     struct srcu_struct *ssp, int *flags,  in srcutorture_get_gp_data()  argument
1820 	*gp_seq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcutorture_get_gp_data()
1837 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)  in srcu_torture_stats_print()  argument
1842 	int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state);  in srcu_torture_stats_print()
1845 	idx = ssp->srcu_idx & 0x1;  in srcu_torture_stats_print()
1849 		 tt, tf, rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq), ss_state,  in srcu_torture_stats_print()
1851 	if (!ssp->sda) {  in srcu_torture_stats_print()
1862 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_torture_stats_print()
1886 		srcu_transition_to_big(ssp);  in srcu_torture_stats_print()
1942 	struct srcu_struct *ssp;  in srcu_module_coming()  local
1946 		ssp = *(sspp++);  in srcu_module_coming()
1947 		ssp->sda = alloc_percpu(struct srcu_data);  in srcu_module_coming()
1948 		if (WARN_ON_ONCE(!ssp->sda))  in srcu_module_coming()
1958 	struct srcu_struct *ssp;  in srcu_module_going()  local
1962 		ssp = *(sspp++);  in srcu_module_going()
1963 		if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) &&  in srcu_module_going()
1964 		    !WARN_ON_ONCE(!ssp->srcu_sup->sda_is_static))  in srcu_module_going()
1965 			cleanup_srcu_struct(ssp);  in srcu_module_going()
1966 		if (!WARN_ON(srcu_readers_active(ssp)))  in srcu_module_going()
1967 			free_percpu(ssp->sda);  in srcu_module_going()