92 	struct srcu_data *sdp;  in init_srcu_struct_data()  local
99 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_data()
100 		raw_spin_lock_init(&ACCESS_PRIVATE(sdp, lock));  in init_srcu_struct_data()
101 		rcu_segcblist_init(&sdp->srcu_cblist);  in init_srcu_struct_data()
102 		sdp->srcu_cblist_invoking = false;  in init_srcu_struct_data()
103 		sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
104 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
105 		sdp->srcu_barrier_head.next = &sdp->srcu_barrier_head;  in init_srcu_struct_data()
106 		sdp->mynode = NULL;  in init_srcu_struct_data()
107 		sdp->cpu = cpu;  in init_srcu_struct_data()
108 		INIT_WORK(&sdp->work, srcu_invoke_callbacks);  in init_srcu_struct_data()
109 		timer_setup(&sdp->delay_work, srcu_delay_timer, 0);  in init_srcu_struct_data()
110 		sdp->ssp = ssp;  in init_srcu_struct_data()
136 	struct srcu_data *sdp;  in init_srcu_struct_nodes()  local
186 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_nodes()
187 		sdp->mynode = &snp_first[cpu / levelspread[level]];  in init_srcu_struct_nodes()
188 		for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {  in init_srcu_struct_nodes()
193 		sdp->grpmask = 1UL << (cpu - sdp->mynode->grplo);  in init_srcu_struct_nodes()
402 static void raw_spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned long *flags)  in raw_spin_lock_irqsave_sdp_contention()  argument
404 	struct srcu_struct *ssp = sdp->ssp;  in raw_spin_lock_irqsave_sdp_contention()
406 	if (raw_spin_trylock_irqsave_rcu_node(sdp, *flags))  in raw_spin_lock_irqsave_sdp_contention()
411 	raw_spin_lock_irqsave_rcu_node(sdp, *flags);  in raw_spin_lock_irqsave_sdp_contention()
475 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_lock_idx()  local
477 		sum += atomic_long_read(&sdp->srcu_ctrs[idx].srcu_locks);  in srcu_readers_lock_idx()
479 			mask = mask | READ_ONCE(sdp->srcu_reader_flavor);  in srcu_readers_lock_idx()
499 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_unlock_idx()  local
501 		sum += atomic_long_read(&sdp->srcu_ctrs[idx].srcu_unlocks);  in srcu_readers_unlock_idx()
502 		mask = mask | READ_ONCE(sdp->srcu_reader_flavor);  in srcu_readers_unlock_idx()
616 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_active()  local
618 		sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks);  in srcu_readers_active()
619 		sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks);  in srcu_readers_active()
620 		sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks);  in srcu_readers_active()
621 		sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks);  in srcu_readers_active()
726 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in cleanup_srcu_struct()  local
728 		timer_delete_sync(&sdp->delay_work);  in cleanup_srcu_struct()
729 		flush_work(&sdp->work);  in cleanup_srcu_struct()
730 		if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist)))  in cleanup_srcu_struct()
762 	struct srcu_data *sdp;  in __srcu_check_read_flavor()  local
769 	sdp = raw_cpu_ptr(ssp->sda);  in __srcu_check_read_flavor()
770 	old_read_flavor = READ_ONCE(sdp->srcu_reader_flavor);  in __srcu_check_read_flavor()
776 		old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor);  in __srcu_check_read_flavor()
780 …WARN_ONCE(old_read_flavor != read_flavor, "CPU %d old state %d new state %d\n", sdp->cpu, old_read…  in __srcu_check_read_flavor()
864 	struct srcu_data *sdp = container_of(t, struct srcu_data, delay_work);  in srcu_delay_timer()  local
866 	queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);  in srcu_delay_timer()
869 static void srcu_queue_delayed_work_on(struct srcu_data *sdp,  in srcu_queue_delayed_work_on()  argument
873 		queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);  in srcu_queue_delayed_work_on()
877 	timer_reduce(&sdp->delay_work, jiffies + delay);  in srcu_queue_delayed_work_on()
884 static void srcu_schedule_cbs_sdp(struct srcu_data *sdp, unsigned long delay)  in srcu_schedule_cbs_sdp()  argument
886 	srcu_queue_delayed_work_on(sdp, delay);  in srcu_schedule_cbs_sdp()
925 	struct srcu_data *sdp;  in srcu_gp_end()  local
982 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_gp_end()
983 			raw_spin_lock_irq_rcu_node(sdp);  in srcu_gp_end()
984 			if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed + 100))  in srcu_gp_end()
985 				sdp->srcu_gp_seq_needed = gpseq;  in srcu_gp_end()
986 			if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed_exp + 100))  in srcu_gp_end()
987 				sdp->srcu_gp_seq_needed_exp = gpseq;  in srcu_gp_end()
988 			raw_spin_unlock_irq_rcu_node(sdp);  in srcu_gp_end()
1062 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,  in srcu_funnel_gp_start()  argument
1066 	int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);  in srcu_funnel_gp_start()
1077 		snp_leaf = sdp->mynode;  in srcu_funnel_gp_start()
1088 					snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
1091 					srcu_schedule_cbs_sdp(sdp, do_norm ? SRCU_INTERVAL : 0);  in srcu_funnel_gp_start()
1100 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
1254 	struct srcu_data *sdp;  in srcu_should_expedite()  local
1263 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_should_expedite()
1264 	raw_spin_lock_irqsave_rcu_node(sdp, flags);  in srcu_should_expedite()
1265 	if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) {  in srcu_should_expedite()
1266 		raw_spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_should_expedite()
1269 	raw_spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_should_expedite()
1313 	struct srcu_data *sdp;  in srcu_gp_start_if_needed()  local
1326 		sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());  in srcu_gp_start_if_needed()
1328 		sdp = raw_cpu_ptr(ssp->sda);  in srcu_gp_start_if_needed()
1329 	raw_spin_lock_irqsave_sdp_contention(sdp, &flags);  in srcu_gp_start_if_needed()
1331 		rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp);  in srcu_gp_start_if_needed()
1370 		rcu_segcblist_advance(&sdp->srcu_cblist,  in srcu_gp_start_if_needed()
1379 		WARN_ON_ONCE(!rcu_segcblist_accelerate(&sdp->srcu_cblist, s));  in srcu_gp_start_if_needed()
1381 	if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) {  in srcu_gp_start_if_needed()
1382 		sdp->srcu_gp_seq_needed = s;  in srcu_gp_start_if_needed()
1385 	if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) {  in srcu_gp_start_if_needed()
1386 		sdp->srcu_gp_seq_needed_exp = s;  in srcu_gp_start_if_needed()
1389 	raw_spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_gp_start_if_needed()
1395 		sdp_mynode = sdp->mynode;  in srcu_gp_start_if_needed()
1398 		srcu_funnel_gp_start(ssp, sdp, s, do_norm);  in srcu_gp_start_if_needed()
1660 	struct srcu_data *sdp;  in srcu_barrier_cb()  local
1664 	sdp = container_of(rhp, struct srcu_data, srcu_barrier_head);  in srcu_barrier_cb()
1665 	ssp = sdp->ssp;  in srcu_barrier_cb()
1678 static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp)  in srcu_barrier_one_cpu()  argument
1680 	raw_spin_lock_irq_rcu_node(sdp);  in srcu_barrier_one_cpu()
1682 	sdp->srcu_barrier_head.func = srcu_barrier_cb;  in srcu_barrier_one_cpu()
1683 	debug_rcu_head_queue(&sdp->srcu_barrier_head);  in srcu_barrier_one_cpu()
1684 	if (!rcu_segcblist_entrain(&sdp->srcu_cblist,  in srcu_barrier_one_cpu()
1685 				   &sdp->srcu_barrier_head)) {  in srcu_barrier_one_cpu()
1686 		debug_rcu_head_unqueue(&sdp->srcu_barrier_head);  in srcu_barrier_one_cpu()
1689 	raw_spin_unlock_irq_rcu_node(sdp);  in srcu_barrier_one_cpu()
1738 	struct srcu_data *sdp = container_of(rhp, struct srcu_data, srcu_ec_head);  in srcu_expedite_current_cb()  local
1740 	raw_spin_lock_irqsave_sdp_contention(sdp, &flags);  in srcu_expedite_current_cb()
1741 	if (sdp->srcu_ec_state == SRCU_EC_IDLE) {  in srcu_expedite_current_cb()
1743 	} else if (sdp->srcu_ec_state == SRCU_EC_PENDING) {  in srcu_expedite_current_cb()
1744 		sdp->srcu_ec_state = SRCU_EC_IDLE;  in srcu_expedite_current_cb()
1746 		WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST);  in srcu_expedite_current_cb()
1747 		sdp->srcu_ec_state = SRCU_EC_PENDING;  in srcu_expedite_current_cb()
1750 	raw_spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_expedite_current_cb()
1753 		__call_srcu(sdp->ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false);  in srcu_expedite_current_cb()
1770 	struct srcu_data *sdp;  in srcu_expedite_current()  local
1773 	sdp = this_cpu_ptr(ssp->sda);  in srcu_expedite_current()
1774 	raw_spin_lock_irqsave_sdp_contention(sdp, &flags);  in srcu_expedite_current()
1775 	if (sdp->srcu_ec_state == SRCU_EC_IDLE) {  in srcu_expedite_current()
1776 		sdp->srcu_ec_state = SRCU_EC_PENDING;  in srcu_expedite_current()
1778 	} else if (sdp->srcu_ec_state == SRCU_EC_PENDING) {  in srcu_expedite_current()
1779 		sdp->srcu_ec_state = SRCU_EC_REPOST;  in srcu_expedite_current()
1781 		WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST);  in srcu_expedite_current()
1783 	raw_spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_expedite_current()
1786 		__call_srcu(ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false);  in srcu_expedite_current()
1885 	struct srcu_data *sdp;  in srcu_invoke_callbacks()  local
1888 	sdp = container_of(work, struct srcu_data, work);  in srcu_invoke_callbacks()
1890 	ssp = sdp->ssp;  in srcu_invoke_callbacks()
1892 	raw_spin_lock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1893 	WARN_ON_ONCE(!rcu_segcblist_segempty(&sdp->srcu_cblist, RCU_NEXT_TAIL));  in srcu_invoke_callbacks()
1894 	rcu_segcblist_advance(&sdp->srcu_cblist,  in srcu_invoke_callbacks()
1901 	if (sdp->srcu_cblist_invoking ||  in srcu_invoke_callbacks()
1902 	    !rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) {  in srcu_invoke_callbacks()
1903 		raw_spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1908 	sdp->srcu_cblist_invoking = true;  in srcu_invoke_callbacks()
1909 	rcu_segcblist_extract_done_cbs(&sdp->srcu_cblist, &ready_cbs);  in srcu_invoke_callbacks()
1911 	raw_spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1926 	raw_spin_lock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1927 	rcu_segcblist_add_len(&sdp->srcu_cblist, -len);  in srcu_invoke_callbacks()
1928 	sdp->srcu_cblist_invoking = false;  in srcu_invoke_callbacks()
1929 	more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist);  in srcu_invoke_callbacks()
1930 	raw_spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1933 		srcu_schedule_cbs_sdp(sdp, 0);  in srcu_invoke_callbacks()
2055 			struct srcu_data *sdp;  in srcu_torture_stats_print()  local
2057 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_torture_stats_print()
2058 			u0 = data_race(atomic_long_read(&sdp->srcu_ctrs[!idx].srcu_unlocks));  in srcu_torture_stats_print()
2059 			u1 = data_race(atomic_long_read(&sdp->srcu_ctrs[idx].srcu_unlocks));  in srcu_torture_stats_print()
2067 			l0 = data_race(atomic_long_read(&sdp->srcu_ctrs[!idx].srcu_locks));  in srcu_torture_stats_print()
2068 			l1 = data_race(atomic_long_read(&sdp->srcu_ctrs[idx].srcu_locks));  in srcu_torture_stats_print()
2074 				"C."[rcu_segcblist_empty(&sdp->srcu_cblist)]);  in srcu_torture_stats_print()