31 static int init_srcu_struct_fields(struct srcu_struct *ssp)  in init_srcu_struct_fields()  argument
33 	ssp->srcu_lock_nesting[0] = 0;  in init_srcu_struct_fields()
34 	ssp->srcu_lock_nesting[1] = 0;  in init_srcu_struct_fields()
35 	init_swait_queue_head(&ssp->srcu_wq);  in init_srcu_struct_fields()
36 	ssp->srcu_cb_head = NULL;  in init_srcu_struct_fields()
37 	ssp->srcu_cb_tail = &ssp->srcu_cb_head;  in init_srcu_struct_fields()
38 	ssp->srcu_gp_running = false;  in init_srcu_struct_fields()
39 	ssp->srcu_gp_waiting = false;  in init_srcu_struct_fields()
40 	ssp->srcu_idx = 0;  in init_srcu_struct_fields()
41 	ssp->srcu_idx_max = 0;  in init_srcu_struct_fields()
42 	INIT_WORK(&ssp->srcu_work, srcu_drive_gp);  in init_srcu_struct_fields()
43 	INIT_LIST_HEAD(&ssp->srcu_work.entry);  in init_srcu_struct_fields()
49 int __init_srcu_struct(struct srcu_struct *ssp, const char *name,  in __init_srcu_struct()  argument
53 	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));  in __init_srcu_struct()
54 	lockdep_init_map(&ssp->dep_map, name, key, 0);  in __init_srcu_struct()
55 	return init_srcu_struct_fields(ssp);  in __init_srcu_struct()
63  * @ssp: structure to initialize.
69 int init_srcu_struct(struct srcu_struct *ssp)  in init_srcu_struct()  argument
71 	return init_srcu_struct_fields(ssp);  in init_srcu_struct()
79  * @ssp: structure to clean up.
84 void cleanup_srcu_struct(struct srcu_struct *ssp)  in cleanup_srcu_struct()  argument
86 	WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]);  in cleanup_srcu_struct()
87 	flush_work(&ssp->srcu_work);  in cleanup_srcu_struct()
88 	WARN_ON(ssp->srcu_gp_running);  in cleanup_srcu_struct()
89 	WARN_ON(ssp->srcu_gp_waiting);  in cleanup_srcu_struct()
90 	WARN_ON(ssp->srcu_cb_head);  in cleanup_srcu_struct()
91 	WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);  in cleanup_srcu_struct()
92 	WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max);  in cleanup_srcu_struct()
93 	WARN_ON(ssp->srcu_idx & 0x1);  in cleanup_srcu_struct()
101 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock()  argument
106 	newval = READ_ONCE(ssp->srcu_lock_nesting[idx]) - 1;  in __srcu_read_unlock()
107 	WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);  in __srcu_read_unlock()
109 	if (!newval && READ_ONCE(ssp->srcu_gp_waiting) && in_task())  in __srcu_read_unlock()
110 		swake_up_one(&ssp->srcu_wq);  in __srcu_read_unlock()
124 	struct srcu_struct *ssp;  in srcu_drive_gp()  local
126 	ssp = container_of(wp, struct srcu_struct, srcu_work);  in srcu_drive_gp()
128 	if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) {  in srcu_drive_gp()
134 	WRITE_ONCE(ssp->srcu_gp_running, true);  in srcu_drive_gp()
136 	lh = ssp->srcu_cb_head;  in srcu_drive_gp()
137 	ssp->srcu_cb_head = NULL;  in srcu_drive_gp()
138 	ssp->srcu_cb_tail = &ssp->srcu_cb_head;  in srcu_drive_gp()
140 	idx = (ssp->srcu_idx & 0x2) / 2;  in srcu_drive_gp()
141 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);  in srcu_drive_gp()
142 	WRITE_ONCE(ssp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */  in srcu_drive_gp()
144 	swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));  in srcu_drive_gp()
146 	WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */  in srcu_drive_gp()
147 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);  in srcu_drive_gp()
167 	WRITE_ONCE(ssp->srcu_gp_running, false);  in srcu_drive_gp()
168 	idx = ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max));  in srcu_drive_gp()
171 		schedule_work(&ssp->srcu_work);  in srcu_drive_gp()
175 static void srcu_gp_start_if_needed(struct srcu_struct *ssp)  in srcu_gp_start_if_needed()  argument
180 	cookie = get_state_synchronize_srcu(ssp);  in srcu_gp_start_if_needed()
181 	if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) {  in srcu_gp_start_if_needed()
185 	WRITE_ONCE(ssp->srcu_idx_max, cookie);  in srcu_gp_start_if_needed()
186 	if (!READ_ONCE(ssp->srcu_gp_running)) {  in srcu_gp_start_if_needed()
188 			schedule_work(&ssp->srcu_work);  in srcu_gp_start_if_needed()
189 		else if (list_empty(&ssp->srcu_work.entry))  in srcu_gp_start_if_needed()
190 			list_add(&ssp->srcu_work.entry, &srcu_boot_list);  in srcu_gp_start_if_needed()
199 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in call_srcu()  argument
208 	*ssp->srcu_cb_tail = rhp;  in call_srcu()
209 	ssp->srcu_cb_tail = &rhp->next;  in call_srcu()
211 	srcu_gp_start_if_needed(ssp);  in call_srcu()
219 void synchronize_srcu(struct srcu_struct *ssp)  in synchronize_srcu()  argument
223 	srcu_lock_sync(&ssp->dep_map);  in synchronize_srcu()
225 	RCU_LOCKDEP_WARN(lockdep_is_held(ssp) ||  in synchronize_srcu()
237 	call_srcu(ssp, &rs.head, wakeme_after_rcu);  in synchronize_srcu()
246 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)  in get_state_synchronize_srcu()  argument
251 	ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;  in get_state_synchronize_srcu()
264 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)  in start_poll_synchronize_srcu()  argument
269 	ret = get_state_synchronize_srcu(ssp);  in start_poll_synchronize_srcu()
270 	srcu_gp_start_if_needed(ssp);  in start_poll_synchronize_srcu()
279 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)  in poll_state_synchronize_srcu()  argument
281 	unsigned long cur_s = READ_ONCE(ssp->srcu_idx);  in poll_state_synchronize_srcu()
304 	struct srcu_struct *ssp;  in srcu_init()  local
308 		ssp = list_first_entry(&srcu_boot_list,  in srcu_init()
310 		list_del_init(&ssp->srcu_work.entry);  in srcu_init()
311 		schedule_work(&ssp->srcu_work);  in srcu_init()