8  * RCU-protected list version
10 #include <linux/list.h>
16  * and compares it to the address of the list head, but neither dereferences
24  * @list: list to be initialized
27  * cleanup tasks, when readers have no access to the list being initialized.
28  * However, if the list being initialized is visible to readers, you
31 static inline void INIT_LIST_HEAD_RCU(struct list_head *list)  in INIT_LIST_HEAD_RCU()  argument
33 	WRITE_ONCE(list->next, list);  in INIT_LIST_HEAD_RCU()
34 	WRITE_ONCE(list->prev, list);  in INIT_LIST_HEAD_RCU()
41 #define list_next_rcu(list)	(*((struct list_head __rcu **)(&(list)->next)))  argument
44  * list_tail_rcu - returns the prev pointer of the head of the list
45  * @head: the head of the list
47  * Note: This should only be used with the list header, and even then
49  * list header.
54  * Check during list traversal that we are within an RCU reader
64 			 "RCU-list traversed in non-reader section!");	\
70 		"RCU-list traversed without holding the required lock!");\
82  * This is only for internal list manipulation where we know
98  * list_add_rcu - add a new entry to rcu-protected list
100  * @head: list head to add it after
107  * with another list-mutation primitive, such as list_add_rcu()
108  * or list_del_rcu(), running on this same list.
110  * the _rcu list-traversal primitives, such as
119  * list_add_tail_rcu - add a new entry to rcu-protected list
121  * @head: list head to add it before
128  * with another list-mutation primitive, such as list_add_tail_rcu()
129  * or list_del_rcu(), running on this same list.
131  * the _rcu list-traversal primitives, such as
141  * list_del_rcu - deletes entry from list without re-initialization
142  * @entry: the element to delete from the list.
149  * pointers that may still be used for walking the list.
153  * with another list-mutation primitive, such as list_del_rcu()
154  * or list_add_rcu(), running on this same list.
156  * the _rcu list-traversal primitives, such as
171  * hlist_del_init_rcu - deletes entry from hash list with re-initialization
172  * @n: the element to delete from the hash list.
176  * must know if the list entry is still hashed or already unhashed.
179  * that may still be used for walking the hash list and we can only
185  * list-mutation primitive, such as hlist_add_head_rcu() or
186  * hlist_del_rcu(), running on this same list.  However, it is
187  * perfectly legal to run concurrently with the _rcu list-traversal
217  * __list_splice_init_rcu - join an RCU-protected list into an existing list.
218  * @list:	the RCU-protected list to splice
219  * @prev:	points to the last element of the existing list
220  * @next:	points to the first element of the existing list
223  * The list pointed to by @prev and @next can be RCU-read traversed
229  * any other updates to the existing list.  In principle, it is possible to
230  * modify the list as soon as sync() begins execution. If this sort of thing
235 static inline void __list_splice_init_rcu(struct list_head *list,  in __list_splice_init_rcu()  argument
240 	struct list_head *first = list->next;  in __list_splice_init_rcu()
241 	struct list_head *last = list->prev;  in __list_splice_init_rcu()
244 	 * "first" and "last" tracking list, so initialize it.  RCU readers  in __list_splice_init_rcu()
245 	 * have access to this list, so we must use INIT_LIST_HEAD_RCU()  in __list_splice_init_rcu()
249 	INIT_LIST_HEAD_RCU(list);  in __list_splice_init_rcu()
252 	 * At this point, the list body still points to the source list.  in __list_splice_init_rcu()
253 	 * Wait for any readers to finish using the list before splicing  in __list_splice_init_rcu()
254 	 * the list body into the new list.  Any new readers will see  in __list_splice_init_rcu()
255 	 * an empty list.  in __list_splice_init_rcu()
263 	 * Readers are finished with the source list, so perform splice.  in __list_splice_init_rcu()
264 	 * The order is important if the new list is global and accessible  in __list_splice_init_rcu()
277  * list_splice_init_rcu - splice an RCU-protected list into an existing list,
279  * @list:	the RCU-protected list to splice
280  * @head:	the place in the existing list to splice the first list into
283 static inline void list_splice_init_rcu(struct list_head *list,  in list_splice_init_rcu()  argument
287 	if (!list_empty(list))  in list_splice_init_rcu()
288 		__list_splice_init_rcu(list, head, head->next, sync);  in list_splice_init_rcu()
292  * list_splice_tail_init_rcu - splice an RCU-protected list into an existing
293  *                             list, designed for queues.
294  * @list:	the RCU-protected list to splice
295  * @head:	the place in the existing list to splice the first list into
298 static inline void list_splice_tail_init_rcu(struct list_head *list,  in list_splice_tail_init_rcu()  argument
302 	if (!list_empty(list))  in list_splice_tail_init_rcu()
303 		__list_splice_init_rcu(list, head->prev, head, sync);  in list_splice_tail_init_rcu()
312  * This primitive may safely run concurrently with the _rcu list-mutation
330  * The list may not be empty when list_empty_rcu checks it, but it may be when
341  * list_first_or_null_rcu - get the first element from a list
342  * @ptr:        the list head to take the element from.
346  * Note that if the list is empty, it returns NULL.
348  * This primitive may safely run concurrently with the _rcu list-mutation
359  * list_next_or_null_rcu - get the first element from a list
360  * @head:	the head for the list.
361  * @ptr:        the list head to take the next element from.
365  * Note that if the ptr is at the end of the list, NULL is returned.
367  * This primitive may safely run concurrently with the _rcu list-mutation
380  * list_for_each_entry_rcu	-	iterate over rcu list of given type
382  * @head:	the head for your list.
386  * This list-traversal primitive may safely run concurrently with
387  * the _rcu list-mutation primitives such as list_add_rcu()
397  * list_for_each_entry_srcu	-	iterate over rcu list of given type
399  * @head:	the head for your list.
401  * @cond:	lockdep expression for the lock required to traverse the list.
403  * This list-traversal primitive may safely run concurrently with
404  * the _rcu list-mutation primitives such as list_add_rcu()
422  * list-mutation primitives such as list_add_rcu(), but requires some
425  * cannot be invoked.  Another example is when items are added to the list,
432  * list_for_each_entry_lockless - iterate over rcu list of given type
434  * @head:	the head for your list.
438  * list-mutation primitives such as list_add_rcu(), but requires some
441  * cannot be invoked.  Another example is when items are added to the list,
450  * list_for_each_entry_continue_rcu - continue iteration over list of given type
452  * @head:	the head for your list.
455  * Continue to iterate over list of given type, continuing after
456  * the current position which must have been in the list when the RCU read
459  * previous walk of the list in the same RCU read-side critical section, or
461  * to keep the node alive *and* in the list.
473  * list_for_each_entry_from_rcu - iterate over a list from current point
475  * @head:	the head for your list.
478  * Iterate over the tail of a list starting from a given position,
479  * which must have been in the list when the RCU read lock was taken.
481  * previous walk of the list in the same RCU read-side critical section, or
483  * to keep the node alive *and* in the list.
494  * hlist_del_rcu - deletes entry from hash list without re-initialization
495  * @n: the element to delete from the hash list.
502  * pointers that may still be used for walking the hash list.
506  * with another list-mutation primitive, such as hlist_add_head_rcu()
507  * or hlist_del_rcu(), running on this same list.
509  * the _rcu list-traversal primitives, such as
568  * @n: the element to add to the hash list.
569  * @h: the list to add to.
577  * with another list-mutation primitive, such as hlist_add_head_rcu()
578  * or hlist_del_rcu(), running on this same list.
580  * the _rcu list-traversal primitives, such as
583  * list-traversal primitive must be guarded by rcu_read_lock().
599  * @n: the element to add to the hash list.
600  * @h: the list to add to.
608  * with another list-mutation primitive, such as hlist_add_head_rcu()
609  * or hlist_del_rcu(), running on this same list.
611  * the _rcu list-traversal primitives, such as
614  * list-traversal primitive must be guarded by rcu_read_lock().
636  * @n: the new element to add to the hash list.
645  * with another list-mutation primitive, such as hlist_add_head_rcu()
646  * or hlist_del_rcu(), running on this same list.
648  * the _rcu list-traversal primitives, such as
663  * @n: the new element to add to the hash list.
672  * with another list-mutation primitive, such as hlist_add_head_rcu()
673  * or hlist_del_rcu(), running on this same list.
675  * the _rcu list-traversal primitives, such as
695  * hlist_for_each_entry_rcu - iterate over rcu list of given type
697  * @head:	the head for your list.
701  * This list-traversal primitive may safely run concurrently with
702  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
714  * hlist_for_each_entry_srcu - iterate over rcu list of given type
716  * @head:	the head for your list.
718  * @cond:	lockdep expression for the lock required to traverse the list.
720  * This list-traversal primitive may safely run concurrently with
721  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
735  * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
737  * @head:	the head for your list.
740  * This list-traversal primitive may safely run concurrently with
741  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
755  * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
757  * @head:	the head for your list.
760  * This list-traversal primitive may safely run concurrently with
761  * the _rcu list-mutation primitives such as hlist_add_head_rcu()