fs/bcachefs/rcu_pending.c

87 	GENRADIX(struct rcu_head *)	objs;
109 	DARRAY_PREALLOCATED(struct rcu_pending_seq, 4) objs;
121 	if (p->objs.nr)
192 	struct rcu_pending_seq objs = {};
195 	if (p->objs.nr &&
196 	    __poll_state_synchronize_rcu(pending->srcu, p->objs.data[0].seq)) {
197 		objs = p->objs.data[0];
198 		darray_remove_item(&p->objs, p->objs.data);
210 		for (size_t i = 0; i < objs.nr; ) {
211 			size_t nr_this_node = min(GENRADIX_NODE_SIZE / sizeof(void *), objs.nr - i);
213 			kfree_bulk(nr_this_node, (void **) genradix_ptr(&objs.objs, i));
216 		genradix_free(&objs.objs);
243 		for (size_t i = 0; i < objs.nr; i++) {
244 			struct rcu_head *obj = *genradix_ptr(&objs.objs, i);
247 		genradix_free(&objs.objs);
261 		for (size_t i = 0; i < objs.nr; i++)
262 			pending->process(pending, *genradix_ptr(&objs.objs, i));
263 		genradix_free(&objs.objs);
287 	if ((p->objs.nr && __poll_state_synchronize_rcu(pending->srcu, p->objs.data[0].seq)) ||
332 	darray_for_each_reverse(p->objs, objs)
333 		if (rcu_gp_poll_cookie_eq(objs->seq, seq))
334 			return objs;
336 	if (darray_push_gfp(&p->objs, ((struct rcu_pending_seq) { .seq = seq }), GFP_ATOMIC))
339 	return &darray_last(p->objs);
420 	struct rcu_pending_seq *objs;
450 	objs = get_object_radix(p, seq);
451 	if (unlikely(!objs))
454 	if (unlikely(!objs->cursor)) {
461 		objs->cursor = genradix_ptr_alloc_preallocated_inlined(&objs->objs,
462 						objs->nr, &new_node, GFP_ATOMIC|__GFP_NOWARN);
463 		if (unlikely(!objs->cursor)) {
482 	*objs->cursor++ = ptr ?: head;
484 	if (!(((ulong) objs->cursor) & (GENRADIX_NODE_SIZE - 1)))
485 		objs->cursor = NULL;
486 	start_gp = !objs->nr;
487 	objs->nr++;
539 	darray_for_each(p->objs, objs)
540 		if (objs->nr) {
541 			ret = *genradix_ptr(&objs->objs, --objs->nr);
542 			objs->cursor = NULL;
543 			if (!objs->nr)
544 				genradix_free(&objs->objs);
629 		WARN_ON(p->objs.nr);
630 		darray_exit(&p->objs);
658 		darray_init(&p->objs);