19 objpool_init_percpu_slot(struct objpool_head *pool,  in objpool_init_percpu_slot()  argument
24 	void *obj = (void *)&slot->entries[pool->capacity];  in objpool_init_percpu_slot()
28 	slot->mask = pool->capacity - 1;  in objpool_init_percpu_slot()
37 		obj = obj + pool->obj_size;  in objpool_init_percpu_slot()
40 		pool->nr_objs++;  in objpool_init_percpu_slot()
48 objpool_init_percpu_slots(struct objpool_head *pool, int nr_objs,  in objpool_init_percpu_slots()  argument
53 	for (i = 0; i < pool->nr_cpus; i++) {  in objpool_init_percpu_slots()
68 		size = struct_size(slot, entries, pool->capacity) +  in objpool_init_percpu_slots()
69 			pool->obj_size * nodes;  in objpool_init_percpu_slots()
79 		if (pool->gfp & GFP_ATOMIC)  in objpool_init_percpu_slots()
80 			slot = kmalloc_node(size, pool->gfp, cpu_to_node(i));  in objpool_init_percpu_slots()
82 			slot = __vmalloc_node(size, sizeof(void *), pool->gfp,  in objpool_init_percpu_slots()
87 		pool->cpu_slots[i] = slot;  in objpool_init_percpu_slots()
90 		rc = objpool_init_percpu_slot(pool, slot, nodes, context, objinit);  in objpool_init_percpu_slots()
98 /* cleanup all percpu slots of the object pool */
99 static void objpool_fini_percpu_slots(struct objpool_head *pool)  in objpool_fini_percpu_slots()  argument
103 	if (!pool->cpu_slots)  in objpool_fini_percpu_slots()
106 	for (i = 0; i < pool->nr_cpus; i++)  in objpool_fini_percpu_slots()
107 		kvfree(pool->cpu_slots[i]);  in objpool_fini_percpu_slots()
108 	kfree(pool->cpu_slots);  in objpool_fini_percpu_slots()
111 /* initialize object pool and pre-allocate objects */
112 int objpool_init(struct objpool_head *pool, int nr_objs, int object_size,  in objpool_init()  argument
131 	/* initialize objpool pool */  in objpool_init()
132 	memset(pool, 0, sizeof(struct objpool_head));  in objpool_init()
133 	pool->nr_cpus = nr_cpu_ids;  in objpool_init()
134 	pool->obj_size = object_size;  in objpool_init()
135 	pool->capacity = capacity;  in objpool_init()
136 	pool->gfp = gfp & ~__GFP_ZERO;  in objpool_init()
137 	pool->context = context;  in objpool_init()
138 	pool->release = release;  in objpool_init()
139 	slot_size = pool->nr_cpus * sizeof(struct objpool_slot);  in objpool_init()
140 	pool->cpu_slots = kzalloc(slot_size, pool->gfp);  in objpool_init()
141 	if (!pool->cpu_slots)  in objpool_init()
145 	rc = objpool_init_percpu_slots(pool, nr_objs, context, objinit);  in objpool_init()
147 		objpool_fini_percpu_slots(pool);  in objpool_init()
149 		refcount_set(&pool->ref, pool->nr_objs + 1);  in objpool_init()
157 objpool_try_add_slot(void *obj, struct objpool_head *pool, int cpu)  in objpool_try_add_slot()  argument
159 	struct objpool_slot *slot = pool->cpu_slots[cpu];  in objpool_try_add_slot()
168 		WARN_ON_ONCE(tail - head > pool->nr_objs);  in objpool_try_add_slot()
179 /* reclaim an object to object pool */
180 int objpool_push(void *obj, struct objpool_head *pool)  in objpool_push()  argument
187 	rc = objpool_try_add_slot(obj, pool, raw_smp_processor_id());  in objpool_push()
195 static inline void *objpool_try_get_slot(struct objpool_head *pool, int cpu)  in objpool_try_get_slot()  argument
197 	struct objpool_slot *slot = pool->cpu_slots[cpu];  in objpool_try_get_slot()
216 		if (READ_ONCE(slot->last) - head - 1 >= pool->nr_objs) {  in objpool_try_get_slot()
232 /* allocate an object from object pool */
233 void *objpool_pop(struct objpool_head *pool)  in objpool_pop()  argument
244 		obj = objpool_try_get_slot(pool, cpu);  in objpool_pop()
256 void objpool_free(struct objpool_head *pool)  in objpool_free()  argument
258 	if (!pool->cpu_slots)  in objpool_free()
262 	objpool_fini_percpu_slots(pool);  in objpool_free()
265 	if (pool->release)  in objpool_free()
266 		pool->release(pool, pool->context);  in objpool_free()
271 int objpool_drop(void *obj, struct objpool_head *pool)  in objpool_drop()  argument
273 	if (!obj || !pool)  in objpool_drop()
276 	if (refcount_dec_and_test(&pool->ref)) {  in objpool_drop()
277 		objpool_free(pool);  in objpool_drop()
286 void objpool_fini(struct objpool_head *pool)  in objpool_fini()  argument
291 	while (objpool_pop(pool))  in objpool_fini()
294 	if (refcount_sub_and_test(count, &pool->ref))  in objpool_fini()
295 		objpool_free(pool);  in objpool_fini()