5  *  memory buffer pool support. Such pools are mostly used
25 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
28 	const int nr = pool->curr_nr;  in poison_error()
34 	pr_err("Mempool %p size %zu\n", pool, size);  in poison_error()
42 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
51 			poison_error(pool, element, size, i);  in __check_element()
58 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
61 	if (pool->free == mempool_free_slab || pool->free == mempool_kfree) {  in check_element()
62 		__check_element(pool, element, ksize(element));  in check_element()
63 	} else if (pool->free == mempool_free_pages) {  in check_element()
65 		int order = (int)(long)pool->pool_data;  in check_element()
68 		__check_element(pool, addr, 1UL << (PAGE_SHIFT + order));  in check_element()
81 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
84 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) {  in poison_element()
86 	} else if (pool->alloc == mempool_alloc_pages) {  in poison_element()
88 		int order = (int)(long)pool->pool_data;  in poison_element()
96 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
99 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
104 static __always_inline void kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
106 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_poison_element()
108 	else if (pool->alloc == mempool_alloc_pages)  in kasan_poison_element()
109 		kasan_free_pages(element, (unsigned long)pool->pool_data);  in kasan_poison_element()
112 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
114 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_unpoison_element()
116 	else if (pool->alloc == mempool_alloc_pages)  in kasan_unpoison_element()
117 		kasan_alloc_pages(element, (unsigned long)pool->pool_data);  in kasan_unpoison_element()
120 static __always_inline void add_element(mempool_t *pool, void *element)  in add_element()  argument
122 	BUG_ON(pool->curr_nr >= pool->min_nr);  in add_element()
123 	poison_element(pool, element);  in add_element()
124 	kasan_poison_element(pool, element);  in add_element()
125 	pool->elements[pool->curr_nr++] = element;  in add_element()
128 static void *remove_element(mempool_t *pool)  in remove_element()  argument
130 	void *element = pool->elements[--pool->curr_nr];  in remove_element()
132 	BUG_ON(pool->curr_nr < 0);  in remove_element()
133 	kasan_unpoison_element(pool, element);  in remove_element()
134 	check_element(pool, element);  in remove_element()
140  * @pool:      pointer to the memory pool which was initialized with
143  * Free all reserved elements in @pool and @pool itself.  This function
149 void mempool_exit(mempool_t *pool)  in mempool_exit()  argument
151 	while (pool->curr_nr) {  in mempool_exit()
152 		void *element = remove_element(pool);  in mempool_exit()
153 		pool->free(element, pool->pool_data);  in mempool_exit()
155 	kfree(pool->elements);  in mempool_exit()
156 	pool->elements = NULL;  in mempool_exit()
161  * mempool_destroy - deallocate a memory pool
162  * @pool:      pointer to the memory pool which was allocated via
165  * Free all reserved elements in @pool and @pool itself.  This function
168 void mempool_destroy(mempool_t *pool)  in mempool_destroy()  argument
170 	if (unlikely(!pool))  in mempool_destroy()
173 	mempool_exit(pool);  in mempool_destroy()
174 	kfree(pool);  in mempool_destroy()
178 int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init_node()  argument
182 	spin_lock_init(&pool->lock);  in mempool_init_node()
183 	pool->min_nr	= min_nr;  in mempool_init_node()
184 	pool->pool_data = pool_data;  in mempool_init_node()
185 	pool->alloc	= alloc_fn;  in mempool_init_node()
186 	pool->free	= free_fn;  in mempool_init_node()
187 	init_waitqueue_head(&pool->wait);  in mempool_init_node()
189 	pool->elements = kmalloc_array_node(min_nr, sizeof(void *),  in mempool_init_node()
191 	if (!pool->elements)  in mempool_init_node()
197 	while (pool->curr_nr < pool->min_nr) {  in mempool_init_node()
200 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_init_node()
202 			mempool_exit(pool);  in mempool_init_node()
205 		add_element(pool, element);  in mempool_init_node()
213  * mempool_init - initialize a memory pool
214  * @pool:      pointer to the memory pool that should be initialized
216  *             allocated for this pool.
221  * Like mempool_create(), but initializes the pool in (i.e. embedded in another
226 int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init()  argument
229 	return mempool_init_node(pool, min_nr, alloc_fn, free_fn,  in mempool_init()
236  * mempool_create - create a memory pool
238  *             allocated for this pool.
244  * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
249  * Return: pointer to the created memory pool object or %NULL on error.
263 	mempool_t *pool;  in mempool_create_node()  local
265 	pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);  in mempool_create_node()
266 	if (!pool)  in mempool_create_node()
269 	if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,  in mempool_create_node()
271 		kfree(pool);  in mempool_create_node()
275 	return pool;  in mempool_create_node()
280  * mempool_resize - resize an existing memory pool
281  * @pool:       pointer to the memory pool which was allocated via
284  *              allocated for this pool.
286  * This function shrinks/grows the pool. In the case of growing,
287  * it cannot be guaranteed that the pool will be grown to the new
297 int mempool_resize(mempool_t *pool, int new_min_nr)  in mempool_resize()  argument
306 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
307 	if (new_min_nr <= pool->min_nr) {  in mempool_resize()
308 		while (new_min_nr < pool->curr_nr) {  in mempool_resize()
309 			element = remove_element(pool);  in mempool_resize()
310 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
311 			pool->free(element, pool->pool_data);  in mempool_resize()
312 			spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
314 		pool->min_nr = new_min_nr;  in mempool_resize()
317 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
319 	/* Grow the pool */  in mempool_resize()
325 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
326 	if (unlikely(new_min_nr <= pool->min_nr)) {  in mempool_resize()
328 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
332 	memcpy(new_elements, pool->elements,  in mempool_resize()
333 			pool->curr_nr * sizeof(*new_elements));  in mempool_resize()
334 	kfree(pool->elements);  in mempool_resize()
335 	pool->elements = new_elements;  in mempool_resize()
336 	pool->min_nr = new_min_nr;  in mempool_resize()
338 	while (pool->curr_nr < pool->min_nr) {  in mempool_resize()
339 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
340 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
343 		spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
344 		if (pool->curr_nr < pool->min_nr) {  in mempool_resize()
345 			add_element(pool, element);  in mempool_resize()
347 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
348 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
353 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
360  * mempool_alloc - allocate an element from a specific memory pool
361  * @pool:      pointer to the memory pool which was allocated via
373 void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)  in mempool_alloc()  argument
391 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc()
395 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc()
396 	if (likely(pool->curr_nr)) {  in mempool_alloc()
397 		element = remove_element(pool);  in mempool_alloc()
398 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
411 	 * alloc failed with that and @pool was empty, retry immediately.  in mempool_alloc()
414 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
421 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
425 	/* Let's wait for someone else to return an element to @pool */  in mempool_alloc()
427 	prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);  in mempool_alloc()
429 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
437 	finish_wait(&pool->wait, &wait);  in mempool_alloc()
443  * mempool_free - return an element to the pool.
444  * @element:   pool element pointer.
445  * @pool:      pointer to the memory pool which was allocated via
450 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
459 	 * for @element and the following @pool->curr_nr.  This ensures  in mempool_free()
460 	 * that the visible value of @pool->curr_nr is from after the  in mempool_free()
488 	 * pool waking up the waiters.  in mempool_free()
490 	if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {  in mempool_free()
491 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free()
492 		if (likely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
493 			add_element(pool, element);  in mempool_free()
494 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
495 			wake_up(&pool->wait);  in mempool_free()
498 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
500 	pool->free(element, pool->pool_data);  in mempool_free()