3  *  memory buffer pool support. Such pools are mostly used
41 static void poison_error(struct mempool *pool, void *element, size_t size,  in poison_error()  argument
44 	const int nr = pool->curr_nr;  in poison_error()
50 	pr_err("Mempool %p size %zu\n", pool, size);  in poison_error()
58 static void __check_element(struct mempool *pool, void *element, size_t size)  in __check_element()  argument
67 			poison_error(pool, element, size, i);  in __check_element()
74 static void check_element(struct mempool *pool, void *element)  in check_element()  argument
81 	if (pool->free == mempool_kfree) {  in check_element()
82 		__check_element(pool, element, (size_t)pool->pool_data);  in check_element()
83 	} else if (pool->free == mempool_free_slab) {  in check_element()
84 		__check_element(pool, element, kmem_cache_size(pool->pool_data));  in check_element()
85 	} else if (pool->free == mempool_free_pages) {  in check_element()
87 		int order = (int)(long)pool->pool_data;  in check_element()
94 			__check_element(pool, addr, PAGE_SIZE);  in check_element()
100 		__check_element(pool, addr, PAGE_SIZE << order);  in check_element()
113 static void poison_element(struct mempool *pool, void *element)  in poison_element()  argument
120 	if (pool->alloc == mempool_kmalloc) {  in poison_element()
121 		__poison_element(element, (size_t)pool->pool_data);  in poison_element()
122 	} else if (pool->alloc == mempool_alloc_slab) {  in poison_element()
123 		__poison_element(element, kmem_cache_size(pool->pool_data));  in poison_element()
124 	} else if (pool->alloc == mempool_alloc_pages) {  in poison_element()
126 		int order = (int)(long)pool->pool_data;  in poison_element()
144 static inline void check_element(struct mempool *pool, void *element)  in check_element()  argument
147 static inline void poison_element(struct mempool *pool, void *element)  in poison_element()  argument
152 static __always_inline bool kasan_poison_element(struct mempool *pool,  in kasan_poison_element()  argument
155 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_poison_element()
157 	else if (pool->alloc == mempool_alloc_pages)  in kasan_poison_element()
159 						(unsigned long)pool->pool_data);  in kasan_poison_element()
163 static void kasan_unpoison_element(struct mempool *pool, void *element)  in kasan_unpoison_element()  argument
165 	if (pool->alloc == mempool_kmalloc)  in kasan_unpoison_element()
166 		kasan_mempool_unpoison_object(element, (size_t)pool->pool_data);  in kasan_unpoison_element()
167 	else if (pool->alloc == mempool_alloc_slab)  in kasan_unpoison_element()
169 					      kmem_cache_size(pool->pool_data));  in kasan_unpoison_element()
170 	else if (pool->alloc == mempool_alloc_pages)  in kasan_unpoison_element()
172 					     (unsigned long)pool->pool_data);  in kasan_unpoison_element()
175 static __always_inline void add_element(struct mempool *pool, void *element)  in add_element()  argument
177 	BUG_ON(pool->min_nr != 0 && pool->curr_nr >= pool->min_nr);  in add_element()
178 	poison_element(pool, element);  in add_element()
179 	if (kasan_poison_element(pool, element))  in add_element()
180 		pool->elements[pool->curr_nr++] = element;  in add_element()
183 static void *remove_element(struct mempool *pool)  in remove_element()  argument
185 	void *element = pool->elements[--pool->curr_nr];  in remove_element()
187 	BUG_ON(pool->curr_nr < 0);  in remove_element()
188 	kasan_unpoison_element(pool, element);  in remove_element()
189 	check_element(pool, element);  in remove_element()
195  * @pool:      pointer to the memory pool which was initialized with
198  * Free all reserved elements in @pool and @pool itself.  This function
204 void mempool_exit(struct mempool *pool)  in mempool_exit()  argument
206 	while (pool->curr_nr) {  in mempool_exit()
207 		void *element = remove_element(pool);  in mempool_exit()
208 		pool->free(element, pool->pool_data);  in mempool_exit()
210 	kfree(pool->elements);  in mempool_exit()
211 	pool->elements = NULL;  in mempool_exit()
216  * mempool_destroy - deallocate a memory pool
217  * @pool:      pointer to the memory pool which was allocated via
220  * Free all reserved elements in @pool and @pool itself.  This function
223 void mempool_destroy(struct mempool *pool)  in mempool_destroy()  argument
225 	if (unlikely(!pool))  in mempool_destroy()
228 	mempool_exit(pool);  in mempool_destroy()
229 	kfree(pool);  in mempool_destroy()
233 int mempool_init_node(struct mempool *pool, int min_nr,  in mempool_init_node()  argument
237 	spin_lock_init(&pool->lock);  in mempool_init_node()
238 	pool->min_nr	= min_nr;  in mempool_init_node()
239 	pool->pool_data = pool_data;  in mempool_init_node()
240 	pool->alloc	= alloc_fn;  in mempool_init_node()
241 	pool->free	= free_fn;  in mempool_init_node()
242 	init_waitqueue_head(&pool->wait);  in mempool_init_node()
245 	 * zero minimum pool  in mempool_init_node()
247 	pool->elements = kmalloc_array_node(max(1, min_nr), sizeof(void *),  in mempool_init_node()
249 	if (!pool->elements)  in mempool_init_node()
254 	 * also pre-allocate 1 element for zero minimum pool.  in mempool_init_node()
256 	while (pool->curr_nr < max(1, pool->min_nr)) {  in mempool_init_node()
259 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_init_node()
261 			mempool_exit(pool);  in mempool_init_node()
264 		add_element(pool, element);  in mempool_init_node()
272  * mempool_init - initialize a memory pool
273  * @pool:      pointer to the memory pool that should be initialized
275  *             allocated for this pool.
280  * Like mempool_create(), but initializes the pool in (i.e. embedded in another
285 int mempool_init_noprof(struct mempool *pool, int min_nr,  in mempool_init_noprof()  argument
289 	return mempool_init_node(pool, min_nr, alloc_fn, free_fn,  in mempool_init_noprof()
296  * mempool_create_node - create a memory pool
298  *             allocated for this pool.
306  * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
311  * Return: pointer to the created memory pool object or %NULL on error.
317 	struct mempool *pool;  in mempool_create_node_noprof()  local
319 	pool = kmalloc_node_noprof(sizeof(*pool), gfp_mask | __GFP_ZERO, node_id);  in mempool_create_node_noprof()
320 	if (!pool)  in mempool_create_node_noprof()
323 	if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,  in mempool_create_node_noprof()
325 		kfree(pool);  in mempool_create_node_noprof()
329 	return pool;  in mempool_create_node_noprof()
334  * mempool_resize - resize an existing memory pool
335  * @pool:       pointer to the memory pool which was allocated via
338  *              allocated for this pool.
340  * This function shrinks/grows the pool. In the case of growing,
341  * it cannot be guaranteed that the pool will be grown to the new
351 int mempool_resize(struct mempool *pool, int new_min_nr)  in mempool_resize()  argument
360 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
361 	if (new_min_nr <= pool->min_nr) {  in mempool_resize()
362 		while (new_min_nr < pool->curr_nr) {  in mempool_resize()
363 			element = remove_element(pool);  in mempool_resize()
364 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
365 			pool->free(element, pool->pool_data);  in mempool_resize()
366 			spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
368 		pool->min_nr = new_min_nr;  in mempool_resize()
371 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
373 	/* Grow the pool */  in mempool_resize()
378 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
379 	if (unlikely(new_min_nr <= pool->min_nr)) {  in mempool_resize()
381 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
385 	memcpy(new_elements, pool->elements,  in mempool_resize()
386 			pool->curr_nr * sizeof(*new_elements));  in mempool_resize()
387 	kfree(pool->elements);  in mempool_resize()
388 	pool->elements = new_elements;  in mempool_resize()
389 	pool->min_nr = new_min_nr;  in mempool_resize()
391 	while (pool->curr_nr < pool->min_nr) {  in mempool_resize()
392 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
393 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
396 		spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
397 		if (pool->curr_nr < pool->min_nr) {  in mempool_resize()
398 			add_element(pool, element);  in mempool_resize()
400 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
401 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
406 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
412 static unsigned int mempool_alloc_from_pool(struct mempool *pool, void **elems,  in mempool_alloc_from_pool()  argument
419 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc_from_pool()
420 	if (unlikely(pool->curr_nr < count - allocated))  in mempool_alloc_from_pool()
424 			elems[i] = remove_element(pool);  in mempool_alloc_from_pool()
428 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc_from_pool()
445 		prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);  in mempool_alloc_from_pool()
446 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc_from_pool()
449 		 * Wait for someone else to return an element to @pool, but wake  in mempool_alloc_from_pool()
455 		finish_wait(&pool->wait, &wait);  in mempool_alloc_from_pool()
458 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc_from_pool()
478  * mempool_alloc_bulk - allocate multiple elements from a memory pool
479  * @pool:	pointer to the memory pool
485  * first calling into the alloc_fn supplied at pool initialization time, and
486  * dipping into the reserved pool when alloc_fn fails to allocate an element.
492 int mempool_alloc_bulk_noprof(struct mempool *pool, void **elems,  in mempool_alloc_bulk_noprof()  argument
499 	VM_WARN_ON_ONCE(count > pool->min_nr);  in mempool_alloc_bulk_noprof()
520 		elems[i] = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc_bulk_noprof()
529 	allocated = mempool_alloc_from_pool(pool, elems, count, allocated,  in mempool_alloc_bulk_noprof()
537  * mempool_alloc - allocate an element from a memory pool
538  * @pool:	pointer to the memory pool
541  * Allocate an element from @pool.  This is done by first calling into the
542  * alloc_fn supplied at pool initialization time, and dipping into the reserved
543  * pool when alloc_fn fails to allocate an element.
546  * for elements to become available in the pool.
552 void *mempool_alloc_noprof(struct mempool *pool, gfp_t gfp_mask)  in mempool_alloc_noprof()  argument
566 		element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc_noprof()
571 		 * Try to allocate an element from the pool.  in mempool_alloc_noprof()
577 		if (!mempool_alloc_from_pool(pool, &element, 1, 0, gfp_temp)) {  in mempool_alloc_noprof()
594  *                              belonging to a memory pool
595  * @pool:	pointer to the memory pool
604 void *mempool_alloc_preallocated(struct mempool *pool)  in mempool_alloc_preallocated()  argument
608 	mempool_alloc_from_pool(pool, &element, 1, 0, GFP_NOWAIT);  in mempool_alloc_preallocated()
615  * @pool:	pointer to the memory pool
619  * Returns a number of elements from the start of @elem to @pool if @pool needs
623  * Return: number of elements transferred to @pool.  Elements are always
627 unsigned int mempool_free_bulk(struct mempool *pool, void **elems,  in mempool_free_bulk()  argument
636 	 * for @element and the following @pool->curr_nr.  This ensures  in mempool_free_bulk()
637 	 * that the visible value of @pool->curr_nr is from after the  in mempool_free_bulk()
665 	 * pool waking up the waiters.  in mempool_free_bulk()
668 	 * so waiters sleeping on pool->wait would never be woken by the  in mempool_free_bulk()
673 	if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {  in mempool_free_bulk()
674 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free_bulk()
675 		while (pool->curr_nr < pool->min_nr && freed < count) {  in mempool_free_bulk()
676 			add_element(pool, elems[freed++]);  in mempool_free_bulk()
679 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free_bulk()
680 	} else if (unlikely(pool->min_nr == 0 &&  in mempool_free_bulk()
681 		     READ_ONCE(pool->curr_nr) == 0)) {  in mempool_free_bulk()
683 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free_bulk()
684 		if (likely(pool->curr_nr == 0)) {  in mempool_free_bulk()
685 			add_element(pool, elems[freed++]);  in mempool_free_bulk()
688 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free_bulk()
691 	if (unlikely(added) && wq_has_sleeper(&pool->wait))  in mempool_free_bulk()
692 		wake_up(&pool->wait);  in mempool_free_bulk()
699  * mempool_free - return an element to the pool.
701  * @pool:	pointer to the memory pool
703  * Returns @element to @pool if it needs replenishing, else frees it using
704  * the free_fn callback in @pool.
708 void mempool_free(void *element, struct mempool *pool)  in mempool_free()  argument
710 	if (likely(element) && !mempool_free_bulk(pool, &element, 1))  in mempool_free()
711 		pool->free(element, pool->pool_data);  in mempool_free()