1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk).
4 *
5 * (C) SGI 2006, Christoph Lameter
6 * Cleaned up and restructured to ease the addition of alternative
7 * implementations of SLAB allocators.
8 * (C) Linux Foundation 2008-2013
9 * Unified interface for all slab allocators
10 */
11
12 #ifndef _LINUX_SLAB_H
13 #define _LINUX_SLAB_H
14
15 #include <linux/cache.h>
16 #include <linux/gfp.h>
17 #include <linux/overflow.h>
18 #include <linux/types.h>
19 #include <linux/rcupdate.h>
20 #include <linux/workqueue.h>
21 #include <linux/percpu-refcount.h>
22 #include <linux/cleanup.h>
23 #include <linux/hash.h>
24
25 enum _slab_flag_bits {
26 _SLAB_CONSISTENCY_CHECKS,
27 _SLAB_RED_ZONE,
28 _SLAB_POISON,
29 _SLAB_KMALLOC,
30 _SLAB_HWCACHE_ALIGN,
31 _SLAB_CACHE_DMA,
32 _SLAB_CACHE_DMA32,
33 _SLAB_STORE_USER,
34 _SLAB_PANIC,
35 _SLAB_TYPESAFE_BY_RCU,
36 _SLAB_TRACE,
37 #ifdef CONFIG_DEBUG_OBJECTS
38 _SLAB_DEBUG_OBJECTS,
39 #endif
40 _SLAB_NOLEAKTRACE,
41 _SLAB_NO_MERGE,
42 #ifdef CONFIG_FAILSLAB
43 _SLAB_FAILSLAB,
44 #endif
45 #ifdef CONFIG_MEMCG
46 _SLAB_ACCOUNT,
47 #endif
48 #ifdef CONFIG_KASAN_GENERIC
49 _SLAB_KASAN,
50 #endif
51 _SLAB_NO_USER_FLAGS,
52 #ifdef CONFIG_KFENCE
53 _SLAB_SKIP_KFENCE,
54 #endif
55 #ifndef CONFIG_SLUB_TINY
56 _SLAB_RECLAIM_ACCOUNT,
57 #endif
58 _SLAB_OBJECT_POISON,
59 _SLAB_CMPXCHG_DOUBLE,
60 #ifdef CONFIG_SLAB_OBJ_EXT
61 _SLAB_NO_OBJ_EXT,
62 #endif
63 _SLAB_FLAGS_LAST_BIT
64 };
65
66 #define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr)))
67 #define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U))
68
69 /*
70 * Flags to pass to kmem_cache_create().
71 * The ones marked DEBUG need CONFIG_SLUB_DEBUG enabled, otherwise are no-op
72 */
73 /* DEBUG: Perform (expensive) checks on alloc/free */
74 #define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS)
75 /* DEBUG: Red zone objs in a cache */
76 #define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE)
77 /* DEBUG: Poison objects */
78 #define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON)
79 /* Indicate a kmalloc slab */
80 #define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC)
81 /**
82 * define SLAB_HWCACHE_ALIGN - Align objects on cache line boundaries.
83 *
84 * Sufficiently large objects are aligned on cache line boundary. For object
85 * size smaller than a half of cache line size, the alignment is on the half of
86 * cache line size. In general, if object size is smaller than 1/2^n of cache
87 * line size, the alignment is adjusted to 1/2^n.
88 *
89 * If explicit alignment is also requested by the respective
90 * &struct kmem_cache_args field, the greater of both is alignments is applied.
91 */
92 #define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN)
93 /* Use GFP_DMA memory */
94 #define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA)
95 /* Use GFP_DMA32 memory */
96 #define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32)
97 /* DEBUG: Store the last owner for bug hunting */
98 #define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER)
99 /* Panic if kmem_cache_create() fails */
100 #define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC)
101 /**
102 * define SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS!
103 *
104 * This delays freeing the SLAB page by a grace period, it does _NOT_
105 * delay object freeing. This means that if you do kmem_cache_free()
106 * that memory location is free to be reused at any time. Thus it may
107 * be possible to see another object there in the same RCU grace period.
108 *
109 * This feature only ensures the memory location backing the object
110 * stays valid, the trick to using this is relying on an independent
111 * object validation pass. Something like:
112 *
113 * ::
114 *
115 * begin:
116 * rcu_read_lock();
117 * obj = lockless_lookup(key);
118 * if (obj) {
119 * if (!try_get_ref(obj)) // might fail for free objects
120 * rcu_read_unlock();
121 * goto begin;
122 *
123 * if (obj->key != key) { // not the object we expected
124 * put_ref(obj);
125 * rcu_read_unlock();
126 * goto begin;
127 * }
128 * }
129 * rcu_read_unlock();
130 *
131 * This is useful if we need to approach a kernel structure obliquely,
132 * from its address obtained without the usual locking. We can lock
133 * the structure to stabilize it and check it's still at the given address,
134 * only if we can be sure that the memory has not been meanwhile reused
135 * for some other kind of object (which our subsystem's lock might corrupt).
136 *
137 * rcu_read_lock before reading the address, then rcu_read_unlock after
138 * taking the spinlock within the structure expected at that address.
139 *
140 * Note that object identity check has to be done *after* acquiring a
141 * reference, therefore user has to ensure proper ordering for loads.
142 * Similarly, when initializing objects allocated with SLAB_TYPESAFE_BY_RCU,
143 * the newly allocated object has to be fully initialized *before* its
144 * refcount gets initialized and proper ordering for stores is required.
145 * refcount_{add|inc}_not_zero_acquire() and refcount_set_release() are
146 * designed with the proper fences required for reference counting objects
147 * allocated with SLAB_TYPESAFE_BY_RCU.
148 *
149 * Note that it is not possible to acquire a lock within a structure
150 * allocated with SLAB_TYPESAFE_BY_RCU without first acquiring a reference
151 * as described above. The reason is that SLAB_TYPESAFE_BY_RCU pages
152 * are not zeroed before being given to the slab, which means that any
153 * locks must be initialized after each and every kmem_struct_alloc().
154 * Alternatively, make the ctor passed to kmem_cache_create() initialize
155 * the locks at page-allocation time, as is done in __i915_request_ctor(),
156 * sighand_ctor(), and anon_vma_ctor(). Such a ctor permits readers
157 * to safely acquire those ctor-initialized locks under rcu_read_lock()
158 * protection.
159 *
160 * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU.
161 */
162 #define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU)
163 /* Trace allocations and frees */
164 #define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE)
165
166 /* Flag to prevent checks on free */
167 #ifdef CONFIG_DEBUG_OBJECTS
168 # define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS)
169 #else
170 # define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED
171 #endif
172
173 /* Avoid kmemleak tracing */
174 #define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE)
175
176 /*
177 * Prevent merging with compatible kmem caches. This flag should be used
178 * cautiously. Valid use cases:
179 *
180 * - caches created for self-tests (e.g. kunit)
181 * - general caches created and used by a subsystem, only when a
182 * (subsystem-specific) debug option is enabled
183 * - performance critical caches, should be very rare and consulted with slab
184 * maintainers, and not used together with CONFIG_SLUB_TINY
185 */
186 #define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE)
187
188 /* Fault injection mark */
189 #ifdef CONFIG_FAILSLAB
190 # define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB)
191 #else
192 # define SLAB_FAILSLAB __SLAB_FLAG_UNUSED
193 #endif
194 /**
195 * define SLAB_ACCOUNT - Account allocations to memcg.
196 *
197 * All object allocations from this cache will be memcg accounted, regardless of
198 * __GFP_ACCOUNT being or not being passed to individual allocations.
199 */
200 #ifdef CONFIG_MEMCG
201 # define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT)
202 #else
203 # define SLAB_ACCOUNT __SLAB_FLAG_UNUSED
204 #endif
205
206 #ifdef CONFIG_KASAN_GENERIC
207 #define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN)
208 #else
209 #define SLAB_KASAN __SLAB_FLAG_UNUSED
210 #endif
211
212 /*
213 * Ignore user specified debugging flags.
214 * Intended for caches created for self-tests so they have only flags
215 * specified in the code and other flags are ignored.
216 */
217 #define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS)
218
219 #ifdef CONFIG_KFENCE
220 #define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE)
221 #else
222 #define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED
223 #endif
224
225 /* The following flags affect the page allocator grouping pages by mobility */
226 /**
227 * define SLAB_RECLAIM_ACCOUNT - Objects are reclaimable.
228 *
229 * Use this flag for caches that have an associated shrinker. As a result, slab
230 * pages are allocated with __GFP_RECLAIMABLE, which affects grouping pages by
231 * mobility, and are accounted in SReclaimable counter in /proc/meminfo
232 */
233 #ifndef CONFIG_SLUB_TINY
234 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT)
235 #else
236 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED
237 #endif
238 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */
239
240 /* Slab created using create_boot_cache */
241 #ifdef CONFIG_SLAB_OBJ_EXT
242 #define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT)
243 #else
244 #define SLAB_NO_OBJ_EXT __SLAB_FLAG_UNUSED
245 #endif
246
247 /*
248 * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
249 *
250 * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault.
251 *
252 * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can.
253 * Both make kfree a no-op.
254 */
255 #define ZERO_SIZE_PTR ((void *)16)
256
257 #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \
258 (unsigned long)ZERO_SIZE_PTR)
259
260 #include <linux/kasan.h>
261
262 struct list_lru;
263 struct mem_cgroup;
264 /*
265 * struct kmem_cache related prototypes
266 */
267 bool slab_is_available(void);
268
269 /**
270 * struct kmem_cache_args - Less common arguments for kmem_cache_create()
271 *
272 * Any uninitialized fields of the structure are interpreted as unused. The
273 * exception is @freeptr_offset where %0 is a valid value, so
274 * @use_freeptr_offset must be also set to %true in order to interpret the field
275 * as used. For @useroffset %0 is also valid, but only with non-%0
276 * @usersize.
277 *
278 * When %NULL args is passed to kmem_cache_create(), it is equivalent to all
279 * fields unused.
280 */
281 struct kmem_cache_args {
282 /**
283 * @align: The required alignment for the objects.
284 *
285 * %0 means no specific alignment is requested.
286 */
287 unsigned int align;
288 /**
289 * @useroffset: Usercopy region offset.
290 *
291 * %0 is a valid offset, when @usersize is non-%0
292 */
293 unsigned int useroffset;
294 /**
295 * @usersize: Usercopy region size.
296 *
297 * %0 means no usercopy region is specified.
298 */
299 unsigned int usersize;
300 /**
301 * @freeptr_offset: Custom offset for the free pointer
302 * in &SLAB_TYPESAFE_BY_RCU caches
303 *
304 * By default &SLAB_TYPESAFE_BY_RCU caches place the free pointer
305 * outside of the object. This might cause the object to grow in size.
306 * Cache creators that have a reason to avoid this can specify a custom
307 * free pointer offset in their struct where the free pointer will be
308 * placed.
309 *
310 * Note that placing the free pointer inside the object requires the
311 * caller to ensure that no fields are invalidated that are required to
312 * guard against object recycling (See &SLAB_TYPESAFE_BY_RCU for
313 * details).
314 *
315 * Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset
316 * is specified, %use_freeptr_offset must be set %true.
317 *
318 * Note that @ctor currently isn't supported with custom free pointers
319 * as a @ctor requires an external free pointer.
320 */
321 unsigned int freeptr_offset;
322 /**
323 * @use_freeptr_offset: Whether a @freeptr_offset is used.
324 */
325 bool use_freeptr_offset;
326 /**
327 * @ctor: A constructor for the objects.
328 *
329 * The constructor is invoked for each object in a newly allocated slab
330 * page. It is the cache user's responsibility to free object in the
331 * same state as after calling the constructor, or deal appropriately
332 * with any differences between a freshly constructed and a reallocated
333 * object.
334 *
335 * %NULL means no constructor.
336 */
337 void (*ctor)(void *);
338 };
339
340 struct kmem_cache *__kmem_cache_create_args(const char *name,
341 unsigned int object_size,
342 struct kmem_cache_args *args,
343 slab_flags_t flags);
344 static inline struct kmem_cache *
__kmem_cache_create(const char * name,unsigned int size,unsigned int align,slab_flags_t flags,void (* ctor)(void *))345 __kmem_cache_create(const char *name, unsigned int size, unsigned int align,
346 slab_flags_t flags, void (*ctor)(void *))
347 {
348 struct kmem_cache_args kmem_args = {
349 .align = align,
350 .ctor = ctor,
351 };
352
353 return __kmem_cache_create_args(name, size, &kmem_args, flags);
354 }
355
356 /**
357 * kmem_cache_create_usercopy - Create a kmem cache with a region suitable
358 * for copying to userspace.
359 * @name: A string which is used in /proc/slabinfo to identify this cache.
360 * @size: The size of objects to be created in this cache.
361 * @align: The required alignment for the objects.
362 * @flags: SLAB flags
363 * @useroffset: Usercopy region offset
364 * @usersize: Usercopy region size
365 * @ctor: A constructor for the objects, or %NULL.
366 *
367 * This is a legacy wrapper, new code should use either KMEM_CACHE_USERCOPY()
368 * if whitelisting a single field is sufficient, or kmem_cache_create() with
369 * the necessary parameters passed via the args parameter (see
370 * &struct kmem_cache_args)
371 *
372 * Return: a pointer to the cache on success, NULL on failure.
373 */
374 static inline struct kmem_cache *
kmem_cache_create_usercopy(const char * name,unsigned int size,unsigned int align,slab_flags_t flags,unsigned int useroffset,unsigned int usersize,void (* ctor)(void *))375 kmem_cache_create_usercopy(const char *name, unsigned int size,
376 unsigned int align, slab_flags_t flags,
377 unsigned int useroffset, unsigned int usersize,
378 void (*ctor)(void *))
379 {
380 struct kmem_cache_args kmem_args = {
381 .align = align,
382 .ctor = ctor,
383 .useroffset = useroffset,
384 .usersize = usersize,
385 };
386
387 return __kmem_cache_create_args(name, size, &kmem_args, flags);
388 }
389
390 /* If NULL is passed for @args, use this variant with default arguments. */
391 static inline struct kmem_cache *
__kmem_cache_default_args(const char * name,unsigned int size,struct kmem_cache_args * args,slab_flags_t flags)392 __kmem_cache_default_args(const char *name, unsigned int size,
393 struct kmem_cache_args *args,
394 slab_flags_t flags)
395 {
396 struct kmem_cache_args kmem_default_args = {};
397
398 /* Make sure we don't get passed garbage. */
399 if (WARN_ON_ONCE(args))
400 return ERR_PTR(-EINVAL);
401
402 return __kmem_cache_create_args(name, size, &kmem_default_args, flags);
403 }
404
405 /**
406 * kmem_cache_create - Create a kmem cache.
407 * @__name: A string which is used in /proc/slabinfo to identify this cache.
408 * @__object_size: The size of objects to be created in this cache.
409 * @__args: Optional arguments, see &struct kmem_cache_args. Passing %NULL
410 * means defaults will be used for all the arguments.
411 *
412 * This is currently implemented as a macro using ``_Generic()`` to call
413 * either the new variant of the function, or a legacy one.
414 *
415 * The new variant has 4 parameters:
416 * ``kmem_cache_create(name, object_size, args, flags)``
417 *
418 * See __kmem_cache_create_args() which implements this.
419 *
420 * The legacy variant has 5 parameters:
421 * ``kmem_cache_create(name, object_size, align, flags, ctor)``
422 *
423 * The align and ctor parameters map to the respective fields of
424 * &struct kmem_cache_args
425 *
426 * Context: Cannot be called within a interrupt, but can be interrupted.
427 *
428 * Return: a pointer to the cache on success, NULL on failure.
429 */
430 #define kmem_cache_create(__name, __object_size, __args, ...) \
431 _Generic((__args), \
432 struct kmem_cache_args *: __kmem_cache_create_args, \
433 void *: __kmem_cache_default_args, \
434 default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__)
435
436 void kmem_cache_destroy(struct kmem_cache *s);
437 int kmem_cache_shrink(struct kmem_cache *s);
438
439 /*
440 * Please use this macro to create slab caches. Simply specify the
441 * name of the structure and maybe some flags that are listed above.
442 *
443 * The alignment of the struct determines object alignment. If you
444 * f.e. add ____cacheline_aligned_in_smp to the struct declaration
445 * then the objects will be properly aligned in SMP configurations.
446 */
447 #define KMEM_CACHE(__struct, __flags) \
448 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \
449 &(struct kmem_cache_args) { \
450 .align = __alignof__(struct __struct), \
451 }, (__flags))
452
453 /*
454 * To whitelist a single field for copying to/from usercopy, use this
455 * macro instead for KMEM_CACHE() above.
456 */
457 #define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \
458 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \
459 &(struct kmem_cache_args) { \
460 .align = __alignof__(struct __struct), \
461 .useroffset = offsetof(struct __struct, __field), \
462 .usersize = sizeof_field(struct __struct, __field), \
463 }, (__flags))
464
465 /*
466 * Common kmalloc functions provided by all allocators
467 */
468 void * __must_check krealloc_noprof(const void *objp, size_t new_size,
469 gfp_t flags) __realloc_size(2);
470 #define krealloc(...) alloc_hooks(krealloc_noprof(__VA_ARGS__))
471
472 void kfree(const void *objp);
473 void kfree_sensitive(const void *objp);
474 size_t __ksize(const void *objp);
475
476 DEFINE_FREE(kfree, void *, if (!IS_ERR_OR_NULL(_T)) kfree(_T))
477 DEFINE_FREE(kfree_sensitive, void *, if (_T) kfree_sensitive(_T))
478
479 /**
480 * ksize - Report actual allocation size of associated object
481 *
482 * @objp: Pointer returned from a prior kmalloc()-family allocation.
483 *
484 * This should not be used for writing beyond the originally requested
485 * allocation size. Either use krealloc() or round up the allocation size
486 * with kmalloc_size_roundup() prior to allocation. If this is used to
487 * access beyond the originally requested allocation size, UBSAN_BOUNDS
488 * and/or FORTIFY_SOURCE may trip, since they only know about the
489 * originally allocated size via the __alloc_size attribute.
490 */
491 size_t ksize(const void *objp);
492
493 #ifdef CONFIG_PRINTK
494 bool kmem_dump_obj(void *object);
495 #else
kmem_dump_obj(void * object)496 static inline bool kmem_dump_obj(void *object) { return false; }
497 #endif
498
499 /*
500 * Some archs want to perform DMA into kmalloc caches and need a guaranteed
501 * alignment larger than the alignment of a 64-bit integer.
502 * Setting ARCH_DMA_MINALIGN in arch headers allows that.
503 */
504 #ifdef ARCH_HAS_DMA_MINALIGN
505 #if ARCH_DMA_MINALIGN > 8 && !defined(ARCH_KMALLOC_MINALIGN)
506 #define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
507 #endif
508 #endif
509
510 #ifndef ARCH_KMALLOC_MINALIGN
511 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
512 #elif ARCH_KMALLOC_MINALIGN > 8
513 #define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN
514 #define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
515 #endif
516
517 /*
518 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.
519 * Intended for arches that get misalignment faults even for 64 bit integer
520 * aligned buffers.
521 */
522 #ifndef ARCH_SLAB_MINALIGN
523 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
524 #endif
525
526 /*
527 * Arches can define this function if they want to decide the minimum slab
528 * alignment at runtime. The value returned by the function must be a power
529 * of two and >= ARCH_SLAB_MINALIGN.
530 */
531 #ifndef arch_slab_minalign
arch_slab_minalign(void)532 static inline unsigned int arch_slab_minalign(void)
533 {
534 return ARCH_SLAB_MINALIGN;
535 }
536 #endif
537
538 /*
539 * kmem_cache_alloc and friends return pointers aligned to ARCH_SLAB_MINALIGN.
540 * kmalloc and friends return pointers aligned to both ARCH_KMALLOC_MINALIGN
541 * and ARCH_SLAB_MINALIGN, but here we only assume the former alignment.
542 */
543 #define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN)
544 #define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN)
545 #define __assume_page_alignment __assume_aligned(PAGE_SIZE)
546
547 /*
548 * Kmalloc array related definitions
549 */
550
551 /*
552 * SLUB directly allocates requests fitting in to an order-1 page
553 * (PAGE_SIZE*2). Larger requests are passed to the page allocator.
554 */
555 #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)
556 #define KMALLOC_SHIFT_MAX (MAX_PAGE_ORDER + PAGE_SHIFT)
557 #ifndef KMALLOC_SHIFT_LOW
558 #define KMALLOC_SHIFT_LOW 3
559 #endif
560
561 /* Maximum allocatable size */
562 #define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX)
563 /* Maximum size for which we actually use a slab cache */
564 #define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH)
565 /* Maximum order allocatable via the slab allocator */
566 #define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT)
567
568 /*
569 * Kmalloc subsystem.
570 */
571 #ifndef KMALLOC_MIN_SIZE
572 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
573 #endif
574
575 /*
576 * This restriction comes from byte sized index implementation.
577 * Page size is normally 2^12 bytes and, in this case, if we want to use
578 * byte sized index which can represent 2^8 entries, the size of the object
579 * should be equal or greater to 2^12 / 2^8 = 2^4 = 16.
580 * If minimum size of kmalloc is less than 16, we use it as minimum object
581 * size and give up to use byte sized index.
582 */
583 #define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \
584 (KMALLOC_MIN_SIZE) : 16)
585
586 #ifdef CONFIG_RANDOM_KMALLOC_CACHES
587 #define RANDOM_KMALLOC_CACHES_NR 15 // # of cache copies
588 #else
589 #define RANDOM_KMALLOC_CACHES_NR 0
590 #endif
591
592 /*
593 * Whenever changing this, take care of that kmalloc_type() and
594 * create_kmalloc_caches() still work as intended.
595 *
596 * KMALLOC_NORMAL can contain only unaccounted objects whereas KMALLOC_CGROUP
597 * is for accounted but unreclaimable and non-dma objects. All the other
598 * kmem caches can have both accounted and unaccounted objects.
599 */
600 enum kmalloc_cache_type {
601 KMALLOC_NORMAL = 0,
602 #ifndef CONFIG_ZONE_DMA
603 KMALLOC_DMA = KMALLOC_NORMAL,
604 #endif
605 #ifndef CONFIG_MEMCG
606 KMALLOC_CGROUP = KMALLOC_NORMAL,
607 #endif
608 KMALLOC_RANDOM_START = KMALLOC_NORMAL,
609 KMALLOC_RANDOM_END = KMALLOC_RANDOM_START + RANDOM_KMALLOC_CACHES_NR,
610 #ifdef CONFIG_SLUB_TINY
611 KMALLOC_RECLAIM = KMALLOC_NORMAL,
612 #else
613 KMALLOC_RECLAIM,
614 #endif
615 #ifdef CONFIG_ZONE_DMA
616 KMALLOC_DMA,
617 #endif
618 #ifdef CONFIG_MEMCG
619 KMALLOC_CGROUP,
620 #endif
621 NR_KMALLOC_TYPES
622 };
623
624 typedef struct kmem_cache * kmem_buckets[KMALLOC_SHIFT_HIGH + 1];
625
626 extern kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES];
627
628 /*
629 * Define gfp bits that should not be set for KMALLOC_NORMAL.
630 */
631 #define KMALLOC_NOT_NORMAL_BITS \
632 (__GFP_RECLAIMABLE | \
633 (IS_ENABLED(CONFIG_ZONE_DMA) ? __GFP_DMA : 0) | \
634 (IS_ENABLED(CONFIG_MEMCG) ? __GFP_ACCOUNT : 0))
635
636 extern unsigned long random_kmalloc_seed;
637
kmalloc_type(gfp_t flags,unsigned long caller)638 static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, unsigned long caller)
639 {
640 /*
641 * The most common case is KMALLOC_NORMAL, so test for it
642 * with a single branch for all the relevant flags.
643 */
644 if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0))
645 #ifdef CONFIG_RANDOM_KMALLOC_CACHES
646 /* RANDOM_KMALLOC_CACHES_NR (=15) copies + the KMALLOC_NORMAL */
647 return KMALLOC_RANDOM_START + hash_64(caller ^ random_kmalloc_seed,
648 ilog2(RANDOM_KMALLOC_CACHES_NR + 1));
649 #else
650 return KMALLOC_NORMAL;
651 #endif
652
653 /*
654 * At least one of the flags has to be set. Their priorities in
655 * decreasing order are:
656 * 1) __GFP_DMA
657 * 2) __GFP_RECLAIMABLE
658 * 3) __GFP_ACCOUNT
659 */
660 if (IS_ENABLED(CONFIG_ZONE_DMA) && (flags & __GFP_DMA))
661 return KMALLOC_DMA;
662 if (!IS_ENABLED(CONFIG_MEMCG) || (flags & __GFP_RECLAIMABLE))
663 return KMALLOC_RECLAIM;
664 else
665 return KMALLOC_CGROUP;
666 }
667
668 /*
669 * Figure out which kmalloc slab an allocation of a certain size
670 * belongs to.
671 * 0 = zero alloc
672 * 1 = 65 .. 96 bytes
673 * 2 = 129 .. 192 bytes
674 * n = 2^(n-1)+1 .. 2^n
675 *
676 * Note: __kmalloc_index() is compile-time optimized, and not runtime optimized;
677 * typical usage is via kmalloc_index() and therefore evaluated at compile-time.
678 * Callers where !size_is_constant should only be test modules, where runtime
679 * overheads of __kmalloc_index() can be tolerated. Also see kmalloc_slab().
680 */
__kmalloc_index(size_t size,bool size_is_constant)681 static __always_inline unsigned int __kmalloc_index(size_t size,
682 bool size_is_constant)
683 {
684 if (!size)
685 return 0;
686
687 if (size <= KMALLOC_MIN_SIZE)
688 return KMALLOC_SHIFT_LOW;
689
690 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
691 return 1;
692 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
693 return 2;
694 if (size <= 8) return 3;
695 if (size <= 16) return 4;
696 if (size <= 32) return 5;
697 if (size <= 64) return 6;
698 if (size <= 128) return 7;
699 if (size <= 256) return 8;
700 if (size <= 512) return 9;
701 if (size <= 1024) return 10;
702 if (size <= 2 * 1024) return 11;
703 if (size <= 4 * 1024) return 12;
704 if (size <= 8 * 1024) return 13;
705 if (size <= 16 * 1024) return 14;
706 if (size <= 32 * 1024) return 15;
707 if (size <= 64 * 1024) return 16;
708 if (size <= 128 * 1024) return 17;
709 if (size <= 256 * 1024) return 18;
710 if (size <= 512 * 1024) return 19;
711 if (size <= 1024 * 1024) return 20;
712 if (size <= 2 * 1024 * 1024) return 21;
713
714 if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant)
715 BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()");
716 else
717 BUG();
718
719 /* Will never be reached. Needed because the compiler may complain */
720 return -1;
721 }
722 static_assert(PAGE_SHIFT <= 20);
723 #define kmalloc_index(s) __kmalloc_index(s, true)
724
725 #include <linux/alloc_tag.h>
726
727 /**
728 * kmem_cache_alloc - Allocate an object
729 * @cachep: The cache to allocate from.
730 * @flags: See kmalloc().
731 *
732 * Allocate an object from this cache.
733 * See kmem_cache_zalloc() for a shortcut of adding __GFP_ZERO to flags.
734 *
735 * Return: pointer to the new object or %NULL in case of error
736 */
737 void *kmem_cache_alloc_noprof(struct kmem_cache *cachep,
738 gfp_t flags) __assume_slab_alignment __malloc;
739 #define kmem_cache_alloc(...) alloc_hooks(kmem_cache_alloc_noprof(__VA_ARGS__))
740
741 void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
742 gfp_t gfpflags) __assume_slab_alignment __malloc;
743 #define kmem_cache_alloc_lru(...) alloc_hooks(kmem_cache_alloc_lru_noprof(__VA_ARGS__))
744
745 /**
746 * kmem_cache_charge - memcg charge an already allocated slab memory
747 * @objp: address of the slab object to memcg charge
748 * @gfpflags: describe the allocation context
749 *
750 * kmem_cache_charge allows charging a slab object to the current memcg,
751 * primarily in cases where charging at allocation time might not be possible
752 * because the target memcg is not known (i.e. softirq context)
753 *
754 * The objp should be pointer returned by the slab allocator functions like
755 * kmalloc (with __GFP_ACCOUNT in flags) or kmem_cache_alloc. The memcg charge
756 * behavior can be controlled through gfpflags parameter, which affects how the
757 * necessary internal metadata can be allocated. Including __GFP_NOFAIL denotes
758 * that overcharging is requested instead of failure, but is not applied for the
759 * internal metadata allocation.
760 *
761 * There are several cases where it will return true even if the charging was
762 * not done:
763 * More specifically:
764 *
765 * 1. For !CONFIG_MEMCG or cgroup_disable=memory systems.
766 * 2. Already charged slab objects.
767 * 3. For slab objects from KMALLOC_NORMAL caches - allocated by kmalloc()
768 * without __GFP_ACCOUNT
769 * 4. Allocating internal metadata has failed
770 *
771 * Return: true if charge was successful otherwise false.
772 */
773 bool kmem_cache_charge(void *objp, gfp_t gfpflags);
774 void kmem_cache_free(struct kmem_cache *s, void *objp);
775
776 kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags,
777 unsigned int useroffset, unsigned int usersize,
778 void (*ctor)(void *));
779
780 /*
781 * Bulk allocation and freeing operations. These are accelerated in an
782 * allocator specific way to avoid taking locks repeatedly or building
783 * metadata structures unnecessarily.
784 *
785 * Note that interrupts must be enabled when calling these functions.
786 */
787 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
788
789 int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p);
790 #define kmem_cache_alloc_bulk(...) alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
791
kfree_bulk(size_t size,void ** p)792 static __always_inline void kfree_bulk(size_t size, void **p)
793 {
794 kmem_cache_free_bulk(NULL, size, p);
795 }
796
797 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags,
798 int node) __assume_slab_alignment __malloc;
799 #define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__))
800
801 /*
802 * These macros allow declaring a kmem_buckets * parameter alongside size, which
803 * can be compiled out with CONFIG_SLAB_BUCKETS=n so that a large number of call
804 * sites don't have to pass NULL.
805 */
806 #ifdef CONFIG_SLAB_BUCKETS
807 #define DECL_BUCKET_PARAMS(_size, _b) size_t (_size), kmem_buckets *(_b)
808 #define PASS_BUCKET_PARAMS(_size, _b) (_size), (_b)
809 #define PASS_BUCKET_PARAM(_b) (_b)
810 #else
811 #define DECL_BUCKET_PARAMS(_size, _b) size_t (_size)
812 #define PASS_BUCKET_PARAMS(_size, _b) (_size)
813 #define PASS_BUCKET_PARAM(_b) NULL
814 #endif
815
816 /*
817 * The following functions are not to be used directly and are intended only
818 * for internal use from kmalloc() and kmalloc_node()
819 * with the exception of kunit tests
820 */
821
822 void *__kmalloc_noprof(size_t size, gfp_t flags)
823 __assume_kmalloc_alignment __alloc_size(1);
824
825 void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
826 __assume_kmalloc_alignment __alloc_size(1);
827
828 void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size)
829 __assume_kmalloc_alignment __alloc_size(3);
830
831 void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags,
832 int node, size_t size)
833 __assume_kmalloc_alignment __alloc_size(4);
834
835 void *__kmalloc_large_noprof(size_t size, gfp_t flags)
836 __assume_page_alignment __alloc_size(1);
837
838 void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
839 __assume_page_alignment __alloc_size(1);
840
841 /**
842 * kmalloc - allocate kernel memory
843 * @size: how many bytes of memory are required.
844 * @flags: describe the allocation context
845 *
846 * kmalloc is the normal method of allocating memory
847 * for objects smaller than page size in the kernel.
848 *
849 * The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN
850 * bytes. For @size of power of two bytes, the alignment is also guaranteed
851 * to be at least to the size. For other sizes, the alignment is guaranteed to
852 * be at least the largest power-of-two divisor of @size.
853 *
854 * The @flags argument may be one of the GFP flags defined at
855 * include/linux/gfp_types.h and described at
856 * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>`
857 *
858 * The recommended usage of the @flags is described at
859 * :ref:`Documentation/core-api/memory-allocation.rst <memory_allocation>`
860 *
861 * Below is a brief outline of the most useful GFP flags
862 *
863 * %GFP_KERNEL
864 * Allocate normal kernel ram. May sleep.
865 *
866 * %GFP_NOWAIT
867 * Allocation will not sleep.
868 *
869 * %GFP_ATOMIC
870 * Allocation will not sleep. May use emergency pools.
871 *
872 * Also it is possible to set different flags by OR'ing
873 * in one or more of the following additional @flags:
874 *
875 * %__GFP_ZERO
876 * Zero the allocated memory before returning. Also see kzalloc().
877 *
878 * %__GFP_HIGH
879 * This allocation has high priority and may use emergency pools.
880 *
881 * %__GFP_NOFAIL
882 * Indicate that this allocation is in no way allowed to fail
883 * (think twice before using).
884 *
885 * %__GFP_NORETRY
886 * If memory is not immediately available,
887 * then give up at once.
888 *
889 * %__GFP_NOWARN
890 * If allocation fails, don't issue any warnings.
891 *
892 * %__GFP_RETRY_MAYFAIL
893 * Try really hard to succeed the allocation but fail
894 * eventually.
895 */
kmalloc_noprof(size_t size,gfp_t flags)896 static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t flags)
897 {
898 if (__builtin_constant_p(size) && size) {
899 unsigned int index;
900
901 if (size > KMALLOC_MAX_CACHE_SIZE)
902 return __kmalloc_large_noprof(size, flags);
903
904 index = kmalloc_index(size);
905 return __kmalloc_cache_noprof(
906 kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
907 flags, size);
908 }
909 return __kmalloc_noprof(size, flags);
910 }
911 #define kmalloc(...) alloc_hooks(kmalloc_noprof(__VA_ARGS__))
912
913 #define kmem_buckets_alloc(_b, _size, _flags) \
914 alloc_hooks(__kmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE))
915
916 #define kmem_buckets_alloc_track_caller(_b, _size, _flags) \
917 alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE, _RET_IP_))
918
kmalloc_node_noprof(size_t size,gfp_t flags,int node)919 static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gfp_t flags, int node)
920 {
921 if (__builtin_constant_p(size) && size) {
922 unsigned int index;
923
924 if (size > KMALLOC_MAX_CACHE_SIZE)
925 return __kmalloc_large_node_noprof(size, flags, node);
926
927 index = kmalloc_index(size);
928 return __kmalloc_cache_node_noprof(
929 kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
930 flags, node, size);
931 }
932 return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node);
933 }
934 #define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__))
935
936 /**
937 * kmalloc_array - allocate memory for an array.
938 * @n: number of elements.
939 * @size: element size.
940 * @flags: the type of memory to allocate (see kmalloc).
941 */
kmalloc_array_noprof(size_t n,size_t size,gfp_t flags)942 static inline __alloc_size(1, 2) void *kmalloc_array_noprof(size_t n, size_t size, gfp_t flags)
943 {
944 size_t bytes;
945
946 if (unlikely(check_mul_overflow(n, size, &bytes)))
947 return NULL;
948 return kmalloc_noprof(bytes, flags);
949 }
950 #define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
951
952 /**
953 * krealloc_array - reallocate memory for an array.
954 * @p: pointer to the memory chunk to reallocate
955 * @new_n: new number of elements to alloc
956 * @new_size: new size of a single member of the array
957 * @flags: the type of memory to allocate (see kmalloc)
958 *
959 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
960 * initial memory allocation, every subsequent call to this API for the same
961 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
962 * __GFP_ZERO is not fully honored by this API.
963 *
964 * See krealloc_noprof() for further details.
965 *
966 * In any case, the contents of the object pointed to are preserved up to the
967 * lesser of the new and old sizes.
968 */
krealloc_array_noprof(void * p,size_t new_n,size_t new_size,gfp_t flags)969 static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(void *p,
970 size_t new_n,
971 size_t new_size,
972 gfp_t flags)
973 {
974 size_t bytes;
975
976 if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
977 return NULL;
978
979 return krealloc_noprof(p, bytes, flags);
980 }
981 #define krealloc_array(...) alloc_hooks(krealloc_array_noprof(__VA_ARGS__))
982
983 /**
984 * kcalloc - allocate memory for an array. The memory is set to zero.
985 * @n: number of elements.
986 * @size: element size.
987 * @flags: the type of memory to allocate (see kmalloc).
988 */
989 #define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
990
991 void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node,
992 unsigned long caller) __alloc_size(1);
993 #define kmalloc_node_track_caller_noprof(size, flags, node, caller) \
994 __kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node, caller)
995 #define kmalloc_node_track_caller(...) \
996 alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_))
997
998 /*
999 * kmalloc_track_caller is a special version of kmalloc that records the
1000 * calling function of the routine calling it for slab leak tracking instead
1001 * of just the calling function (confusing, eh?).
1002 * It's useful when the call to kmalloc comes from a widely-used standard
1003 * allocator where we care about the real place the memory allocation
1004 * request comes from.
1005 */
1006 #define kmalloc_track_caller(...) kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE)
1007
1008 #define kmalloc_track_caller_noprof(...) \
1009 kmalloc_node_track_caller_noprof(__VA_ARGS__, NUMA_NO_NODE, _RET_IP_)
1010
kmalloc_array_node_noprof(size_t n,size_t size,gfp_t flags,int node)1011 static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags,
1012 int node)
1013 {
1014 size_t bytes;
1015
1016 if (unlikely(check_mul_overflow(n, size, &bytes)))
1017 return NULL;
1018 if (__builtin_constant_p(n) && __builtin_constant_p(size))
1019 return kmalloc_node_noprof(bytes, flags, node);
1020 return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(bytes, NULL), flags, node);
1021 }
1022 #define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__))
1023
1024 #define kcalloc_node(_n, _size, _flags, _node) \
1025 kmalloc_array_node(_n, _size, (_flags) | __GFP_ZERO, _node)
1026
1027 /*
1028 * Shortcuts
1029 */
1030 #define kmem_cache_zalloc(_k, _flags) kmem_cache_alloc(_k, (_flags)|__GFP_ZERO)
1031
1032 /**
1033 * kzalloc - allocate memory. The memory is set to zero.
1034 * @size: how many bytes of memory are required.
1035 * @flags: the type of memory to allocate (see kmalloc).
1036 */
kzalloc_noprof(size_t size,gfp_t flags)1037 static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags)
1038 {
1039 return kmalloc_noprof(size, flags | __GFP_ZERO);
1040 }
1041 #define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__))
1042 #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
1043
1044 void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) __alloc_size(1);
1045 #define kvmalloc_node_noprof(size, flags, node) \
1046 __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node)
1047 #define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__))
1048
1049 #define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE)
1050 #define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE)
1051 #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO)
1052
1053 #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
1054 #define kmem_buckets_valloc(_b, _size, _flags) \
1055 alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE))
1056
1057 static inline __alloc_size(1, 2) void *
kvmalloc_array_node_noprof(size_t n,size_t size,gfp_t flags,int node)1058 kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node)
1059 {
1060 size_t bytes;
1061
1062 if (unlikely(check_mul_overflow(n, size, &bytes)))
1063 return NULL;
1064
1065 return kvmalloc_node_noprof(bytes, flags, node);
1066 }
1067
1068 #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
1069 #define kvcalloc_node_noprof(_n,_s,_f,_node) kvmalloc_array_node_noprof(_n,_s,(_f)|__GFP_ZERO,_node)
1070 #define kvcalloc_noprof(...) kvcalloc_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
1071
1072 #define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__))
1073 #define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__))
1074 #define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__))
1075
1076 void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags)
1077 __realloc_size(2);
1078 #define kvrealloc(...) alloc_hooks(kvrealloc_noprof(__VA_ARGS__))
1079
1080 extern void kvfree(const void *addr);
1081 DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T))
1082
1083 extern void kvfree_sensitive(const void *addr, size_t len);
1084
1085 unsigned int kmem_cache_size(struct kmem_cache *s);
1086
1087 #ifndef CONFIG_KVFREE_RCU_BATCHED
kvfree_rcu_barrier(void)1088 static inline void kvfree_rcu_barrier(void)
1089 {
1090 rcu_barrier();
1091 }
1092
kfree_rcu_scheduler_running(void)1093 static inline void kfree_rcu_scheduler_running(void) { }
1094 #else
1095 void kvfree_rcu_barrier(void);
1096
1097 void kfree_rcu_scheduler_running(void);
1098 #endif
1099
1100 /**
1101 * kmalloc_size_roundup - Report allocation bucket size for the given size
1102 *
1103 * @size: Number of bytes to round up from.
1104 *
1105 * This returns the number of bytes that would be available in a kmalloc()
1106 * allocation of @size bytes. For example, a 126 byte request would be
1107 * rounded up to the next sized kmalloc bucket, 128 bytes. (This is strictly
1108 * for the general-purpose kmalloc()-based allocations, and is not for the
1109 * pre-sized kmem_cache_alloc()-based allocations.)
1110 *
1111 * Use this to kmalloc() the full bucket size ahead of time instead of using
1112 * ksize() to query the size after an allocation.
1113 */
1114 size_t kmalloc_size_roundup(size_t size);
1115
1116 void __init kmem_cache_init_late(void);
1117 void __init kvfree_rcu_init(void);
1118
1119 #endif /* _LINUX_SLAB_H */
1120