1 #ifndef JEMALLOC_INTERNAL_BASE_H
2 #define JEMALLOC_INTERNAL_BASE_H
3
4 #include "jemalloc/internal/edata.h"
5 #include "jemalloc/internal/ehooks.h"
6 #include "jemalloc/internal/mutex.h"
7
8 enum metadata_thp_mode_e {
9 metadata_thp_disabled = 0,
10 /*
11 * Lazily enable hugepage for metadata. To avoid high RSS caused by THP
12 * + low usage arena (i.e. THP becomes a significant percentage), the
13 * "auto" option only starts using THP after a base allocator used up
14 * the first THP region. Starting from the second hugepage (in a single
15 * arena), "auto" behaves the same as "always", i.e. madvise hugepage
16 * right away.
17 */
18 metadata_thp_auto = 1,
19 metadata_thp_always = 2,
20 metadata_thp_mode_limit = 3
21 };
22 typedef enum metadata_thp_mode_e metadata_thp_mode_t;
23
24 #define METADATA_THP_DEFAULT metadata_thp_disabled
25 extern metadata_thp_mode_t opt_metadata_thp;
26 extern const char *metadata_thp_mode_names[];
27
28
29 /* Embedded at the beginning of every block of base-managed virtual memory. */
30 typedef struct base_block_s base_block_t;
31 struct base_block_s {
32 /* Total size of block's virtual memory mapping. */
33 size_t size;
34
35 /* Next block in list of base's blocks. */
36 base_block_t *next;
37
38 /* Tracks unused trailing space. */
39 edata_t edata;
40 };
41
42 typedef struct base_s base_t;
43 struct base_s {
44 /*
45 * User-configurable extent hook functions.
46 */
47 ehooks_t ehooks;
48
49 /*
50 * User-configurable extent hook functions for metadata allocations.
51 */
52 ehooks_t ehooks_base;
53
54 /* Protects base_alloc() and base_stats_get() operations. */
55 malloc_mutex_t mtx;
56
57 /* Using THP when true (metadata_thp auto mode). */
58 bool auto_thp_switched;
59 /*
60 * Most recent size class in the series of increasingly large base
61 * extents. Logarithmic spacing between subsequent allocations ensures
62 * that the total number of distinct mappings remains small.
63 */
64 pszind_t pind_last;
65
66 /* Serial number generation state. */
67 size_t extent_sn_next;
68
69 /* Chain of all blocks associated with base. */
70 base_block_t *blocks;
71
72 /* Heap of extents that track unused trailing space within blocks. */
73 edata_heap_t avail[SC_NSIZES];
74
75 /* Stats, only maintained if config_stats. */
76 size_t allocated;
77 size_t resident;
78 size_t mapped;
79 /* Number of THP regions touched. */
80 size_t n_thp;
81 };
82
83 static inline unsigned
base_ind_get(const base_t * base)84 base_ind_get(const base_t *base) {
85 return ehooks_ind_get(&base->ehooks);
86 }
87
88 static inline bool
metadata_thp_enabled(void)89 metadata_thp_enabled(void) {
90 return (opt_metadata_thp != metadata_thp_disabled);
91 }
92
93 base_t *b0get(void);
94 base_t *base_new(tsdn_t *tsdn, unsigned ind,
95 const extent_hooks_t *extent_hooks, bool metadata_use_hooks);
96 void base_delete(tsdn_t *tsdn, base_t *base);
97 ehooks_t *base_ehooks_get(base_t *base);
98 ehooks_t *base_ehooks_get_for_metadata(base_t *base);
99 extent_hooks_t *base_extent_hooks_set(base_t *base,
100 extent_hooks_t *extent_hooks);
101 void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);
102 edata_t *base_alloc_edata(tsdn_t *tsdn, base_t *base);
103 void base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated,
104 size_t *resident, size_t *mapped, size_t *n_thp);
105 void base_prefork(tsdn_t *tsdn, base_t *base);
106 void base_postfork_parent(tsdn_t *tsdn, base_t *base);
107 void base_postfork_child(tsdn_t *tsdn, base_t *base);
108 bool base_boot(tsdn_t *tsdn);
109
110 #endif /* JEMALLOC_INTERNAL_BASE_H */
111