1 #ifdef JEMALLOC_INTERNAL_TSD_GENERIC_H
2 #error This file should be included only once, by tsd.h.
3 #endif
4 #define JEMALLOC_INTERNAL_TSD_GENERIC_H
5
6 typedef struct tsd_init_block_s tsd_init_block_t;
7 struct tsd_init_block_s {
8 ql_elm(tsd_init_block_t) link;
9 pthread_t thread;
10 void *data;
11 };
12
13 /* Defined in tsd.c, to allow the mutex headers to have tsd dependencies. */
14 typedef struct tsd_init_head_s tsd_init_head_t;
15
16 typedef struct {
17 bool initialized;
18 tsd_t val;
19 } tsd_wrapper_t;
20
21 void *tsd_init_check_recursion(tsd_init_head_t *head,
22 tsd_init_block_t *block);
23 void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
24
25 extern pthread_key_t tsd_tsd;
26 extern tsd_init_head_t tsd_init_head;
27 extern tsd_wrapper_t tsd_boot_wrapper;
28 extern bool tsd_booted;
29
30 /* Initialization/cleanup. */
31 JEMALLOC_ALWAYS_INLINE void
tsd_cleanup_wrapper(void * arg)32 tsd_cleanup_wrapper(void *arg) {
33 tsd_wrapper_t *wrapper = (tsd_wrapper_t *)arg;
34
35 if (wrapper->initialized) {
36 wrapper->initialized = false;
37 tsd_cleanup(&wrapper->val);
38 if (wrapper->initialized) {
39 /* Trigger another cleanup round. */
40 if (pthread_setspecific(tsd_tsd, (void *)wrapper) != 0)
41 {
42 malloc_write("<jemalloc>: Error setting TSD\n");
43 if (opt_abort) {
44 abort();
45 }
46 }
47 return;
48 }
49 }
50 malloc_tsd_dalloc(wrapper);
51 }
52
53 JEMALLOC_ALWAYS_INLINE void
tsd_wrapper_set(tsd_wrapper_t * wrapper)54 tsd_wrapper_set(tsd_wrapper_t *wrapper) {
55 if (unlikely(!tsd_booted)) {
56 return;
57 }
58 if (pthread_setspecific(tsd_tsd, (void *)wrapper) != 0) {
59 malloc_write("<jemalloc>: Error setting TSD\n");
60 abort();
61 }
62 }
63
64 JEMALLOC_ALWAYS_INLINE tsd_wrapper_t *
tsd_wrapper_get(bool init)65 tsd_wrapper_get(bool init) {
66 tsd_wrapper_t *wrapper;
67
68 if (unlikely(!tsd_booted)) {
69 return &tsd_boot_wrapper;
70 }
71
72 wrapper = (tsd_wrapper_t *)pthread_getspecific(tsd_tsd);
73
74 if (init && unlikely(wrapper == NULL)) {
75 tsd_init_block_t block;
76 wrapper = (tsd_wrapper_t *)
77 tsd_init_check_recursion(&tsd_init_head, &block);
78 if (wrapper) {
79 return wrapper;
80 }
81 wrapper = (tsd_wrapper_t *)
82 malloc_tsd_malloc(sizeof(tsd_wrapper_t));
83 block.data = (void *)wrapper;
84 if (wrapper == NULL) {
85 malloc_write("<jemalloc>: Error allocating TSD\n");
86 abort();
87 } else {
88 wrapper->initialized = false;
89 JEMALLOC_DIAGNOSTIC_PUSH
90 JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
91 tsd_t initializer = TSD_INITIALIZER;
92 JEMALLOC_DIAGNOSTIC_POP
93 wrapper->val = initializer;
94 }
95 tsd_wrapper_set(wrapper);
96 tsd_init_finish(&tsd_init_head, &block);
97 }
98 return wrapper;
99 }
100
101 JEMALLOC_ALWAYS_INLINE bool
tsd_boot0(void)102 tsd_boot0(void) {
103 tsd_wrapper_t *wrapper;
104 tsd_init_block_t block;
105
106 wrapper = (tsd_wrapper_t *)
107 tsd_init_check_recursion(&tsd_init_head, &block);
108 if (wrapper) {
109 return false;
110 }
111 block.data = &tsd_boot_wrapper;
112 if (pthread_key_create(&tsd_tsd, tsd_cleanup_wrapper) != 0) {
113 return true;
114 }
115 tsd_booted = true;
116 tsd_wrapper_set(&tsd_boot_wrapper);
117 tsd_init_finish(&tsd_init_head, &block);
118 return false;
119 }
120
121 JEMALLOC_ALWAYS_INLINE void
tsd_boot1(void)122 tsd_boot1(void) {
123 tsd_wrapper_t *wrapper;
124 wrapper = (tsd_wrapper_t *)malloc_tsd_malloc(sizeof(tsd_wrapper_t));
125 if (wrapper == NULL) {
126 malloc_write("<jemalloc>: Error allocating TSD\n");
127 abort();
128 }
129 tsd_boot_wrapper.initialized = false;
130 tsd_cleanup(&tsd_boot_wrapper.val);
131 wrapper->initialized = false;
132 JEMALLOC_DIAGNOSTIC_PUSH
133 JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
134 tsd_t initializer = TSD_INITIALIZER;
135 JEMALLOC_DIAGNOSTIC_POP
136 wrapper->val = initializer;
137 tsd_wrapper_set(wrapper);
138 }
139
140 JEMALLOC_ALWAYS_INLINE bool
tsd_boot(void)141 tsd_boot(void) {
142 if (tsd_boot0()) {
143 return true;
144 }
145 tsd_boot1();
146 return false;
147 }
148
149 JEMALLOC_ALWAYS_INLINE bool
tsd_booted_get(void)150 tsd_booted_get(void) {
151 return tsd_booted;
152 }
153
154 JEMALLOC_ALWAYS_INLINE bool
tsd_get_allocates(void)155 tsd_get_allocates(void) {
156 return true;
157 }
158
159 /* Get/set. */
160 JEMALLOC_ALWAYS_INLINE tsd_t *
tsd_get(bool init)161 tsd_get(bool init) {
162 tsd_wrapper_t *wrapper;
163
164 assert(tsd_booted);
165 wrapper = tsd_wrapper_get(init);
166 if (tsd_get_allocates() && !init && wrapper == NULL) {
167 return NULL;
168 }
169 return &wrapper->val;
170 }
171
172 JEMALLOC_ALWAYS_INLINE void
tsd_set(tsd_t * val)173 tsd_set(tsd_t *val) {
174 tsd_wrapper_t *wrapper;
175
176 assert(tsd_booted);
177 wrapper = tsd_wrapper_get(true);
178 if (likely(&wrapper->val != val)) {
179 wrapper->val = *(val);
180 }
181 wrapper->initialized = true;
182 }
183