1 #ifndef JEMALLOC_INTERNAL_PROF_INLINES_H
2 #define JEMALLOC_INTERNAL_PROF_INLINES_H
3
4 #include "jemalloc/internal/safety_check.h"
5 #include "jemalloc/internal/sz.h"
6 #include "jemalloc/internal/thread_event.h"
7
8 JEMALLOC_ALWAYS_INLINE void
prof_active_assert()9 prof_active_assert() {
10 cassert(config_prof);
11 /*
12 * If opt_prof is off, then prof_active must always be off, regardless
13 * of whether prof_active_mtx is in effect or not.
14 */
15 assert(opt_prof || !prof_active_state);
16 }
17
18 JEMALLOC_ALWAYS_INLINE bool
prof_active_get_unlocked(void)19 prof_active_get_unlocked(void) {
20 prof_active_assert();
21 /*
22 * Even if opt_prof is true, sampling can be temporarily disabled by
23 * setting prof_active to false. No locking is used when reading
24 * prof_active in the fast path, so there are no guarantees regarding
25 * how long it will take for all threads to notice state changes.
26 */
27 return prof_active_state;
28 }
29
30 JEMALLOC_ALWAYS_INLINE bool
prof_gdump_get_unlocked(void)31 prof_gdump_get_unlocked(void) {
32 /*
33 * No locking is used when reading prof_gdump_val in the fast path, so
34 * there are no guarantees regarding how long it will take for all
35 * threads to notice state changes.
36 */
37 return prof_gdump_val;
38 }
39
40 JEMALLOC_ALWAYS_INLINE prof_tdata_t *
prof_tdata_get(tsd_t * tsd,bool create)41 prof_tdata_get(tsd_t *tsd, bool create) {
42 prof_tdata_t *tdata;
43
44 cassert(config_prof);
45
46 tdata = tsd_prof_tdata_get(tsd);
47 if (create) {
48 assert(tsd_reentrancy_level_get(tsd) == 0);
49 if (unlikely(tdata == NULL)) {
50 if (tsd_nominal(tsd)) {
51 tdata = prof_tdata_init(tsd);
52 tsd_prof_tdata_set(tsd, tdata);
53 }
54 } else if (unlikely(tdata->expired)) {
55 tdata = prof_tdata_reinit(tsd, tdata);
56 tsd_prof_tdata_set(tsd, tdata);
57 }
58 assert(tdata == NULL || tdata->attached);
59 }
60
61 return tdata;
62 }
63
64 JEMALLOC_ALWAYS_INLINE void
prof_info_get(tsd_t * tsd,const void * ptr,emap_alloc_ctx_t * alloc_ctx,prof_info_t * prof_info)65 prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
66 prof_info_t *prof_info) {
67 cassert(config_prof);
68 assert(ptr != NULL);
69 assert(prof_info != NULL);
70
71 arena_prof_info_get(tsd, ptr, alloc_ctx, prof_info, false);
72 }
73
74 JEMALLOC_ALWAYS_INLINE void
prof_info_get_and_reset_recent(tsd_t * tsd,const void * ptr,emap_alloc_ctx_t * alloc_ctx,prof_info_t * prof_info)75 prof_info_get_and_reset_recent(tsd_t *tsd, const void *ptr,
76 emap_alloc_ctx_t *alloc_ctx, prof_info_t *prof_info) {
77 cassert(config_prof);
78 assert(ptr != NULL);
79 assert(prof_info != NULL);
80
81 arena_prof_info_get(tsd, ptr, alloc_ctx, prof_info, true);
82 }
83
84 JEMALLOC_ALWAYS_INLINE void
prof_tctx_reset(tsd_t * tsd,const void * ptr,emap_alloc_ctx_t * alloc_ctx)85 prof_tctx_reset(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
86 cassert(config_prof);
87 assert(ptr != NULL);
88
89 arena_prof_tctx_reset(tsd, ptr, alloc_ctx);
90 }
91
92 JEMALLOC_ALWAYS_INLINE void
prof_tctx_reset_sampled(tsd_t * tsd,const void * ptr)93 prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
94 cassert(config_prof);
95 assert(ptr != NULL);
96
97 arena_prof_tctx_reset_sampled(tsd, ptr);
98 }
99
100 JEMALLOC_ALWAYS_INLINE void
prof_info_set(tsd_t * tsd,edata_t * edata,prof_tctx_t * tctx,size_t size)101 prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
102 cassert(config_prof);
103 assert(edata != NULL);
104 assert((uintptr_t)tctx > (uintptr_t)1U);
105
106 arena_prof_info_set(tsd, edata, tctx, size);
107 }
108
109 JEMALLOC_ALWAYS_INLINE bool
prof_sample_should_skip(tsd_t * tsd,bool sample_event)110 prof_sample_should_skip(tsd_t *tsd, bool sample_event) {
111 cassert(config_prof);
112
113 /* Fastpath: no need to load tdata */
114 if (likely(!sample_event)) {
115 return true;
116 }
117
118 /*
119 * sample_event is always obtained from the thread event module, and
120 * whenever it's true, it means that the thread event module has
121 * already checked the reentrancy level.
122 */
123 assert(tsd_reentrancy_level_get(tsd) == 0);
124
125 prof_tdata_t *tdata = prof_tdata_get(tsd, true);
126 if (unlikely(tdata == NULL)) {
127 return true;
128 }
129
130 return !tdata->active;
131 }
132
133 JEMALLOC_ALWAYS_INLINE prof_tctx_t *
prof_alloc_prep(tsd_t * tsd,bool prof_active,bool sample_event)134 prof_alloc_prep(tsd_t *tsd, bool prof_active, bool sample_event) {
135 prof_tctx_t *ret;
136
137 if (!prof_active ||
138 likely(prof_sample_should_skip(tsd, sample_event))) {
139 ret = (prof_tctx_t *)(uintptr_t)1U;
140 } else {
141 ret = prof_tctx_create(tsd);
142 }
143
144 return ret;
145 }
146
147 JEMALLOC_ALWAYS_INLINE void
prof_malloc(tsd_t * tsd,const void * ptr,size_t size,size_t usize,emap_alloc_ctx_t * alloc_ctx,prof_tctx_t * tctx)148 prof_malloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize,
149 emap_alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) {
150 cassert(config_prof);
151 assert(ptr != NULL);
152 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
153
154 if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) {
155 prof_malloc_sample_object(tsd, ptr, size, usize, tctx);
156 } else {
157 prof_tctx_reset(tsd, ptr, alloc_ctx);
158 }
159 }
160
161 JEMALLOC_ALWAYS_INLINE void
prof_realloc(tsd_t * tsd,const void * ptr,size_t size,size_t usize,prof_tctx_t * tctx,bool prof_active,const void * old_ptr,size_t old_usize,prof_info_t * old_prof_info,bool sample_event)162 prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize,
163 prof_tctx_t *tctx, bool prof_active, const void *old_ptr, size_t old_usize,
164 prof_info_t *old_prof_info, bool sample_event) {
165 bool sampled, old_sampled, moved;
166
167 cassert(config_prof);
168 assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);
169
170 if (prof_active && ptr != NULL) {
171 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
172 if (prof_sample_should_skip(tsd, sample_event)) {
173 /*
174 * Don't sample. The usize passed to prof_alloc_prep()
175 * was larger than what actually got allocated, so a
176 * backtrace was captured for this allocation, even
177 * though its actual usize was insufficient to cross the
178 * sample threshold.
179 */
180 prof_alloc_rollback(tsd, tctx);
181 tctx = (prof_tctx_t *)(uintptr_t)1U;
182 }
183 }
184
185 sampled = ((uintptr_t)tctx > (uintptr_t)1U);
186 old_sampled = ((uintptr_t)old_prof_info->alloc_tctx > (uintptr_t)1U);
187 moved = (ptr != old_ptr);
188
189 if (unlikely(sampled)) {
190 prof_malloc_sample_object(tsd, ptr, size, usize, tctx);
191 } else if (moved) {
192 prof_tctx_reset(tsd, ptr, NULL);
193 } else if (unlikely(old_sampled)) {
194 /*
195 * prof_tctx_reset() would work for the !moved case as well,
196 * but prof_tctx_reset_sampled() is slightly cheaper, and the
197 * proper thing to do here in the presence of explicit
198 * knowledge re: moved state.
199 */
200 prof_tctx_reset_sampled(tsd, ptr);
201 } else {
202 prof_info_t prof_info;
203 prof_info_get(tsd, ptr, NULL, &prof_info);
204 assert((uintptr_t)prof_info.alloc_tctx == (uintptr_t)1U);
205 }
206
207 /*
208 * The prof_free_sampled_object() call must come after the
209 * prof_malloc_sample_object() call, because tctx and old_tctx may be
210 * the same, in which case reversing the call order could cause the tctx
211 * to be prematurely destroyed as a side effect of momentarily zeroed
212 * counters.
213 */
214 if (unlikely(old_sampled)) {
215 prof_free_sampled_object(tsd, old_usize, old_prof_info);
216 }
217 }
218
219 JEMALLOC_ALWAYS_INLINE size_t
prof_sample_align(size_t orig_align)220 prof_sample_align(size_t orig_align) {
221 /*
222 * Enforce page alignment, so that sampled allocations can be identified
223 * w/o metadata lookup.
224 */
225 assert(opt_prof);
226 return (opt_cache_oblivious && orig_align < PAGE) ? PAGE :
227 orig_align;
228 }
229
230 JEMALLOC_ALWAYS_INLINE bool
prof_sample_aligned(const void * ptr)231 prof_sample_aligned(const void *ptr) {
232 return ((uintptr_t)ptr & PAGE_MASK) == 0;
233 }
234
235 JEMALLOC_ALWAYS_INLINE bool
prof_sampled(tsd_t * tsd,const void * ptr)236 prof_sampled(tsd_t *tsd, const void *ptr) {
237 prof_info_t prof_info;
238 prof_info_get(tsd, ptr, NULL, &prof_info);
239 bool sampled = (uintptr_t)prof_info.alloc_tctx > (uintptr_t)1U;
240 if (sampled) {
241 assert(prof_sample_aligned(ptr));
242 }
243 return sampled;
244 }
245
246 JEMALLOC_ALWAYS_INLINE void
prof_free(tsd_t * tsd,const void * ptr,size_t usize,emap_alloc_ctx_t * alloc_ctx)247 prof_free(tsd_t *tsd, const void *ptr, size_t usize,
248 emap_alloc_ctx_t *alloc_ctx) {
249 prof_info_t prof_info;
250 prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info);
251
252 cassert(config_prof);
253 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
254
255 if (unlikely((uintptr_t)prof_info.alloc_tctx > (uintptr_t)1U)) {
256 assert(prof_sample_aligned(ptr));
257 prof_free_sampled_object(tsd, usize, &prof_info);
258 }
259 }
260
261 #endif /* JEMALLOC_INTERNAL_PROF_INLINES_H */
262