| /src/contrib/jemalloc/src/ |
| H A D | tsd.c | 66 tsd_in_nominal_list(tsd_t *tsd) { in tsd_in_nominal_list() argument
75 if (tsd == tsd_list) { in tsd_in_nominal_list()
85 tsd_add_nominal(tsd_t *tsd) { in tsd_add_nominal() argument
86 assert(!tsd_in_nominal_list(tsd)); in tsd_add_nominal()
87 assert(tsd_state_get(tsd) <= tsd_state_nominal_max); in tsd_add_nominal()
88 ql_elm_new(tsd, TSD_MANGLE(tsd_link)); in tsd_add_nominal()
89 malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); in tsd_add_nominal()
90 ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tsd_link)); in tsd_add_nominal()
91 malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); in tsd_add_nominal()
95 tsd_remove_nominal(tsd_t *tsd) { in tsd_remove_nominal() argument
[all …]
|
| H A D | prof_recent.c | 30 prof_recent_alloc_max_get(tsd_t *tsd) { in prof_recent_alloc_max_get() argument
31 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); in prof_recent_alloc_max_get()
36 prof_recent_alloc_max_update(tsd_t *tsd, ssize_t max) { in prof_recent_alloc_max_update() argument
37 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); in prof_recent_alloc_max_update()
38 ssize_t old_max = prof_recent_alloc_max_get(tsd); in prof_recent_alloc_max_update()
58 increment_recent_count(tsd_t *tsd, prof_tctx_t *tctx) { in increment_recent_count() argument
59 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); in increment_recent_count()
65 prof_recent_alloc_prepare(tsd_t *tsd, prof_tctx_t *tctx) { in prof_recent_alloc_prepare() argument
68 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); in prof_recent_alloc_prepare()
69 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); in prof_recent_alloc_prepare()
[all …]
|
| H A D | thread_event.c | 22 uint64_t event##_new_event_wait(tsd_t *tsd); \
23 uint64_t event##_postponed_event_wait(tsd_t *tsd); \
24 void event##_event_handler(tsd_t *tsd, uint64_t elapsed);
31 static uint64_t event##_fetch_elapsed(tsd_t *tsd);
37 tcache_gc_fetch_elapsed(tsd_t *tsd) { in tcache_gc_fetch_elapsed() argument
42 tcache_gc_dalloc_fetch_elapsed(tsd_t *tsd) { in tcache_gc_dalloc_fetch_elapsed() argument
47 prof_sample_fetch_elapsed(tsd_t *tsd) { in prof_sample_fetch_elapsed() argument
48 uint64_t last_event = thread_allocated_last_event_get(tsd); in prof_sample_fetch_elapsed()
49 uint64_t last_sample_event = prof_sample_last_event_get(tsd); in prof_sample_fetch_elapsed()
50 prof_sample_last_event_set(tsd, last_event); in prof_sample_fetch_elapsed()
[all …]
|
| H A D | peak_event.c | 19 peak_event_update(tsd_t *tsd) { in peak_event_update() argument
20 uint64_t alloc = tsd_thread_allocated_get(tsd); in peak_event_update()
21 uint64_t dalloc = tsd_thread_deallocated_get(tsd); in peak_event_update()
22 peak_t *peak = tsd_peakp_get(tsd); in peak_event_update()
27 peak_event_activity_callback(tsd_t *tsd) { in peak_event_activity_callback() argument
29 tsd); in peak_event_activity_callback()
30 uint64_t alloc = tsd_thread_allocated_get(tsd); in peak_event_activity_callback()
31 uint64_t dalloc = tsd_thread_deallocated_get(tsd); in peak_event_activity_callback()
39 peak_event_zero(tsd_t *tsd) { in peak_event_zero() argument
40 uint64_t alloc = tsd_thread_allocated_get(tsd); in peak_event_zero()
[all …]
|
| H A D | prof.c | 83 prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx) { in prof_alloc_rollback() argument
86 if (tsd_reentrancy_level_get(tsd) > 0) { in prof_alloc_rollback()
92 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); in prof_alloc_rollback()
94 prof_tctx_try_destroy(tsd, tctx); in prof_alloc_rollback()
99 prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t size, in prof_malloc_sample_object() argument
104 prof_sys_thread_name_fetch(tsd); in prof_malloc_sample_object()
107 edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, in prof_malloc_sample_object()
109 prof_info_set(tsd, edata, tctx, size); in prof_malloc_sample_object()
113 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); in prof_malloc_sample_object()
136 bool record_recent = prof_recent_alloc_prepare(tsd, tctx); in prof_malloc_sample_object()
[all …]
|
| H A D | prof_data.c | 140 prof_data_init(tsd_t *tsd) { in prof_data_init() argument
142 return ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, in prof_data_init()
147 prof_enter(tsd_t *tsd, prof_tdata_t *tdata) { in prof_enter() argument
149 assert(tdata == prof_tdata_get(tsd, false)); in prof_enter()
156 malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx); in prof_enter()
160 prof_leave(tsd_t *tsd, prof_tdata_t *tdata) { in prof_leave() argument
162 assert(tdata == prof_tdata_get(tsd, false)); in prof_leave()
164 malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx); in prof_leave()
177 prof_idump(tsd_tsdn(tsd)); in prof_leave()
180 prof_gdump(tsd_tsdn(tsd)); in prof_leave()
[all …]
|
| H A D | tcache.c | 90 tcache_gc_new_event_wait(tsd_t *tsd) { in tcache_gc_new_event_wait() argument
95 tcache_gc_postponed_event_wait(tsd_t *tsd) { in tcache_gc_postponed_event_wait() argument
100 tcache_gc_dalloc_new_event_wait(tsd_t *tsd) { in tcache_gc_dalloc_new_event_wait() argument
105 tcache_gc_dalloc_postponed_event_wait(tsd_t *tsd) { in tcache_gc_dalloc_postponed_event_wait() argument
123 tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, in tcache_gc_small() argument
148 tcache_bin_flush_small(tsd, tcache, cache_bin, szind, in tcache_gc_small()
162 tcache_gc_large(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, in tcache_gc_large() argument
171 tcache_bin_flush_large(tsd, tcache, cache_bin, szind, in tcache_gc_large()
176 tcache_event(tsd_t *tsd) { in tcache_event() argument
177 tcache_t *tcache = tcache_get(tsd); in tcache_event()
[all …]
|
| H A D | jemalloc.c | 456 arena_bind(tsd_t *tsd, unsigned ind, bool internal) { in arena_bind() argument
457 arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false); in arena_bind()
461 tsd_iarena_set(tsd, arena); in arena_bind()
463 tsd_arena_set(tsd, arena); in arena_bind()
466 tsd_binshards_t *bins = tsd_binshardsp_get(tsd); in arena_bind()
476 arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) { in arena_migrate() argument
482 tsd_arena_set(tsd, newarena); in arena_migrate()
486 arena_decay(tsd_tsdn(tsd), oldarena, in arena_migrate()
492 arena_unbind(tsd_t *tsd, unsigned ind, bool internal) { in arena_unbind() argument
495 arena = arena_get(tsd_tsdn(tsd), ind, false); in arena_unbind()
[all …]
|
| H A D | background_thread.c | 53 bool background_thread_create(tsd_t *tsd, unsigned arena_ind) NOT_REACHED in background_thread_create() argument
54 bool background_threads_enable(tsd_t *tsd) NOT_REACHED in background_thread_create()
55 bool background_threads_disable(tsd_t *tsd) NOT_REACHED in background_thread_create()
233 background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) {
235 malloc_mutex_assert_owner(tsd_tsdn(tsd),
238 malloc_mutex_assert_not_owner(tsd_tsdn(tsd),
242 pre_reentrancy(tsd, NULL);
243 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
253 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
256 post_reentrancy(tsd);
[all …]
|
| H A D | prof_stats.c | 12 prof_stats_enter(tsd_t *tsd, szind_t ind) { in prof_stats_enter() argument
15 malloc_mutex_lock(tsd_tsdn(tsd), &prof_stats_mtx); in prof_stats_enter()
19 prof_stats_leave(tsd_t *tsd) { in prof_stats_leave() argument
20 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_stats_mtx); in prof_stats_leave()
24 prof_stats_inc(tsd_t *tsd, szind_t ind, size_t size) { in prof_stats_inc() argument
26 prof_stats_enter(tsd, ind); in prof_stats_inc()
31 prof_stats_leave(tsd); in prof_stats_inc()
35 prof_stats_dec(tsd_t *tsd, szind_t ind, size_t size) { in prof_stats_dec() argument
37 prof_stats_enter(tsd, ind); in prof_stats_dec()
40 prof_stats_leave(tsd); in prof_stats_dec()
[all …]
|
| H A D | prof_sys.c | 278 prof_backtrace(tsd_t *tsd, prof_bt_t *bt) { in prof_backtrace() argument
283 pre_reentrancy(tsd, NULL); in prof_backtrace()
285 post_reentrancy(tsd); in prof_backtrace()
320 prof_sys_thread_name_fetch(tsd_t *tsd) { in prof_sys_thread_name_fetch() argument
324 prof_thread_name_set_impl(tsd, buf); in prof_sys_thread_name_fetch()
491 prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, in prof_dump() argument
494 assert(tsd_reentrancy_level_get(tsd) == 0); in prof_dump()
496 prof_tdata_t * tdata = prof_tdata_get(tsd, true); in prof_dump()
504 pre_reentrancy(tsd, NULL); in prof_dump()
505 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx); in prof_dump()
[all …]
|
| H A D | prof_log.c | 124 prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) { in prof_log_bt_index() argument
126 malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx); in prof_log_bt_index()
138 iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, in prof_log_bt_index()
159 ckh_insert(tsd, &log_bt_node_set, (void *)new_node, NULL); in prof_log_bt_index()
167 prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) { in prof_log_thr_index() argument
169 malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx); in prof_log_thr_index()
180 iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, in prof_log_thr_index()
196 ckh_insert(tsd, &log_thr_node_set, (void *)new_node, NULL); in prof_log_thr_index()
205 prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info) { in prof_try_log() argument
208 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); in prof_try_log()
[all …]
|
| H A D | ctl.c | 57 static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
982 arenas_i_impl(tsd_t *tsd, size_t i, bool compat, bool init) { in arenas_i_impl() argument
995 (struct container_s *)base_alloc(tsd_tsdn(tsd), in arenas_i_impl()
1003 ret = (ctl_arena_t *)base_alloc(tsd_tsdn(tsd), b0get(), in arenas_i_impl()
1262 ctl_arena_init(tsd_t *tsd, const arena_config_t *config) { in ctl_arena_init() argument
1275 if (arenas_i_impl(tsd, arena_ind, false, true) == NULL) { in ctl_arena_init()
1280 if (arena_init(tsd_tsdn(tsd), arena_ind, config) == NULL) { in ctl_arena_init()
1387 ctl_init(tsd_t *tsd) { in ctl_init() argument
1389 tsdn_t *tsdn = tsd_tsdn(tsd); in ctl_init()
1423 if ((ctl_sarena = arenas_i_impl(tsd, MALLCTL_ARENAS_ALL, false, in ctl_init()
[all …]
|
| /src/contrib/jemalloc/include/jemalloc/internal/ |
| H A D | tsd.h | 165 tsd_t *tsd_fetch_slow(tsd_t *tsd, bool internal);
166 void tsd_state_set(tsd_t *tsd, uint8_t new_state);
167 void tsd_slow_update(tsd_t *tsd);
168 void tsd_prefork(tsd_t *tsd);
169 void tsd_postfork_parent(tsd_t *tsd);
170 void tsd_postfork_child(tsd_t *tsd);
263 tsd_state_get(tsd_t *tsd) { in tsd_state_get() argument
270 return *(uint8_t *)&tsd->state; in tsd_state_get()
279 tsd_t tsd; member
283 tsd_tsdn(tsd_t *tsd) { in tsd_tsdn() argument
[all …]
|
| H A D | thread_event.h | 46 void te_assert_invariants_debug(tsd_t *tsd);
47 void te_event_trigger(tsd_t *tsd, te_ctx_t *ctx);
48 void te_recompute_fast_threshold(tsd_t *tsd);
49 void tsd_te_init(tsd_t *tsd);
77 counter##_get(tsd_t *tsd) { \
78 return tsd_##counter##_get(tsd); \
93 counter##_set(tsd_t *tsd, uint64_t v) { \
94 *tsd_##counter##p_get(tsd) = v; \
114 te_malloc_fastpath_ctx(tsd_t *tsd, uint64_t *allocated, uint64_t *threshold) { in te_malloc_fastpath_ctx() argument
115 *allocated = *tsd_thread_allocatedp_get_unsafe(tsd); in te_malloc_fastpath_ctx()
[all …]
|
| H A D | prof_inlines.h | 41 prof_tdata_get(tsd_t *tsd, bool create) { in prof_tdata_get() argument
46 tdata = tsd_prof_tdata_get(tsd); in prof_tdata_get()
48 assert(tsd_reentrancy_level_get(tsd) == 0); in prof_tdata_get()
50 if (tsd_nominal(tsd)) { in prof_tdata_get()
51 tdata = prof_tdata_init(tsd); in prof_tdata_get()
52 tsd_prof_tdata_set(tsd, tdata); in prof_tdata_get()
55 tdata = prof_tdata_reinit(tsd, tdata); in prof_tdata_get()
56 tsd_prof_tdata_set(tsd, tdata); in prof_tdata_get()
65 prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx, in prof_info_get() argument
71 arena_prof_info_get(tsd, ptr, alloc_ctx, prof_info, false); in prof_info_get()
[all …]
|
| H A D | jemalloc_internal_inlines_b.h | 7 percpu_arena_update(tsd_t *tsd, unsigned cpu) { in percpu_arena_update() argument
9 arena_t *oldarena = tsd_arena_get(tsd); in percpu_arena_update()
15 arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true); in percpu_arena_update()
19 arena_migrate(tsd, oldarena, newarena); in percpu_arena_update()
20 tcache_t *tcache = tcache_get(tsd); in percpu_arena_update()
22 tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd); in percpu_arena_update()
23 tcache_arena_reassociate(tsd_tsdn(tsd), tcache_slow, in percpu_arena_update()
32 arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { in arena_choose_impl() argument
40 if (unlikely(tsd_reentrancy_level_get(tsd) > 0)) { in arena_choose_impl()
41 return arena_get(tsd_tsdn(tsd), 0, true); in arena_choose_impl()
[all …]
|
| H A D | tcache_inlines.h | 12 tcache_enabled_get(tsd_t *tsd) { in tcache_enabled_get() argument
13 return tsd_tcache_enabled_get(tsd); in tcache_enabled_get()
17 tcache_enabled_set(tsd_t *tsd, bool enabled) { in tcache_enabled_set() argument
18 bool was_enabled = tsd_tcache_enabled_get(tsd); in tcache_enabled_set()
21 tsd_tcache_data_init(tsd); in tcache_enabled_set()
23 tcache_cleanup(tsd); in tcache_enabled_set()
26 tsd_tcache_enabled_set(tsd, enabled); in tcache_enabled_set()
27 tsd_slow_update(tsd); in tcache_enabled_set()
45 tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, in tcache_alloc_small() argument
56 arena = arena_choose(tsd, arena); in tcache_alloc_small()
[all …]
|
| H A D | tcache_externs.h | 41 void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
43 void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
45 void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *bin,
49 tcache_t *tcache_create_explicit(tsd_t *tsd);
50 void tcache_cleanup(tsd_t *tsd);
52 bool tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
53 void tcaches_flush(tsd_t *tsd, unsigned ind);
54 void tcaches_destroy(tsd_t *tsd, unsigned ind);
61 void tcache_flush(tsd_t *tsd);
62 bool tsd_tcache_data_init(tsd_t *tsd);
[all …]
|
| H A D | prof_externs.h | 59 prof_tdata_t *prof_tdata_init(tsd_t *tsd);
60 prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
62 void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx);
63 void prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t size,
65 void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_info_t *prof_info);
66 prof_tctx_t *prof_tctx_create(tsd_t *tsd);
68 bool prof_mdump(tsd_t *tsd, const char *filename);
71 void prof_tdata_cleanup(tsd_t *tsd);
74 const char *prof_thread_name_get(tsd_t *tsd);
75 int prof_thread_name_set(tsd_t *tsd, const char *thread_name);
[all …]
|
| H A D | jemalloc_internal_inlines_a.h | 75 tcache_available(tsd_t *tsd) { in tcache_available() argument
81 if (likely(tsd_tcache_enabled_get(tsd))) { in tcache_available()
83 if (config_debug && tsd_tcache_slowp_get(tsd)->arena != NULL) { in tcache_available()
84 tcache_assert_initialized(tsd_tcachep_get(tsd)); in tcache_available()
93 tcache_get(tsd_t *tsd) { in tcache_get() argument
94 if (!tcache_available(tsd)) { in tcache_get()
98 return tsd_tcachep_get(tsd); in tcache_get()
102 tcache_slow_get(tsd_t *tsd) { in tcache_slow_get() argument
103 if (!tcache_available(tsd)) { in tcache_slow_get()
107 return tsd_tcache_slowp_get(tsd); in tcache_slow_get()
[all …]
|
| H A D | peak_event.h | 10 void peak_event_update(tsd_t *tsd);
12 void peak_event_zero(tsd_t *tsd);
13 uint64_t peak_event_max(tsd_t *tsd);
17 uint64_t peak_alloc_new_event_wait(tsd_t *tsd);
18 uint64_t peak_alloc_postponed_event_wait(tsd_t *tsd);
19 void peak_alloc_event_handler(tsd_t *tsd, uint64_t elapsed);
20 uint64_t peak_dalloc_new_event_wait(tsd_t *tsd);
21 uint64_t peak_dalloc_postponed_event_wait(tsd_t *tsd);
22 void peak_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed);
|
| H A D | prof_data.h | 19 bool prof_data_init(tsd_t *tsd);
20 prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt);
21 char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name);
22 int prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name);
24 void prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque,
26 prof_tdata_t * prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid,
28 void prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata);
29 void prof_reset(tsd_t *tsd, size_t lg_sample);
30 void prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx);
|
| H A D | jemalloc_internal_inlines_c.h | 63 ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) { in ialloc() argument
64 return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd), false, in ialloc()
95 ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { in ipalloc() argument
96 return ipallocztm(tsd_tsdn(tsd), usize, alignment, zero, in ipalloc()
97 tcache_get(tsd), false, NULL); in ipalloc()
124 idalloc(tsd_t *tsd, void *ptr) { in idalloc() argument
125 idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd), NULL, false, true); in idalloc()
199 iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, in iralloc() argument
201 return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, in iralloc()
202 tcache_get(tsd), NULL, hook_args); in iralloc()
[all …]
|
| /src/contrib/llvm-project/compiler-rt/lib/asan/ |
| H A D | asan_posix.cpp | 76 static void (*tsd_destructor)(void *tsd) = nullptr;
90 void AsanTSDInit(void (*destructor)(void *tsd)) { in AsanTSDInit() argument
100 void AsanTSDSet(void *tsd) { in AsanTSDSet() argument
102 CHECK(tsd); in AsanTSDSet()
104 key.key = tsd; in AsanTSDSet()
107 void PlatformTSDDtor(void *tsd) { in PlatformTSDDtor() argument
109 CHECK_EQ(key.key, tsd); in PlatformTSDDtor()
113 AsanThread::TSDDtor(tsd); in PlatformTSDDtor()
118 void AsanTSDInit(void (*destructor)(void *tsd)) { in AsanTSDInit() argument
129 void AsanTSDSet(void *tsd) { in AsanTSDSet() argument
[all …]
|