Skip to content

Commit c2fcf9c

Browse files
interwqinterwq
committed
Switch to fine-grained reentrancy support.
Previously we had a general detection and support of reentrancy, at the cost of having branches and inc / dec operations on fast paths. To avoid taxing fast paths, we move the reentrancy operations onto tsd slow state, and only modify reentrancy level around external calls (that might trigger reentrancy).
1 parent b348ba2 commit c2fcf9c

File tree

8 files changed

+87
-83
lines changed

8 files changed

+87
-83
lines changed

‎include/jemalloc/internal/jemalloc_internal_inlines_a.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ tcache_t *tcache_get(tsd_t *tsd);
3333
malloc_cpuid_t malloc_getcpu(void);
3434
unsigned percpu_arena_choose(void);
3535
unsigned percpu_arena_ind_limit(void);
36+
void pre_reentrancy(tsd_t *tsd);
37+
void post_reentrancy(tsd_t *tsd);
3638
#endif
3739

3840
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
@@ -445,6 +447,27 @@ tcache_get(tsd_t *tsd) {
445447

446448
return tsd_tcachep_get(tsd);
447449
}
450+
451+
JEMALLOC_INLINE void
452+
pre_reentrancy(tsd_t *tsd) {
453+
bool fast = tsd_fast(tsd);
454+
++*tsd_reentrancy_levelp_get(tsd);
455+
if (fast) {
456+
/* Prepare slow path for reentrancy. */
457+
tsd_slow_update(tsd);
458+
assert(tsd->state == tsd_state_nominal_slow);
459+
}
460+
}
461+
462+
JEMALLOC_INLINE void
463+
post_reentrancy(tsd_t *tsd) {
464+
int8_t *reentrancy_level = tsd_reentrancy_levelp_get(tsd);
465+
assert(*reentrancy_level > 0);
466+
if (--*reentrancy_level == 0) {
467+
tsd_slow_update(tsd);
468+
}
469+
}
470+
448471
#endif
449472

450473
#endif /* JEMALLOC_INTERNAL_INLINES_A_H */

‎include/jemalloc/internal/jemalloc_internal_inlines_b.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
1616
}
1717

1818
/* During reentrancy, arena 0 is the safest bet. */
19-
if (*tsd_reentrancy_levelp_get(tsd) > 1) {
19+
if (unlikely(tsd_reentrancy_level_get(tsd) > 0)) {
2020
return arena_get(tsd_tsdn(tsd), 0, true);
2121
}
2222

‎include/jemalloc/internal/jemalloc_internal_inlines_c.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,8 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, alloc_ctx_t *alloc_ctx,
117117
if (config_stats && is_internal) {
118118
arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr));
119119
}
120-
if (!is_internal && *tsd_reentrancy_levelp_get(tsdn_tsd(tsdn)) != 0) {
121-
tcache = NULL;
120+
if (!is_internal && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
121+
assert(tcache == NULL);
122122
}
123123
arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
124124
}

‎include/jemalloc/internal/tsd_inlines.h‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ tsd_t *tsdn_tsd(tsdn_t *tsdn);
2020
rtree_ctx_t *tsd_rtree_ctx(tsd_t *tsd);
2121
rtree_ctx_t *tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback);
2222
bool tsd_fast(tsd_t *tsd);
23-
void tsd_assert_fast(tsd_t *tsd);
23+
bool tsd_assert_fast(tsd_t *tsd);
2424
#endif
2525

2626
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
@@ -52,9 +52,11 @@ MALLOC_TSD
5252
#undef MALLOC_TSD_getset_no
5353
#undef O
5454

55-
JEMALLOC_ALWAYS_INLINE void
55+
JEMALLOC_ALWAYS_INLINE bool
5656
tsd_assert_fast(tsd_t *tsd) {
57-
assert(!malloc_slow && tsd_tcache_enabled_get(tsd));
57+
assert(!malloc_slow && tsd_tcache_enabled_get(tsd) &&
58+
tsd_reentrancy_level_get(tsd) == 0);
59+
return true;
5860
}
5961

6062
JEMALLOC_ALWAYS_INLINE bool

‎include/jemalloc/internal/tsd_structs.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ struct tsd_init_head_s {
5555
/* O(name, type, [gs]et, init, cleanup) */ \
5656
O(tcache_enabled, bool, yes, yes, no) \
5757
O(arenas_tdata_bypass, bool, no, no, no) \
58-
O(reentrancy_level, int8_t, no, no, no) \
58+
O(reentrancy_level, int8_t, yes, no, no) \
5959
O(narenas_tdata, uint32_t, yes, no, no) \
6060
O(thread_allocated, uint64_t, yes, no, no) \
6161
O(thread_deallocated, uint64_t, yes, no, no) \

‎src/arena.c‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1966,11 +1966,9 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
19661966
* If we're here, then arena 0 already exists, so bootstrapping
19671967
* is done enough that we should have tsd.
19681968
*/
1969-
int8_t *reentrancy_level = tsd_reentrancy_levelp_get(tsdn_tsd(
1970-
tsdn));
1971-
++*reentrancy_level;
1969+
pre_reentrancy(tsdn_tsd(tsdn));
19721970
hooks_arena_new_hook();
1973-
--*reentrancy_level;
1971+
post_reentrancy(tsdn_tsd(tsdn));
19741972
}
19751973

19761974
return arena;

‎src/jemalloc.c‎

Lines changed: 50 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1663,13 +1663,8 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
16631663
szind_t ind = 0;
16641664
size_t usize = 0;
16651665

1666-
/*
1667-
* For reentrancy checking, we get the old reentrancy level from tsd and
1668-
* reset it once we're done. In case of early bailout though, we never
1669-
* bother getting the old level, so we shouldn't try to reset it. This
1670-
* is indicated by leaving the pointer as NULL.
1671-
*/
1672-
int8_t *reentrancy_level = NULL;
1666+
/* Reentrancy is only checked on slow path. */
1667+
int8_t reentrancy_level;
16731668

16741669
/* Compute the amount of memory the user wants. */
16751670
if (unlikely(compute_size_with_overflow(sopts->may_overflow, dopts,
@@ -1716,12 +1711,11 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
17161711
* If we need to handle reentrancy, we can do it out of a
17171712
* known-initialized arena (i.e. arena 0).
17181713
*/
1719-
reentrancy_level = tsd_reentrancy_levelp_get(tsd);
1720-
++*reentrancy_level;
1721-
if (*reentrancy_level == 1) {
1714+
reentrancy_level = tsd_reentrancy_level_get(tsd);
1715+
if (reentrancy_level == 0) {
17221716
witness_assert_lockless(tsd_tsdn(tsd));
17231717
}
1724-
if (unlikely(*reentrancy_level > 1)) {
1718+
if (sopts->slow && unlikely(reentrancy_level > 0)) {
17251719
/*
17261720
* We should never specify particular arenas or tcaches from
17271721
* within our internal allocations.
@@ -1795,14 +1789,9 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
17951789
}
17961790

17971791
/* Success! */
1798-
if (*reentrancy_level == 1) {
1792+
if (reentrancy_level == 0) {
17991793
witness_assert_lockless(tsd_tsdn(tsd));
18001794
}
1801-
/*
1802-
* If we got here, we never bailed out on a failure path, so
1803-
* reentrancy_level is non-null.
1804-
*/
1805-
--*reentrancy_level;
18061795
*dopts->result = allocation;
18071796
return 0;
18081797

@@ -1826,10 +1815,6 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
18261815
*dopts->result = NULL;
18271816
}
18281817

1829-
if (reentrancy_level != NULL) {
1830-
--*reentrancy_level;
1831-
}
1832-
18331818
return ENOMEM;
18341819

18351820
/*
@@ -1857,10 +1842,6 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
18571842
*dopts->result = NULL;
18581843
}
18591844

1860-
if (reentrancy_level != NULL) {
1861-
--*reentrancy_level;
1862-
}
1863-
18641845
return EINVAL;
18651846
}
18661847

@@ -2053,8 +2034,11 @@ irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
20532034

20542035
JEMALLOC_ALWAYS_INLINE_C void
20552036
ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
2056-
if (*tsd_reentrancy_levelp_get(tsd) == 0) {
2037+
assert(slow_path || tsd_assert_fast(tsd));
2038+
if (tsd_reentrancy_level_get(tsd) == 0) {
20572039
witness_assert_lockless(tsd_tsdn(tsd));
2040+
} else {
2041+
assert(slow_path);
20582042
}
20592043

20602044
assert(ptr != NULL);
@@ -2088,8 +2072,11 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
20882072

20892073
JEMALLOC_ALWAYS_INLINE_C void
20902074
isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
2091-
if (*tsd_reentrancy_levelp_get(tsd) == 0) {
2075+
assert(slow_path || tsd_assert_fast(tsd));
2076+
if (tsd_reentrancy_level_get(tsd) == 0) {
20922077
witness_assert_lockless(tsd_tsdn(tsd));
2078+
} else {
2079+
assert(slow_path);
20932080
}
20942081

20952082
assert(ptr != NULL);
@@ -2129,14 +2116,14 @@ je_realloc(void *ptr, size_t size) {
21292116

21302117
if (unlikely(size == 0)) {
21312118
if (ptr != NULL) {
2132-
tsd_t *tsd;
2133-
21342119
/* realloc(ptr, 0) is equivalent to free(ptr). */
21352120
UTRACE(ptr, 0, 0);
2136-
tsd = tsd_fetch();
2137-
tcache_t *tcache = NULL;
2138-
if (likely(*tsd_reentrancy_levelp_get(tsd) == 0)) {
2121+
tcache_t *tcache;
2122+
tsd_t *tsd = tsd_fetch();
2123+
if (tsd_reentrancy_level_get(tsd) == 0) {
21392124
tcache = tcache_get(tsd);
2125+
} else {
2126+
tcache = NULL;
21402127
}
21412128
ifree(tsd, ptr, tcache, true);
21422129
return NULL;
@@ -2200,29 +2187,25 @@ je_free(void *ptr) {
22002187
UTRACE(ptr, 0, 0);
22012188
if (likely(ptr != NULL)) {
22022189
tsd_t *tsd = tsd_fetch();
2203-
if (*tsd_reentrancy_levelp_get(tsd) == 0) {
2190+
if (tsd_reentrancy_level_get(tsd) == 0) {
22042191
witness_assert_lockless(tsd_tsdn(tsd));
22052192
}
2193+
22062194
tcache_t *tcache;
22072195
if (likely(tsd_fast(tsd))) {
22082196
tsd_assert_fast(tsd);
2209-
if (likely(*tsd_reentrancy_levelp_get(tsd) == 0)) {
2210-
/* Getting tcache ptr unconditionally. */
2211-
tcache = tsd_tcachep_get(tsd);
2212-
assert(tcache == tcache_get(tsd));
2213-
} else {
2214-
tcache = NULL;
2215-
}
2197+
/* Unconditionally get tcache ptr on fast path. */
2198+
tcache = tsd_tcachep_get(tsd);
22162199
ifree(tsd, ptr, tcache, false);
22172200
} else {
2218-
if (likely(*tsd_reentrancy_levelp_get(tsd) == 0)) {
2201+
if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
22192202
tcache = tcache_get(tsd);
22202203
} else {
22212204
tcache = NULL;
22222205
}
22232206
ifree(tsd, ptr, tcache, true);
22242207
}
2225-
if (*tsd_reentrancy_levelp_get(tsd) == 0) {
2208+
if (tsd_reentrancy_level_get(tsd) == 0) {
22262209
witness_assert_lockless(tsd_tsdn(tsd));
22272210
}
22282211
}
@@ -2707,33 +2690,32 @@ je_sallocx(const void *ptr, int flags) {
27072690

27082691
JEMALLOC_EXPORT void JEMALLOC_NOTHROW
27092692
je_dallocx(void *ptr, int flags) {
2710-
tsd_t *tsd;
2711-
tcache_t *tcache;
2712-
27132693
assert(ptr != NULL);
27142694
assert(malloc_initialized() || IS_INITIALIZER);
27152695

2716-
tsd = tsd_fetch();
2696+
tsd_t *tsd = tsd_fetch();
27172697
bool fast = tsd_fast(tsd);
27182698
witness_assert_lockless(tsd_tsdn(tsd));
2699+
2700+
tcache_t *tcache;
27192701
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
27202702
/* Not allowed to be reentrant and specify a custom tcache. */
2721-
assert(*tsd_reentrancy_levelp_get(tsd) == 0);
2703+
assert(tsd_reentrancy_level_get(tsd) == 0);
27222704
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
27232705
tcache = NULL;
27242706
} else {
27252707
tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
27262708
}
27272709
} else {
2728-
if (likely(*tsd_reentrancy_levelp_get(tsd) == 0)) {
2729-
if (likely(fast)) {
2730-
tcache = tsd_tcachep_get(tsd);
2731-
assert(tcache == tcache_get(tsd));
2732-
} else {
2710+
if (likely(fast)) {
2711+
tcache = tsd_tcachep_get(tsd);
2712+
assert(tcache == tcache_get(tsd));
2713+
} else {
2714+
if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
27332715
tcache = tcache_get(tsd);
2716+
} else {
2717+
tcache = NULL;
27342718
}
2735-
} else {
2736-
tcache = NULL;
27372719
}
27382720
}
27392721

@@ -2749,10 +2731,9 @@ je_dallocx(void *ptr, int flags) {
27492731

27502732
JEMALLOC_ALWAYS_INLINE_C size_t
27512733
inallocx(tsdn_t *tsdn, size_t size, int flags) {
2752-
size_t usize;
2753-
27542734
witness_assert_lockless(tsdn);
27552735

2736+
size_t usize;
27562737
if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) {
27572738
usize = s2u(size);
27582739
} else {
@@ -2764,36 +2745,34 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) {
27642745

27652746
JEMALLOC_EXPORT void JEMALLOC_NOTHROW
27662747
je_sdallocx(void *ptr, size_t size, int flags) {
2767-
tsd_t *tsd;
2768-
size_t usize;
2769-
tcache_t *tcache;
2770-
27712748
assert(ptr != NULL);
27722749
assert(malloc_initialized() || IS_INITIALIZER);
2773-
tsd = tsd_fetch();
2750+
2751+
tsd_t *tsd = tsd_fetch();
27742752
bool fast = tsd_fast(tsd);
2775-
usize = inallocx(tsd_tsdn(tsd), size, flags);
2753+
size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
27762754
assert(usize == isalloc(tsd_tsdn(tsd), ptr));
2777-
27782755
witness_assert_lockless(tsd_tsdn(tsd));
2756+
2757+
tcache_t *tcache;
27792758
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
27802759
/* Not allowed to be reentrant and specify a custom tcache. */
2781-
assert(*tsd_reentrancy_levelp_get(tsd) == 0);
2760+
assert(tsd_reentrancy_level_get(tsd) == 0);
27822761
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
27832762
tcache = NULL;
27842763
} else {
27852764
tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
27862765
}
27872766
} else {
2788-
if (likely(*tsd_reentrancy_levelp_get(tsd) == 0)) {
2789-
if (likely(fast)) {
2790-
tcache = tsd_tcachep_get(tsd);
2791-
assert(tcache == tcache_get(tsd));
2792-
} else {
2767+
if (likely(fast)) {
2768+
tcache = tsd_tcachep_get(tsd);
2769+
assert(tcache == tcache_get(tsd));
2770+
} else {
2771+
if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
27932772
tcache = tcache_get(tsd);
2773+
} else {
2774+
tcache = NULL;
27942775
}
2795-
} else {
2796-
tcache = NULL;
27972776
}
27982777
}
27992778

‎src/tsd.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ malloc_tsd_data(, , tsd_t, TSD_INITIALIZER)
1515
void
1616
tsd_slow_update(tsd_t *tsd) {
1717
if (tsd_nominal(tsd)) {
18-
if (malloc_slow || !tsd->tcache_enabled) {
18+
if (malloc_slow || !tsd->tcache_enabled ||
19+
tsd_reentrancy_level_get(tsd) > 0) {
1920
tsd->state = tsd_state_nominal_slow;
2021
} else {
2122
tsd->state = tsd_state_nominal;
@@ -28,6 +29,7 @@ tsd_fetch_slow(tsd_t *tsd) {
2829
if (tsd->state == tsd_state_nominal_slow) {
2930
/* On slow path but no work needed. */
3031
assert(malloc_slow || !tsd_tcache_enabled_get(tsd) ||
32+
tsd_reentrancy_level_get(tsd) > 0 ||
3133
*tsd_arenas_tdata_bypassp_get(tsd));
3234
} else if (tsd->state == tsd_state_uninitialized) {
3335
tsd->state = tsd_state_nominal;

0 commit comments

Comments
 (0)