Skip to content

Commit

Permalink
Adding support for MMTk (non-moving Immix)
Browse files Browse the repository at this point in the history
  • Loading branch information
@udesou
udesou committed Dec 12, 2024
1 parent 582585b commit 8ddb110
Show file tree
Hide file tree
Showing 22 changed files with 1,679 additions and 210 deletions.
44 changes: 44 additions & 0 deletions Make.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ HAVE_SSP := 0
WITH_GC_VERIFY := 0
WITH_GC_DEBUG_ENV := 0

# Use stock if MMTK_PLAN hasn't been defined
MMTK_PLAN ?= None

# Enable DTrace support
WITH_DTRACE := 0

Expand Down Expand Up @@ -829,6 +832,41 @@ JCXXFLAGS += -DGC_DEBUG_ENV
JCFLAGS += -DGC_DEBUG_ENV
endif

ifneq (${MMTK_PLAN},None)
ifeq (${MMTK_JULIA_DIR},)
$(error MMTK_JULIA_DIR must be set to use MMTk)
endif
JCXXFLAGS += -DMMTK_GC
JCFLAGS += -DMMTK_GC
ifeq (${MMTK_BUILD},)
ifeq (debug,$(findstring debug,$(MAKECMDGOALS)))
MMTK_BUILD = debug
else
MMTK_BUILD = release
endif
endif
ifeq (${MMTK_PLAN},Immix)
JCXXFLAGS += -DMMTK_PLAN_IMMIX
JCFLAGS += -DMMTK_PLAN_IMMIX
else
$(error "Unsupported MMTk plan: $(MMTK_PLAN)")
endif
MMTK_DIR = ${MMTK_JULIA_DIR}/mmtk
MMTK_API_INC = $(MMTK_DIR)/api
ifeq ($(OS),Linux)
MMTK_LIB_NAME := libmmtk_julia.so
else
$(error "Unsupported OS for MMTk")
endif
MMTK_LIB_SRC := $(MMTK_DIR)/target/$(MMTK_BUILD)/$(MMTK_LIB_NAME)
MMTK_LIB_DST := $(BUILDROOT)/usr/lib/$(MMTK_LIB_NAME)
MMTK_LIB := -lmmtk_julia
LDFLAGS += -Wl,-rpath=$(MMTK_DIR)/target/$(MMTK_BUILD)/
else
MMTK_JULIA_INC :=
MMTK_LIB :=
endif

ifeq ($(WITH_DTRACE), 1)
JCXXFLAGS += -DUSE_DTRACE
JCFLAGS += -DUSE_DTRACE
Expand Down Expand Up @@ -1823,6 +1861,9 @@ PRINT_PERL = printf ' %b %b\n' $(PERLCOLOR)PERL$(ENDCOLOR) $(BINCOLOR)$(GOAL)
PRINT_FLISP = printf ' %b %b\n' $(FLISPCOLOR)FLISP$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1)
PRINT_JULIA = printf ' %b %b\n' $(JULIACOLOR)JULIA$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1)
PRINT_DTRACE = printf ' %b %b\n' $(DTRACECOLOR)DTRACE$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1)
ifneq (${MMTK_PLAN},None)
PRINT_MMTK = printf ' %b %b\n' $(LINKCOLOR)MMTK$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1)
endif

else
QUIET_MAKE =
Expand All @@ -1833,6 +1874,9 @@ PRINT_PERL = echo '$(subst ','\'',$(1))'; $(1)
PRINT_FLISP = echo '$(subst ','\'',$(1))'; $(1)
PRINT_JULIA = echo '$(subst ','\'',$(1))'; $(1)
PRINT_DTRACE = echo '$(subst ','\'',$(1))'; $(1)
ifneq (${MMTK_PLAN},None)
PRINT_MMTK = echo '$(subst ','\'',$(1))'; $(1)
endif

endif # VERBOSE

Expand Down
17 changes: 13 additions & 4 deletions base/timing.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,14 @@ function gc_page_utilization_data()
return Base.unsafe_wrap(Array, page_utilization_raw, JL_GC_N_MAX_POOLS, own=false)
end


const USING_STOCK_GC = occursin("stock", unsafe_string(ccall(:jl_gc_active_impl, Ptr{UInt8}, ())))
# Full sweep reasons are currently only available for the stock GC
@static if USING_STOCK_GC
# must be kept in sync with `src/gc-stock.h``
const FULL_SWEEP_REASONS = [:FULL_SWEEP_REASON_SWEEP_ALWAYS_FULL, :FULL_SWEEP_REASON_FORCED_FULL_SWEEP,
:FULL_SWEEP_REASON_USER_MAX_EXCEEDED, :FULL_SWEEP_REASON_LARGE_PROMOTION_RATE]
end

"""
Base.full_sweep_reasons()
Expand All @@ -124,11 +129,15 @@ The reasons are:
Note that the set of reasons is not guaranteed to be stable across minor versions of Julia.
"""
function full_sweep_reasons()
reason = cglobal(:jl_full_sweep_reasons, UInt64)
reasons_as_array = Base.unsafe_wrap(Vector{UInt64}, reason, length(FULL_SWEEP_REASONS), own=false)
d = Dict{Symbol, Int64}()
for (i, r) in enumerate(FULL_SWEEP_REASONS)
d[r] = reasons_as_array[i]
# populate the dictionary according to the reasons above for the stock GC
# otherwise return an empty dictionary for now
@static if USING_STOCK_GC
reason = cglobal(:jl_full_sweep_reasons, UInt64)
reasons_as_array = Base.unsafe_wrap(Vector{UInt64}, reason, length(FULL_SWEEP_REASONS), own=false)
for (i, r) in enumerate(FULL_SWEEP_REASONS)
d[r] = reasons_as_array[i]
end
end
return d
end
Expand Down
56 changes: 45 additions & 11 deletions src/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ ifeq ($(USECLANG),1)
FLAGS += -Wno-return-type-c-linkage -Wno-atomic-alignment
endif

ifeq ($(WITH_MMTK), 1)
FLAGS += -I$(MMTK_API_INC)
endif

FLAGS += -DJL_BUILD_ARCH='"$(ARCH)"'
ifeq ($(OS),WINNT)
FLAGS += -DJL_BUILD_UNAME='"NT"'
Expand All @@ -40,23 +44,41 @@ ifeq ($(OS),FreeBSD)
FLAGS += -I$(LOCALBASE)/include
endif

# GC source code. It depends on which GC implementation to use.
GC_SRCS := gc-common gc-stacks gc-alloc-profiler gc-heap-snapshot
ifneq (${MMTK_PLAN},None)
GC_SRCS += gc-mmtk
else
GC_SRCS += gc-stock gc-debug gc-pages gc-page-profiler
endif

SRCS := \
jltypes gf typemap smallintset ast builtins module interpreter symbol \
dlload sys init task array genericmemory staticdata toplevel jl_uv datatype \
simplevector runtime_intrinsics precompile jloptions mtarraylist \
threading scheduler stackwalk gc-common gc-stock gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \
jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \
crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall engine
threading scheduler stackwalk \
method jlapi signal-handling safepoint timing subtype rtutils \
crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall engine \
$(GC_SRCS)

RT_LLVMLINK :=
CG_LLVMLINK :=

ifeq ($(JULIACODEGEN),LLVM)
# Currently these files are used by both GCs. But we should make the list specific to stock, and MMTk should have its own implementation.
GC_CODEGEN_SRCS := llvm-final-gc-lowering llvm-late-gc-lowering llvm-gc-invariant-verifier
ifneq (${MMTK_PLAN},None)
FLAGS += -I$(MMTK_API_INC)
GC_CODEGEN_SRCS += llvm-late-gc-lowering-mmtk
else
GC_CODEGEN_SRCS += llvm-late-gc-lowering-stock
endif
CODEGEN_SRCS := codegen jitlayers aotcompile debuginfo disasm llvm-simdloop \
llvm-final-gc-lowering llvm-pass-helpers llvm-late-gc-lowering llvm-ptls \
llvm-lower-handlers llvm-gc-invariant-verifier llvm-propagate-addrspaces \
llvm-pass-helpers llvm-ptls \
llvm-lower-handlers llvm-propagate-addrspaces \
llvm-multiversioning llvm-alloc-opt llvm-alloc-helpers cgmemmgr llvm-remove-addrspaces \
llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures pipeline llvm_api
llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures pipeline llvm_api \
$(GC_CODEGEN_SRCS)
FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --includedir)
CG_LLVM_LIBS := all
ifeq ($(USE_POLLY),1)
Expand Down Expand Up @@ -99,7 +121,12 @@ ifeq ($(USE_SYSTEM_LIBUV),0)
UV_HEADERS += uv.h
UV_HEADERS += uv/*.h
endif
PUBLIC_HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,work-stealing-queue.h gc-interface.h gc-tls.h gc-tls-common.h julia.h julia_assert.h julia_threads.h julia_fasttls.h julia_locks.h julia_atomics.h jloptions.h)
PUBLIC_HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,work-stealing-queue.h gc-interface.h gc-tls-common.h julia.h julia_assert.h julia_threads.h julia_fasttls.h julia_locks.h julia_atomics.h jloptions.h)
ifneq (${MMTK_PLAN},None)
PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-mmtk.h)
else
PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-stock.h)
endif
ifeq ($(OS),WINNT)
PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,win32_ucontext.h)
endif
Expand Down Expand Up @@ -164,8 +191,8 @@ LIBJULIA_PATH_REL := libjulia
endif

COMMON_LIBPATHS := -L$(build_libdir) -L$(build_shlibdir)
RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI)
CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI)
RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) $(MMTK_LIB)
CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) $(MMTK_LIB)
RT_DEBUG_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp-debug.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport-debug.a -ljulia-debug $(RT_LIBS)
CG_DEBUG_LIBS := $(COMMON_LIBPATHS) $(CG_LIBS) -ljulia-debug -ljulia-internal-debug
RT_RELEASE_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport.a -ljulia $(RT_LIBS)
Expand Down Expand Up @@ -222,6 +249,12 @@ $(BUILDDIR)/%.h.gen : $(SRCDIR)/%.d
sed 's/JULIA_/JL_PROBE_/' $@ > $@.tmp
mv $@.tmp $@

# Compile files from the binding side and copy so file into lib folder
ifneq (${MMTK_PLAN},None)
$(MMTK_LIB_DST): $(MMTK_LIB_SRC)
@$(call PRINT_MMTK, cp $< $@)
endif

$(BUILDDIR)/jl_internal_funcs.inc: $(SRCDIR)/jl_exported_funcs.inc
# Generate `.inc` file that contains a list of `#define` macros to rename functions defined in `libjulia-internal`
# to have a `ijl_` prefix instead of `jl_`, to denote that they are coming from `libjulia-internal`. This avoids
Expand Down Expand Up @@ -314,6 +347,7 @@ $(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: $(addprefix $(SRCDIR)/,de
$(BUILDDIR)/disasm.o $(BUILDDIR)/disasm.dbg.obj: $(SRCDIR)/debuginfo.h $(SRCDIR)/processor.h
$(BUILDDIR)/gc-debug.o $(BUILDDIR)/gc-debug.dbg.obj: $(SRCDIR)/gc-common.h $(SRCDIR)/gc-stock.h
$(BUILDDIR)/gc-pages.o $(BUILDDIR)/gc-pages.dbg.obj: $(SRCDIR)/gc-common.h $(SRCDIR)/gc-stock.h
$(BUILDDIR)/gc-mmtk.o $(BUILDDIR)/gc-mmtk.dbg.obj: $(SRCDIR)/gc-common.h $(SRCDIR)/gc-heap-snapshot.h $(SRCDIR)/gc-alloc-profiler.h
$(BUILDDIR)/gc-stacks.o $(BUILDDIR)/gc-stacks.dbg.obj: $(SRCDIR)/gc-common.h $(SRCDIR)/gc-stock.h
$(BUILDDIR)/gc-stock.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc-common.h $(SRCDIR)/gc-stock.h $(SRCDIR)/gc-heap-snapshot.h $(SRCDIR)/gc-alloc-profiler.h $(SRCDIR)/gc-page-profiler.h
$(BUILDDIR)/gc-heap-snapshot.o $(BUILDDIR)/gc-heap-snapshot.dbg.obj: $(SRCDIR)/gc-heap-snapshot.h
Expand Down Expand Up @@ -386,13 +420,13 @@ $(BUILDDIR)/julia.expmap: $(SRCDIR)/julia.expmap.in $(JULIAHOME)/VERSION $(LLVM_
sed <'$<' >'$@' -e "s/@JULIA_SHLIB_SYMBOL_VERSION@/JL_LIBJULIA_$(SOMAJOR)/" \
-e "s/@LLVM_SHLIB_SYMBOL_VERSION@/$(LLVM_SHLIB_SYMBOL_VERSION)/"

$(build_shlibdir)/libjulia-internal.$(JL_MAJOR_MINOR_SHLIB_EXT): $(BUILDDIR)/julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV)
$(build_shlibdir)/libjulia-internal.$(JL_MAJOR_MINOR_SHLIB_EXT): $(BUILDDIR)/julia.expmap $(OBJS) $(MMTK_LIB_DST) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV)
@$(call PRINT_LINK, $(CXXLD) $(call IMPLIB_FLAGS,$@) $(JCXXFLAGS) $(JL_CXXFLAGS) $(CXXLDFLAGS) $(SHIPFLAGS) $(OBJS) $(RPATH_LIB) -o $@ \
$(JLDFLAGS) $(BOLT_LDFLAGS) $(JLIBLDFLAGS) $(RT_RELEASE_LIBS) $(call SONAME_FLAGS,libjulia-internal.$(JL_MAJOR_SHLIB_EXT)))
@$(INSTALL_NAME_CMD)libjulia-internal.$(SHLIB_EXT) $@
$(DSYMUTIL) $@

$(build_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_MINOR_SHLIB_EXT): $(BUILDDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a $(LIBUV)
$(build_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_MINOR_SHLIB_EXT): $(BUILDDIR)/julia.expmap $(DOBJS) $(MMTK_LIB_DST) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a $(LIBUV)
@$(call PRINT_LINK, $(CXXLD) $(call IMPLIB_FLAGS,$@) $(JCXXFLAGS) $(JL_CXXFLAGS) $(CXXLDFLAGS) $(DEBUGFLAGS) $(DOBJS) $(RPATH_LIB) -o $@ \
$(JLDFLAGS) $(JLIBLDFLAGS) $(RT_DEBUG_LIBS) $(call SONAME_FLAGS,libjulia-internal-debug.$(JL_MAJOR_SHLIB_EXT)))
@$(INSTALL_NAME_CMD)libjulia-internal-debug.$(SHLIB_EXT) $@
Expand Down
50 changes: 50 additions & 0 deletions src/gc-common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,38 @@ JL_DLLEXPORT jl_value_t *(jl_gc_alloc)(jl_ptls_t ptls, size_t sz, void *ty)
return jl_gc_alloc_(ptls, sz, ty);
}

JL_DLLEXPORT void *jl_malloc(size_t sz)
{
return jl_gc_counted_malloc(sz);
}

//_unchecked_calloc does not check for potential overflow of nm*sz
STATIC_INLINE void *_unchecked_calloc(size_t nm, size_t sz) {
size_t nmsz = nm*sz;
return jl_gc_counted_calloc(nmsz, 1);
}

JL_DLLEXPORT void *jl_calloc(size_t nm, size_t sz)
{
if (nm > SSIZE_MAX/sz)
return NULL;
return _unchecked_calloc(nm, sz);
}

JL_DLLEXPORT void jl_free(void *p)
{
if (p != NULL) {
size_t sz = memory_block_usable_size(p, 0);
return jl_gc_counted_free_with_size(p, sz);
}
}

JL_DLLEXPORT void *jl_realloc(void *p, size_t sz)
{
size_t old = p ? memory_block_usable_size(p, 0) : 0;
return jl_gc_counted_realloc_with_old_size(p, old, sz);
}

// =========================================================================== //
// Generic Memory
// =========================================================================== //
Expand Down Expand Up @@ -677,6 +709,24 @@ JL_DLLEXPORT void jl_throw_out_of_memory_error(void)
jl_throw(jl_memory_exception);
}

// Sweeping mtarraylist_buffers:
// These buffers are made unreachable via `mtarraylist_resizeto` from mtarraylist.c
// and are freed at the end of GC via jl_gc_sweep_stack_pools_and_mtarraylist_buffers
void sweep_mtarraylist_buffers(void) JL_NOTSAFEPOINT
{
for (int i = 0; i < gc_n_threads; i++) {
jl_ptls_t ptls = gc_all_tls_states[i];
if (ptls == NULL) {
continue;
}
small_arraylist_t *buffers = &ptls->lazily_freed_mtarraylist_buffers;
void *buf;
while ((buf = small_arraylist_pop(buffers)) != NULL) {
free(buf);
}
}
}

#ifdef __cplusplus
}
#endif
35 changes: 35 additions & 0 deletions src/gc-common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,31 @@
extern "C" {
#endif

// =========================================================================== //
// GC Big objects
// =========================================================================== //

JL_EXTENSION typedef struct _bigval_t {
struct _bigval_t *next;
struct _bigval_t *prev;
size_t sz;
#ifdef _P64 // Add padding so that the value is 64-byte aligned
// (8 pointers of 8 bytes each) - (4 other pointers in struct)
void *_padding[8 - 4];
#else
// (16 pointers of 4 bytes each) - (4 other pointers in struct)
void *_padding[16 - 4];
#endif
//struct jl_taggedvalue_t <>;
union {
uintptr_t header;
struct {
uintptr_t gc:2;
} bits;
};
// must be 64-byte aligned here, in 32 & 64 bit modes
} bigval_t;

// =========================================================================== //
// GC Callbacks
// =========================================================================== //
Expand Down Expand Up @@ -193,4 +218,14 @@ extern jl_ptls_t* gc_all_tls_states;

extern int gc_logging_enabled;

// =========================================================================== //
// MISC
// =========================================================================== //

// number of stacks to always keep available per pool
#define MIN_STACK_MAPPINGS_PER_POOL 5

void _jl_free_stack(jl_ptls_t ptls, void *stkbuf, size_t bufsz) JL_NOTSAFEPOINT;
void sweep_mtarraylist_buffers(void) JL_NOTSAFEPOINT;

#endif // JL_GC_COMMON_H
12 changes: 11 additions & 1 deletion src/gc-interface.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,13 @@ JL_DLLEXPORT void jl_gc_set_max_memory(uint64_t max_mem);
JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection);
// Returns whether the thread with `tid` is a collector thread
JL_DLLEXPORT int gc_is_collector_thread(int tid) JL_NOTSAFEPOINT;
// Returns which GC implementation is being used and possibly its version according to the list of supported GCs
// NB: it should clearly identify the GC by including e.g. ‘stock’ or ‘mmtk’ as a substring.
JL_DLLEXPORT const char* jl_gc_active_impl(void);
// Sweep Julia's stack pools and mtarray buffers. Note that this function has been added to the interface as
// each GC should implement it but it will most likely not be used by other code in the runtime.
// It still needs to be annotated with JL_DLLEXPORT since it is called from Rust by MMTk.
JL_DLLEXPORT void jl_gc_sweep_stack_pools_and_mtarraylist_buffers(jl_ptls_t ptls) JL_NOTSAFEPOINT;

// ========================================================================= //
// Metrics
Expand Down Expand Up @@ -138,7 +145,6 @@ JL_DLLEXPORT uint64_t jl_gc_total_hrtime(void);
// **must** also set the type of the returning object to be `ty`. The type `ty` may also be used to record
// an allocation of that type in the allocation profiler.
struct _jl_value_t *jl_gc_alloc_(struct _jl_tls_states_t * ptls, size_t sz, void *ty);

// Allocates small objects and increments Julia allocation counterst. Size of the object
// header must be included in the object size. The (possibly unused in some implementations)
// offset to the arena in which we're allocating is passed in the second parameter, and the
Expand Down Expand Up @@ -198,6 +204,10 @@ JL_DLLEXPORT void *jl_gc_perm_alloc(size_t sz, int zero, unsigned align,
// the allocated object. All objects stored in fields of this object
// must be either permanently allocated or have other roots.
struct _jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT;
// This function notifies the GC about memory addresses that are set when loading the boot image.
// The GC may use that information to, for instance, determine that such objects should
// be treated as marked and belonged to the old generation in nursery collections.
void jl_gc_notify_image_load(const char* img_data, size_t len);

// ========================================================================= //
// Runtime Write-Barriers
Expand Down
Loading

0 comments on commit 8ddb110

Please sign in to comment.