diff --git a/src/Makefile b/src/Makefile index ae7e36b401d95..cfe4ebf874d21 100644 --- a/src/Makefile +++ b/src/Makefile @@ -81,7 +81,7 @@ endif # headers are used for dependency tracking, while public headers will be part of the dist UV_HEADERS := HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,julia.h julia_assert.h julia_threads.h tls.h locks.h atomics.h julia_internal.h options.h timing.h) -PUBLIC_HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,julia.h julia_assert.h julia_threads.h tls.h locks.h atomics.h) +PUBLIC_HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,julia.h julia_assert.h julia_threads.h tls.h locks.h atomics.h julia_gcext.h) ifeq ($(USE_SYSTEM_LIBUV),0) UV_HEADERS += uv.h UV_HEADERS += uv/*.h diff --git a/src/datatype.c b/src/datatype.c index e83e2fb30eb8b..8a9a883889905 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -11,6 +11,7 @@ #include "julia.h" #include "julia_internal.h" #include "julia_assert.h" +#include "julia_gcext.h" #ifdef __cplusplus extern "C" { @@ -526,6 +527,34 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t * return bt; } +JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name, + jl_module_t *module, + jl_datatype_t *super, + jl_markfunc_t markfunc, + jl_sweepfunc_t sweepfunc, + int haspointers, + int large) +{ + jl_datatype_t *bt = jl_new_datatype(name, module, super, + jl_emptysvec, jl_emptysvec, jl_emptysvec, 0, 1, 0); + bt->size = large ? GC_MAX_SZCLASS+1 : 0; + jl_datatype_layout_t *layout = (jl_datatype_layout_t *) + jl_gc_perm_alloc(sizeof(jl_datatype_layout_t) + sizeof(jl_fielddescdyn_t), + 0, 4, 0); + layout->nfields = 0; + layout->alignment = sizeof(void *); + layout->haspadding = 1; + layout->npointers = haspointers; + layout->fielddesc_type = 3; + jl_fielddescdyn_t * desc = + (jl_fielddescdyn_t *) ((char *)layout + sizeof(*layout)); + desc->markfunc = markfunc; + desc->sweepfunc = sweepfunc; + bt->layout = layout; + bt->instance = NULL; + return bt; +} + // bits constructors ---------------------------------------------------------- JL_DLLEXPORT jl_value_t *jl_new_bits(jl_value_t *dt, void *data) diff --git a/src/gc-debug.c b/src/gc-debug.c index 78709118aa910..ac35e47a8b1da 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -200,7 +200,7 @@ static void gc_verify_track(jl_ptls_t ptls) { jl_gc_mark_cache_t *gc_cache = &ptls->gc_cache; do { - gc_mark_sp_t sp; + jl_gc_mark_sp_t sp; gc_mark_sp_init(gc_cache, &sp); arraylist_push(&lostval_parents_done, lostval); jl_printf(JL_STDERR, "Now looking for %p =======\n", lostval); @@ -247,7 +247,7 @@ static void gc_verify_track(jl_ptls_t ptls) void gc_verify(jl_ptls_t ptls) { jl_gc_mark_cache_t *gc_cache = &ptls->gc_cache; - gc_mark_sp_t sp; + jl_gc_mark_sp_t sp; gc_mark_sp_init(gc_cache, &sp); lostval = NULL; lostval_parents.len = 0; @@ -1242,7 +1242,7 @@ int gc_slot_to_arrayidx(void *obj, void *_slot) // Print a backtrace from the bottom (start) of the mark stack up to `sp` // `pc_offset` will be added to `sp` for convenience in the debugger. -NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, gc_mark_sp_t sp, int pc_offset) +NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, jl_gc_mark_sp_t sp, int pc_offset) { jl_jmp_buf *old_buf = ptls->safe_restore; jl_jmp_buf buf; diff --git a/src/gc.c b/src/gc.c index 773612b425eb7..f1ebed6840434 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1,12 +1,125 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "gc.h" +#include "julia_gcext.h" #include "julia_assert.h" #ifdef __cplusplus extern "C" { #endif +// Linked list of callback functions + +typedef void (*jl_gc_cb_func_t)(void); + +typedef struct jl_gc_callback_list_t { + struct jl_gc_callback_list_t *next; + jl_gc_cb_func_t func; +} jl_gc_callback_list_t; + +static jl_gc_callback_list_t *gc_cblist_root_scanner; +static jl_gc_callback_list_t *gc_cblist_task_scanner; +static jl_gc_callback_list_t *gc_cblist_pre_gc; +static jl_gc_callback_list_t *gc_cblist_post_gc; +static jl_gc_callback_list_t *gc_cblist_notify_external_alloc; +static jl_gc_callback_list_t *gc_cblist_notify_external_free; + +#define gc_invoke_callbacks(ty, list, args) \ + do { \ + for (jl_gc_callback_list_t *cb = list; \ + cb != NULL; \ + cb = cb->next) \ + { \ + ((ty)(cb->func)) args; \ + } \ + } while (0) + +static void jl_gc_register_callback(jl_gc_callback_list_t **list, + jl_gc_cb_func_t func) +{ + while (*list != NULL) { + if ((*list)->func == func) + return; + list = &((*list)->next); + } + *list = (jl_gc_callback_list_t *)malloc(sizeof(jl_gc_callback_list_t)); + (*list)->next = NULL; + (*list)->func = func; +} + +static void jl_gc_deregister_callback(jl_gc_callback_list_t **list, + jl_gc_cb_func_t func) +{ + while (*list != NULL) { + if ((*list)->func == func) { + jl_gc_callback_list_t *tmp = *list; + (*list) = (*list)->next; + free(tmp); + return; + } + list = &((*list)->next); + } +} + +JL_DLLEXPORT void jl_gc_set_cb_root_scanner(jl_gc_cb_root_scanner_t cb, int enable) +{ + if (enable) + jl_gc_register_callback(&gc_cblist_root_scanner, (jl_gc_cb_func_t)cb); + else + jl_gc_deregister_callback(&gc_cblist_root_scanner, (jl_gc_cb_func_t)cb); +} + +JL_DLLEXPORT void jl_gc_set_cb_task_scanner(jl_gc_cb_task_scanner_t cb, int enable) +{ + if (enable) + jl_gc_register_callback(&gc_cblist_task_scanner, (jl_gc_cb_func_t)cb); + else + jl_gc_deregister_callback(&gc_cblist_task_scanner, (jl_gc_cb_func_t)cb); +} + +JL_DLLEXPORT void jl_gc_set_cb_pre_gc(jl_gc_cb_pre_gc_t cb, int enable) +{ + if (enable) + jl_gc_register_callback(&gc_cblist_pre_gc, (jl_gc_cb_func_t)cb); + else + jl_gc_deregister_callback(&gc_cblist_pre_gc, (jl_gc_cb_func_t)cb); +} + +JL_DLLEXPORT void jl_gc_set_cb_post_gc(jl_gc_cb_post_gc_t cb, int enable) +{ + if (enable) + jl_gc_register_callback(&gc_cblist_post_gc, (jl_gc_cb_func_t)cb); + else + jl_gc_deregister_callback(&gc_cblist_post_gc, (jl_gc_cb_func_t)cb); +} + +JL_DLLEXPORT void jl_gc_set_cb_notify_external_alloc(jl_gc_cb_notify_external_alloc_t cb, int enable) +{ + if (enable) + jl_gc_register_callback(&gc_cblist_notify_external_alloc, (jl_gc_cb_func_t)cb); + else + jl_gc_deregister_callback(&gc_cblist_notify_external_alloc, (jl_gc_cb_func_t)cb); +} + +JL_DLLEXPORT void jl_gc_set_cb_notify_external_free(jl_gc_cb_notify_external_free_t cb, int enable) +{ + if (enable) + jl_gc_register_callback(&gc_cblist_notify_external_free, (jl_gc_cb_func_t)cb); + else + jl_gc_deregister_callback(&gc_cblist_notify_external_free, (jl_gc_cb_func_t)cb); +} + +// Save/restore local mark stack to/from thread-local storage. + +STATIC_INLINE void export_gc_state(jl_ptls_t ptls, jl_gc_mark_sp_t *sp) { + ptls->gc_mark_sp = *sp; +} + +STATIC_INLINE void import_gc_state(jl_ptls_t ptls, jl_gc_mark_sp_t *sp) { + // Has the stack been reallocated in the meantime? + *sp = ptls->gc_mark_sp; +} + // Protect all access to `finalizer_list_marked` and `to_finalize`. // For accessing `ptls->finalizers`, the lock is needed if a thread // is going to realloc the buffer (of its own list) or accessing the @@ -14,6 +127,10 @@ extern "C" { static jl_mutex_t finalizers_lock; static jl_mutex_t gc_cache_lock; +// Flag that tells us whether we need to support conservative marking +// of objects. +static int support_conservative_marking = 0; + /** * Note about GC synchronization: * @@ -351,6 +468,32 @@ JL_DLLEXPORT void jl_finalize_th(jl_ptls_t ptls, jl_value_t *o) arraylist_free(&copied_list); } +static void gc_sweep_foreign_objs_in_list(arraylist_t *objs) +{ + size_t p = 0; + for (size_t i = 0; i < objs->len; i++) { + jl_value_t *v = (jl_value_t *)(objs->items[i]); + jl_datatype_t *t = (jl_datatype_t *)(jl_typeof(v)); + const jl_datatype_layout_t *layout = t->layout; + jl_fielddescdyn_t *desc = (jl_fielddescdyn_t*)jl_dt_layout_fields(layout); + if (!gc_ptr_tag(v, 1)) { + desc->sweepfunc(v); + } + else { + objs->items[p++] = v; + } + } + objs->len = p; +} + +static void gc_sweep_foreign_objs(void) +{ + for (int i = 0;i < jl_n_threads; i++) { + jl_ptls_t ptls2 = jl_all_tls_states[i]; + gc_sweep_foreign_objs_in_list(&ptls2->sweep_objs); + } +} + // GC knobs and self-measurement variables static int64_t last_gc_total_bytes = 0; @@ -745,6 +888,8 @@ JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz) bigval_t *v = (bigval_t*)malloc_cache_align(allocsz); if (v == NULL) jl_throw(jl_memory_exception); + gc_invoke_callbacks(jl_gc_cb_notify_external_alloc_t, + gc_cblist_notify_external_alloc, (v, allocsz)); #ifdef JULIA_ENABLE_THREADING jl_atomic_fetch_add(&gc_num.allocd, allocsz); #else @@ -793,6 +938,8 @@ static bigval_t **sweep_big_list(int sweep_full, bigval_t **pv) JL_NOTSAFEPOINT #ifdef MEMDEBUG memset(v, 0xbb, v->sz&~3); #endif + gc_invoke_callbacks(jl_gc_cb_notify_external_free_t, + gc_cblist_notify_external_free, (v)); jl_free_aligned(v); } gc_time_count_big(old_bits, bits); @@ -913,7 +1060,22 @@ static inline jl_taggedvalue_t *reset_page(const jl_gc_pool_t *p, jl_gc_pagemeta memset(pg->ages, 0, GC_PAGE_SZ / 8 / p->osize + 1); jl_taggedvalue_t *beg = (jl_taggedvalue_t*)(pg->data + GC_PAGE_OFFSET); jl_taggedvalue_t *next = (jl_taggedvalue_t*)pg->data; - next->next = fl; + if (fl == NULL) { + next->next = NULL; + } + else { + // Insert free page after first page. + // This prevents unnecessary fragmentation from multiple pages + // being allocated from at the same time. Instead, objects will + // only ever be allocated from the first object in the list. + // This is specifically being relied on by the implementation + // of jl_gc_internal_obj_base_ptr() so that the function does + // not have to traverse the entire list. + jl_taggedvalue_t *flpage = (jl_taggedvalue_t *)gc_page_data(fl); + next->next = flpage->next; + flpage->next = beg; + beg = fl; + } pg->has_young = 0; pg->has_marked = 0; pg->fl_begin_offset = -1; @@ -1301,7 +1463,6 @@ static void gc_sweep_perm_alloc(void) gc_time_sysimg_end(t0); } - // mark phase JL_DLLEXPORT void jl_gc_queue_root(jl_value_t *ptr) @@ -1367,7 +1528,7 @@ STATIC_INLINE uintptr_t gc_read_stack(void *_addr, uintptr_t offset, } JL_NORETURN NOINLINE void gc_assert_datatype_fail(jl_ptls_t ptls, jl_datatype_t *vt, - gc_mark_sp_t sp) + jl_gc_mark_sp_t sp) { jl_printf(JL_STDOUT, "GC error (probable corruption) :\n"); gc_debug_print_status(); @@ -1383,7 +1544,7 @@ JL_NORETURN NOINLINE void gc_assert_datatype_fail(jl_ptls_t ptls, jl_datatype_t void *gc_mark_label_addrs[_GC_MARK_L_MAX]; // Double the mark stack (both pc and data) with the lock held. -static void NOINLINE gc_mark_stack_resize(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp) +static void NOINLINE gc_mark_stack_resize(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp) { jl_gc_mark_data_t *old_data = gc_cache->data_stack; void **pc_stack = sp->pc_start; @@ -1404,7 +1565,7 @@ static void NOINLINE gc_mark_stack_resize(jl_gc_mark_cache_t *gc_cache, gc_mark_ // lock held. The caller should invalidate any local cache of the stack addresses that's not // in `gc_cache` or `sp` // The `sp` will be updated on return if `inc` is true. -STATIC_INLINE void gc_mark_stack_push(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, +STATIC_INLINE void gc_mark_stack_push(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp, void *pc, void *data, size_t data_size, int inc) JL_NOTSAFEPOINT { assert(data_size <= sizeof(jl_gc_mark_data_t)); @@ -1445,7 +1606,7 @@ STATIC_INLINE int gc_try_setmark(jl_value_t *obj, uintptr_t *nptr, } // Queue a finalizer list to be scanned in the mark loop. Start marking from index `start`. -void gc_mark_queue_finlist(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, +void gc_mark_queue_finlist(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp, arraylist_t *list, size_t start) { size_t len = list->len; @@ -1459,7 +1620,7 @@ void gc_mark_queue_finlist(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, // Queue a object to be scanned. The object should already be marked and the GC metadata // should already be updated for it. Only scanning of the object should be performed. -STATIC_INLINE void gc_mark_queue_scan_obj(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, +STATIC_INLINE void gc_mark_queue_scan_obj(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp, jl_value_t *obj) { jl_taggedvalue_t *o = jl_astaggedvalue(obj); @@ -1475,7 +1636,7 @@ STATIC_INLINE void gc_mark_queue_scan_obj(jl_gc_mark_cache_t *gc_cache, gc_mark_ // The object will be marked atomically which can also happen concurrently. // It will be queued if the object wasn't marked already (or concurrently by another thread) // Returns whether the object is young. -STATIC_INLINE int gc_mark_queue_obj(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, void *_obj) JL_NOTSAFEPOINT +STATIC_INLINE int gc_mark_queue_obj(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp, void *_obj) JL_NOTSAFEPOINT { jl_value_t *obj = (jl_value_t*)jl_assume(_obj); uintptr_t nptr = 0; @@ -1489,6 +1650,22 @@ STATIC_INLINE int gc_mark_queue_obj(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t * return (int)nptr; } +JL_DLLEXPORT int jl_gc_mark_queue_obj(jl_ptls_t ptls, jl_value_t *obj) +{ + return gc_mark_queue_obj(&ptls->gc_cache, &ptls->gc_mark_sp, obj); +} + +JL_DLLEXPORT void jl_gc_mark_queue_objarray(jl_ptls_t ptls, jl_value_t *parent, + jl_value_t **objs, size_t nobjs) +{ + gc_mark_objarray_t data = { parent, objs, objs + nobjs, + jl_astaggedvalue(parent)->bits.gc & 2 }; + gc_mark_stack_push(&ptls->gc_cache, &ptls->gc_mark_sp, + gc_mark_label_addrs[GC_MARK_L_objarray], + &data, sizeof(data), 1); +} + + // Check if `nptr` is tagged for `old + refyoung`, // Push the object to the remset and update the `nptr` counter if necessary. STATIC_INLINE void gc_mark_push_remset(jl_ptls_t ptls, jl_value_t *obj, uintptr_t nptr) JL_NOTSAFEPOINT @@ -1508,7 +1685,7 @@ STATIC_INLINE void gc_mark_push_remset(jl_ptls_t ptls, jl_value_t *obj, uintptr_ } // Scan a dense array of object references, see `gc_mark_objarray_t` -STATIC_INLINE int gc_mark_scan_objarray(jl_ptls_t ptls, gc_mark_sp_t *sp, +STATIC_INLINE int gc_mark_scan_objarray(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, gc_mark_objarray_t *objary, jl_value_t **begin, jl_value_t **end, jl_value_t **pnew_obj, uintptr_t *ptag, uint8_t *pbits) @@ -1540,7 +1717,7 @@ STATIC_INLINE int gc_mark_scan_objarray(jl_ptls_t ptls, gc_mark_sp_t *sp, } // Scan an object with 8bits field descriptors. see `gc_mark_obj8_t` -STATIC_INLINE int gc_mark_scan_obj8(jl_ptls_t ptls, gc_mark_sp_t *sp, gc_mark_obj8_t *obj8, +STATIC_INLINE int gc_mark_scan_obj8(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, gc_mark_obj8_t *obj8, char *parent, jl_fielddesc8_t *begin, jl_fielddesc8_t *end, jl_value_t **pnew_obj, uintptr_t *ptag, uint8_t *pbits) { @@ -1575,7 +1752,7 @@ STATIC_INLINE int gc_mark_scan_obj8(jl_ptls_t ptls, gc_mark_sp_t *sp, gc_mark_ob } // Scan an object with 16bits field descriptors. see `gc_mark_obj16_t` -STATIC_INLINE int gc_mark_scan_obj16(jl_ptls_t ptls, gc_mark_sp_t *sp, gc_mark_obj16_t *obj16, +STATIC_INLINE int gc_mark_scan_obj16(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, gc_mark_obj16_t *obj16, char *parent, jl_fielddesc16_t *begin, jl_fielddesc16_t *end, jl_value_t **pnew_obj, uintptr_t *ptag, uint8_t *pbits) JL_NOTSAFEPOINT { @@ -1610,7 +1787,7 @@ STATIC_INLINE int gc_mark_scan_obj16(jl_ptls_t ptls, gc_mark_sp_t *sp, gc_mark_o } // Scan an object with 32bits field descriptors. see `gc_mark_obj32_t` -STATIC_INLINE int gc_mark_scan_obj32(jl_ptls_t ptls, gc_mark_sp_t *sp, gc_mark_obj32_t *obj32, +STATIC_INLINE int gc_mark_scan_obj32(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, gc_mark_obj32_t *obj32, char *parent, jl_fielddesc32_t *begin, jl_fielddesc32_t *end, jl_value_t **pnew_obj, uintptr_t *ptag, uint8_t *pbits) { @@ -1741,7 +1918,7 @@ STATIC_INLINE int gc_mark_scan_obj32(jl_ptls_t ptls, gc_mark_sp_t *sp, gc_mark_o // Additional optimizations are done for some of the common cases by skipping // the unnecessary data stack pointer increment and the load from the stack // (i.e. store to load forwaring). See `objary_loaded`, `obj8_loaded` and `obj16_loaded`. -JL_EXTENSION NOINLINE void gc_mark_loop(jl_ptls_t ptls, gc_mark_sp_t sp) +JL_EXTENSION NOINLINE void gc_mark_loop(jl_ptls_t ptls, jl_gc_mark_sp_t sp) { if (__unlikely(ptls == NULL)) { gc_mark_label_addrs[GC_MARK_L_marked_obj] = gc_mark_laddr(marked_obj); @@ -2128,6 +2305,12 @@ mark: { void *stkbuf = ta->stkbuf; int16_t tid = ta->tid; jl_ptls_t ptls2 = jl_all_tls_states[tid]; + if (gc_cblist_task_scanner) { + export_gc_state(ptls, &sp); + gc_invoke_callbacks(jl_gc_cb_task_scanner_t, + gc_cblist_task_scanner, (ta, ta == ptls2->root_task)); + import_gc_state(ptls, &sp); + } #ifdef COPY_STACKS if (stkbuf && ta->copy_stack) gc_setmark_buf_(ptls, stkbuf, bits, ta->bufsz); @@ -2222,10 +2405,9 @@ mark: { obj16 = (gc_mark_obj16_t*)sp.data; goto obj16_loaded; } - else { + else if (layout->fielddesc_type == 2) { // This is very uncommon // Do not do store to load forwarding to save some code size - assert(layout->fielddesc_type == 2); jl_fielddesc32_t *desc = (jl_fielddesc32_t*)jl_dt_layout_fields(layout); assert(first < nfields); gc_mark_obj32_t markdata = {new_obj, desc + first, desc + nfields, nptr}; @@ -2234,6 +2416,18 @@ mark: { sp.data = (jl_gc_mark_data_t *)(((char*)sp.data) + sizeof(markdata)); goto obj32; } + else { + assert(layout->fielddesc_type == 3); + jl_fielddescdyn_t *desc = (jl_fielddescdyn_t*)jl_dt_layout_fields(layout); + + int old = jl_astaggedvalue(new_obj)->bits.gc & 2; + export_gc_state(ptls, &sp); + uintptr_t young = desc->markfunc(ptls, new_obj); + import_gc_state(ptls, &sp); + if (old && young) + gc_mark_push_remset(ptls, new_obj, young * 4 + 3); + goto pop; + } } } } @@ -2242,7 +2436,7 @@ extern jl_array_t *jl_module_init_order; extern jl_typemap_entry_t *call_cache[N_CALL_CACHE]; extern jl_array_t *jl_all_methods; -static void jl_gc_queue_thread_local(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, +static void jl_gc_queue_thread_local(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp, jl_ptls_t ptls2) { // `current_module` might not have a value when the thread is not @@ -2255,7 +2449,7 @@ static void jl_gc_queue_thread_local(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t } // mark the initial root set -static void mark_roots(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp) +static void mark_roots(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp) { // modules gc_mark_queue_obj(gc_cache, sp, jl_main_module); @@ -2403,7 +2597,7 @@ static void jl_gc_premark(jl_ptls_t ptls2) } } -static void jl_gc_queue_remset(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, jl_ptls_t ptls2) +static void jl_gc_queue_remset(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp, jl_ptls_t ptls2) { size_t len = ptls2->heap.last_remset->len; void **items = ptls2->heap.last_remset->items; @@ -2425,7 +2619,7 @@ static void jl_gc_queue_remset(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, j ptls2->heap.rem_bindings.len = n_bnd_refyoung; } -static void jl_gc_queue_bt_buf(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, jl_ptls_t ptls2) +static void jl_gc_queue_bt_buf(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp, jl_ptls_t ptls2) { size_t n = 0; while (n+2 < ptls2->bt_size) { @@ -2441,7 +2635,7 @@ static void jl_gc_queue_bt_buf(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, j static int _jl_gc_collect(jl_ptls_t ptls, int full) { jl_gc_mark_cache_t *gc_cache = &ptls->gc_cache; - gc_mark_sp_t sp; + jl_gc_mark_sp_t sp; gc_mark_sp_init(gc_cache, &sp); uint64_t t0 = jl_hrtime(); @@ -2463,6 +2657,12 @@ static int _jl_gc_collect(jl_ptls_t ptls, int full) // 3. walk roots mark_roots(gc_cache, &sp); + if (gc_cblist_root_scanner) { + export_gc_state(ptls, &sp); + gc_invoke_callbacks(jl_gc_cb_root_scanner_t, + gc_cblist_root_scanner, (full)); + import_gc_state(ptls, &sp); + } gc_mark_loop(ptls, sp); gc_mark_sp_init(gc_cache, &sp); gc_num.since_sweep += gc_num.allocd + (int64_t)gc_num.interval; @@ -2493,7 +2693,9 @@ static int _jl_gc_collect(jl_ptls_t ptls, int full) // so that the objects are not incorrectly reset. gc_mark_loop(ptls, sp); gc_mark_sp_init(gc_cache, &sp); - mark_reset_age = 1; + // Conservative marking relies on age to tell allocated objects + // and freelist entries apart. + mark_reset_age = !support_conservative_marking; // Reset the age and old bit for any unmarked objects referenced by the // `to_finalize` list. These objects are only reachable from this list // and should not be referenced by any old objects so this won't break @@ -2558,6 +2760,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, int full) // 5. start sweeping sweep_weak_refs(); sweep_stack_pools(); + gc_sweep_foreign_objs(); gc_sweep_other(ptls, sweep_full); gc_scrub(); gc_verify_tags(); @@ -2626,6 +2829,8 @@ JL_DLLEXPORT void jl_gc_collect(int full) // TODO (concurrently queue objects) // no-op for non-threading jl_gc_wait_for_the_world(); + gc_invoke_callbacks(jl_gc_cb_pre_gc_t, + gc_cblist_pre_gc, (full)); if (!jl_gc_disable_counter) { JL_LOCK_NOGC(&finalizers_lock); @@ -2650,9 +2855,11 @@ JL_DLLEXPORT void jl_gc_collect(int full) run_finalizers(ptls); ptls->in_finalizer = was_in_finalizer; } + gc_invoke_callbacks(jl_gc_cb_post_gc_t, + gc_cblist_post_gc, (full)); } -void gc_mark_queue_all_roots(jl_ptls_t ptls, gc_mark_sp_t *sp) +void gc_mark_queue_all_roots(jl_ptls_t ptls, jl_gc_mark_sp_t *sp) { jl_gc_mark_cache_t *gc_cache = &ptls->gc_cache; for (size_t i = 0; i < jl_n_threads; i++) @@ -2691,6 +2898,7 @@ void jl_init_thread_heap(jl_ptls_t ptls) arraylist_new(heap->remset, 0); arraylist_new(heap->last_remset, 0); arraylist_new(&ptls->finalizers, 0); + arraylist_new(&ptls->sweep_objs, 0); jl_gc_mark_cache_t *gc_cache = &ptls->gc_cache; gc_cache->perm_scanned_bytes = 0; @@ -2722,7 +2930,7 @@ void jl_gc_init(void) if (maxmem > max_collect_interval) max_collect_interval = maxmem; #endif - gc_mark_sp_t sp = {NULL, NULL, NULL, NULL}; + jl_gc_mark_sp_t sp = {NULL, NULL, NULL, NULL}; gc_mark_loop(NULL, sp); } @@ -3037,6 +3245,146 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) return jl_gc_alloc(ptls, sizeof(void*) * 3, NULL); } +JL_DLLEXPORT int jl_gc_enable_conservative_gc_support(void) +{ + static_assert(jl_buff_tag % GC_PAGE_SZ == 0, + "jl_buff_tag must be a multiple of GC_PAGE_SZ"); + if (jl_is_initialized()) { + int result = jl_atomic_fetch_or(&support_conservative_marking, 1); + if (!result) { + // Do a full collection to ensure that age bits are updated + // properly. We don't have to worry about race conditions + // for this part, as allocation itself is unproblematic and + // a collection will wait for safepoints. + jl_gc_collect(1); + } + return result; + } else { + int result = support_conservative_marking; + support_conservative_marking = 1; + return result; + } +} + +JL_DLLEXPORT int jl_gc_conservative_gc_support_enabled(void) +{ + return jl_atomic_load(&support_conservative_marking); +} + +JL_DLLEXPORT jl_value_t *jl_gc_internal_obj_base_ptr(void *p) +{ + p = (char *) p - 1; + jl_gc_pagemeta_t *meta = page_metadata(p); + if (meta && meta->ages) { + char *page = gc_page_data(p); + // offset within page. + size_t off = (char *)p - page; + if (off < GC_PAGE_OFFSET) + return NULL; + // offset within object + size_t off2 = (off - GC_PAGE_OFFSET); + size_t osize = meta->osize; + off2 %= osize; + if (off - off2 + osize > GC_PAGE_SZ) + return NULL; + jl_taggedvalue_t *cell = (jl_taggedvalue_t *)((char *)p - off2); + // We have to distinguish between three cases: + // 1. We are on a page where every cell is allocated. + // 2. We are on a page where objects are currently bump-allocated + // from the corresponding pool->newpages list. + // 3. We are on a page with a freelist that is used for object + // allocation. + if (meta->nfree == 0) { + // case 1: full page; `cell` must be an object + goto valid_object; + } + jl_gc_pool_t *pool = + jl_all_tls_states[meta->thread_n]->heap.norm_pools + + meta->pool_n; + if (meta->fl_begin_offset == (uint16_t) -1) { + // case 2: this is a page on the newpages list + jl_taggedvalue_t *newpages = pool->newpages; + // Check if the page is being allocated from via newpages + if (!newpages) + return NULL; + char *data = gc_page_data(newpages); + if (data != meta->data) { + // Pages on newpages form a linked list where only the + // first one is allocated from (see reset_page()). + // All other pages are empty. + return NULL; + } + // This is the first page on the newpages list, where objects + // are allocated from. + if ((char *)cell >= (char *)newpages) // past allocation pointer + return NULL; + goto valid_object; + } + // case 3: this is a page with a freelist + // marked or old objects can't be on the freelist + if (cell->bits.gc) + goto valid_object; + // When allocating from a freelist, three subcases are possible: + // * The freelist of a page has been exhausted; this was handled + // under case 1, as nfree == 0. + // * The freelist of the page has not been used, and the age bits + // reflect whether a cell is on the freelist or an object. + // * The freelist is currently being allocated from. In this case, + // pool->freelist will point to the current page; any cell with + // a lower address will be an allocated object, and for cells + // with the same or a higher address, the corresponding age + // bit will reflect whether it's on the freelist. + // Age bits are set in sweep_page() and are 0 for freelist + // entries and 1 for live objects. The above subcases arise + // because allocating a cell will not update the age bit, so we + // need extra logic for pages that have been allocated from. + unsigned obj_id = (off - off2) / osize; + // We now distinguish between the second and third subcase. + // Freelist entries are consumed in ascending order. Anything + // before the freelist pointer was either live during the last + // sweep or has been allocated since. + if (gc_page_data(cell) == gc_page_data(pool->freelist) + && (char *)cell < (char *)pool->freelist) + goto valid_object; + // We know now that the age bit reflects liveness status during + // the last sweep and that the cell has not been reused since. + if (!(meta->ages[obj_id / 8] & (1 << (obj_id % 8)))) { + return NULL; + } + // Not a freelist entry, therefore a valid object. + valid_object: + // We have to treat objects with type `jl_buff_tag` differently, + // as they must not be passed to the usual marking functions. + // Note that jl_buff_tag is a multiple of GC_PAGE_SZ, thus it + // cannot be a type reference. + if ((cell->header & ~(uintptr_t) 3) == jl_buff_tag) + return NULL; + return jl_valueof(cell); + } + return NULL; +} + +JL_DLLEXPORT size_t jl_gc_max_internal_obj_size(void) +{ + return GC_MAX_SZCLASS; +} + +JL_DLLEXPORT size_t jl_gc_external_obj_hdr_size(void) +{ + return sizeof(bigval_t); +} + + +JL_DLLEXPORT void * jl_gc_alloc_typed(jl_ptls_t ptls, size_t sz, void *ty) +{ + return jl_gc_alloc(ptls, sz, ty); +} + +JL_DLLEXPORT void jl_gc_schedule_foreign_sweepfunc(jl_ptls_t ptls, jl_value_t *obj) +{ + arraylist_push(&ptls->sweep_objs, obj); +} + #ifdef __cplusplus } #endif diff --git a/src/gc.h b/src/gc.h index 51b8fca58c466..b3d5af52d581b 100644 --- a/src/gc.h +++ b/src/gc.h @@ -75,13 +75,6 @@ typedef struct { int full_sweep; } jl_gc_num_t; -typedef struct { - void **pc; // Current stack address for the pc (up growing) - jl_gc_mark_data_t *data; // Current stack address for the data (up growing) - void **pc_start; // Cached value of `gc_cache->pc_stack` - void **pc_end; // Cached value of `gc_cache->pc_stack_end` -} gc_mark_sp_t; - enum { GC_MARK_L_marked_obj, GC_MARK_L_scan_only, @@ -190,7 +183,7 @@ union _jl_gc_mark_data { // Pop a data struct from the mark data stack (i.e. decrease the stack pointer) // This should be used after dispatch and therefore the pc stack pointer is already popped from // the stack. -STATIC_INLINE void *gc_pop_markdata_(gc_mark_sp_t *sp, size_t size) +STATIC_INLINE void *gc_pop_markdata_(jl_gc_mark_sp_t *sp, size_t size) { jl_gc_mark_data_t *data = (jl_gc_mark_data_t *)(((char*)sp->data) - size); sp->data = data; @@ -201,7 +194,7 @@ STATIC_INLINE void *gc_pop_markdata_(gc_mark_sp_t *sp, size_t size) // Re-push a frame to the mark stack (both data and pc) // The data and pc are expected to be on the stack (or updated in place) already. // Mainly useful to pause the current scanning in order to scan an new object. -STATIC_INLINE void *gc_repush_markdata_(gc_mark_sp_t *sp, size_t size) +STATIC_INLINE void *gc_repush_markdata_(jl_gc_mark_sp_t *sp, size_t size) { jl_gc_mark_data_t *data = sp->data; sp->pc++; @@ -483,7 +476,7 @@ STATIC_INLINE void gc_big_object_link(bigval_t *hdr, bigval_t **list) JL_NOTSAFE *list = hdr; } -STATIC_INLINE void gc_mark_sp_init(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp) +STATIC_INLINE void gc_mark_sp_init(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp) { sp->pc = gc_cache->pc_stack; sp->data = gc_cache->data_stack; @@ -491,10 +484,10 @@ STATIC_INLINE void gc_mark_sp_init(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *s sp->pc_end = gc_cache->pc_stack_end; } -void gc_mark_queue_all_roots(jl_ptls_t ptls, gc_mark_sp_t *sp); -void gc_mark_queue_finlist(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, +void gc_mark_queue_all_roots(jl_ptls_t ptls, jl_gc_mark_sp_t *sp); +void gc_mark_queue_finlist(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp, arraylist_t *list, size_t start); -void gc_mark_loop(jl_ptls_t ptls, gc_mark_sp_t sp); +void gc_mark_loop(jl_ptls_t ptls, jl_gc_mark_sp_t sp); void sweep_stack_pools(void); void gc_debug_init(void); @@ -620,7 +613,7 @@ extern int gc_verifying; #endif int gc_slot_to_fieldidx(void *_obj, void *slot); int gc_slot_to_arrayidx(void *_obj, void *begin); -NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, gc_mark_sp_t sp, int pc_offset); +NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, jl_gc_mark_sp_t sp, int pc_offset); #ifdef GC_DEBUG_ENV JL_DLLEXPORT extern jl_gc_debug_env_t jl_gc_debug_env; diff --git a/src/julia_gcext.h b/src/julia_gcext.h new file mode 100644 index 0000000000000..eb7e450362189 --- /dev/null +++ b/src/julia_gcext.h @@ -0,0 +1,124 @@ +#ifndef JL_GCEXT_H +#define JL_GCEXT_H + +// requires including "julia.h" beforehand. + +// Callbacks that allow C code to hook into the GC. + +// Marking callbacks for global roots and tasks, respectively. These, +// along with custom mark functions must not alter the GC state except +// through calling jl_gc_mark_queue_obj() and jl_gc_mark_queue_objarray(). +typedef void (*jl_gc_cb_root_scanner_t)(int full); +typedef void (*jl_gc_cb_task_scanner_t)(jl_task_t *task, int full); + +// Callbacks that are invoked before and after a collection. +typedef void (*jl_gc_cb_pre_gc_t)(int full); +typedef void (*jl_gc_cb_post_gc_t)(int full); + +// Callbacks to track external object allocations. +typedef void (*jl_gc_cb_notify_external_alloc_t)(void *addr, size_t size); +typedef void (*jl_gc_cb_notify_external_free_t)(void *addr); + +JL_DLLEXPORT void jl_gc_set_cb_root_scanner(jl_gc_cb_root_scanner_t cb, int enable); +JL_DLLEXPORT void jl_gc_set_cb_task_scanner(jl_gc_cb_task_scanner_t cb, int enable); +JL_DLLEXPORT void jl_gc_set_cb_pre_gc(jl_gc_cb_pre_gc_t cb, int enable); +JL_DLLEXPORT void jl_gc_set_cb_post_gc(jl_gc_cb_post_gc_t cb, int enable); +JL_DLLEXPORT void jl_gc_set_cb_notify_external_alloc(jl_gc_cb_notify_external_alloc_t cb, + int enable); +JL_DLLEXPORT void jl_gc_set_cb_notify_external_free(jl_gc_cb_notify_external_free_t cb, + int enable); + +// Types for custom mark and sweep functions. +typedef uintptr_t (*jl_markfunc_t)(jl_ptls_t, jl_value_t *obj); +typedef void (*jl_sweepfunc_t)(jl_value_t *obj); + +// Function to create a new foreign type with custom +// mark and sweep functions. +JL_DLLEXPORT jl_datatype_t *jl_new_foreign_type( + jl_sym_t *name, + jl_module_t *module, + jl_datatype_t *super, + jl_markfunc_t markfunc, + jl_sweepfunc_t sweepfunc, + int haspointers, + int large); + +JL_DLLEXPORT size_t jl_gc_max_internal_obj_size(void); +JL_DLLEXPORT size_t jl_gc_external_obj_hdr_size(void); + +// Field layout descriptor for custom types that do +// not fit Julia layout conventions. This is associated with +// jl_datatype_t instances where fielddesc_type == 3. + +typedef struct { + jl_markfunc_t markfunc; + jl_sweepfunc_t sweepfunc; +} jl_fielddescdyn_t; + +// Allocate an object of a foreign type. +JL_DLLEXPORT void *jl_gc_alloc_typed(jl_ptls_t ptls, size_t sz, void *ty); + +// Queue an object or array of objects for scanning by the garbage collector. +// These functions must only be called from within a root scanner callback +// or from within a custom mark function. +JL_DLLEXPORT int jl_gc_mark_queue_obj(jl_ptls_t ptls, jl_value_t *obj); +JL_DLLEXPORT void jl_gc_mark_queue_objarray(jl_ptls_t ptls, jl_value_t *parent, + jl_value_t **objs, size_t nobjs); + +// Sweep functions will not automatically be called for objects of +// foreign types, as that may not always be desired. Only calling +// jl_gc_schedule_foreign_sweepfunc() on an object of a foreign type +// will result in the custome sweep function actually being called. +// This must be done at most once per object and should usually be +// done right after allocating the object. +JL_DLLEXPORT void jl_gc_schedule_foreign_sweepfunc(jl_ptls_t ptls, jl_value_t * bj); + +// The following functions enable support for conservative marking. This +// functionality allows the user to determine if a machine word can be +// interpreted as a pointer to an object (including the interior of an +// object). It can be used to, for example, scan foreign stack frames or +// data structures with an unknown layout. It is called conservative +// marking, because it can lead to false positives, as non-pointer data +// can mistakenly be interpreted as a pointer to a Julia object. + +// CAUTION: This is a sharp tool and should only be used as a measure of +// last resort. The user should be familiar with the risk of memory +// leaks (especially on 32-bit machines) if used inappropriately and how +// optimizing compilers can hide references from conservative stack +// scanning. In particular, arrays must be kept explicitly visible to +// the GC (by using JL_GC_PUSH1(), storing them in a location marked by +// the Julia GC, etc.) while their contents are being accessed. The +// reason is that array contents aren't marked separately, so if the +// object itself is not visible to the GC, neither are the contents. + +// Enable support for conservative marking. The function returns +// whether support was already enabled. The function may implicitly +// trigger a full garbage collection to properly update all internal +// data structures. +JL_DLLEXPORT int jl_gc_enable_conservative_gc_support(void); + +// This function returns whether support for conservative scanning has +// been enabled. The return values are the same as for +// jl_gc_enable_conservative_gc_support(). +JL_DLLEXPORT int jl_gc_conservative_gc_support_enabled(void); + +// Returns the base address of a memory block, assuming it is stored in +// a julia memory pool. Return NULL otherwise. Conservative support +// *must* have been enabled for this to work reliably. +// +// NOTE: This will only work for internal pool allocations. For external +// allocations, the user must track allocations using the notification +// callbacks above and verify that they are valid objects. Note that +// external allocations may not all be valid objects and that for those, +// the user *must* validate that they have a proper type, i.e. that +// jl_typeof(obj) is an actual type object. +JL_DLLEXPORT jl_value_t *jl_gc_internal_obj_base_ptr(void *p); + +// Return a non-null pointer to the start of the stack area if the task +// has an associated stack buffer. In that case, *size will also contain +// the size of that stack buffer upon return. Also, if task is a thread's +// current task, that thread's id will be stored in *tid; otherwise, +// *tid will be set to -1. +JL_DLLEXPORT void *jl_task_stack_buffer(jl_task_t *task, size_t *size, int *tid); + +#endif // _JULIA_GCEXT_H diff --git a/src/julia_internal.h b/src/julia_internal.h index 3b83792e49652..c228c2871d40d 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -281,7 +281,9 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc(jl_ptls_t ptls, size_t sz, void *ty); # define jl_gc_alloc(ptls, sz, ty) jl_gc_alloc_(ptls, sz, ty) #endif -#define jl_buff_tag ((uintptr_t)0x4eade800) +// jl_buff_tag must be a multiple of GC_PAGE_SZ so that it can't be +// confused for an actual type reference. +#define jl_buff_tag ((uintptr_t)0x4eadc000) typedef void jl_gc_tracked_buffer_t; // For the benefit of the static analyzer STATIC_INLINE jl_gc_tracked_buffer_t *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz) { diff --git a/src/julia_threads.h b/src/julia_threads.h index b2f7c27fae84b..447416061cff9 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -102,6 +102,14 @@ typedef struct { // Cache of thread local change to global metadata during GC // This is sync'd after marking. typedef union _jl_gc_mark_data jl_gc_mark_data_t; + +typedef struct { + void **pc; // Current stack address for the pc (up growing) + jl_gc_mark_data_t *data; // Current stack address for the data (up growing) + void **pc_start; // Cached value of `gc_cache->pc_stack` + void **pc_end; // Cached value of `gc_cache->pc_stack_end` +} jl_gc_mark_sp_t; + typedef struct { // thread local increment of `perm_scanned_bytes` size_t perm_scanned_bytes; @@ -176,6 +184,8 @@ struct _jl_tls_states_t { int finalizers_inhibited; arraylist_t finalizers; jl_gc_mark_cache_t gc_cache; + arraylist_t sweep_objs; + jl_gc_mark_sp_t gc_mark_sp; }; // Update codegen version in `ccall.cpp` after changing either `pause` or `wake` diff --git a/src/task.c b/src/task.c index 04d6157b06d0b..402e0e052dd6d 100644 --- a/src/task.c +++ b/src/task.c @@ -57,6 +57,8 @@ volatile int jl_in_stackwalk = 0; #define MINSTKSZ 131072 #endif +#define ROOT_TASK_STACK_ADJUSTMENT 3000000 + static jl_sym_t *done_sym; static jl_sym_t *failed_sym; static jl_sym_t *runnable_sym; @@ -179,6 +181,36 @@ static void JL_NORETURN finish_task(jl_task_t *t, jl_value_t *resultval JL_MAYBE abort(); } +JL_DLLEXPORT void *jl_task_stack_buffer(jl_task_t *task, size_t *size, int *tid) +{ + size_t off = 0; +#ifndef _OS_WINDOWS_ + if (jl_all_tls_states[0]->root_task == task) { + // See jl_init_root_task(). The root task of the main thread + // has its buffer enlarged by an artificial 3000000 bytes, but + // that means that the start of the buffer usually points to + // inaccessible memory. We need to correct for this. + off = ROOT_TASK_STACK_ADJUSTMENT; + } +#endif + *tid = -1; + for (int i = 0; i < jl_n_threads; i++) { + jl_ptls_t ptls = jl_all_tls_states[i]; + if (ptls->current_task == task) { + *tid = i; +#ifdef COPY_STACKS + if (task->copy_stack) { + *size = ptls->stacksize; + return (char *)ptls->stackbase - *size; + } +#endif + break; // continue with normal return + } + } + *size = task->bufsz - off; + return (void *)((char *)task->stkbuf + off); +} + static void record_backtrace(void) JL_NOTSAFEPOINT { jl_ptls_t ptls = jl_get_ptls_states(); @@ -813,8 +845,8 @@ void jl_init_root_task(void *stack_lo, void *stack_hi) size_t ssize = (char*)stack_hi - (char*)stack_lo; #ifndef _OS_WINDOWS_ if (ptls->tid == 0) { - stack = (void*)((char*)stack - 3000000); // offset our guess of the address of the bottom of stack to cover the guard pages too - ssize += 3000000; // sizeof stack is known exactly, but not where we are in that stack + stack = (void*)((char*)stack - ROOT_TASK_STACK_ADJUSTMENT); // offset our guess of the address of the bottom of stack to cover the guard pages too + ssize += ROOT_TASK_STACK_ADJUSTMENT; // sizeof stack is known exactly, but not where we are in that stack } #endif ptls->current_task->stkbuf = stack; diff --git a/test/Makefile b/test/Makefile index b4eb4dc1362d3..0eb7aab046a28 100644 --- a/test/Makefile +++ b/test/Makefile @@ -16,6 +16,8 @@ TESTS = all stdlib $(TESTGROUPS) \ EMBEDDING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/embedding" "CC=$(CC)" +GCEXT_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/gcext" "CC=$(CC)" + default: all $(TESTS): @@ -25,7 +27,11 @@ $(TESTS): embedding: @$(MAKE) -C $(SRCDIR)/$@ check $(EMBEDDING_ARGS) +gcext: + @$(MAKE) -C $(SRCDIR)/$@ check $(GCEXT_ARGS) + clean: @$(MAKE) -C embedding $@ $(EMBEDDING_ARGS) + @$(MAKE) -C gcext $@ $(GCEXT_ARGS) -.PHONY: $(TESTS) embedding clean +.PHONY: $(TESTS) embedding gcext clean diff --git a/test/gcext/.gitignore b/test/gcext/.gitignore new file mode 100644 index 0000000000000..0f8c848e5cea6 --- /dev/null +++ b/test/gcext/.gitignore @@ -0,0 +1,2 @@ +/gcext +/gcext-debug diff --git a/test/gcext/LocalTest.jl b/test/gcext/LocalTest.jl new file mode 100644 index 0000000000000..f73b4b47e8023 --- /dev/null +++ b/test/gcext/LocalTest.jl @@ -0,0 +1,102 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +const Stack = Main.TestGCExt.Stack + +function make() + ccall(:stk_make, Stack, ()) +end + +function push(stack :: Stack, val :: String) + ccall(:stk_push, Nothing, (Stack, String), stack, val) +end + +function top(stack :: Stack) + return ccall(:stk_top, String, (Stack,), stack) +end + +function pop(stack :: Stack) + return ccall(:stk_pop, String, (Stack,), stack) +end + +function size(stack :: Stack) + return ccall(:stk_size, UInt, (Stack,), stack) +end + +function empty(stack :: Stack) + return size(stack) == 0 +end + +function blob(stack :: Stack) + return ccall(:stk_blob, Any, (Stack,), stack) +end + +function gc_counter_full() + return ccall(:get_gc_counter, UInt, (Cint,), 1) +end + +function gc_counter_inc() + return ccall(:get_gc_counter, UInt, (Cint,), 0) +end + +function gc_counter() + return gc_counter_full() + gc_counter_inc() +end + +function num_obj_sweeps() + return ccall(:get_obj_sweeps, UInt, ()) +end + +function get_aux_root(n :: Int) + return ccall(:get_aux_root, String, (UInt,), n) +end + +function set_aux_root(n :: Int, x :: String) + return ccall(:set_aux_root, Nothing, (UInt, String), n, x) +end + +function internal_obj_scan(p :: Any) + if ccall(:internal_obj_scan, Cint, (Any,), p) == 0 + global internal_obj_scan_failures += 1 + end +end + +global internal_obj_scan_failures = 0 + +for i in 0:1000 + set_aux_root(i, string(i)) +end + +function test() + local stack = make() + for i in 1:100000 + push(stack, string(i, base=2)) + internal_obj_scan(top(stack)) + end + for i in 1:1000 + local stack2 = make() + internal_obj_scan(stack2) + internal_obj_scan(blob(stack2)) + while !empty(stack) + push(stack2, pop(stack)) + end + stack = stack2 + if i % 100 == 0 + GC.gc() + end + end +end + +@time test() + +global corrupted_roots = 0 +for i in 0:1000 + if get_aux_root(i) != string(i) + global corrupted_roots += 1 + end +end + +print(gc_counter_full(), " full collections.\n") +print(gc_counter_inc(), " partial collections.\n") +print(num_obj_sweeps(), " object sweeps.\n") +print(internal_obj_scan_failures, " internal object scan failures.\n") +print(corrupted_roots, " corrupted auxiliary roots.\n") diff --git a/test/gcext/Makefile b/test/gcext/Makefile new file mode 100644 index 0000000000000..7cb602572e3c5 --- /dev/null +++ b/test/gcext/Makefile @@ -0,0 +1,60 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# This Makefile template requires the following variables to be set +# in the environment or on the command-line: +# JULIA: path to julia[.exe] executable +# BIN: binary build directory + +ifndef JULIA + $(error "Please pass JULIA=[path of target julia binary], or set as environment variable!") +endif +ifndef BIN + $(error "Please pass BIN=[path of build directory], or set as environment variable!") +endif + +#============================================================================= +# this source directory where gcext.c is located +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) + +# get the executable suffix, if any +EXE := $(suffix $(abspath $(JULIA))) + +# get compiler and linker flags. (see: `contrib/julia-config.jl`) +JULIA_CONFIG := $(JULIA) -e 'include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "julia-config.jl"))' -- +CPPFLAGS_ADD := +CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags) +LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) + +DEBUGFLAGS += -g + +#============================================================================= + +release: $(BIN)/gcext$(EXE) +debug: $(BIN)/gcext-debug$(EXE) + +$(BIN)/gcext$(EXE): $(SRCDIR)/gcext.c + $(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) + +$(BIN)/gcext-debug$(EXE): $(SRCDIR)/gcext.c + $(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS) + +ifneq ($(abspath $(BIN)),$(abspath $(SRCDIR))) +# for demonstration purposes, our demo code is also installed +# in $BIN, although this would likely not be typical +$(BIN)/LocalModule.jl: $(SRCDIR)/LocalModule.jl + cp $< $@ +endif + +check: $(BIN)/gcext$(EXE) $(BIN)/LocalTest.jl + $(JULIA) --depwarn=error $(SRCDIR)/gcext-test.jl $< + @echo SUCCESS + +clean: + -rm -f $(BIN)/gcext-debug$(EXE) $(BIN)/gcext$(EXE) + +.PHONY: release debug clean check + +# Makefile debugging trick: +# call print-VARIABLE to see the runtime value of any variable +print-%: + @echo '$*=$($*)' diff --git a/test/gcext/gcext-test.jl b/test/gcext/gcext-test.jl new file mode 100644 index 0000000000000..e6f3e3663ff0e --- /dev/null +++ b/test/gcext/gcext-test.jl @@ -0,0 +1,42 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# tests the output of the embedding example is correct +using Test + +if Sys.iswindows() + # libjulia needs to be in the same directory as the embedding executable or in path + ENV["PATH"] = string(Sys.BINDIR, ";", ENV["PATH"]) +end + +function checknum(s, rx, cond) + m = match(rx, s) + if m === nothing + return false + else + num = m[1] + return cond(parse(UInt, num)) + end +end + +@test length(ARGS) == 1 +@testset "gcext example" begin + out = Pipe() + err = Pipe() + p = run(pipeline(Cmd(ARGS), stdin=devnull, stdout=out, stderr=err), wait=false) + close(out.in) + close(err.in) + out_task = @async readlines(out) + err_task = @async readlines(err) + # @test success(p) + errlines = fetch(err_task) + lines = fetch(out_task) + @test length(errlines) == 0 + @test length(lines) == 6 + @test checknum(lines[2], r"([0-9]+) full collections", n -> n >= 10) + @test checknum(lines[3], r"([0-9]+) partial collections", n -> n > 0) + @test checknum(lines[4], r"([0-9]+) object sweeps", n -> n > 0) + @test checknum(lines[5], r"([0-9]+) internal object scan failures", + n -> n == 0) + @test checknum(lines[6], r"([0-9]+) corrupted auxiliary roots", + n -> n == 0) +end diff --git a/test/gcext/gcext.c b/test/gcext/gcext.c new file mode 100644 index 0000000000000..93715ed6a5bab --- /dev/null +++ b/test/gcext/gcext.c @@ -0,0 +1,664 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include +#include + +#include "julia.h" +#include "julia_gcext.h" + +// Comparing pointers in C without triggering undefined behavior +// can be difficult. As the GC already assumes that the memory +// range goes from 0 to 2^k-1 (region tables), we simply convert +// to uintptr_t and compare those. + +static inline int cmp_ptr(void *p, void *q) +{ + uintptr_t paddr = (uintptr_t)p; + uintptr_t qaddr = (uintptr_t)q; + if (paddr < qaddr) + return -1; + else if (paddr > qaddr) + return 1; + else + return 0; +} + +static inline int lt_ptr(void *a, void *b) +{ + return (uintptr_t)a < (uintptr_t)b; +} + +/* align pointer to full word if mis-aligned */ +static inline void *align_ptr(void *p) +{ + uintptr_t u = (uintptr_t)p; + u &= ~(sizeof(p) - 1); + return (void *)u; +} + +// We use treaps -- a form of balanced trees -- to be able to +// find allocations based on their address. + +typedef struct treap_t { + struct treap_t *left, *right; + size_t prio; + void *addr; + size_t size; +} treap_t; + +static treap_t *treap_free_list; + +treap_t *alloc_treap(void) +{ + treap_t *result; + if (treap_free_list) { + result = treap_free_list; + treap_free_list = treap_free_list->right; + } + else + result = malloc(sizeof(treap_t)); + result->left = NULL; + result->right = NULL; + result->addr = NULL; + result->size = 0; + return result; +} + +void free_treap(treap_t *t) +{ + t->right = treap_free_list; + treap_free_list = t; +} + +static inline int test_bigval_range(treap_t *node, void *p) +{ + char *l = node->addr; + char *r = l + node->size; + if (lt_ptr(p, l)) + return -1; + if (!lt_ptr(p, r)) + return 1; + return 0; +} + +#define L(t) ((t)->left) +#define R(t) ((t)->right) + +static inline void treap_rot_right(treap_t **treap) +{ + /* t l */ + /* / \ / \ */ + /* l r --> a t */ + /* / \ / \ */ + /* a b b r */ + treap_t *t = *treap; + treap_t *l = L(t); + treap_t *a = L(l); + treap_t *b = R(l); + L(l) = a; + R(l) = t; + L(t) = b; + *treap = l; +} + +static inline void treap_rot_left(treap_t **treap) +{ + /* t r */ + /* / \ / \ */ + /* l r --> t b */ + /* / \ / \ */ + /* a b l a */ + treap_t *t = *treap; + treap_t *r = R(t); + treap_t *a = L(r); + treap_t *b = R(r); + L(r) = t; + R(r) = b; + R(t) = a; + *treap = r; +} + +static treap_t *treap_find(treap_t *treap, void *p) +{ + while (treap) { + int c = test_bigval_range(treap, p); + if (c == 0) + return treap; + else if (c < 0) + treap = L(treap); + else + treap = R(treap); + } + return NULL; +} + +static void treap_insert(treap_t **treap, treap_t *val) +{ + treap_t *t = *treap; + if (t == NULL) { + L(val) = NULL; + R(val) = NULL; + *treap = val; + } + else { + int c = cmp_ptr(val->addr, t->addr); + if (c < 0) { + treap_insert(&L(t), val); + if (L(t)->prio > t->prio) { + treap_rot_right(treap); + } + } + else if (c > 0) { + treap_insert(&R(t), val); + if (R(t)->prio > t->prio) { + treap_rot_left(treap); + } + } + } +} + +static void treap_delete_node(treap_t **treap) +{ + for (;;) { + treap_t *t = *treap; + if (L(t) == NULL) { + *treap = R(t); + free_treap(t); + break; + } + else if (R(t) == NULL) { + *treap = L(t); + free_treap(t); + break; + } + else { + if (L(t)->prio > R(t)->prio) { + treap_rot_right(treap); + treap = &R(*treap); + } + else { + treap_rot_left(treap); + treap = &L(*treap); + } + } + } +} + +static int treap_delete(treap_t **treap, void *addr) +{ + while (*treap != NULL) { + int c = cmp_ptr(addr, (*treap)->addr); + if (c == 0) { + treap_delete_node(treap); + return 1; + } + else if (c < 0) { + treap = &L(*treap); + } + else { + treap = &R(*treap); + } + } + return 0; +} + +static uint64_t xorshift_rng_state = 1; + +static uint64_t xorshift_rng(void) +{ + uint64_t x = xorshift_rng_state; + x = x ^ (x >> 12); + x = x ^ (x << 25); + x = x ^ (x >> 27); + xorshift_rng_state = x; + return x * (uint64_t)0x2545F4914F6CDD1DUL; +} + +static treap_t *bigvals; +static size_t bigval_startoffset; + +// Hooks to allocate and free external objects (bigval_t's). + +void alloc_bigval(void *addr, size_t size) +{ + treap_t *node = alloc_treap(); + node->addr = addr; + node->size = size; + node->prio = xorshift_rng(); + treap_insert(&bigvals, node); +} + +void free_bigval(void *p) +{ + if (p) { + treap_delete(&bigvals, p); + } +} + +// Auxiliary roots. These serve no special purpose, except +// allowing us to verify that root scanning works. + +#define NAUXROOTS 1024 +static jl_value_t *aux_roots[NAUXROOTS]; +size_t gc_counter_full, gc_counter_inc; + +jl_value_t *get_aux_root(size_t n) +{ + if (n >= NAUXROOTS) + jl_error("get_aux_root: index out of range"); + return aux_roots[n]; +} + +void set_aux_root(size_t n, jl_value_t *val) +{ + if (n >= NAUXROOTS) + jl_error("set_aux_root: index out of range"); + aux_roots[n] = val; +} + +size_t get_gc_counter(int full) +{ + if (full) + return gc_counter_full; + else + return gc_counter_inc; +} + +static size_t obj_sweeps = 0; + +size_t get_obj_sweeps() +{ + return obj_sweeps; +} + +typedef struct { + size_t size; + size_t capacity; + jl_value_t *data[1]; +} dynstack_t; + +static jl_datatype_t *datatype_stack_internal; +static jl_datatype_t *datatype_stack_external; +static jl_datatype_t *datatype_stack; +static jl_ptls_t ptls; + +static size_t gc_alloc_size(jl_value_t *val) +{ + size_t size; + if (jl_typeis(val, datatype_stack)) + size = sizeof(jl_value_t *); + else if (jl_typeis(val, datatype_stack_internal) || jl_typeis(val, datatype_stack_external)) + size = offsetof(dynstack_t, data) + + ((dynstack_t *)val)->capacity * sizeof(jl_value_t *); + else if (jl_typeis(val, jl_string_type)) { + size = jl_string_len(val) + sizeof(size_t) + 1; + // Round up to whole word size + if (size % sizeof(void *) != 0) + size += sizeof(void *) - size % sizeof(void *); + } + else + size = 0; + return size; +} + +int internal_obj_scan(jl_value_t *val) +{ + if (jl_gc_internal_obj_base_ptr(val) == val) { + size_t size = gc_alloc_size(val); + char *addr = (char *)val; + for (size_t i = 0; i <= size; i++) { + if (jl_gc_internal_obj_base_ptr(addr + i) != val) + return 0; + } + return 1; + } + else { + treap_t *node = treap_find(bigvals, val); + if (!node) + return 0; + char *addr = node->addr; + if ((jl_value_t *)addr != val) + return 0; + size_t size = node->size; + for (size_t i = 0; i <= size; i++) { + if (treap_find(bigvals, addr + i) != node) + return 0; + } + return 1; + } +} + +dynstack_t *allocate_stack_mem(size_t capacity) +{ + size_t size = offsetof(dynstack_t, data) + capacity * sizeof(jl_value_t *); + jl_datatype_t *type = datatype_stack_internal; + if (size > jl_gc_max_internal_obj_size()) + type = datatype_stack_external; + dynstack_t *result = (dynstack_t *)jl_gc_alloc_typed(ptls, size, type); + result->size = 0; + result->capacity = capacity; + return result; +} + +void check_stack(const char *name, jl_value_t *p) +{ + if (jl_typeis(p, datatype_stack)) + return; + jl_type_error(name, (jl_value_t *)datatype_stack, p); +} + +void check_stack_notempty(const char *name, jl_value_t *p) +{ + check_stack(name, p); + dynstack_t *stk = *(dynstack_t **)p; + if (stk->size == 0) + jl_errorf("%s: dynstack_t empty", name); +} + +// Stacks use double indirection in order to be resizable. +// The outer object is a single word containing a pointer to +// a `dynstack_t`, which can contain a variable number of +// Julia objects; the `capacity` field denotes the number of objects +// that can be stored without resizing storage, the `size` field +// denotes the actual number of objects. GC scanning should ignore +// any storage past those. + +// Return the type of stacks + +jl_value_t *stk_type() +{ + return (jl_value_t *)datatype_stack; +} + +// Create a new stack object + +jl_value_t *stk_make() +{ + jl_value_t *hdr = + jl_gc_alloc_typed(ptls, sizeof(jl_value_t *), datatype_stack); + JL_GC_PUSH1(hdr); + *(dynstack_t **)hdr = NULL; + dynstack_t *stk = allocate_stack_mem(8); + *(dynstack_t **)hdr = stk; + jl_gc_schedule_foreign_sweepfunc(ptls, (jl_value_t *)(stk)); + JL_GC_POP(); + return hdr; +} + +// Return a pointer to the inner `dynstack_t` struct. + +jl_value_t *stk_blob(jl_value_t *s) +{ + return (jl_value_t *)(*(dynstack_t **)s); +} + +// Push `v` on `s`. + +void stk_push(jl_value_t *s, jl_value_t *v) +{ + check_stack("push", s); + dynstack_t *stk = *(dynstack_t **)s; + if (stk->size < stk->capacity) { + stk->data[stk->size++] = v; + jl_gc_wb((jl_value_t *)stk, v); + } + else { + dynstack_t *newstk = allocate_stack_mem(stk->capacity * 3 / 2 + 1); + newstk->size = stk->size; + memcpy(newstk->data, stk->data, sizeof(jl_value_t *) * stk->size); + *(dynstack_t **)s = newstk; + newstk->data[newstk->size++] = v; + jl_gc_schedule_foreign_sweepfunc(ptls, (jl_value_t *)(newstk)); + jl_gc_wb_back((jl_value_t *)newstk); + jl_gc_wb(s, (jl_value_t *)newstk); + } +} + +// Return top value from `s`. Raise error if not empty. + +jl_value_t *stk_top(jl_value_t *s) +{ + check_stack_notempty("top", s); + dynstack_t *stk = *(dynstack_t **)s; + return stk->data[stk->size - 1]; +} + +// Pop a value from `s` and return it. Raise error if not empty. + +jl_value_t *stk_pop(jl_value_t *s) +{ + check_stack_notempty("pop", s); + dynstack_t *stk = *(dynstack_t **)s; + stk->size--; + return stk->data[stk->size]; +} + +// Number of objects on the stack. + +size_t stk_size(jl_value_t *s) +{ + check_stack("empty", s); + dynstack_t *stk = *(dynstack_t **)s; + return stk->size; +} + +static jl_module_t *module; + +// Mark auxiliary roots. + +void root_scanner(int full) +{ + for (int i = 0; i < NAUXROOTS; i++) { + if (aux_roots[i]) + jl_gc_mark_queue_obj(ptls, aux_roots[i]); + } +} + +// Test stack direction +static int is_lower_stack_frame(volatile char *frame_addr) { + volatile char buf[1]; + return (uintptr_t) buf < (uintptr_t) frame_addr; +} + +typedef volatile int (*volatile test_frame_func)(volatile char *frame_addr); + +// To prevent inlining, we make this a volatile function pointer. + +static test_frame_func is_lower_stack_frame_ptr = + (test_frame_func) is_lower_stack_frame; + +static int stack_grows_down(void) { + volatile char buf[1]; + return is_lower_stack_frame_ptr(buf); +} + +void task_scanner(jl_task_t *task, int root_task) +{ + // The task scanner is not necessary for liveness, as the + // corresponding task stack is already part of the stack. + // Its purpose is simply to test that the task scanner + // doing actual work does not trigger a problem. + size_t size; + int tid; + void *stack = jl_task_stack_buffer(task, &size, &tid); + if (tid >= 0) { + // this is the live stack of a thread. Is it ours? + if (stack && tid == jl_threadid()) { + // only scan the live portion of the stack. + char *end_stack = (char *) stack + size; + if (lt_ptr(stack, &size) && lt_ptr(&size, (char *)stack + size)) { + if (stack_grows_down()) { + size = end_stack - (char *)&size; + stack = (void *)&size; + } + else { + size = (char *) end_stack - (char *) &size; + } + } else { + // error, current stack frame must be on the live stack. + jl_error("stack frame not part of the current task"); + } + } + else + stack = NULL; + } + if (stack) { + void **start = (void **) stack; + void **end = start + size / sizeof(void *); + while (start < end) { + void *p = *start++; + void *q = jl_gc_internal_obj_base_ptr(p); + if (q) { + jl_gc_mark_queue_obj(ptls, q); + } + } + } +} + +// Hooks to run before and after GC. +// +// As a simple example, we only track counters for full +// and partial collections. + +void pre_gc_func(int full) +{ + if (full) + gc_counter_full++; + else + gc_counter_inc++; +} + +void post_gc_func(int full) {} + +// Mark the outer stack object (containing only a pointer to the data). + +uintptr_t mark_stack(jl_ptls_t ptls, jl_value_t *p) +{ + if (!*(void **)p) + return 0; + return jl_gc_mark_queue_obj(ptls, *(jl_value_t **)p) != 0; +} + +// Mark the actual stack data. +// This is used both for `StackData` and `StackDataLarge`. + +uintptr_t mark_stack_data(jl_ptls_t ptls, jl_value_t *p) +{ + dynstack_t *stk = (dynstack_t *)p; + // Alternate between two marking approaches for testing so + // that we test both. + if (gc_counter_full & 1) { + jl_gc_mark_queue_objarray(ptls, p, stk->data, stk->size); + return 0; + } + else { + uintptr_t n = 0; + for (size_t i = 0; i < stk->size; i++) { + if (jl_gc_mark_queue_obj(ptls, stk->data[i])) + n++; + } + return n; + } +} + +void sweep_stack_data(jl_value_t *p) +{ + obj_sweeps++; + dynstack_t *stk = (dynstack_t *)p; + if (stk->size > stk->capacity) + jl_error("internal error during sweeping"); +} + +// Safely execute Julia code + +jl_value_t *checked_eval_string(const char *code) +{ + jl_value_t *result = jl_eval_string(code); + if (jl_exception_occurred()) { + // none of these allocate, so a gc-root (JL_GC_PUSH) is not necessary + jl_call2( + jl_get_function(jl_base_module, "showerror"), + jl_stderr_obj(), + jl_exception_occurred()); + jl_printf(jl_stderr_stream(), "\n"); + jl_atexit_hook(1); + exit(1); + } + assert(result && "Missing return value but no exception occurred!"); + return result; +} + +void abort_with_error(int full) +{ + abort(); +} + +int main() +{ + // Install callbacks. This should happen before `jl_init()` and + // before any GC is called. + + jl_gc_set_cb_notify_external_alloc(alloc_bigval, 1); + jl_gc_set_cb_notify_external_free(free_bigval, 1); + + jl_init(); + if (jl_gc_enable_conservative_gc_support() < 0) + abort(); + ptls = jl_get_ptls_states(); + jl_gc_set_cb_root_scanner(root_scanner, 1); + jl_gc_set_cb_task_scanner(task_scanner, 1); + jl_gc_set_cb_pre_gc(pre_gc_func, 1); + jl_gc_set_cb_post_gc(post_gc_func, 1); + // Test that deregistration works + jl_gc_set_cb_root_scanner(abort_with_error, 1); + jl_gc_set_cb_root_scanner(abort_with_error, 0); + // Create module to store types in. + module = jl_new_module(jl_symbol("TestGCExt")); + module->parent = jl_main_module; + jl_set_const(jl_main_module, jl_symbol("TestGCExt"), (jl_value_t *)module); + // Define Julia types for our stack implementation. + datatype_stack = jl_new_foreign_type( + jl_symbol("Stack"), module, jl_any_type, mark_stack, NULL, 1, 0); + jl_set_const(module, jl_symbol("Stack"), (jl_value_t *)datatype_stack); + datatype_stack_internal = jl_new_foreign_type( + jl_symbol("StackData"), + module, + jl_any_type, + mark_stack_data, + sweep_stack_data, + 1, + 0); + jl_set_const( + module, + jl_symbol("StackData"), + (jl_value_t *)datatype_stack_internal); + datatype_stack_external = jl_new_foreign_type( + jl_symbol("StackDataLarge"), + module, + jl_any_type, + mark_stack_data, + sweep_stack_data, + 1, + 1); + jl_set_const( + module, + jl_symbol("StackDataLarge"), + (jl_value_t *)datatype_stack_external); + // Remember the offset of external objects + bigval_startoffset = jl_gc_external_obj_hdr_size(); + // Run the actual tests + checked_eval_string( + "let dir = dirname(unsafe_string(Base.JLOptions().julia_bin))\n" + // disable the package manager + " ENV[\"JULIA_PKGDIR\"] = joinpath(dir, \"disabled\")\n" + // locate files relative to the "embedding" executable + " stdlib = filter(env -> startswith(Base.find_package(Base, " + "\"Distributed\"), env), Base.load_path())[end]\n" + " push!(empty!(LOAD_PATH), dir, stdlib)\n" + "end"); + + checked_eval_string( + "module LocalTest\n" + " include(\"LocalTest.jl\")\n" + "end"); +}