From 7600b14f36fc42da42789181740fd3c63d4b4c45 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 30 Oct 2019 15:12:51 -0400 Subject: [PATCH 1/3] datatype: reorder layout computation [NFCI] --- src/datatype.c | 284 ++++++++++++++++++++++++++----------------------- src/jltypes.c | 26 ++--- 2 files changed, 165 insertions(+), 145 deletions(-) diff --git a/src/datatype.c b/src/datatype.c index 2b21e67669308..6599e1cee5528 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -236,7 +236,7 @@ STATIC_INLINE int jl_is_datatype_make_singleton(jl_datatype_t *d) d->uid != 0 && !d->mutabl); } -STATIC_INLINE void jl_allocate_singleton_instance(jl_datatype_t *st) +STATIC_INLINE void jl_maybe_allocate_singleton_instance(jl_datatype_t *st) { if (jl_is_datatype_make_singleton(st)) { st->instance = jl_gc_alloc(jl_get_ptls_states(), 0, st); @@ -244,18 +244,18 @@ STATIC_INLINE void jl_allocate_singleton_instance(jl_datatype_t *st) } } -static unsigned union_isbits(jl_value_t *ty, size_t *nbytes, size_t *align) JL_NOTSAFEPOINT +static unsigned union_isinlinable(jl_value_t *ty, int pointerfree, size_t *nbytes, size_t *align) JL_NOTSAFEPOINT { if (jl_is_uniontype(ty)) { - unsigned na = union_isbits(((jl_uniontype_t*)ty)->a, nbytes, align); + unsigned na = union_isinlinable(((jl_uniontype_t*)ty)->a, 1, nbytes, align); if (na == 0) return 0; - unsigned nb = union_isbits(((jl_uniontype_t*)ty)->b, nbytes, align); + unsigned nb = union_isinlinable(((jl_uniontype_t*)ty)->b, 1, nbytes, align); if (nb == 0) return 0; return na + nb; } - if (jl_is_datatype(ty) && jl_datatype_isinlinealloc(ty)) { + if (jl_is_datatype(ty) && jl_datatype_isinlinealloc(ty) && (!pointerfree || ((jl_datatype_t*)ty)->layout->npointers == 0)) { size_t sz = jl_datatype_size(ty); size_t al = jl_datatype_align(ty); if (*nbytes < sz) @@ -269,7 +269,7 @@ static unsigned union_isbits(jl_value_t *ty, size_t *nbytes, size_t *align) JL_N JL_DLLEXPORT int jl_islayout_inline(jl_value_t *eltype, size_t *fsz, size_t *al) JL_NOTSAFEPOINT { - unsigned countbits = union_isbits(eltype, fsz, al); + unsigned countbits = union_isinlinable(eltype, 0, fsz, al); return (countbits > 0 && countbits < 127) ? countbits : 0; } @@ -296,165 +296,183 @@ static int references_name(jl_value_t *p, jl_typename_t *name) JL_NOTSAFEPOINT return 0; } +static void throw_ovf(int should_malloc, void *desc, jl_datatype_t* st, int offset) +{ + if (should_malloc) + free(desc); + jl_errorf("type %s has field offset %d that exceeds the page size", jl_symbol_name(st->name->name), offset); +} + void jl_compute_field_offsets(jl_datatype_t *st) { - size_t sz = 0, alignm = 1; - size_t fldsz = 0, fldal = 0; - int homogeneous = 1; - jl_value_t *lastty = NULL; - uint64_t max_offset = (((uint64_t)1) << 32) - 1; - uint64_t max_size = max_offset >> 1; - - if (st->name->wrapper) { - jl_datatype_t *w = (jl_datatype_t*)jl_unwrap_unionall(st->name->wrapper); - // compute whether this type can be inlined - // based on whether its definition is self-referential - if (w->types != NULL) { - st->isbitstype = st->isinlinealloc = st->isconcretetype && !st->mutabl; - if (st->isinlinealloc) { - size_t i, nf = jl_svec_len(w->types); - for (i = 0; i < nf; i++) { - jl_value_t *fld = jl_svecref(w->types, i); - if (references_name(fld, w->name)) { - st->isinlinealloc = 0; - st->isbitstype = 0; - st->zeroinit = 1; - break; - } - } - } - size_t i, nf = jl_svec_len(st->types); - for (i = 0; i < nf; i++) { - jl_value_t *fld = jl_svecref(st->types, i); - if (st->isbitstype) - st->isbitstype = jl_is_datatype(fld) && ((jl_datatype_t*)fld)->isbitstype; - if (st->isinlinealloc) - st->isinlinealloc = (jl_is_datatype(fld) && ((jl_datatype_t*)fld)->isbitstype) || jl_islayout_inline(fld, &fldsz, &fldal); - if (!st->zeroinit) - st->zeroinit = (jl_is_datatype(fld) && ((jl_datatype_t*)fld)->isinlinealloc) ? ((jl_datatype_t*)fld)->zeroinit : 1; - if (i < st->ninitialized) { - if (fld == jl_bottom_type) - st->has_concrete_subtype = 0; - else - st->has_concrete_subtype &= !jl_is_datatype(fld) || ((jl_datatype_t *)fld)->has_concrete_subtype; - } - } - } - // If layout doesn't depend on type parameters, it's stored in st->name->wrapper - // and reused by all subtypes. - if (st != w && // this check allows us to re-compute layout for some types during init - w->layout) { - st->layout = w->layout; - st->size = w->size; - jl_allocate_singleton_instance(st); + const uint64_t max_offset = (((uint64_t)1) << 32) - 1; + const uint64_t max_size = max_offset >> 1; + + if (st->types == NULL || st->name->wrapper == NULL || (jl_is_namedtuple_type(st) && !jl_is_concrete_type((jl_value_t*)st))) + return; + jl_datatype_t *w = (jl_datatype_t*)jl_unwrap_unionall(st->name->wrapper); + if (w->types == NULL) // we got called too early--we'll be back + return; + size_t i, nfields = jl_svec_len(st->types); + int isinlinealloc = st->isconcretetype && !st->mutabl; + int isbitstype = isinlinealloc; + assert(st->ninitialized <= nfields); + if (st == w && st->layout) { + // this check allows us to force re-computation of the layout for some types during init + st->layout = NULL; + st->size = 0; + st->zeroinit = 0; + st->has_concrete_subtype = 1; + } + // If layout doesn't depend on type parameters, it's stored in st->name->wrapper + // and reused by all subtypes. + if (w->layout) { + st->layout = w->layout; + st->size = w->size; + st->zeroinit = w->zeroinit; + st->has_concrete_subtype = w->has_concrete_subtype; + if (jl_is_layout_opaque(st->layout)) { // e.g. jl_array_typename return; } } - if (st->types == NULL || (jl_is_namedtuple_type(st) && !jl_is_concrete_type((jl_value_t*)st))) - return; - uint32_t nfields = jl_svec_len(st->types); - if (nfields == 0) { + else if (nfields == 0) { + // if we have no fields, we can trivially skip the rest if (st == jl_symbol_type || st == jl_string_type) { // opaque layout - heap-allocated blob static const jl_datatype_layout_t opaque_byte_layout = {0, 1, 0, 1, 0}; st->layout = &opaque_byte_layout; + return; } else if (st == jl_simplevector_type || st->name == jl_array_typename) { static const jl_datatype_layout_t opaque_ptr_layout = {0, sizeof(void*), 0, 1, 0}; st->layout = &opaque_ptr_layout; + return; } else { // reuse the same layout for all singletons static const jl_datatype_layout_t singleton_layout = {0, 1, 0, 0, 0}; st->layout = &singleton_layout; - jl_allocate_singleton_instance(st); } - return; } - if (!jl_is_concrete_type((jl_value_t*)st)) { - // compute layout whenever field types have no free variables - for (size_t i = 0; i < nfields; i++) { - if (jl_has_free_typevars(jl_field_type(st, i))) - return; + else { + // compute a conservative estimate of whether there could exist an instance of a subtype of this + for (i = 0; st->has_concrete_subtype && i < st->ninitialized; i++) { + jl_value_t *fld = jl_svecref(st->types, i); + if (fld == jl_bottom_type) + st->has_concrete_subtype = 0; + else + st->has_concrete_subtype = !jl_is_datatype(fld) || ((jl_datatype_t *)fld)->has_concrete_subtype; + } + // compute layout for the wrapper object if the field types have no free variables + if (!st->isconcretetype) { + for (i = 0; i < nfields; i++) { + if (jl_has_free_typevars(jl_field_type(st, i))) + return; // not worthwhile computing the rest + } } } - size_t descsz = nfields * sizeof(jl_fielddesc32_t); - jl_fielddesc32_t *desc; - int should_malloc = descsz >= jl_page_size; - if (should_malloc) - desc = (jl_fielddesc32_t*)malloc_s(descsz); - else - desc = (jl_fielddesc32_t*)alloca(descsz); - int haspadding = 0; - assert(st->name == jl_tuple_typename || - st == jl_symbol_type || - st == jl_simplevector_type || - nfields != 0); + // compute whether this type may ever be inlined + // based solely on whether its definition is self-referential + if (isinlinealloc) { + size_t i, nf = jl_svec_len(w->types); + for (i = 0; i < nf; i++) { + jl_value_t *fld = jl_svecref(w->types, i); + if (references_name(fld, w->name)) { + isinlinealloc = 0; + break; + } + } + for (i = 0; isbitstype && i < nfields; i++) { + jl_value_t *fld = jl_field_type(st, i); + isbitstype = jl_isbits(fld); + } + } - for (size_t i = 0; i < nfields; i++) { - jl_value_t *ty = jl_field_type(st, i); - size_t fsz = 0, al = 1; - if (jl_islayout_inline(ty, &fsz, &al)) { - if (__unlikely(fsz > max_size)) - // Should never happen - goto throw_ovf; - desc[i].isptr = 0; - if (jl_is_uniontype(ty)) { - haspadding = 1; - fsz += 1; // selector byte + // if we didn't reuse the layout above, compute it now + if (st->layout == NULL) { + size_t descsz = nfields * sizeof(jl_fielddesc32_t); + jl_fielddesc32_t *desc; + int should_malloc = descsz >= jl_page_size; + if (should_malloc) + desc = (jl_fielddesc32_t*)malloc_s(descsz); + else + desc = (jl_fielddesc32_t*)alloca(descsz); + size_t sz = 0; + size_t alignm = 1; + int zeroinit = 0; + int haspadding = 0; + int homogeneous = 1; + jl_value_t *firstty = jl_field_type(st, 0); + for (i = 0; i < nfields; i++) { + jl_value_t *fld = jl_field_type(st, i); + size_t fsz = 0, al = 1; + if (jl_islayout_inline(fld, &fsz, &al)) { // aka jl_datatype_isinlinealloc + if (__unlikely(fsz > max_size)) + // Should never happen + throw_ovf(should_malloc, desc, st, fsz); + desc[i].isptr = 0; + if (jl_is_uniontype(fld)) { + haspadding = 1; + fsz += 1; // selector byte + zeroinit = 1; + } + else { + if (((jl_datatype_t*)fld)->layout->haspadding) + haspadding = 1; + if (!zeroinit) + zeroinit = ((jl_datatype_t*)fld)->zeroinit; + } + } + else { + fsz = sizeof(void*); + if (fsz > MAX_ALIGN) + fsz = MAX_ALIGN; + al = fsz; + desc[i].isptr = 1; + zeroinit = 1; } - else { // isbits struct - if (((jl_datatype_t*)ty)->layout->haspadding) + assert(al <= JL_HEAP_ALIGNMENT && (JL_HEAP_ALIGNMENT % al) == 0); + if (al != 0) { + size_t alsz = LLT_ALIGN(sz, al); + if (sz & (al - 1)) haspadding = 1; + sz = alsz; + if (al > alignm) + alignm = al; } + homogeneous &= firstty == fld; + desc[i].offset = sz; + desc[i].size = fsz; + if (__unlikely(max_offset - sz < fsz)) + throw_ovf(should_malloc, desc, st, sz); + sz += fsz; } - else { - fsz = sizeof(void*); - if (fsz > MAX_ALIGN) - fsz = MAX_ALIGN; - al = fsz; - desc[i].isptr = 1; - } - assert(al <= JL_HEAP_ALIGNMENT && (JL_HEAP_ALIGNMENT % al) == 0); - if (al != 0) { - size_t alsz = LLT_ALIGN(sz, al); - if (sz & (al - 1)) - haspadding = 1; - sz = alsz; - if (al > alignm) + if (homogeneous && jl_is_tuple_type(st)) { + // Some tuples become LLVM vectors with stronger alignment than what was calculated above. + unsigned al = jl_special_vector_alignment(nfields, firstty); + assert(al % alignm == 0); + // JL_HEAP_ALIGNMENT is the biggest alignment we can guarantee on the heap. + if (al > JL_HEAP_ALIGNMENT) + alignm = JL_HEAP_ALIGNMENT; + else if (al) alignm = al; } - homogeneous &= lastty==NULL || lastty==ty; - lastty = ty; - desc[i].offset = sz; - desc[i].size = fsz; - if (__unlikely(max_offset - sz < fsz)) - goto throw_ovf; - sz += fsz; + st->size = LLT_ALIGN(sz, alignm); + if (st->size > sz) + haspadding = 1; + st->layout = jl_get_layout(nfields, alignm, haspadding, desc); + if (should_malloc) + free(desc); } - if (homogeneous && lastty != NULL && jl_is_tuple_type(st)) { - // Some tuples become LLVM vectors with stronger alignment than what was calculated above. - unsigned al = jl_special_vector_alignment(nfields, lastty); - assert(al % alignm == 0); - // JL_HEAP_ALIGNMENT is the biggest alignment we can guarantee on the heap. - if (al > JL_HEAP_ALIGNMENT) - alignm = JL_HEAP_ALIGNMENT; - else if (al) - alignm = al; - } - st->size = LLT_ALIGN(sz, alignm); - if (st->size > sz) - haspadding = 1; - st->layout = jl_get_layout(nfields, alignm, haspadding, desc); - if (should_malloc) - free(desc); - jl_allocate_singleton_instance(st); + // now finish deciding if this instantiation qualifies for special properties + assert(!isbitstype || st->layout->npointers == 0); // the definition of isbits + if (st->layout->npointers != 0) + isinlinealloc = 0; + st->isbitstype = isbitstype; + st->isinlinealloc = isinlinealloc; + jl_maybe_allocate_singleton_instance(st); return; - throw_ovf: - if (should_malloc) - free(desc); - jl_errorf("type %s has field offset %d that exceeds the page size", jl_symbol_name(st->name->name), descsz); } static int is_anonfn_typename(char *name) diff --git a/src/jltypes.c b/src/jltypes.c index 03c00d9dfa347..da2ff343305a6 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -995,17 +995,19 @@ void jl_precompute_memoized_dt(jl_datatype_t *dt) size_t i, l = jl_nparams(dt); for (i = 0; i < l; i++) { jl_value_t *p = jl_tparam(dt, i); - if (!dt->hasfreetypevars) + if (!dt->hasfreetypevars) { dt->hasfreetypevars = jl_has_free_typevars(p); + if (dt->hasfreetypevars) + dt->isconcretetype = 0; + } if (istuple && dt->isconcretetype) dt->isconcretetype = (jl_is_datatype(p) && ((jl_datatype_t*)p)->isconcretetype) || p == jl_bottom_type; - if (dt->isdispatchtuple) + if (dt->isdispatchtuple) { dt->isdispatchtuple = jl_is_datatype(p) && ((!jl_is_kind(p) && ((jl_datatype_t*)p)->isconcretetype) || (((jl_datatype_t*)p)->name == jl_type_typename && !((jl_datatype_t*)p)->hasfreetypevars)); + } } - if (dt->hasfreetypevars) - dt->isconcretetype = 0; } static void check_datatype_parameters(jl_typename_t *tn, jl_value_t **params, size_t np) @@ -1273,6 +1275,12 @@ static jl_value_t *inst_datatype_inner(jl_datatype_t *dt, jl_svec_t *p, jl_value ndt->ditype = NULL; ndt->size = 0; jl_precompute_memoized_dt(ndt); + if (istuple) + ndt->ninitialized = ntp - isvatuple; + else if (isnamedtuple) + ndt->ninitialized = jl_svec_len(ndt->types); + else + ndt->ninitialized = dt->ninitialized; // assign uid as early as possible if (cacheable && !ndt->abstract) @@ -1313,22 +1321,16 @@ static jl_value_t *inst_datatype_inner(jl_datatype_t *dt, jl_svec_t *p, jl_value jl_gc_wb(ndt, ndt->types); } } + if (jl_is_primitivetype(dt)) { ndt->size = dt->size; ndt->layout = dt->layout; - ndt->isbitstype = ndt->isinlinealloc = (!ndt->hasfreetypevars); + ndt->isbitstype = ndt->isinlinealloc = ndt->isconcretetype; } else if (cacheable && ndt->types != NULL && !ndt->abstract) { jl_compute_field_offsets(ndt); } - if (istuple) - ndt->ninitialized = ntp - isvatuple; - else if (isnamedtuple) - ndt->ninitialized = jl_svec_len(ndt->types); - else - ndt->ninitialized = dt->ninitialized; - if (cacheable) { JL_UNLOCK(&typecache_lock); // Might GC } From b7d5328d8519f1b523725bf9bfcff2f778dc74f1 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 30 Oct 2019 15:24:16 -0400 Subject: [PATCH 2/3] datatype: fix memory leak in layout of non-concrete types --- src/datatype.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/datatype.c b/src/datatype.c index 6599e1cee5528..cde5e7c48a2f0 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -138,6 +138,7 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t nfields, } // allocate a new descriptor + // TODO: lots of these are the same--take advantage of the fact these are immutable to combine them uint32_t fielddesc_size = jl_fielddesc_size(fielddesc_type); int has_padding = nfields && npointers; jl_datatype_layout_t *flddesc = @@ -365,6 +366,8 @@ void jl_compute_field_offsets(jl_datatype_t *st) } // compute layout for the wrapper object if the field types have no free variables if (!st->isconcretetype) { + if (st != w) + return; // otherwise we would leak memory for (i = 0; i < nfields; i++) { if (jl_has_free_typevars(jl_field_type(st, i))) return; // not worthwhile computing the rest From 4e0d3533eb27d166e4d64fa18e7880e155e78f7e Mon Sep 17 00:00:00 2001 From: Oscar Blumberg Date: Thu, 31 Oct 2019 22:18:10 -0400 Subject: [PATCH 3/3] datatype: generalize the format used for isptr handling This lets us scan a datatype slightly easier, and opens up a future possibility where we don't have a one-to-one relationship between fields and contained pointers. --- base/reflection.jl | 10 +++--- src/datatype.c | 81 ++++++++++++++++++++++++-------------------- src/dump.c | 39 +++++++-------------- src/gc-debug.c | 6 ++-- src/gc.c | 62 ++++++++++++++------------------- src/gc.h | 12 +++---- src/julia.h | 46 +++++++++++++++---------- src/julia_internal.h | 6 ---- src/staticdata.c | 18 ++-------- 9 files changed, 126 insertions(+), 154 deletions(-) diff --git a/base/reflection.jl b/base/reflection.jl index 813ab0f4bc1ee..7a563eaac031c 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -317,10 +317,10 @@ datatype_fieldtypes(x::DataType) = ccall(:jl_get_fieldtypes, Any, (Any,), x) struct DataTypeLayout nfields::UInt32 + npointers::UInt32 alignment::UInt32 - # alignment : 28; + # alignment : 9; # haspadding : 1; - # pointerfree : 1; # fielddesc_type : 2; end @@ -380,8 +380,8 @@ Can be called on any `isconcretetype`. function datatype_pointerfree(dt::DataType) @_pure_meta dt.layout == C_NULL && throw(UndefRefError()) - alignment = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).alignment - return (alignment >> 10) & 0xFFFFF == 0 + npointers = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).npointers + return npointers == 0 end """ @@ -397,7 +397,7 @@ function datatype_fielddesc_type(dt::DataType) @_pure_meta dt.layout == C_NULL && throw(UndefRefError()) alignment = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).alignment - return (alignment >> 30) & 3 + return (alignment >> 10) & 3 end """ diff --git a/src/datatype.c b/src/datatype.c index cde5e7c48a2f0..17a780f59d7c8 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -99,29 +99,22 @@ jl_datatype_t *jl_new_uninitialized_datatype(void) } static jl_datatype_layout_t *jl_get_layout(uint32_t nfields, + uint32_t npointers, uint32_t alignment, int haspadding, - jl_fielddesc32_t desc[]) JL_NOTSAFEPOINT + jl_fielddesc32_t desc[], + uint32_t pointers[]) JL_NOTSAFEPOINT { // compute the smallest fielddesc type that can hold the layout description int fielddesc_type = 0; - uint32_t npointers = 0; - // First pointer field - uint32_t first_ptr = (uint32_t)-1; - // Last pointer field - uint32_t last_ptr = 0; if (nfields > 0) { uint32_t max_size = 0; uint32_t max_offset = desc[nfields - 1].offset; + if (npointers > 0 && pointers[npointers - 1] > max_offset) + max_offset = pointers[npointers - 1]; for (size_t i = 0; i < nfields; i++) { if (desc[i].size > max_size) max_size = desc[i].size; - if (desc[i].isptr) { - npointers++; - if (first_ptr == (uint32_t)-1) - first_ptr = i; - last_ptr = i; - } } jl_fielddesc8_t maxdesc8 = { 0, max_size, max_offset }; jl_fielddesc16_t maxdesc16 = { 0, max_size, max_offset }; @@ -140,24 +133,14 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t nfields, // allocate a new descriptor // TODO: lots of these are the same--take advantage of the fact these are immutable to combine them uint32_t fielddesc_size = jl_fielddesc_size(fielddesc_type); - int has_padding = nfields && npointers; - jl_datatype_layout_t *flddesc = - (jl_datatype_layout_t*)jl_gc_perm_alloc(sizeof(jl_datatype_layout_t) + - nfields * fielddesc_size + - (has_padding ? sizeof(uint32_t) : 0), 0, 4, 0); - if (has_padding) { - if (first_ptr > UINT16_MAX) - first_ptr = UINT16_MAX; - last_ptr = nfields - last_ptr - 1; - if (last_ptr > UINT16_MAX) - last_ptr = UINT16_MAX; - flddesc = (jl_datatype_layout_t*)(((char*)flddesc) + sizeof(uint32_t)); - jl_datatype_layout_n_nonptr(flddesc) = (first_ptr << 16) | last_ptr; - } + jl_datatype_layout_t *flddesc = (jl_datatype_layout_t*)jl_gc_perm_alloc( + sizeof(jl_datatype_layout_t) + nfields * fielddesc_size + (npointers << fielddesc_type), + 0, 4, 0); flddesc->nfields = nfields; flddesc->alignment = alignment; flddesc->haspadding = haspadding; flddesc->fielddesc_type = fielddesc_type; + flddesc->npointers = npointers; // fill out the fields of the new descriptor jl_fielddesc8_t* desc8 = (jl_fielddesc8_t*)jl_dt_layout_fields(flddesc); @@ -180,12 +163,20 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t nfields, desc32[i].isptr = desc[i].isptr; } } - uint32_t nexp = 0; - while (npointers >= 0x10000) { - nexp++; - npointers = npointers >> 1; + uint8_t* ptrs8 = (uint8_t*)jl_dt_layout_ptrs(flddesc); + uint16_t* ptrs16 = (uint16_t*)jl_dt_layout_ptrs(flddesc); + uint32_t* ptrs32 = (uint32_t*)jl_dt_layout_ptrs(flddesc); + for (size_t i = 0; i < npointers; i++) { + if (fielddesc_type == 0) { + ptrs8[i] = pointers[i]; + } + else if (fielddesc_type == 1) { + ptrs16[i] = pointers[i]; + } + else { + ptrs32[i] = pointers[i]; + } } - flddesc->npointers = npointers | (nexp << 16); return flddesc; } @@ -340,18 +331,18 @@ void jl_compute_field_offsets(jl_datatype_t *st) // if we have no fields, we can trivially skip the rest if (st == jl_symbol_type || st == jl_string_type) { // opaque layout - heap-allocated blob - static const jl_datatype_layout_t opaque_byte_layout = {0, 1, 0, 1, 0}; + static const jl_datatype_layout_t opaque_byte_layout = {0, 1, 1, 0, 0}; st->layout = &opaque_byte_layout; return; } else if (st == jl_simplevector_type || st->name == jl_array_typename) { - static const jl_datatype_layout_t opaque_ptr_layout = {0, sizeof(void*), 0, 1, 0}; + static const jl_datatype_layout_t opaque_ptr_layout = {0, 1, sizeof(void*), 0, 0}; st->layout = &opaque_ptr_layout; return; } else { // reuse the same layout for all singletons - static const jl_datatype_layout_t singleton_layout = {0, 1, 0, 0, 0}; + static const jl_datatype_layout_t singleton_layout = {0, 0, 1, 0, 0}; st->layout = &singleton_layout; } } @@ -396,6 +387,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) if (st->layout == NULL) { size_t descsz = nfields * sizeof(jl_fielddesc32_t); jl_fielddesc32_t *desc; + uint32_t *pointers; int should_malloc = descsz >= jl_page_size; if (should_malloc) desc = (jl_fielddesc32_t*)malloc_s(descsz); @@ -406,6 +398,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) int zeroinit = 0; int haspadding = 0; int homogeneous = 1; + uint32_t npointers = 0; jl_value_t *firstty = jl_field_type(st, 0); for (i = 0; i < nfields; i++) { jl_value_t *fld = jl_field_type(st, i); @@ -434,6 +427,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) al = fsz; desc[i].isptr = 1; zeroinit = 1; + npointers++; } assert(al <= JL_HEAP_ALIGNMENT && (JL_HEAP_ALIGNMENT % al) == 0); if (al != 0) { @@ -464,9 +458,22 @@ void jl_compute_field_offsets(jl_datatype_t *st) st->size = LLT_ALIGN(sz, alignm); if (st->size > sz) haspadding = 1; - st->layout = jl_get_layout(nfields, alignm, haspadding, desc); - if (should_malloc) + if (should_malloc && npointers) + pointers = (uint32_t*)malloc_s(npointers * sizeof(uint32_t)); + else + pointers = (uint32_t*)alloca(npointers * sizeof(uint32_t)); + size_t ptr_i = 0; + for (i = 0; i < nfields; i++) { + if (desc[i].isptr) + pointers[ptr_i++] = desc[i].offset / sizeof(jl_value_t**); + } + assert(ptr_i == npointers); + st->layout = jl_get_layout(nfields, npointers, alignm, haspadding, desc, pointers); + if (should_malloc) { free(desc); + if (npointers) + free(pointers); + } } // now finish deciding if this instantiation qualifies for special properties assert(!isbitstype || st->layout->npointers == 0); // the definition of isbits @@ -577,7 +584,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t * alignm = MAX_ALIGN; bt->isbitstype = bt->isinlinealloc = (parameters == jl_emptysvec); bt->size = nbytes; - bt->layout = jl_get_layout(0, alignm, 0, NULL); + bt->layout = jl_get_layout(0, 0, alignm, 0, NULL, NULL); bt->instance = NULL; return bt; } diff --git a/src/dump.c b/src/dump.c index b1c5fa5f77602..9fd00e5f521c3 100644 --- a/src/dump.c +++ b/src/dump.c @@ -393,13 +393,10 @@ static void jl_serialize_datatype(jl_serializer_state *s, jl_datatype_t *dt) JL_ write_uint8(s->s, layout); if (layout == 0) { uint32_t nf = dt->layout->nfields; - write_int32(s->s, nf); - uint32_t alignment = ((uint32_t*)dt->layout)[1]; - write_int32(s->s, alignment); - if (dt->layout->npointers && nf) - write_int32(s->s, ((uint32_t*)dt->layout)[-1]); + uint32_t np = dt->layout->npointers; size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type); - ios_write(s->s, (char*)(&dt->layout[1]), nf * fieldsize); + ios_write(s->s, (const char*)dt->layout, sizeof(*dt->layout)); + ios_write(s->s, (const char*)(dt->layout + 1), nf * fieldsize + (np << dt->layout->fielddesc_type)); } } @@ -1441,29 +1438,17 @@ static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_v } else { assert(layout == 0); - uint32_t nf = read_int32(s->s); - uint32_t alignment = read_int32(s->s); - union { - struct { - uint32_t nf; - uint32_t alignment; - } buffer; - jl_datatype_layout_t layout; - } header; - header.buffer.nf = nf; - header.buffer.alignment = alignment; - int has_padding = header.layout.npointers && nf; - uint8_t fielddesc_type = header.layout.fielddesc_type; + jl_datatype_layout_t buffer; + ios_read(s->s, (char*)&buffer, sizeof(buffer)); + uint32_t nf = buffer.nfields; + uint32_t np = buffer.npointers; + uint8_t fielddesc_type = buffer.fielddesc_type; size_t fielddesc_size = nf > 0 ? jl_fielddesc_size(fielddesc_type) : 0; jl_datatype_layout_t *layout = (jl_datatype_layout_t*)jl_gc_perm_alloc( - sizeof(jl_datatype_layout_t) + nf * fielddesc_size + - (has_padding ? sizeof(uint32_t) : 0), 0, 4, 0); - if (has_padding) { - layout = (jl_datatype_layout_t*)(((char*)layout) + sizeof(uint32_t)); - jl_datatype_layout_n_nonptr(layout) = read_int32(s->s); - } - *layout = header.layout; - ios_read(s->s, (char*)&layout[1], nf * fielddesc_size); + sizeof(jl_datatype_layout_t) + nf * fielddesc_size + (np << fielddesc_type), + 0, 4, 0); + *layout = buffer; + ios_read(s->s, (char*)(layout + 1), nf * fielddesc_size + (np << fielddesc_type)); dt->layout = layout; } } diff --git a/src/gc-debug.c b/src/gc-debug.c index 9bae5cd249c0a..d841236d22e7c 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -1310,7 +1310,7 @@ NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, jl_gc_mark_sp_t sp, int pc_off break; } jl_datatype_t *vt = (jl_datatype_t*)jl_typeof(data->parent); - jl_fielddesc8_t *desc = (jl_fielddesc8_t*)jl_dt_layout_fields(vt->layout); + uint8_t *desc = (uint8_t*)jl_dt_layout_ptrs(vt->layout); jl_safe_printf("%p: %s Object (8bit) %p :: %p -- [%d, %d)\n of type ", (void*)data, prefix, (void*)data->parent, ((void**)data->parent)[-1], (int)(data->begin - desc), (int)(data->end - desc)); @@ -1323,7 +1323,7 @@ NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, jl_gc_mark_sp_t sp, int pc_off break; } jl_datatype_t *vt = (jl_datatype_t*)jl_typeof(data->parent); - jl_fielddesc16_t *desc = (jl_fielddesc16_t*)jl_dt_layout_fields(vt->layout); + uint16_t *desc = (uint16_t*)jl_dt_layout_ptrs(vt->layout); jl_safe_printf("%p: %s Object (16bit) %p :: %p -- [%d, %d)\n of type ", (void*)data, prefix, (void*)data->parent, ((void**)data->parent)[-1], (int)(data->begin - desc), (int)(data->end - desc)); @@ -1336,7 +1336,7 @@ NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, jl_gc_mark_sp_t sp, int pc_off break; } jl_datatype_t *vt = (jl_datatype_t*)jl_typeof(data->parent); - jl_fielddesc32_t *desc = (jl_fielddesc32_t*)jl_dt_layout_fields(vt->layout); + uint32_t *desc = (uint32_t*)jl_dt_layout_ptrs(vt->layout); jl_safe_printf("%p: %s Object (32bit) %p :: %p -- [%d, %d)\n of type ", (void*)data, prefix, (void*)data->parent, ((void**)data->parent)[-1], (int)(data->begin - desc), (int)(data->end - desc)); diff --git a/src/gc.c b/src/gc.c index 4f28af0e12734..45a60be04765d 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1734,15 +1734,13 @@ STATIC_INLINE int gc_mark_scan_objarray(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, // Scan an object with 8bits field descriptors. see `gc_mark_obj8_t` STATIC_INLINE int gc_mark_scan_obj8(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, gc_mark_obj8_t *obj8, - char *parent, jl_fielddesc8_t *begin, jl_fielddesc8_t *end, + char *parent, uint8_t *begin, uint8_t *end, jl_value_t **pnew_obj, uintptr_t *ptag, uint8_t *pbits) { (void)jl_assume(obj8 == (gc_mark_obj8_t*)sp->data); (void)jl_assume(begin < end); for (; begin < end; begin++) { - if (!begin->isptr) - continue; - jl_value_t **slot = (jl_value_t**)(parent + begin->offset); + jl_value_t **slot = &((jl_value_t**)parent)[*begin]; *pnew_obj = *slot; if (*pnew_obj) verify_parent2("object", parent, slot, "field(%d)", @@ -1769,15 +1767,13 @@ STATIC_INLINE int gc_mark_scan_obj8(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, gc_mark // Scan an object with 16bits field descriptors. see `gc_mark_obj16_t` STATIC_INLINE int gc_mark_scan_obj16(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, gc_mark_obj16_t *obj16, - char *parent, jl_fielddesc16_t *begin, jl_fielddesc16_t *end, + char *parent, uint16_t *begin, uint16_t *end, jl_value_t **pnew_obj, uintptr_t *ptag, uint8_t *pbits) JL_NOTSAFEPOINT { (void)jl_assume(obj16 == (gc_mark_obj16_t*)sp->data); (void)jl_assume(begin < end); for (; begin < end; begin++) { - if (!begin->isptr) - continue; - jl_value_t **slot = (jl_value_t**)(parent + begin->offset); + jl_value_t **slot = &((jl_value_t**)parent)[*begin]; *pnew_obj = *slot; if (*pnew_obj) verify_parent2("object", parent, slot, "field(%d)", @@ -1804,15 +1800,13 @@ STATIC_INLINE int gc_mark_scan_obj16(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, gc_mar // Scan an object with 32bits field descriptors. see `gc_mark_obj32_t` STATIC_INLINE int gc_mark_scan_obj32(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, gc_mark_obj32_t *obj32, - char *parent, jl_fielddesc32_t *begin, jl_fielddesc32_t *end, + char *parent, uint32_t *begin, uint32_t *end, jl_value_t **pnew_obj, uintptr_t *ptag, uint8_t *pbits) { (void)jl_assume(obj32 == (gc_mark_obj32_t*)sp->data); (void)jl_assume(begin < end); for (; begin < end; begin++) { - if (!begin->isptr) - continue; - jl_value_t **slot = (jl_value_t**)(parent + begin->offset); + jl_value_t **slot = &((jl_value_t**)parent)[*begin]; *pnew_obj = *slot; if (*pnew_obj) verify_parent2("object", parent, slot, "field(%d)", @@ -1963,13 +1957,13 @@ JL_EXTENSION NOINLINE void gc_mark_loop(jl_ptls_t ptls, jl_gc_mark_sp_t sp) gc_mark_obj8_t *obj8; char *obj8_parent; - jl_fielddesc8_t *obj8_begin; - jl_fielddesc8_t *obj8_end; + uint8_t *obj8_begin; + uint8_t *obj8_end; gc_mark_obj16_t *obj16; char *obj16_parent; - jl_fielddesc16_t *obj16_begin; - jl_fielddesc16_t *obj16_end; + uint16_t *obj16_begin; + uint16_t *obj16_end; pop: if (sp.pc == sp.pc_start) { @@ -2033,8 +2027,8 @@ scan_only: { obj32: { gc_mark_obj32_t *obj32 = gc_pop_markdata(&sp, gc_mark_obj32_t); char *parent = (char*)obj32->parent; - jl_fielddesc32_t *begin = obj32->begin; - jl_fielddesc32_t *end = obj32->end; + uint32_t *begin = obj32->begin; + uint32_t *end = obj32->end; if (gc_mark_scan_obj32(ptls, &sp, obj32, parent, begin, end, &new_obj, &tag, &bits)) goto mark; goto pop; @@ -2420,10 +2414,11 @@ mark: { assert(layout->fielddesc_type == 0); size_t nfields = layout->nfields; assert(nfields > 0); - obj8_begin = (jl_fielddesc8_t*)jl_dt_layout_fields(layout); - obj8_end = obj8_begin + nfields; + uint32_t npointers = layout->npointers; + obj8_begin = (uint8_t*)jl_dt_layout_ptrs(layout); + obj8_end = obj8_begin + npointers; // assume tasks always reference young objects: set lowest bit - uintptr_t nptr = (9 << 2) | 1 | bits; + uintptr_t nptr = (npointers << 2) | 1 | bits; gc_mark_obj8_t markdata = {new_obj, obj8_begin, obj8_end, nptr}; gc_mark_stack_push(&ptls->gc_cache, &sp, gc_mark_laddr(obj8), &markdata, sizeof(markdata), 0); @@ -2453,17 +2448,12 @@ mark: { uint32_t npointers = layout->npointers; if (npointers == 0) goto pop; - uintptr_t nptr = ((npointers & 0xff) << (npointers & 0x300)) << 2; - nptr = nptr | (bits & GC_OLD); - uint32_t offsets = jl_datatype_layout_n_nonptr(layout); - size_t nfields = layout->nfields; - nfields -= offsets & 0xffff; - size_t first = offsets >> 16; + uintptr_t nptr = npointers << 2 | (bits & GC_OLD); + assert(layout->nfields > 0 && layout->fielddesc_type != 3 && "opaque types should have been handled specially"); if (layout->fielddesc_type == 0) { - jl_fielddesc8_t *desc = (jl_fielddesc8_t*)jl_dt_layout_fields(layout); obj8_parent = (char*)new_obj; - obj8_begin = desc + first; - obj8_end = desc + nfields; + obj8_begin = (uint8_t*)jl_dt_layout_ptrs(layout); + obj8_end = obj8_begin + npointers; assert(obj8_begin < obj8_end); gc_mark_obj8_t markdata = {new_obj, obj8_begin, obj8_end, nptr}; gc_mark_stack_push(&ptls->gc_cache, &sp, gc_mark_laddr(obj8), @@ -2472,10 +2462,9 @@ mark: { goto obj8_loaded; } else if (layout->fielddesc_type == 1) { - jl_fielddesc16_t *desc = (jl_fielddesc16_t*)jl_dt_layout_fields(layout); obj16_parent = (char*)new_obj; - obj16_begin = desc + first; - obj16_end = desc + nfields; + obj16_begin = (uint16_t*)jl_dt_layout_ptrs(layout); + obj16_end = obj16_begin + npointers; assert(obj16_begin < obj16_end); gc_mark_obj16_t markdata = {new_obj, obj16_begin, obj16_end, nptr}; gc_mark_stack_push(&ptls->gc_cache, &sp, gc_mark_laddr(obj16), @@ -2486,9 +2475,9 @@ mark: { else if (layout->fielddesc_type == 2) { // This is very uncommon // Do not do store to load forwarding to save some code size - jl_fielddesc32_t *desc = (jl_fielddesc32_t*)jl_dt_layout_fields(layout); - assert(first < nfields); - gc_mark_obj32_t markdata = {new_obj, desc + first, desc + nfields, nptr}; + uint32_t *obj32_begin = (uint32_t*)jl_dt_layout_ptrs(layout); + uint32_t *obj32_end = obj32_begin + npointers; + gc_mark_obj32_t markdata = {new_obj, obj32_begin, obj32_end, nptr}; gc_mark_stack_push(&ptls->gc_cache, &sp, gc_mark_laddr(obj32), &markdata, sizeof(markdata), 0); sp.data = (jl_gc_mark_data_t *)(((char*)sp.data) + sizeof(markdata)); @@ -2497,7 +2486,6 @@ mark: { else { assert(layout->fielddesc_type == 3); jl_fielddescdyn_t *desc = (jl_fielddescdyn_t*)jl_dt_layout_fields(layout); - int old = jl_astaggedvalue(new_obj)->bits.gc & 2; export_gc_state(ptls, &sp); uintptr_t young = desc->markfunc(ptls, new_obj); diff --git a/src/gc.h b/src/gc.h index 1dea38863a2fe..3e81193b67b5c 100644 --- a/src/gc.h +++ b/src/gc.h @@ -119,24 +119,24 @@ typedef struct { // A normal object with 8bits field descriptors typedef struct { jl_value_t *parent; // The parent object to trigger write barrier on. - jl_fielddesc8_t *begin; // Current field descriptor. - jl_fielddesc8_t *end; // End of field descriptor. + uint8_t *begin; // Current field descriptor. + uint8_t *end; // End of field descriptor. uintptr_t nptr; // See notes about `nptr` above. } gc_mark_obj8_t; // A normal object with 16bits field descriptors typedef struct { jl_value_t *parent; // The parent object to trigger write barrier on. - jl_fielddesc16_t *begin; // Current field descriptor. - jl_fielddesc16_t *end; // End of field descriptor. + uint16_t *begin; // Current field descriptor. + uint16_t *end; // End of field descriptor. uintptr_t nptr; // See notes about `nptr` above. } gc_mark_obj16_t; // A normal object with 32bits field descriptors typedef struct { jl_value_t *parent; // The parent object to trigger write barrier on. - jl_fielddesc32_t *begin; // Current field descriptor. - jl_fielddesc32_t *end; // End of field descriptor. + uint32_t *begin; // Current field descriptor. + uint32_t *end; // End of field descriptor. uintptr_t nptr; // See notes about `nptr` above. } gc_mark_obj32_t; diff --git a/src/julia.h b/src/julia.h index 64c2d5456a668..b90675566f987 100644 --- a/src/julia.h +++ b/src/julia.h @@ -425,14 +425,19 @@ typedef struct { typedef struct { uint32_t nfields; + uint32_t npointers; // number of pointer uint32_t alignment : 9; // strictest alignment over all fields uint32_t haspadding : 1; // has internal undefined bytes - uint32_t npointers : 20; // number of pointer fields, top 4 bits are exponent (under-approximation) uint32_t fielddesc_type : 2; // 0 -> 8, 1 -> 16, 2 -> 32 // union { - // jl_fielddesc8_t field8[]; - // jl_fielddesc16_t field16[]; - // jl_fielddesc32_t field32[]; + // jl_fielddesc8_t field8[nfields]; + // jl_fielddesc16_t field16[nfields]; + // jl_fielddesc32_t field32[nfields]; + // }; + // union { // offsets relative to data start in words + // uint8_t ptr8[npointers]; + // uint16_t ptr16[npointers]; + // uint32_t ptr32[npointers]; // }; } jl_datatype_layout_t; @@ -955,7 +960,25 @@ STATIC_INLINE char *jl_symbol_name_(jl_sym_t *s) JL_NOTSAFEPOINT } #define jl_symbol_name(s) jl_symbol_name_(s) +static inline uint32_t jl_fielddesc_size(int8_t fielddesc_type) JL_NOTSAFEPOINT +{ + return 2 << fielddesc_type; + //if (fielddesc_type == 0) { + // return sizeof(jl_fielddesc8_t); + //} + //else if (fielddesc_type == 1) { + // return sizeof(jl_fielddesc16_t); + //} + //else { + // return sizeof(jl_fielddesc32_t); + //} +} + #define jl_dt_layout_fields(d) ((const char*)(d) + sizeof(jl_datatype_layout_t)) +static inline const char *jl_dt_layout_ptrs(const jl_datatype_layout_t *l) JL_NOTSAFEPOINT +{ + return jl_dt_layout_fields(l) + jl_fielddesc_size(l->fielddesc_type) * l->nfields; +} #define DEFINE_FIELD_ACCESSORS(f) \ static inline uint32_t jl_field_##f(jl_datatype_t *st, \ @@ -980,20 +1003,7 @@ static inline int jl_field_isptr(jl_datatype_t *st, int i) JL_NOTSAFEPOINT { const jl_datatype_layout_t *ly = st->layout; assert(i >= 0 && (size_t)i < ly->nfields); - return ((const jl_fielddesc8_t*)(jl_dt_layout_fields(ly) + (i << (ly->fielddesc_type + 1))))->isptr; -} - -static inline uint32_t jl_fielddesc_size(int8_t fielddesc_type) JL_NOTSAFEPOINT -{ - if (fielddesc_type == 0) { - return sizeof(jl_fielddesc8_t); - } - else if (fielddesc_type == 1) { - return sizeof(jl_fielddesc16_t); - } - else { - return sizeof(jl_fielddesc32_t); - } + return ((const jl_fielddesc8_t*)(jl_dt_layout_fields(ly) + jl_fielddesc_size(ly->fielddesc_type) * i))->isptr; } #undef DEFINE_FIELD_ACCESSORS diff --git a/src/julia_internal.h b/src/julia_internal.h index 2c914761c6b8a..f9de00f95058d 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -308,12 +308,6 @@ jl_svec_t *jl_perm_symsvec(size_t n, ...); }), __VA_ARGS__) #endif -// Returns a int32 where the high 16 bits are a lower bound of the number of non-pointer fields -// at the beginning of the type and the low 16 bits are a lower bound on the number of non-pointer -// fields at the end of the type. This field only exists for a layout that has at least one -// pointer fields. -#define jl_datatype_layout_n_nonptr(layout) ((uint32_t*)(layout))[-1] - jl_value_t *jl_gc_realloc_string(jl_value_t *s, size_t sz); JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz); diff --git a/src/staticdata.c b/src/staticdata.c index 2e327defa9608..737ffff03ee74 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -808,24 +808,12 @@ static void jl_write_values(jl_serializer_state *s) newdt->ditype = NULL; if (dt->layout != NULL) { size_t nf = dt->layout->nfields; + size_t np = dt->layout->npointers; size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type); - int has_padding = dt->layout->npointers && nf; char *flddesc = (char*)dt->layout; - size_t fldsize = sizeof(jl_datatype_layout_t) + nf * fieldsize; - uintptr_t layout_unaligned = LLT_ALIGN(ios_pos(s->const_data), sizeof(uint32_t)); + size_t fldsize = sizeof(jl_datatype_layout_t) + nf * fieldsize + (np << dt->layout->fielddesc_type); uintptr_t layout = LLT_ALIGN(ios_pos(s->const_data), sizeof(void*)); - if (has_padding) { - if (layout == layout_unaligned) { - layout += sizeof(void*); - layout_unaligned = layout - sizeof(uint32_t); - } - flddesc -= sizeof(uint32_t); - fldsize += sizeof(uint32_t); - write_padding(s->const_data, layout_unaligned - ios_pos(s->const_data)); // realign stream - } - else { - write_padding(s->const_data, layout - ios_pos(s->const_data)); // realign stream - } + write_padding(s->const_data, layout - ios_pos(s->const_data)); // realign stream newdt->layout = NULL; // relocation offset layout /= sizeof(void*); arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_datatype_t, layout))); // relocation location