implement Memory{T} as a new backend for Array{T}

TODO: add more tests for Memory specifically
TODO: add to julia-licm the gc_loaded
TODO: make sure all tests are passing (updated as needed)
TODO: write API documentation for Memory (as needed) and devdoc updates (as needed)
TODO: test Memory owner relationship (avoid implied unalias copy) of Memory in staticdata
TODO: can we detect ownership for dataids for more accurate range checking?
TODO: add memoryowner intrinsic (since this is a hidden field)
TODO: add memoryoffset intrinsic (since this is computed better with exact idiv)
TODO: implement dataids for Memory aliasing check and fix dataids for Array

also makes memory allocation ccalls safer, for catching Serialization
and deepcopy bugs in packages.

more things to do

TODO: rename ptr to ptr_or_offset
TODO: don't mutate Memory length when making a String, but set Array ref field to empty Memory
TODO: add alias `AtomicMemory{T} = GenericMemory{:atomic,T}`
TODO: document jl_memoryt_slice
TODO: add AddrSpace::Int parameter to GenericMemory

Future possible kinds of possible Memories:
GenericMemory{:atomic, 0, Int}
GenericMemory{:not_atomic, 0, Int}
GenericMemory{:unsync, 0, Int}
GenericMemory{:normal, 0, Int}
GenericMemory{:default, 0, Int}
GenericMemory{:racy, 0, Int}
GenericMemory{:local, 0, Int}
GenericMemory{:const, 0, Int}
GenericMemory{:ntuple, 0, Int}
GenericMemory{:value, 0, Int}

Author: vtjnash

base/Base.jl

@@ -9,7 +9,8 @@ using Core.Intrinsics, Core.IR
 const _included_files = Array{Tuple{Module,String},1}(Core.undef, 1)
 function include(mod::Module, path::String)
     ccall(:jl_array_grow_end, Cvoid, (Any, UInt), _included_files, UInt(1))
-    Core.arrayset(true, _included_files, (mod, ccall(:jl_prepend_cwd, Any, (Any,), path)), arraylen(_included_files))
+    len = getfield(_included_files.size, 1)
+    Core.arrayset(true, _included_files, (mod, ccall(:jl_prepend_cwd, Any, (Any,), path)), len)
     Core.println(path)
     ccall(:jl_uv_flush, Nothing, (Ptr{Nothing},), Core.io_pointer(Core.stdout))
     Core.include(mod, path)
@@ -186,6 +187,7 @@ include("strings/lazy.jl")
 
 # array structures
 include("indices.jl")
+include("genericmemory.jl")
 include("array.jl")
 include("abstractarray.jl")
 include("subarray.jl")

base/abstractarray.jl

@@ -1285,8 +1285,6 @@ function getindex(A::AbstractArray, I...)
     error_if_canonical_getindex(IndexStyle(A), A, I...)
     _getindex(IndexStyle(A), A, to_indices(A, I)...)
 end
-# To avoid invalidations from multidimensional.jl: getindex(A::Array, i1::Union{Integer, CartesianIndex}, I::Union{Integer, CartesianIndex}...)
-@propagate_inbounds getindex(A::Array, i1::Integer, I::Integer...) = A[to_indices(A, (i1, I...))...]
 
 @inline unsafe_getindex(A::AbstractArray, I...) = @inbounds getindex(A, I...)
 
@@ -1526,7 +1524,8 @@ their component parts.  A typical definition for an array that wraps a parent is
 `Base.dataids(C::CustomArray) = dataids(C.parent)`.
 """
 dataids(A::AbstractArray) = (UInt(objectid(A)),)
-dataids(A::Array) = (UInt(pointer(A)),)
+dataids(A::Memory) = (UInt(pointer(A)),)
+dataids(A::Array) = dataids(A.ref.mem)
 dataids(::AbstractRange) = ()
 dataids(x) = ()
 

base/array.jl

@@ -120,7 +120,7 @@ const DenseVecOrMat{T} = Union{DenseVector{T}, DenseMatrix{T}}
 
 ## Basic functions ##
 
-using Core: arraysize, arrayset, const_arrayref
+using Core: arraysize, arrayset, arrayref, const_arrayref
 
 """
     @_safeindex
@@ -232,6 +232,7 @@ function _unsetindex!(A::Array{T}, i::Int) where {T}
 end
 
 
+# TODO: deprecate this (aligned_sizeof and/or elsize and/or sizeof(Some{T}) are more correct)
 """
     Base.bitsunionsize(U::Union) -> Int
 
@@ -252,7 +253,7 @@ function bitsunionsize(u::Union)
     return sz
 end
 
-elsize(@nospecialize _::Type{A}) where {T,A<:Array{T}} = aligned_sizeof(T)
+elsize(::Type{A}) where {T,A<:Array{T}} = aligned_sizeof(T)
 function elsize(::Type{Ptr{T}}) where T
     # this only must return something valid for values which satisfy is_valid_intrinsic_elptr(T),
     # which includes Any and most concrete datatypes
@@ -261,15 +262,23 @@ function elsize(::Type{Ptr{T}}) where T
     return LLT_ALIGN(Core.sizeof(T), datatype_alignment(T))
 end
 elsize(::Type{Union{}}, slurp...) = 0
-sizeof(a::Array) = Core.sizeof(a)
+
+sizeof(a::Array) = length(a) * elsize(typeof(a)) # TODO(jwn): add bitsunion bytes?
 
 function isassigned(a::Array, i::Int...)
     @inline
     @boundscheck checkbounds(Bool, a, i...) || return false
-    ii = (_sub2ind(size(a), i...) % UInt) - 1
-    ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1
+    ii = _sub2ind(size(a), i...)
+    return @inbounds isassigned(memoryref(a.ref, ii, false))
+end
+
+function isassigned(a::Vector, i::Int) # slight compiler simplification for the most common case
+    @inline
+    @boundscheck checkbounds(Bool, a, i) || return false
+    return @inbounds isassigned(memoryref(a.ref, i, false))
 end
 
+
 ## copy ##
 
 """
@@ -291,9 +300,12 @@ end
 
 
 function _unsafe_copyto!(dest, doffs, src, soffs, n)
+    @_terminates_locally_meta
+    # use pointer math to determine if they are deemed to alias
     destp = pointer(dest, doffs)
     srcp = pointer(src, soffs)
-    @inbounds if destp < srcp || destp > srcp + n
+    endp = pointer(src, soffs + n - 1)
+    @inbounds if destp < srcp || destp > endp
         for i = 1:n
             if isassigned(src, soffs + i - 1)
                 dest[doffs + i - 1] = src[soffs + i - 1]
@@ -324,6 +336,7 @@ that N is inbounds on either array. Incorrect usage may corrupt or segfault your
 the same manner as C.
 """
 function unsafe_copyto!(dest::Array{T}, doffs, src::Array{T}, soffs, n) where T
+    @_terminates_locally_meta
     t1 = @_gc_preserve_begin dest
     t2 = @_gc_preserve_begin src
     destp = pointer(dest, doffs)
@@ -1063,24 +1076,188 @@ function setindex!(A::Array{T}, X::Array{T}, c::Colon) where T
     return A
 end
 
-# efficiently grow an array
+# Pick new memory size for efficiently growing an array
+# TODO: This should know about the size of our GC pools
+# Specifically we are wasting ~10% of memory for small arrays
+# by not picking memory sizes that max out a GC pool
+function overallocation(maxsize)
+    maxsize < 8 && return 8;
+    # compute maxsize = maxsize + 4*maxsize^(7/8) + maxsize/8
+    # for small n, we grow faster than O(n)
+    # for large n, we grow at O(n/8)
+    # and as we reach O(memory) for memory>>1MB,
+    # this means we end by adding about 10% of memory each time
+    exp2 = sizeof(maxsize) * 8 - Core.Intrinsics.ctlz_int(maxsize)
+    maxsize += (1 << div(exp2 * 7, 8)) * 4 + div(maxsize, 8)
+    return maxsize
+end
+
+function array_new_memory(mem::Memory{T}, newlen::Int) where T
+    if ccall(:jl_mem_owner, Any, (Memory{T},), mem) isa String
+        # if data is in a String, keep it that way
+        # TODO: use jl_gc_expand_string(oldstr, newlen)?
+        str = _string_n(newlen)
+        return ccall(:jl_string_to_genericmemory, Memory{T}, (Any,), str)
+    else
+        # TODO: when implimented, this should use a memory growing call
+        mem = Memory{T}(undef, newlen)
+        return mem
+    end
+end
 
-_growbeg!(a::Vector, delta::Integer) =
-    ccall(:jl_array_grow_beg, Cvoid, (Any, UInt), a, delta)
-_growend!(a::Vector, delta::Integer) =
-    ccall(:jl_array_grow_end, Cvoid, (Any, UInt), a, delta)
-_growat!(a::Vector, i::Integer, delta::Integer) =
-    ccall(:jl_array_grow_at, Cvoid, (Any, Int, UInt), a, i - 1, delta)
+function _growbeg!(a::Vector, delta::Integer)
+    delta = Int(delta)
+    delta == 0 && return # avoid attempting to index off the end
+    delta >= 0 || throw(ArgumentError("grow requires delta >= 0"))
+    ref = a.ref
+    mem = ref.mem
+    len = length(a)
+    offset = memoffset(ref)
+    newlen = len + delta
+    a.size = (newlen,)
+    # if offset is far enough advanced to fit data in existing memory without copying
+    if delta <= offset
+        a.ref = MemoryRef(ref, 1 - delta, false)
+    else
+        @noinline (function()
+        memlen = length(mem)
+        # since we will allocate the array in the middle of the memory we need at least 2*delta extra space
+        # the +1 is because I didn't want to have an off by 1 error.
+        newmemlen = max(overallocation(memlen), len+2*delta+1)
+        newoffset = div(newmemlen - newlen, 2)
+        # If there is extra data after the end of the array we can use that space so long as there is enough
+        # space at the end that there won't be quadratic behavior with a mix of growth from both ends.
+        # Specifically, we want to ensure that we will only do this operation once before
+        # increasing the size of the array, and that we leave enough space at both the benining and the end.
+        if newoffset + newlen + 1 < memlen
+            newoffset = div(memlen - newlen, 2)
+            newmem = mem
+        else
+            newmem = array_new_memory(mem, newmemlen)
+        end
+        if len > 0
+            unsafe_copyto!(newmem, newoffset+delta+1, mem, offset+1, len)
+        end
+        a.ref = MemoryRef(newmem, newoffset+1, false)
+        end)()
+    end
+    return
+end
 
-# efficiently delete part of an array
+function _growend!(a::Vector, delta::Integer)
+    delta = Int(delta)
+    delta >= 0 || throw(ArgumentError("grow requires delta >= 0"))
+    ref = a.ref
+    mem = ref.mem
+    memlen = length(mem)
+    len = length(a)
+    newlen = len + delta
+    offset = memoffset(ref)
+    a.size = (newlen,)
+    newmemlen = offset + newlen
+    if memlen < newmemlen
+        @noinline (function()
+        if offset > div(5*newlen, 4)
+            # If the offset is far enough that we can copy without resizing
+            # while maintaining proportional spacing on both ends of the array
+            # note that this branch prevents infinite growth when doing combinations
+            # of push! and popfirst! (i.e. when using a Vector as a queue)
+            newmem = mem
+            newoffset = div(newlen, 8)
+        else
+            # grow either by our computed overallocation factor
+            # or exactly the requested size, whichever is larger
+            # TODO we should possibly increase the offset if the current offset is nonzero,
+            newmemlen2 = max(overallocation(memlen), newmemlen)
+            newmem = array_new_memory(mem, newmemlen2)
+            newoffset = offset
+        end
+        if len > 0
+            unsafe_copyto!(newmem, newoffset+1, mem, offset+1, len)
+        end
+        a.ref = MemoryRef(newmem, newoffset+1, false)
+        end)()
+    end
+    return
+end
 
-_deletebeg!(a::Vector, delta::Integer) =
-    ccall(:jl_array_del_beg, Cvoid, (Any, UInt), a, delta)
-_deleteend!(a::Vector, delta::Integer) =
-    ccall(:jl_array_del_end, Cvoid, (Any, UInt), a, delta)
-_deleteat!(a::Vector, i::Integer, delta::Integer) =
-    ccall(:jl_array_del_at, Cvoid, (Any, Int, UInt), a, i - 1, delta)
+function _growat!(a::Vector, i::Integer, delta::Integer)
+    delta = Int(delta)
+    i == 1 && return _growbeg!(a, delta)
+    len = length(a)
+    i == len + 1 && return _growend!(a, delta)
+    delta >= 0 || throw(ArgumentError("grow requires delta >= 0"))
+    1 < i <= len || throw(BoundsError(a, i))
+    ref = a.ref
+    mem = ref.mem
+    memlen = length(mem)
+    newlen = len + delta
+    offset = memoffset(ref)
+    a.size = (newlen,)
+    newmemlen = offset + newlen
+
+    # which side would we rather grow into?
+    prefer_start = i <= div(len, 2)
+    # if offset is far enough advanced to fit data in begining of the memory
+    if prefer_start && delta <= offset
+        a.ref = MemoryRef(mem, offset-delta+1, false)
+        unsafe_copyto!(mem, offset-delta+1, mem, offset+1, i)
+    elseif !prefer_start && memlen >= newmemlen
+        unsafe_copyto!(mem, offset+delta+i, mem, offset+i, len-i+1)
+    else
+        # since we will allocate the array in the middle of the memory we need at least 2*delta extra space
+        # the +1 is because I didn't want to have an off by 1 error.
+        newmemlen = max(overallocation(memlen), len+2*delta+1)
+        newoffset = (newmemlen - newlen) ÷ 2
+        newmem = array_new_memory(mem, newmemlen)
+        a.ref = MemoryRef(newmem, newoffset+1, false)
+        unsafe_copyto!(newmem, newoffset+1, mem, offset+1, i)
+        unsafe_copyto!(newmem, newoffset+delta+i, mem, offset+i, len-i+1)
+    end
+end
 
+# efficiently delete part of an array
+function _deletebeg!(a::Vector, delta::Integer)
+    delta = Int(delta)
+    len = length(a)
+    0 <= delta <= len || throw(ArgumentError("_deleteat! requires delta in 0:length(a)"))
+    for i in 1:delta
+        _unsetindex!(a, i)
+    end
+    newlen = len - delta
+    if newlen != 0 # if newlen==0 we could accidentally index past the memory
+        a.ref = MemoryRef(a.ref, delta + 1, false)
+    end
+    a.size = (newlen,)
+    return
+end
+function _deleteend!(a::Vector, delta::Integer)
+    delta = Int(delta)
+    len = length(a)
+    0 <= delta <= len || throw(ArgumentError("_deleteat! requires delta in 0:length(a)"))
+    newlen = len - delta
+    for i in newlen+1:len
+        _unsetindex!(a, i)
+    end
+    a.size = (newlen,)
+    return
+end
+function _deleteat!(a::Vector, i::Integer, delta::Integer)
+    i = Int(i)
+    len = length(a)
+    0 <= delta || throw(ArgumentError("_deleteat! requires delta >= 0"))
+    1 <= i <= len || throw(BoundsError(a, i))
+    i + delta <= len + 1 || throw(BoundsError(a, i + delta - 1))
+    newa = a
+    if 2*i + delta <= len
+        unsafe_copyto!(newa, 1 + delta, a, 1, i - 1)
+        _deletebeg!(a, delta)
+    else
+        unsafe_copyto!(newa, i, a, i + delta, len + 1 - delta - i)
+        _deleteend!(a, delta)
+    end
+    return
+end
 ## Dequeue functionality ##
 
 """
@@ -1347,7 +1524,32 @@ function sizehint! end
 
 function sizehint!(a::Vector, sz::Integer)
     ccall(:jl_array_sizehint, Cvoid, (Any, UInt), a, sz)
-    a
+    return a
+    #=n = length(a)
+    len = length(a)
+    ref = a.ref
+    mem = ref.mem
+    memlen = length(mem)
+    newlen = len + delta
+    offset = memoffset(ref)
+    elsz = jl_array_elsize(a)
+    if (elsz == 0)
+        return a
+    end
+    sz = max(sz, offset+len)
+
+    if sz <= memlen
+        size_t dec = memlen - sz;
+        # if we don't save at least an eighth of maxsize then its not worth it to shrink
+        if dec <= div(memlen, 8)
+            return a
+        jl_array_shrink(a, dec)
+    else
+        inc = sz - len;
+        _growend(a, inc);
+        a.size = (len,)
+    end
+    a=#
 end
 
 """

base/bitset.jl

@@ -15,7 +15,7 @@ mutable struct BitSet <: AbstractSet{Int}
     # 1st stored Int equals 64*offset
     offset::Int
 
-    BitSet() = new(resize!(Vector{UInt64}(undef, 4), 0), NO_OFFSET)
+    BitSet() = new(Vector{UInt64}(MemoryRef(Memory{UInt64}(undef, 4)), (0,)), NO_OFFSET)
 end
 
 """