From 9e822189bd629ff5bc7aa4c98188492c91d70142 Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Wed, 18 Sep 2019 09:48:58 -0700 Subject: [PATCH 01/10] Avoid dynamic code in get_ith This uses a generated function to avoid a dynamic call for `getindex` allowing it to be called in a CUDAnative kernel. --- src/structarray.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/structarray.jl b/src/structarray.jl index 8a1ea84c..b4cb3ffe 100644 --- a/src/structarray.jl +++ b/src/structarray.jl @@ -134,11 +134,11 @@ Base.axes(s::StructArray) = axes(fieldarrays(s)[1]) Base.axes(s::StructArray{<:Any, <:Any, <:EmptyTup}) = (1:0,) get_ith(cols::NamedTuple, I...) = get_ith(Tuple(cols), I...) -function get_ith(cols::NTuple{N, Any}, I...) where N - ntuple(N) do i - @inbounds res = getfield(cols, i)[I...] - return res +@generated function get_ith(cols::NTuple{N, Any}, I...) where N + args = ntuple(N) do i + :(@inbounds res = getfield(cols, $i)[I...]) end + :(($(args...),)) end Base.@propagate_inbounds function Base.getindex(x::StructArray{T, <:Any, <:Any, CartesianIndex{N}}, I::Vararg{Int, N}) where {T, N} From 695fbfce5a76c2d5fe98625a3db0ab625dcfc98b Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Mon, 23 Sep 2019 17:50:48 +0000 Subject: [PATCH 02/10] Make foreachfield a static function This allows `setindex!` on `StructArray`s to be used in `CUDAnative` kernels. --- src/utils.jl | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 355d3168..83cc9676 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -28,20 +28,50 @@ else const _getproperty = getproperty end -function _foreachfield(names, xs) +function _sstuple(::Type{<:NTuple{N, Any}}) where {N} + ntuple(j->Symbol(j), N) +end + +function _sstuple(::Type{NT}) where {NT<:NamedTuple} + _map_params(x->_sstuple(staticschema(x)), NT) +end + +function _getcolproperties!(exprs, s, es=[]) + if typeof(s) <: Symbol + push!(exprs, es) + return + end + for key in keys(s) + _getcolproperties!(exprs, getproperty(s,key), vcat(es, key)) + end +end + +@generated function foreachfield(::Type{T}, f, xs...) where {T<:Tup} + # TODO get columnsproperties directly from T without converting to the + # tuple s. + s = _sstuple(T) + columnsproperties = [] + _getcolproperties!(columnsproperties, s) + exprs = Expr[] - for field in names - sym = QuoteNode(field) - args = [Expr(:call, :_getproperty, :(getfield(xs, $j)), sym) for j in 1:length(xs)] + for col in columnsproperties + args = Expr[] + for prop in col + sym = QuoteNode(prop) + if length(args) == 0 + args = [Expr(:call, :_getproperty, :(getfield(xs, $j)), sym) for j in 1:length(xs)] + else + for j in 1:length(xs) + args[j] = Expr(:call, :_getproperty, args[j], sym) + end + end + end push!(exprs, Expr(:call, :f, args...)) end push!(exprs, :(return nothing)) return Expr(:block, exprs...) end -@generated foreachfield(::Type{<:NamedTuple{names}}, f, xs...) where {names} = _foreachfield(names, xs) -@generated foreachfield(::Type{<:NTuple{N, Any}}, f, xs...) where {N} = _foreachfield(Base.OneTo(N), xs) - foreachfield(f, x::T, xs...) where {T} = foreachfield(staticschema(T), f, x, xs...) """ From 3bea6173a732bdeedc37ee364482f7f2cc44d452 Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Thu, 26 Sep 2019 12:05:14 -0700 Subject: [PATCH 03/10] Use Adapt for converting CPU to GPU StructArrays --- src/StructArrays.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/StructArrays.jl b/src/StructArrays.jl index c9f52df1..c0d41349 100644 --- a/src/StructArrays.jl +++ b/src/StructArrays.jl @@ -16,4 +16,8 @@ include("groupjoin.jl") include("lazy.jl") include("tables.jl") +# Use Adapt allows for automatic conversion of CPU to GPU StructArrays +import Adapt +Adapt.adapt_storage(to, s::StructArray) = replace_storage(x->Adapt.adapt(to, x), s) + end # module From 189afa39b4a3942f1fa71398b0a7c36058c968f8 Mon Sep 17 00:00:00 2001 From: Pietro Vertechi Date: Sat, 4 Jan 2020 20:23:16 +0000 Subject: [PATCH 04/10] make getindex generated and revert foreach changes --- src/structarray.jl | 7 +++---- src/utils.jl | 44 +++++++------------------------------------- 2 files changed, 10 insertions(+), 41 deletions(-) diff --git a/src/structarray.jl b/src/structarray.jl index b4cb3ffe..ae779be3 100644 --- a/src/structarray.jl +++ b/src/structarray.jl @@ -135,10 +135,9 @@ Base.axes(s::StructArray{<:Any, <:Any, <:EmptyTup}) = (1:0,) get_ith(cols::NamedTuple, I...) = get_ith(Tuple(cols), I...) @generated function get_ith(cols::NTuple{N, Any}, I...) where N - args = ntuple(N) do i - :(@inbounds res = getfield(cols, $i)[I...]) - end - :(($(args...),)) + args = [:(getfield(cols, $i)[I...]) for i in 1:N] + tup = Expr(:tuple, args...) + return :(@inbounds $tup) end Base.@propagate_inbounds function Base.getindex(x::StructArray{T, <:Any, <:Any, CartesianIndex{N}}, I::Vararg{Int, N}) where {T, N} diff --git a/src/utils.jl b/src/utils.jl index 83cc9676..355d3168 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -28,50 +28,20 @@ else const _getproperty = getproperty end -function _sstuple(::Type{<:NTuple{N, Any}}) where {N} - ntuple(j->Symbol(j), N) -end - -function _sstuple(::Type{NT}) where {NT<:NamedTuple} - _map_params(x->_sstuple(staticschema(x)), NT) -end - -function _getcolproperties!(exprs, s, es=[]) - if typeof(s) <: Symbol - push!(exprs, es) - return - end - for key in keys(s) - _getcolproperties!(exprs, getproperty(s,key), vcat(es, key)) - end -end - -@generated function foreachfield(::Type{T}, f, xs...) where {T<:Tup} - # TODO get columnsproperties directly from T without converting to the - # tuple s. - s = _sstuple(T) - columnsproperties = [] - _getcolproperties!(columnsproperties, s) - +function _foreachfield(names, xs) exprs = Expr[] - for col in columnsproperties - args = Expr[] - for prop in col - sym = QuoteNode(prop) - if length(args) == 0 - args = [Expr(:call, :_getproperty, :(getfield(xs, $j)), sym) for j in 1:length(xs)] - else - for j in 1:length(xs) - args[j] = Expr(:call, :_getproperty, args[j], sym) - end - end - end + for field in names + sym = QuoteNode(field) + args = [Expr(:call, :_getproperty, :(getfield(xs, $j)), sym) for j in 1:length(xs)] push!(exprs, Expr(:call, :f, args...)) end push!(exprs, :(return nothing)) return Expr(:block, exprs...) end +@generated foreachfield(::Type{<:NamedTuple{names}}, f, xs...) where {names} = _foreachfield(names, xs) +@generated foreachfield(::Type{<:NTuple{N, Any}}, f, xs...) where {N} = _foreachfield(Base.OneTo(N), xs) + foreachfield(f, x::T, xs...) where {T} = foreachfield(staticschema(T), f, x, xs...) """ From 166d15d50879a54376872b87740ffa4423296cdb Mon Sep 17 00:00:00 2001 From: Pietro Vertechi Date: Sat, 4 Jan 2020 21:01:45 +0000 Subject: [PATCH 05/10] simpler foreachfield --- src/utils.jl | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 355d3168..55b34954 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -28,19 +28,26 @@ else const _getproperty = getproperty end -function _foreachfield(names, xs) +function _foreachfield(names, vars) exprs = Expr[] for field in names sym = QuoteNode(field) - args = [Expr(:call, :_getproperty, :(getfield(xs, $j)), sym) for j in 1:length(xs)] + args = [Expr(:call, :_getproperty, var, sym) for var in vars] push!(exprs, Expr(:call, :f, args...)) end push!(exprs, :(return nothing)) return Expr(:block, exprs...) end -@generated foreachfield(::Type{<:NamedTuple{names}}, f, xs...) where {names} = _foreachfield(names, xs) -@generated foreachfield(::Type{<:NTuple{N, Any}}, f, xs...) where {N} = _foreachfield(Base.OneTo(N), xs) +@generated foreachfield(::Type{<:NamedTuple{names}}, f, x) where {names} = + _foreachfield(names, (:x,)) +@generated foreachfield(::Type{<:NTuple{N, Any}}, f, x) where {N} = + _foreachfield(Base.OneTo(N)), (:x,) + +@generated foreachfield(::Type{<:NamedTuple{names}}, f, x, y) where {names} = + _foreachfield(names, (:x, :y)) +@generated foreachfield(::Type{<:NTuple{N, Any}}, f, x, y) where {N} = + _foreachfield(Base.OneTo(N), (:x, :y)) foreachfield(f, x::T, xs...) where {T} = foreachfield(staticschema(T), f, x, xs...) From ba63f3a54c7558340db664525e909a63b976c756 Mon Sep 17 00:00:00 2001 From: Pietro Vertechi Date: Sat, 4 Jan 2020 21:18:57 +0000 Subject: [PATCH 06/10] restore foreachfield for an arbitrary number of variables --- src/utils.jl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 55b34954..9ac1f521 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -28,8 +28,12 @@ else const _getproperty = getproperty end -function _foreachfield(names, vars) +function _foreachfield(names, L) + vars = ntuple(i -> gensym(), L) exprs = Expr[] + for (i, v) in enumerate(vars) + push!(exprs, Expr(:(=), v, Expr(:call, :getfield, :xs, i))) + end for field in names sym = QuoteNode(field) args = [Expr(:call, :_getproperty, var, sym) for var in vars] @@ -39,15 +43,10 @@ function _foreachfield(names, vars) return Expr(:block, exprs...) end -@generated foreachfield(::Type{<:NamedTuple{names}}, f, x) where {names} = - _foreachfield(names, (:x,)) -@generated foreachfield(::Type{<:NTuple{N, Any}}, f, x) where {N} = - _foreachfield(Base.OneTo(N)), (:x,) - -@generated foreachfield(::Type{<:NamedTuple{names}}, f, x, y) where {names} = - _foreachfield(names, (:x, :y)) -@generated foreachfield(::Type{<:NTuple{N, Any}}, f, x, y) where {N} = - _foreachfield(Base.OneTo(N), (:x, :y)) +@generated foreachfield(::Type{<:NamedTuple{names}}, f, xs::Vararg{Any, L}) where {names, L} = + _foreachfield(names, L) +@generated foreachfield(::Type{<:NTuple{N, Any}}, f, xs::Vararg{Any, L}) where {N, L} = + _foreachfield(Base.OneTo(N), L) foreachfield(f, x::T, xs...) where {T} = foreachfield(staticschema(T), f, x, xs...) From f9c4c96cc9408db4342f7d580bea8a30a3624086 Mon Sep 17 00:00:00 2001 From: Pietro Vertechi Date: Mon, 6 Jan 2020 14:40:19 +0000 Subject: [PATCH 07/10] remove generated get_ith --- src/structarray.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/structarray.jl b/src/structarray.jl index ae779be3..4d1d517f 100644 --- a/src/structarray.jl +++ b/src/structarray.jl @@ -134,11 +134,11 @@ Base.axes(s::StructArray) = axes(fieldarrays(s)[1]) Base.axes(s::StructArray{<:Any, <:Any, <:EmptyTup}) = (1:0,) get_ith(cols::NamedTuple, I...) = get_ith(Tuple(cols), I...) -@generated function get_ith(cols::NTuple{N, Any}, I...) where N - args = [:(getfield(cols, $i)[I...]) for i in 1:N] - tup = Expr(:tuple, args...) - return :(@inbounds $tup) +function get_ith(cols::Tuple, I...) + @inbounds r = first(cols)[I...] + return (r, get_ith(Base.tail(cols), I...)...) end +get_ith(::Tuple{}, I...) = () Base.@propagate_inbounds function Base.getindex(x::StructArray{T, <:Any, <:Any, CartesianIndex{N}}, I::Vararg{Int, N}) where {T, N} cols = fieldarrays(x) From ce15a14b8c73acf21781f1bfebda20e0153e6d40 Mon Sep 17 00:00:00 2001 From: Pietro Vertechi Date: Mon, 6 Jan 2020 14:49:53 +0000 Subject: [PATCH 08/10] test adapt --- Project.toml | 3 ++- src/compatibility.jl | 24 ++++++++++++++++++++++++ test/runtests.jl | 12 ++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 src/compatibility.jl diff --git a/Project.toml b/Project.toml index 7b267372..14041624 100644 --- a/Project.toml +++ b/Project.toml @@ -12,10 +12,11 @@ Tables = "1" julia = "1" [extras] +GPUArrays = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" WeakRefStrings = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" [targets] -test = ["Test", "OffsetArrays", "PooledArrays", "WeakRefStrings"] +test = ["Test", "GPUArrays", "OffsetArrays", "PooledArrays", "WeakRefStrings"] diff --git a/src/compatibility.jl b/src/compatibility.jl new file mode 100644 index 00000000..f6e56e46 --- /dev/null +++ b/src/compatibility.jl @@ -0,0 +1,24 @@ +# GPU storage +import Adapt +Adapt.adapt_structure(to, s::StructArray) = replace_storage(x->Adapt.adapt(to, x), s) + +# Table interface +import Tables + +Tables.istable(::Type{<:StructVector}) = true +Tables.rowaccess(::Type{<:StructVector}) = true +Tables.columnaccess(::Type{<:StructVector}) = true + +Tables.rows(s::StructVector) = s +Tables.columns(s::StructVector) = fieldarrays(s) + +Tables.schema(s::StructVector) = Tables.Schema(staticschema(eltype(s))) + +# refarray interface +import DataAPI: refarray, refvalue + +refarray(s::StructArray) = StructArray(map(refarray, fieldarrays(s))) + +function refvalue(s::StructArray{T}, v::Tup) where {T} + createinstance(T, map(refvalue, fieldarrays(s), v)...) +end diff --git a/test/runtests.jl b/test/runtests.jl index e9610602..66fb7605 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,6 +3,8 @@ using StructArrays: staticschema, iscompatible, _promote_typejoin, append!! using OffsetArrays: OffsetArray import Tables, PooledArrays, WeakRefStrings using DataAPI: refarray, refvalue +using Adapt: adapt +import GPUArrays using Test @testset "index" begin @@ -700,3 +702,13 @@ end @test vcat(dest, StructVector(makeitr())) == append!!(copy(dest), makeitr()) end end + +@testset "adapt" begin + s = StructArray(a = 1:10, b = StructArray(c = 1:10, d = 1:10)) + t = adapt(Array, s) + @test propertynames(t) == (:a, :b) + @test s == t + @test t.a isa Array + @test t.b.c isa Array + @test t.b.d isa Array +end From faa0f7678d5f74065e9fbe50315b102bdcbe6b80 Mon Sep 17 00:00:00 2001 From: Pietro Vertechi Date: Fri, 14 Feb 2020 12:11:55 +0000 Subject: [PATCH 09/10] remove extra file --- src/compatibility.jl | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 src/compatibility.jl diff --git a/src/compatibility.jl b/src/compatibility.jl deleted file mode 100644 index f6e56e46..00000000 --- a/src/compatibility.jl +++ /dev/null @@ -1,24 +0,0 @@ -# GPU storage -import Adapt -Adapt.adapt_structure(to, s::StructArray) = replace_storage(x->Adapt.adapt(to, x), s) - -# Table interface -import Tables - -Tables.istable(::Type{<:StructVector}) = true -Tables.rowaccess(::Type{<:StructVector}) = true -Tables.columnaccess(::Type{<:StructVector}) = true - -Tables.rows(s::StructVector) = s -Tables.columns(s::StructVector) = fieldarrays(s) - -Tables.schema(s::StructVector) = Tables.Schema(staticschema(eltype(s))) - -# refarray interface -import DataAPI: refarray, refvalue - -refarray(s::StructArray) = StructArray(map(refarray, fieldarrays(s))) - -function refvalue(s::StructArray{T}, v::Tup) where {T} - createinstance(T, map(refvalue, fieldarrays(s), v)...) -end From f38a4696ef477cf8a2e72320037ea605df2b9de9 Mon Sep 17 00:00:00 2001 From: Pietro Vertechi Date: Fri, 14 Feb 2020 12:16:04 +0000 Subject: [PATCH 10/10] fix rebasing issues --- Project.toml | 2 ++ src/StructArrays.jl | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 14041624..9e6ad1dc 100644 --- a/Project.toml +++ b/Project.toml @@ -3,10 +3,12 @@ uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" version = "0.4.2" [deps] +Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] +Adapt = "1" DataAPI = "1" Tables = "1" julia = "1" diff --git a/src/StructArrays.jl b/src/StructArrays.jl index c0d41349..01406726 100644 --- a/src/StructArrays.jl +++ b/src/StructArrays.jl @@ -18,6 +18,6 @@ include("tables.jl") # Use Adapt allows for automatic conversion of CPU to GPU StructArrays import Adapt -Adapt.adapt_storage(to, s::StructArray) = replace_storage(x->Adapt.adapt(to, x), s) +Adapt.adapt_structure(to, s::StructArray) = replace_storage(x->Adapt.adapt(to, x), s) end # module