compress json data arrays (#62)

* compress json data arrays

* make json compression opt-in

* forgot left-over src_inject

Author: marius311

Project.toml

@@ -5,7 +5,9 @@ version = "0.13.2"
 
 [deps]
 Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 Cobweb = "ec354790-cf28-43e8-bb59-b484409b7bad"
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 EasyConfig = "acab07b0-f158-46d4-8913-50acef6d41fe"
@@ -15,7 +17,9 @@ REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 [compat]
 Aqua = "0.8"
 Artifacts = "1"
+Base64 = "1.11.0"
 Cobweb = "0.6, 0.7"
+CodecZlib = "0.7.6"
 Dates = "1"
 Downloads = "1.6"
 EasyConfig = "0.1"

README.md

@@ -15,9 +15,10 @@
 ## ✨ Features
 
 - 🚀 Fastest time-to-first-plot in Julia!
-- 🌐 Use the [Plotly.js Javascript documentation](https://plotly.com/javascript/) directly.  No magic syntax: Just [`JSON3.write`](https://github.com/quinnj/JSON3.jl).
+- 🌐 Use the [Plotly.js Javascript documentation](https://plotly.com/javascript/) directly. No magic syntax.
 - 📂 Set deeply-nested keys easily, e.g. `myplot.layout.xaxis.title.font.family = "Arial"`.
 - 📊 The Same [built-in themes](https://plotly.com/python/templates/) as Plotly's python package.
+- 🗜️ Use `PlotlyLight.preset.display.compress!()` to automatically compress large arrays and produce plots that download and display faster. 
 
 <br><br>
 

src/PlotlyLight.jl

@@ -1,13 +1,15 @@
 module PlotlyLight
 
 using Artifacts: @artifact_str
+using Base64
 using Downloads: download
 using Dates
 using REPL
 
 using JSON3: JSON3
 using EasyConfig: Config
 using Cobweb: Cobweb, h, IFrame, Node
+using CodecZlib
 
 #-----------------------------------------------------------------------------# exports
 export Config, preset, Plot, plot
@@ -42,6 +44,7 @@ Base.@kwdef mutable struct Settings
     use_iframe::Bool        = false
     iframe_style            = "display:block; border:none; min-height:350px; min-width:350px; width:100%; height:100%"
     src_inject::Vector      = []
+    compress::Bool          = false
 end
 settings::Settings = Settings()
 
@@ -63,6 +66,14 @@ function with_settings(f; kw...)
     end
 end
 
+function get_src_inject(s::Settings)
+    src_inject = s.src_inject
+    if s.compress
+        src_inject = union(src_inject, json_compression_src_inject)
+    end
+    return src_inject
+end
+
 #-----------------------------------------------------------------------------# utils/other
 attributes(t::Symbol) = plotly.schema.traces[t].attributes
 check_attribute(trace, attr::Symbol) = haskey(attributes(Symbol(trace)), attr) || @warn("`$trace` does not have attribute `$attr`.")
@@ -122,7 +133,7 @@ end
 rand_id() = "plotlyx-" * join(rand('a':'z', 10))
 
 function html_div(o::Plot, id=rand_id())
-    h.div(class="plotlylight-parent", settings.src_inject..., settings.src, settings.div(; id), NewPlotScript(o, settings, id))
+    h.div(class="plotlylight-parent", get_src_inject(settings)..., settings.src, settings.div(; id), NewPlotScript(o, settings, id))
 end
 
 function html_page(o::Plot, id=rand_id())
@@ -133,7 +144,7 @@ function html_page(o::Plot, id=rand_id())
             h.meta(name="description", content="PlotlyLight.jl Plot"),
             h.title("PlotlyLight.jl"),
             settings.page_css,
-            settings.src_inject...,
+            get_src_inject(settings)...,
             settings.src
         ),
         h.body(h.div(class="plotlylight-parent", settings.div(; id), NewPlotScript(o, settings, id)))
@@ -188,6 +199,7 @@ preset = (
     display = (
         fullscreen!     = () -> (settings.div.style = "height:100vh; width:100vw"),
         mathjax!        = () -> (push!(settings.src_inject, h.script(src="https://cdn.jsdelivr.net/npm/[email protected]/es5/tex-svg.js"))),
+        compress!       = (enabled=true) -> (settings.compress = enabled)
     )
 )
 

src/json.jl

@@ -36,9 +36,92 @@ json(io::IO, ::Union{Missing, Nothing}) = print(io, "null")
 json(io::IO, x::Bool) = print(io, x ? "true" : "false")
 
 # Arrays
-json(io::IO, x::AbstractVector) = json_join(io, x, ',', '[', ']')
-json(io::IO, x::AbstractArray) = json(io, eachslice(x; dims=1))
+_json_generic_arr(io::IO, x::AbstractVector) = json_join(io, x, ',', '[', ']')
+_json_generic_arr(io::IO, x::AbstractArray) = json(io, eachslice(x; dims=1))
+json(io::IO, x::AbstractVector) = _json_generic_arr(io, x)
+json(io::IO, x::AbstractArray) = _json_generic_arr(io, x)
 
 # Objects
 json(io::IO, x::Pair) = json(io, x.first, JSON(':'), x.second)
 json(io::IO, x::Union{NamedTuple, AbstractDict}) = json_join(io, pairs(x), ',', '{', '}')
+
+
+
+# Compress certain array types for some (huge) space savings for large arrays
+
+json_compression_src_inject = [
+    h.script(src="https://cdn.jsdelivr.net/npm/[email protected]/umd/index.js"),
+    h.script(raw"""
+    function numArrFromBase64(T, base64_dat, ...dims) {
+        arr = new T(fflate.unzlibSync(Uint8Array.from(atob(base64_dat), c => c.charCodeAt(0))).buffer)
+        if (dims.length == 1) {
+            return arr; 
+        } else if (dims.length == 2) {
+            arr2d = [];
+            for (let i = 0; i < arr.length; i += dims[1]) {
+                arr2d.push(arr.subarray(i, i + dims[1]));
+            }
+            return arr2d;
+        } else {
+            throw new Error(`>2 dims not implemented.`);
+        }
+    }
+    function strVecFromBase64(base64_dat, lens) {
+        strs = fflate.strFromU8(fflate.unzlibSync(Uint8Array.from(atob(base64_dat), c => c.charCodeAt(0))));
+        arr = [];
+        cur = 0;
+        for (var i = 0; i < lens.length; i++) {
+            arr.push(strs.slice(cur, cur + lens[i]));
+            cur += lens[i];
+        }
+        return arr;
+    }
+    """)
+]
+
+json(io::IO, arr::AbstractVector{<:AbstractFloat}) = _json_num_arr(io, arr)
+json(io::IO, arr::AbstractMatrix{<:AbstractFloat}) = _json_num_arr(io, arr)
+json(io::IO, arr::AbstractVector{<:Integer}) = _json_num_arr(io, arr)
+json(io::IO, arr::AbstractMatrix{<:Integer}) = _json_num_arr(io, arr)
+
+function _to_js_eltype(arr::AbstractArray{<:AbstractFloat})
+    # be opinionated and cap at Float32, which should be enough for
+    # plotting, halving filesize vs Float64
+    T = (eltype(arr) == Float16) ? Float16 : Float32
+    return convert(AbstractArray{T}, arr)
+end
+
+function _to_js_eltype(arr::AbstractArray{<:Integer})
+    # find the smallest integer type that can represent the data
+    mn, mx = extrema(arr)
+    types = (UInt8, Int8, UInt16, Int16, UInt32, Int32)
+    i = findfirst(t -> mn >= typemin(t) && mx <= typemax(t), types)
+    isnothing(i) && error("Integer values in plot data are too large to fit in UInt32 or Int32.")
+    T = types[i]
+    return convert(AbstractArray{T}, arr)
+end
+
+function _json_num_arr(io::IO, arr)
+    if settings.compress
+        js_arr = _to_js_eltype(arr)
+        T = eltype(js_arr)
+        base64_dat = base64encode(transcode(ZlibCompressor, Vector(reinterpret(UInt8, view(transpose(js_arr), :)))))
+        dims = join(size(js_arr), ',')
+        T_js = string(T)[1] * lowercase(string(T)[2:end])
+        print(io, "numArrFromBase64($(T_js)Array,'", base64_dat, "',", dims, ")")
+    else
+        _json_generic_arr(io, arr)
+    end
+end
+
+function json(io::IO, arr::AbstractVector{<:AbstractString})
+    if settings.compress
+        # store a (compressed) contatenation of the strings and indices where each element starts
+        base64_dat = base64encode(transcode(ZlibCompressor, join(arr)))
+        print(io, "strVecFromBase64('", base64_dat, "',")
+        json(io, length.(arr))
+        print(io, ")")
+    else
+        _json_generic_arr(io, arr)
+    end
+end

test/runtests.jl

@@ -21,6 +21,12 @@ html(x) = repr("text/html", x)
     @test json(Inf) == "null"
     @test json(-Inf) == "null"
     @test json(DateTime(2021,1,1)) == "\"2021-01-01 00:00:00\""
+    preset.display.compress!(true)
+    @test json(Int[1, 2]) == "numArrFromBase64(Uint8Array,'eJxjZAIAAAYABA==',2)"
+    @test json(Int[1 2; 3 4]) == "numArrFromBase64(Uint8Array,'eJxjZGJmAQAAGAAL',2,2)"
+    @test json(Float64[1, 2]) == "numArrFromBase64(Float32Array,'eJxjYGiwZ2BgcAAABIMBAA==',2)"
+    @test json(Float64[1 2; 3 4]) == "numArrFromBase64(Float32Array,'eJxjYGiwZ2BgcAAiIG5wAAAQgwJA',2,2)"
+    @test json(["a", "b"]) == "strVecFromBase64('eJxLTAIAASYAxA==',numArrFromBase64(Uint8Array,'eJxjZAQAAAUAAw==',2))"
 end
 
 #-----------------------------------------------------------------------------# Plot methods