Excise REPL and its dependencies from sysimg (#51399)

Author: vchuravy

Makefile

@@ -116,7 +116,7 @@ stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-%
 
 debug release : % : julia-% stdlibs-cache-%
 
-docs: julia-sysimg-$(JULIA_BUILD_MODE)
+docs: julia-sysimg-$(JULIA_BUILD_MODE) stdlibs-cache-$(JULIA_BUILD_MODE)
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/doc JULIA_EXECUTABLE='$(call spawn,$(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE))) --startup-file=no'
 
 docs-revise:
@@ -184,7 +184,7 @@ $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(buil
 	@$(call PRINT_CC, $(HOSTCC) -o $(build_depsbindir)/stringreplace $(JULIAHOME)/contrib/stringreplace.c)
 
 julia-base-cache: julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia
-	@JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \
+	@JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) JULIA_FALLBACK_REPL=1 WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \
 		$(call spawn, $(JULIA_EXECUTABLE) --startup-file=no $(call cygpath_w,$(JULIAHOME)/etc/write_base_cache.jl) \
 		$(call cygpath_w,$(build_datarootdir)/julia/base.cache))
 

base/client.jl

@@ -405,13 +405,28 @@ function load_InteractiveUtils(mod::Module=Main)
     return getfield(mod, :InteractiveUtils)
 end
 
+function load_REPL()
+    # load interactive-only libraries
+    try
+        return Base.require(PkgId(UUID(0x3fa0cd96_eef1_5676_8a61_b3b8758bbffb), "REPL"))
+    catch ex
+        @warn "Failed to import REPL" exception=(ex, catch_backtrace())
+    end
+    return nothing
+end
+
 global active_repl
 
 # run the requested sort of evaluation loop on stdio
 function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_file::Bool, color_set::Bool)
-    load_InteractiveUtils()
-
-    fallback_repl = get_bool_env("JULIA_FALLBACK_REPL", false)
+    fallback_repl = parse(Bool, get(ENV, "JULIA_FALLBACK_REPL", "false"))
+    if !fallback_repl && interactive
+        load_InteractiveUtils()
+        if !isassigned(REPL_MODULE_REF)
+            load_REPL()
+        end
+    end
+    # TODO cleanup REPL_MODULE_REF
 
     if !fallback_repl && interactive && isassigned(REPL_MODULE_REF)
         invokelatest(REPL_MODULE_REF[]) do REPL
@@ -435,8 +450,8 @@ function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_f
         end
     else
         # otherwise provide a simple fallback
-        if interactive && !quiet
-            @warn "REPL provider not available: using basic fallback"
+        if !fallback_repl && interactive && !quiet
+            @warn "REPL provider not available: using basic fallback" LOAD_PATH=join(Base.LOAD_PATH, Sys.iswindows() ? ';' : ':')
         end
         banner == :no || Base.banner(short=banner==:short)
         let input = stdin

base/docs/Docs.jl

@@ -535,7 +535,7 @@ function docm(source::LineNumberNode, mod::Module, ex)
     elseif isassigned(Base.REPL_MODULE_REF)
         # TODO: this is a shim to continue to allow `@doc` for looking up docstrings
         REPL = Base.REPL_MODULE_REF[]
-        return REPL.lookup_doc(ex)
+        return invokelatest(REPL.lookup_doc, ex)
     end
     return nothing
 end

base/loading.jl

@@ -1818,9 +1818,6 @@ function __require_prelocked(uuidkey::PkgId, env=nothing)
         insert_extension_triggers(uuidkey)
         # After successfully loading, notify downstream consumers
         run_package_callbacks(uuidkey)
-        if uuidkey == REPL_PKGID
-            REPL_MODULE_REF[] = newm
-        end
     else
         newm = root_module(uuidkey)
     end

base/sysimg.jl

@@ -31,43 +31,19 @@ let
     # Run with the `--exclude-jlls` option to filter out all JLL packages
     stdlibs = [
         # No dependencies
-        :ArgTools,
-        :Artifacts,
-        :Base64,
-        :CRC32c,
-        :FileWatching,
-        :Libdl,
-        :Logging,
-        :Mmap,
-        :NetworkOptions,
-        :SHA,
-        :Serialization,
-        :Sockets,
-        :Unicode,
+        :FileWatching, # used by loading.jl -- implicit assumption that init runs
+        :Libdl, # Transitive through LinAlg
+        :Artifacts, # Transitive through LinAlg
+        :SHA, # transitive through Random
+        :Sockets, # used by stream.jl
+
+        # Transitive through LingAlg
+        # OpenBLAS_jll
+        # libblastrampoline_jll
 
         # 1-depth packages
-        :LinearAlgebra,
-        :Markdown,
-        :Printf,
-        :Random,
-        :Tar,
-
-        # 2-depth packages
-        :Dates,
-        :Future,
-        :InteractiveUtils,
-        :LibGit2,
-        :UUIDs,
-
-        # 3-depth packages
-        :REPL,
-        :TOML,
-
-        # 4-depth packages
-        :LibCURL,
-
-        # 5-depth packages
-        :Downloads,
+        :LinearAlgebra, # Commits type-piracy and GEMM
+        :Random, # Can't be removed due to rand being exported by Base
     ]
     # PackageCompiler can filter out stdlibs so it can be empty
     maxlen = maximum(textwidth.(string.(stdlibs)); init=0)

base/terminfo.jl

@@ -95,8 +95,8 @@ function read(data::IO, ::Type{TermInfoRaw})
             throw(ArgumentError("Terminfo did not contain a null byte after the flag section, expected to position the start of the numbers section on an even byte"))
     end
     # Numbers, Strings, Table
-    numbers = reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt))) .|> ltoh
-    string_indices = reinterpret(UInt16, read(data, string_count * sizeof(UInt16))) .|> ltoh
+    numbers = map(ltoh, reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt))))
+    string_indices = map(ltoh, reinterpret(UInt16, read(data, string_count * sizeof(UInt16))))
     strings_table = read(data, table_bytes)
     strings = map(string_indices) do idx
         if idx ∉ (0xffff, 0xfffe)
@@ -107,7 +107,7 @@ function read(data::IO, ::Type{TermInfoRaw})
         end
     end
     TermInfoRaw(term_names, flags, numbers, strings,
-                if !eof(data) extendedterminfo(data; NumInt) end)
+                if !eof(data) extendedterminfo(data, NumInt) end)
 end
 
 """
@@ -119,7 +119,7 @@ This will accept any terminfo content that conforms with `term(5)`.
 
 See also: `read(::IO, ::Type{TermInfoRaw})`
 """
-function extendedterminfo(data::IO; NumInt::Union{Type{UInt16}, Type{UInt32}})
+function extendedterminfo(data::IO, NumInt::Union{Type{UInt16}, Type{UInt32}})
     # Extended info
     if position(data) % 2 != 0
         0x00 == read(data, UInt8) ||
@@ -138,12 +138,15 @@ function extendedterminfo(data::IO; NumInt::Union{Type{UInt16}, Type{UInt32}})
             throw(ArgumentError("Terminfo did not contain a null byte after the extended flag section, expected to position the start of the numbers section on an even byte"))
     end
     numbers = map(n -> Int(ltoh(n)), reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt))))
-    table_indices = reinterpret(UInt16, read(data, table_count * sizeof(UInt16))) .|> ltoh
+    table_indices = map(ltoh, reinterpret(UInt16, read(data, table_count * sizeof(UInt16))))
     table_strings = [String(readuntil(data, 0x00)) for _ in 1:length(table_indices)]
+    info = Dict{Symbol, Union{Bool, Int, String}}()
     strings = table_strings[1:string_count]
-    labels = Symbol.(table_strings[string_count+1:end])
-    Dict{Symbol, Union{Bool, Int, String}}(
-        labels .=> vcat(flags, numbers, strings))
+    labels = table_strings[string_count+1:end]
+    for (label, val) in zip(labels, vcat(flags, numbers, strings))
+        info[Symbol(label)] = val
+    end
+    return info
 end
 
 """
@@ -178,7 +181,7 @@ function TermInfo(raw::TermInfoRaw)
         Symbol[]
     end
     TermInfo(raw.names, length(raw.flags),
-             raw.numbers .!= typemax(eltype(raw.numbers)),
+             map(n-> n != typemax(typeof(n)), raw.numbers),
              map(!isnothing, raw.strings),
              extensions, capabilities)
 end

base/util.jl

@@ -700,7 +700,9 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2),
     catch
         buf = PipeBuffer()
         original_load_path = copy(Base.LOAD_PATH); empty!(Base.LOAD_PATH); pushfirst!(Base.LOAD_PATH, "@stdlib")
-        Base.require(Base, :InteractiveUtils).versioninfo(buf)
+        let InteractiveUtils = Base.require(Base, :InteractiveUtils)
+            @invokelatest InteractiveUtils.versioninfo(buf)
+        end
         empty!(Base.LOAD_PATH); append!(Base.LOAD_PATH, original_load_path)
         error("A test has failed. Please submit a bug report (https://github.com/JuliaLang/julia/issues)\n" *
               "including error messages above and the output of versioninfo():\n$(read(buf, String))")