Skip to content

Commit d358228

Browse files
KristofferCKristofferC
authored
Revert "Revert "prepare for sysimage excision by having the precompile code run in toplevel (#3441)" (#3446)"
This reverts commit 1f0e6a7.
1 parent 1f0e6a7 commit d358228

File tree

4 files changed

+124
-147
lines changed

4 files changed

+124
-147
lines changed

‎src/Operations.jl‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -810,7 +810,12 @@ function download_source(ctx::Context; readonly=true)
810810
max_name = maximum(widths; init=0)
811811

812812
# Check what registries the current pkg server tracks
813-
server_registry_info = Registry.pkg_server_registry_info()
813+
# Disable if precompiling to not access internet
814+
server_registry_info = if Base.JLOptions().incremental == 0
815+
Registry.pkg_server_registry_info()
816+
else
817+
nothing
818+
end
814819

815820
@sync begin
816821
jobs = Channel{eltype(pkgs_to_install)}(ctx.num_concurrent_downloads)

‎src/Pkg.jl‎

Lines changed: 1 addition & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -789,98 +789,6 @@ function _auto_precompile(ctx::Types.Context, pkgs::Vector{PackageSpec}=PackageS
789789
end
790790
end
791791

792-
using LibGit2: LibGit2
793-
using Tar: Tar
794-
function _run_precompilation_script_setup()
795-
tmp = mktempdir()
796-
cd(tmp)
797-
empty!(DEPOT_PATH)
798-
pushfirst!(DEPOT_PATH, tmp)
799-
touch("Project.toml")
800-
Pkg.activate(".")
801-
Pkg.generate("TestPkg")
802-
uuid = TOML.parsefile(joinpath("TestPkg", "Project.toml"))["uuid"]
803-
mv("TestPkg", "TestPkg.jl")
804-
tree_hash = cd("TestPkg.jl") do
805-
sig = LibGit2.Signature("TEST", "[email protected]", round(time()), 0)
806-
repo = LibGit2.init(".")
807-
LibGit2.add!(repo, "")
808-
commit = LibGit2.commit(repo, "initial commit"; author=sig, committer=sig)
809-
th = LibGit2.peel(LibGit2.GitTree, LibGit2.GitObject(repo, commit)) |> LibGit2.GitHash |> string
810-
close(repo)
811-
th
812-
end
813-
# Prevent cloning the General registry by adding a fake one
814-
mkpath("registries/Registry/T/TestPkg")
815-
write("registries/Registry/Registry.toml", """
816-
name = "Registry"
817-
uuid = "37c07fec-e54c-4851-934c-2e3885e4053e"
818-
repo = "https://github.com/JuliaRegistries/Registry.git"
819-
[packages]
820-
$uuid = { name = "TestPkg", path = "T/TestPkg" }
821-
""")
822-
write("registries/Registry/T/TestPkg/Compat.toml", """
823-
["0"]
824-
julia = "1"
825-
""")
826-
write("registries/Registry/T/TestPkg/Deps.toml", """
827-
["0"]
828-
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
829-
""")
830-
write("registries/Registry/T/TestPkg/Versions.toml", """
831-
["0.1.0"]
832-
git-tree-sha1 = "$tree_hash"
833-
""")
834-
write("registries/Registry/T/TestPkg/Package.toml", """
835-
name = "TestPkg"
836-
uuid = "$uuid"
837-
repo = "$(escape_string(tmp))/TestPkg.jl"
838-
""")
839-
Tar.create("registries/Registry", "registries/Registry.tar")
840-
cmd = `$(Pkg.PlatformEngines.exe7z()) a "registries/Registry.tar.gz" -tgzip "registries/Registry.tar"`
841-
run(pipeline(cmd, stdout = stdout_f(), stderr = stderr_f()))
842-
write("registries/Registry.toml", """
843-
git-tree-sha1 = "11b5fad51c4f98cfe0c145ceab0b8fb63fed6f81"
844-
uuid = "37c07fec-e54c-4851-934c-2e3885e4053e"
845-
path = "Registry.tar.gz"
846-
""")
847-
Base.rm("registries/Registry"; recursive=true)
848-
return tmp
849-
end
850-
851-
function _run_precompilation_script_artifact()
852-
# Create simple artifact, bind it, then use it:
853-
foo_hash = Pkg.Artifacts.create_artifact(dir -> touch(joinpath(dir, "foo")))
854-
Artifacts.bind_artifact!("./Artifacts.toml", "foo", foo_hash)
855-
# Also create multiple platform-specific ones because that's a codepath we need precompiled
856-
Artifacts.bind_artifact!("./Artifacts.toml", "foo_plat", foo_hash; platform=Base.BinaryPlatforms.HostPlatform())
857-
# Because @artifact_str doesn't work at REPL-level, we JIT out a file that we can include()
858-
write("load_artifact.jl", """
859-
Pkg.Artifacts.artifact"foo"
860-
Pkg.Artifacts.artifact"foo_plat"
861-
""")
862-
foo_path = include("load_artifact.jl")
863-
end
864-
865-
const CTRL_C = '\x03'
866-
const precompile_script = """
867-
import Pkg
868-
_pwd = pwd()
869-
Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true
870-
tmp = Pkg._run_precompilation_script_setup()
871-
$CTRL_C
872-
Pkg.add("TestPkg")
873-
Pkg.develop(Pkg.PackageSpec(path="TestPkg.jl"))
874-
Pkg.add(Pkg.PackageSpec(path="TestPkg.jl/"))
875-
Pkg.REPLMode.try_prompt_pkg_add(Symbol[:notapackage])
876-
Pkg.update(; update_registry=false)
877-
Pkg.precompile()
878-
] add Te\t\t$CTRL_C
879-
] st
880-
$CTRL_C
881-
Pkg._run_precompilation_script_artifact()
882-
rm(tmp; recursive=true)
883-
cd(_pwd)
884-
"""
792+
include("precompile.jl")
885793

886794
end # module

‎src/precompile.jl‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
using LibGit2: LibGit2
2+
using Tar: Tar
3+
using Downloads
4+
5+
let
6+
function _run_precompilation_script_setup()
7+
tmp = mktempdir()
8+
cd(tmp)
9+
empty!(DEPOT_PATH)
10+
pushfirst!(DEPOT_PATH, tmp)
11+
pushfirst!(LOAD_PATH, "@")
12+
write("Project.toml",
13+
"""
14+
name = "Hello"
15+
uuid = "33cfe95a-1eb2-52ea-b672-e2afdf69b78f"
16+
"""
17+
)
18+
mkdir("src")
19+
write("src/Hello.jl",
20+
"""
21+
module Hello
22+
end
23+
"""
24+
)
25+
Pkg.activate(".")
26+
Pkg.generate("TestPkg")
27+
uuid = TOML.parsefile(joinpath("TestPkg", "Project.toml"))["uuid"]
28+
mv("TestPkg", "TestPkg.jl")
29+
tree_hash = cd("TestPkg.jl") do
30+
sig = LibGit2.Signature("TEST", "[email protected]", round(time()), 0)
31+
repo = LibGit2.init(".")
32+
LibGit2.add!(repo, "")
33+
commit = LibGit2.commit(repo, "initial commit"; author=sig, committer=sig)
34+
th = LibGit2.peel(LibGit2.GitTree, LibGit2.GitObject(repo, commit)) |> LibGit2.GitHash |> string
35+
close(repo)
36+
th
37+
end
38+
# Prevent cloning the General registry by adding a fake one
39+
mkpath("registries/Registry/T/TestPkg")
40+
write("registries/Registry/Registry.toml", """
41+
name = "Registry"
42+
uuid = "37c07fec-e54c-4851-934c-2e3885e4053e"
43+
repo = "https://github.com/JuliaRegistries/Registry.git"
44+
[packages]
45+
$uuid = { name = "TestPkg", path = "T/TestPkg" }
46+
""")
47+
write("registries/Registry/T/TestPkg/Compat.toml", """
48+
["0"]
49+
julia = "1"
50+
""")
51+
write("registries/Registry/T/TestPkg/Deps.toml", """
52+
["0"]
53+
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
54+
""")
55+
write("registries/Registry/T/TestPkg/Versions.toml", """
56+
["0.1.0"]
57+
git-tree-sha1 = "$tree_hash"
58+
""")
59+
write("registries/Registry/T/TestPkg/Package.toml", """
60+
name = "TestPkg"
61+
uuid = "$uuid"
62+
repo = "$(escape_string(tmp))/TestPkg.jl"
63+
""")
64+
Tar.create("registries/Registry", "registries/Registry.tar")
65+
cmd = `$(Pkg.PlatformEngines.exe7z()) a "registries/Registry.tar.gz" -tgzip "registries/Registry.tar"`
66+
run(pipeline(cmd, stdout = stdout_f(), stderr = stderr_f()))
67+
write("registries/Registry.toml", """
68+
git-tree-sha1 = "11b5fad51c4f98cfe0c145ceab0b8fb63fed6f81"
69+
uuid = "37c07fec-e54c-4851-934c-2e3885e4053e"
70+
path = "Registry.tar.gz"
71+
""")
72+
Base.rm("registries/Registry"; recursive=true)
73+
return tmp
74+
end
75+
76+
# SnoopPrecompile is useful but not available in Base
77+
# using SnoopPrecompile
78+
function pkg_precompile()
79+
Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true
80+
# Default 30 sec grace period means we hang 30 seconds before precompiling finishes
81+
redirect_stderr(devnull) do
82+
redirect_stdout(devnull) do
83+
Downloads.DOWNLOADER[] = Downloads.Downloader(; grace=1.0)
84+
withenv("JULIA_PKG_SERVER" => nothing) do
85+
# @precompile_setup begin
86+
tmp = _run_precompilation_script_setup()
87+
# @precompile_all_calls begin
88+
withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0) do
89+
@show DEPOT_PATH
90+
@show LOAD_PATH
91+
92+
Pkg.add("TestPkg")
93+
Pkg.develop(Pkg.PackageSpec(path="TestPkg.jl"))
94+
Pkg.add(Pkg.PackageSpec(path="TestPkg.jl/"))
95+
Pkg.REPLMode.try_prompt_pkg_add(Symbol[:notapackage])
96+
Pkg.update(; update_registry=false)
97+
Pkg.status()
98+
end
99+
Pkg.precompile()
100+
try Base.rm(tmp; recursive=true)
101+
catch
102+
end
103+
104+
Base.precompile(Tuple{typeof(Pkg.REPLMode.promptf)})
105+
Base.precompile(Tuple{typeof(Pkg.REPLMode.repl_init), REPL.LineEditREPL})
106+
Base.precompile(Tuple{typeof(Pkg.API.status)})
107+
Base.precompile(Tuple{typeof(Pkg.Types.read_project_compat), Base.Dict{String, Any}, Pkg.Types.Project})
108+
Base.precompile(Tuple{typeof(Pkg.Versions.semver_interval), Base.RegexMatch})
109+
# end
110+
# end
111+
end
112+
end
113+
end
114+
end
115+
116+
pkg_precompile()
117+
end

‎test/api.jl‎

Lines changed: 0 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -46,59 +46,6 @@ end
4646
include("FakeTerminals.jl")
4747
import .FakeTerminals.FakeTerminal
4848

49-
@testset "Pkg.precompile_script" begin
50-
function fake_repl(@nospecialize(f); options::REPL.Options=REPL.Options(confirm_exit=false))
51-
# Use pipes so we can easily do blocking reads
52-
# In the future if we want we can add a test that the right object
53-
# gets displayed by intercepting the display
54-
input = Pipe()
55-
output = Pipe()
56-
err = Pipe()
57-
Base.link_pipe!(input, reader_supports_async=true, writer_supports_async=true)
58-
Base.link_pipe!(output, reader_supports_async=true, writer_supports_async=true)
59-
Base.link_pipe!(err, reader_supports_async=true, writer_supports_async=true)
60-
61-
repl = REPL.LineEditREPL(FakeTerminal(input.out, output.in, err.in), true)
62-
repl.options = options
63-
64-
f(input.in, output.out, repl)
65-
t = @async begin
66-
close(input.in)
67-
close(output.in)
68-
close(err.in)
69-
end
70-
@test read(err.out, String) == ""
71-
#display(read(output.out, String))
72-
Base.wait(t)
73-
nothing
74-
end
75-
pwd_before = pwd()
76-
fake_repl() do stdin_write, stdout_read, repl
77-
repltask = @async REPL.run_repl(repl)
78-
79-
for line in split(Pkg.precompile_script, "\n"; keepempty=false)
80-
sleep(0.1)
81-
# Consume any extra output
82-
if bytesavailable(stdout_read) > 0
83-
copyback = readavailable(stdout_read)
84-
#@info(copyback)
85-
end
86-
87-
# Write the line
88-
write(stdin_write, line, "\n")
89-
90-
# Read until some kind of prompt
91-
readuntil(stdout_read, "\n")
92-
readuntil(stdout_read, ">")
93-
#@info(line)
94-
end
95-
96-
write(stdin_write, "\x04")
97-
wait(repltask)
98-
end
99-
cd(pwd_before) # something in the precompile_script changes the working directory
100-
end
101-
10249
@testset "Pkg.precompile" begin
10350
# sequential precompile, depth-first
10451
isolate() do; cd_tempdir() do tmp

0 commit comments

Comments
 (0)