Skip to content

improve precompile generation script #37918

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 8, 2020
Merged

improve precompile generation script #37918

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 59 additions & 54 deletions contrib/generate_precompile.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ hardcoded_precompile_statements = """
@assert precompile(Tuple{typeof(Base.Experimental.register_error_hint), Any, Type})
"""

precompile_script = """
repl_script = """
2+2
print("")
@time 1+1
Expand All @@ -45,6 +45,9 @@ f(x) = x03
f(1,2)
[][1]
cd("complet_path\t\t$CTRL_C
"""

precompile_script = """
# Used by JuliaInterpreter
push!(Set{Module}(), Main)
push!(Set{Method}(), first(methods(collect)))
Expand Down Expand Up @@ -88,7 +91,7 @@ if Artifacts !== nothing
precompile_script *= """
using Artifacts, Base.BinaryPlatforms, Libdl
artifacts_toml = abspath(joinpath(Sys.STDLIB, "Artifacts", "test", "Artifacts.toml"))
cd(() -> @artifact_str("c_simple"), dirname(artifacts_toml))
# cd(() -> (name = "c_simple"; @artifact_str(name)), dirname(artifacts_toml))
artifacts = Artifacts.load_artifacts_toml(artifacts_toml)
platforms = [Artifacts.unpack_platform(e, "c_simple", artifacts_toml) for e in artifacts["c_simple"]]
best_platform = select_platform(Dict(p => triplet(p) for p in platforms))
Expand All @@ -102,7 +105,8 @@ Pkg = get(Base.loaded_modules,
nothing)

if Pkg !== nothing
precompile_script *= Pkg.precompile_script
# TODO: Split Pkg precompile script into REPL and script part
repl_script *= Pkg.precompile_script
end

FileWatching = get(Base.loaded_modules,
Expand Down Expand Up @@ -141,10 +145,17 @@ function generate_precompile_statements()
debug_output = devnull # or stdout
sysimg = Base.unsafe_string(Base.JLOptions().image_file)

# Precompile a package
global hardcoded_precompile_statements
# Extract the precompile statements from the precompile file
statements = Set{String}()

# From hardcoded statements
for statement in split(hardcoded_precompile_statements::String, '\n')
push!(statements, statement)
end

# Collect statements from running the script
mktempdir() do prec_path
# Also precompile a package here
pkgname = "__PackagePrecompilationStatementModule"
mkpath(joinpath(prec_path, pkgname, "src"))
path = joinpath(prec_path, pkgname, "src", "$pkgname.jl")
Expand All @@ -154,20 +165,20 @@ function generate_precompile_statements()
end
""")
tmp = tempname()
# Running compilecache on buildbots fails with
# `More than one command line CPU targets specified without a `--output-` flag specified`
# so start a new process without a CPU target specified
s = """
push!(DEPOT_PATH, $(repr(prec_path)));
Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp));
Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path)))
$precompile_script
"""
run(`$(julia_exepath()) -O0 --sysimage $sysimg --startup-file=no -Cnative -e $s`)
hardcoded_precompile_statements *= "\n" * read(tmp, String)
for statement in split(read(tmp, String), '\n')
push!(statements, statement)
end
end

mktemp() do precompile_file, precompile_file_h
# Run a repl process and replay our script
# Collect statements from running a REPL process and replaying our REPL script
pts, ptm = open_fake_pty()
blackhole = Sys.isunix() ? "/dev/null" : "nul"
if have_repl
Expand Down Expand Up @@ -212,12 +223,12 @@ function generate_precompile_statements()
readavailable(output_copy)
# Input our script
if have_repl
precompile_lines = split(precompile_script::String, '\n'; keepempty=false)
precompile_lines = split(repl_script::String, '\n'; keepempty=false)
curr = 0
for l in precompile_lines
sleep(0.1)
curr += 1
print("\rGenerating precompile statements... $curr/$(length(precompile_lines))")
print("\rGenerating REPL precompile statements... $curr/$(length(precompile_lines))")
# consume any other output
bytesavailable(output_copy) > 0 && readavailable(output_copy)
# push our input
Expand All @@ -237,57 +248,51 @@ function generate_precompile_statements()
close(ptm)
write(debug_output, "\n#### FINISHED ####\n")

# Extract the precompile statements from the precompile file
statements = Set{String}()
for statement in eachline(precompile_file_h)
for statement in split(read(precompile_file, String), '\n')
# Main should be completely clean
occursin("Main.", statement) && continue
push!(statements, statement)
end
end

for statement in split(hardcoded_precompile_statements::String, '\n')
push!(statements, statement)
end

# Create a staging area where all the loaded packages are available
PrecompileStagingArea = Module()
for (_pkgid, _mod) in Base.loaded_modules
if !(_pkgid.name in ("Main", "Core", "Base"))
eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod))
end
# Create a staging area where all the loaded packages are available
PrecompileStagingArea = Module()
for (_pkgid, _mod) in Base.loaded_modules
if !(_pkgid.name in ("Main", "Core", "Base"))
eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod))
end
end

# Execute the collected precompile statements
n_succeeded = 0
include_time = @elapsed for statement in sort(collect(statements))
# println(statement)
# The compiler has problem caching signatures with `Vararg{?, N}`. Replacing
# N with a large number seems to work around it.
statement = replace(statement, r"Vararg{(.*?), N} where N" => s"Vararg{\1, 100}")
try
Base.include_string(PrecompileStagingArea, statement)
n_succeeded += 1
print("\rExecuting precompile statements... $n_succeeded/$(length(statements))")
catch
# See #28808
# @error "Failed to precompile $statement"
end
# Execute the collected precompile statements
n_succeeded = 0
include_time = @elapsed for statement in sort(collect(statements))
# println(statement)
# The compiler has problem caching signatures with `Vararg{?, N}`. Replacing
# N with a large number seems to work around it.
statement = replace(statement, r"Vararg{(.*?), N} where N" => s"Vararg{\1, 100}")
try
Base.include_string(PrecompileStagingArea, statement)
n_succeeded += 1
print("\rExecuting precompile statements... $n_succeeded/$(length(statements))")
catch
# See #28808
# @error "Failed to precompile $statement"
end
println()
if have_repl
# Seems like a reasonable number right now, adjust as needed
# comment out if debugging script
@assert n_succeeded > 1200
end

tot_time = time_ns() - start_time
include_time *= 1e9
gen_time = tot_time - include_time
println("Precompilation complete. Summary:")
print("Total ─────── "); Base.time_print(tot_time); println()
print("Generation ── "); Base.time_print(gen_time); print(" "); show(IOContext(stdout, :compact=>true), gen_time / tot_time * 100); println("%")
print("Execution ─── "); Base.time_print(include_time); print(" "); show(IOContext(stdout, :compact=>true), include_time / tot_time * 100); println("%")
end
println()
if have_repl
# Seems like a reasonable number right now, adjust as needed
# comment out if debugging script
@assert n_succeeded > 1200
end

tot_time = time_ns() - start_time
include_time *= 1e9
gen_time = tot_time - include_time
println("Precompilation complete. Summary:")
print("Total ─────── "); Base.time_print(tot_time); println()
print("Generation ── "); Base.time_print(gen_time); print(" "); show(IOContext(stdout, :compact=>true), gen_time / tot_time * 100); println("%")
print("Execution ─── "); Base.time_print(include_time); print(" "); show(IOContext(stdout, :compact=>true), include_time / tot_time * 100); println("%")

return
end
Expand Down