build: improve precompile generation script (#37918)

Author: KristofferC

contrib/generate_precompile.jl

@@ -29,7 +29,7 @@ hardcoded_precompile_statements = """
 @assert precompile(Tuple{typeof(Base.Experimental.register_error_hint), Any, Type})
 """
 
-precompile_script = """
+repl_script = """
 2+2
 print("")
 @time 1+1
@@ -45,6 +45,9 @@ f(x) = x03
 f(1,2)
 [][1]
 cd("complet_path\t\t$CTRL_C
+"""
+
+precompile_script = """
 # Used by JuliaInterpreter
 push!(Set{Module}(), Main)
 push!(Set{Method}(), first(methods(collect)))
@@ -88,7 +91,7 @@ if Artifacts !== nothing
     precompile_script *= """
     using Artifacts, Base.BinaryPlatforms, Libdl
     artifacts_toml = abspath(joinpath(Sys.STDLIB, "Artifacts", "test", "Artifacts.toml"))
-    cd(() -> @artifact_str("c_simple"), dirname(artifacts_toml))
+    # cd(() -> (name = "c_simple"; @artifact_str(name)), dirname(artifacts_toml))
     artifacts = Artifacts.load_artifacts_toml(artifacts_toml)
     platforms = [Artifacts.unpack_platform(e, "c_simple", artifacts_toml) for e in artifacts["c_simple"]]
     best_platform = select_platform(Dict(p => triplet(p) for p in platforms))
@@ -102,7 +105,8 @@ Pkg = get(Base.loaded_modules,
           nothing)
 
 if Pkg !== nothing
-    precompile_script *= Pkg.precompile_script
+    # TODO: Split Pkg precompile script into REPL and script part
+    repl_script *= Pkg.precompile_script
 end
 
 FileWatching = get(Base.loaded_modules,
@@ -141,10 +145,17 @@ function generate_precompile_statements()
     debug_output = devnull # or stdout
     sysimg = Base.unsafe_string(Base.JLOptions().image_file)
 
-    # Precompile a package
-    global hardcoded_precompile_statements
+    # Extract the precompile statements from the precompile file
+    statements = Set{String}()
+
+    # From hardcoded statements
+    for statement in split(hardcoded_precompile_statements::String, '\n')
+        push!(statements, statement)
+    end
 
+    # Collect statements from running the script
     mktempdir() do prec_path
+        # Also precompile a package here
         pkgname = "__PackagePrecompilationStatementModule"
         mkpath(joinpath(prec_path, pkgname, "src"))
         path = joinpath(prec_path, pkgname, "src", "$pkgname.jl")
@@ -154,20 +165,20 @@ function generate_precompile_statements()
               end
               """)
         tmp = tempname()
-        # Running compilecache on buildbots fails with
-        # `More than one command line CPU targets specified without a `--output-` flag specified`
-        # so start a new process without a CPU target specified
         s = """
             push!(DEPOT_PATH, $(repr(prec_path)));
             Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp));
             Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path)))
+            $precompile_script
             """
         run(`$(julia_exepath()) -O0 --sysimage $sysimg --startup-file=no -Cnative -e $s`)
-        hardcoded_precompile_statements *= "\n" * read(tmp, String)
+        for statement in split(read(tmp, String), '\n')
+            push!(statements, statement)
+        end
     end
 
     mktemp() do precompile_file, precompile_file_h
-        # Run a repl process and replay our script
+        # Collect statements from running a REPL process and replaying our REPL script
         pts, ptm = open_fake_pty()
         blackhole = Sys.isunix() ? "/dev/null" : "nul"
         if have_repl
@@ -212,12 +223,12 @@ function generate_precompile_statements()
         readavailable(output_copy)
         # Input our script
         if have_repl
-            precompile_lines = split(precompile_script::String, '\n'; keepempty=false)
+            precompile_lines = split(repl_script::String, '\n'; keepempty=false)
             curr = 0
             for l in precompile_lines
                 sleep(0.1)
                 curr += 1
-                print("\rGenerating precompile statements... $curr/$(length(precompile_lines))")
+                print("\rGenerating REPL precompile statements... $curr/$(length(precompile_lines))")
                 # consume any other output
                 bytesavailable(output_copy) > 0 && readavailable(output_copy)
                 # push our input
@@ -237,57 +248,51 @@ function generate_precompile_statements()
         close(ptm)
         write(debug_output, "\n#### FINISHED ####\n")
 
-        # Extract the precompile statements from the precompile file
-        statements = Set{String}()
-        for statement in eachline(precompile_file_h)
+        for statement in split(read(precompile_file, String), '\n')
             # Main should be completely clean
             occursin("Main.", statement) && continue
             push!(statements, statement)
         end
+    end
 
-        for statement in split(hardcoded_precompile_statements::String, '\n')
-            push!(statements, statement)
-        end
-
-        # Create a staging area where all the loaded packages are available
-        PrecompileStagingArea = Module()
-        for (_pkgid, _mod) in Base.loaded_modules
-            if !(_pkgid.name in ("Main", "Core", "Base"))
-                eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod))
-            end
+    # Create a staging area where all the loaded packages are available
+    PrecompileStagingArea = Module()
+    for (_pkgid, _mod) in Base.loaded_modules
+        if !(_pkgid.name in ("Main", "Core", "Base"))
+            eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod))
         end
+    end
 
-        # Execute the collected precompile statements
-        n_succeeded = 0
-        include_time = @elapsed for statement in sort(collect(statements))
-            # println(statement)
-            # The compiler has problem caching signatures with `Vararg{?, N}`. Replacing
-            # N with a large number seems to work around it.
-            statement = replace(statement, r"Vararg{(.*?), N} where N" => s"Vararg{\1, 100}")
-            try
-                Base.include_string(PrecompileStagingArea, statement)
-                n_succeeded += 1
-                print("\rExecuting precompile statements... $n_succeeded/$(length(statements))")
-            catch
-                # See #28808
-                # @error "Failed to precompile $statement"
-            end
+    # Execute the collected precompile statements
+    n_succeeded = 0
+    include_time = @elapsed for statement in sort(collect(statements))
+        # println(statement)
+        # The compiler has problem caching signatures with `Vararg{?, N}`. Replacing
+        # N with a large number seems to work around it.
+        statement = replace(statement, r"Vararg{(.*?), N} where N" => s"Vararg{\1, 100}")
+        try
+            Base.include_string(PrecompileStagingArea, statement)
+            n_succeeded += 1
+            print("\rExecuting precompile statements... $n_succeeded/$(length(statements))")
+        catch
+            # See #28808
+            # @error "Failed to precompile $statement"
         end
-        println()
-        if have_repl
-            # Seems like a reasonable number right now, adjust as needed
-            # comment out if debugging script
-            @assert n_succeeded > 1200
-        end
-
-        tot_time = time_ns() - start_time
-        include_time *= 1e9
-        gen_time = tot_time - include_time
-        println("Precompilation complete. Summary:")
-        print("Total ─────── "); Base.time_print(tot_time); println()
-        print("Generation ── "); Base.time_print(gen_time);     print(" "); show(IOContext(stdout, :compact=>true), gen_time / tot_time * 100); println("%")
-        print("Execution ─── "); Base.time_print(include_time); print(" "); show(IOContext(stdout, :compact=>true), include_time / tot_time * 100); println("%")
     end
+    println()
+    if have_repl
+        # Seems like a reasonable number right now, adjust as needed
+        # comment out if debugging script
+        @assert n_succeeded > 1200
+    end
+
+    tot_time = time_ns() - start_time
+    include_time *= 1e9
+    gen_time = tot_time - include_time
+    println("Precompilation complete. Summary:")
+    print("Total ─────── "); Base.time_print(tot_time); println()
+    print("Generation ── "); Base.time_print(gen_time);     print(" "); show(IOContext(stdout, :compact=>true), gen_time / tot_time * 100); println("%")
+    print("Execution ─── "); Base.time_print(include_time); print(" "); show(IOContext(stdout, :compact=>true), include_time / tot_time * 100); println("%")
 
     return
 end