From d25bd806baff269eee9ef284946756d1e13fe925 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 11 Oct 2025 14:39:26 +0200 Subject: [PATCH 1/3] Use ParallelTestRunner.jl --- test/Project.toml | 8 +- test/runtests.jl | 368 +++------------------------------------------- test/setup.jl | 85 ----------- 3 files changed, 22 insertions(+), 439 deletions(-) delete mode 100644 test/setup.jl diff --git a/test/Project.toml b/test/Project.toml index e6f21d04e..462c8e7b4 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,14 +1,14 @@ [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" -REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +ParallelTestRunner = "0.1.2" diff --git a/test/runtests.jl b/test/runtests.jl index 766c2041a..f674cd58b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,362 +1,30 @@ -using Distributed -using Dates -import REPL -using Printf: @sprintf +using ParallelTestRunner +using JLArrays -# parse some command-line arguments -function extract_flag!(args, flag, default=nothing) - for f in args - if startswith(f, flag) - # Check if it's just `--flag` or if it's `--flag=foo` - if f != flag - val = split(f, '=')[2] - if default !== nothing && !(typeof(default) <: AbstractString) - val = parse(typeof(default), val) - end - else - val = default - end +include("testsuite.jl") - # Drop this value from our args - filter!(x -> x != f, args) - return (true, val) - end - end - return (false, default) -end -do_help, _ = extract_flag!(ARGS, "--help") -if do_help - println(""" - Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...] - - --help Show this text. - --list List all available tests. - --quickfail Fail the entire run as soon as a single test errored. - --jobs=N Launch `N` processes to perform tests (default: Sys.CPU_THREADS). - - Remaining arguments filter the tests that will be executed.""") - exit(0) -end -_, jobs = extract_flag!(ARGS, "--jobs", Sys.CPU_THREADS) -do_quickfail, _ = extract_flag!(ARGS, "--quickfail") - -include("setup.jl") # make sure everything is precompiled - -@info "Running $jobs tests in parallel. If this is too many, specify the `--jobs` argument to the tests, or set the JULIA_CPU_THREADS environment variable." - -# choose tests -const tests = [] -const test_runners = Dict() -for AT in (JLArray, Array), name in keys(TestSuite.tests) - push!(tests, "$(AT)/$name") - test_runners["$(AT)/$name"] = ()->TestSuite.tests[name](AT) -end -unique!(tests) +const init_code = quote + using Test, JLArrays -# parse some more command-line arguments -## --list to list all available tests -do_list, _ = extract_flag!(ARGS, "--list") -if do_list - println("Available tests:") - for test in sort(tests) - println(" - $test") - end - exit(0) -end -## no options should remain -optlike_args = filter(startswith("-"), ARGS) -if !isempty(optlike_args) - error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)") -end -## the remaining args filter tests -if !isempty(ARGS) - filter!(tests) do test - any(arg->startswith(test, arg), ARGS) - end -end + include("testsuite.jl") -# add workers -const test_exeflags = Base.julia_cmd() -filter!(test_exeflags.exec) do c - return !(startswith(c, "--depwarn") || startswith(c, "--check-bounds")) -end -push!(test_exeflags.exec, "--check-bounds=yes") -push!(test_exeflags.exec, "--startup-file=no") -push!(test_exeflags.exec, "--depwarn=yes") -push!(test_exeflags.exec, "--project=$(Base.active_project())") -const test_exename = popfirst!(test_exeflags.exec) -function addworker(X; kwargs...) - withenv("JULIA_NUM_THREADS" => 1, "OPENBLAS_NUM_THREADS" => 1) do - procs = addprocs(X; exename=test_exename, exeflags=test_exeflags, kwargs...) - @everywhere procs include($(joinpath(@__DIR__, "setup.jl"))) - procs + # Disable Float16-related tests until JuliaGPU/KernelAbstractions#600 is resolved + @static if isdefined(JLArrays.KernelAbstractions, :POCL) + TestSuite.supported_eltypes(::Type{<:JLArray}) = + setdiff(TestSuite.supported_eltypes(), [Float16, ComplexF16]) end end -addworker(min(jobs, length(tests))) -# pretty print information about gc and mem usage -testgroupheader = "Test" -workerheader = "(Worker)" -name_align = maximum([textwidth(testgroupheader) + textwidth(" ") + - textwidth(workerheader); map(x -> textwidth(x) + - 3 + ndigits(nworkers()), tests)]) -elapsed_align = textwidth("Time (s)") -gc_align = textwidth("GC (s)") -percent_align = textwidth("GC %") -alloc_align = textwidth("Alloc (MB)") -rss_align = textwidth("RSS (MB)") -printstyled(" "^(name_align + textwidth(testgroupheader) - 3), " | ") -printstyled(" | ---------------- CPU ---------------- |\n", color=:white) -printstyled(testgroupheader, color=:white) -printstyled(lpad(workerheader, name_align - textwidth(testgroupheader) + 1), " | ", color=:white) -printstyled("Time (s) | GC (s) | GC % | Alloc (MB) | RSS (MB) |\n", color=:white) -print_lock = stdout isa Base.LibuvStream ? stdout.lock : ReentrantLock() -if stderr isa Base.LibuvStream - stderr.lock = print_lock -end -function print_testworker_stats(test, wrkr, resp) - @nospecialize resp - lock(print_lock) - try - printstyled(test, color=:white) - printstyled(lpad("($wrkr)", name_align - textwidth(test) + 1, " "), " | ", color=:white) - time_str = @sprintf("%7.2f",resp[2]) - printstyled(lpad(time_str, elapsed_align, " "), " | ", color=:white) - - cpu_gc_str = @sprintf("%5.2f", resp[4]) - printstyled(lpad(cpu_gc_str, gc_align, " "), " | ", color=:white) - # since there may be quite a few digits in the percentage, - # the left-padding here is less to make sure everything fits - cpu_percent_str = @sprintf("%4.1f", 100 * resp[4] / resp[2]) - printstyled(lpad(cpu_percent_str, percent_align, " "), " | ", color=:white) - cpu_alloc_str = @sprintf("%5.2f", resp[3] / 2^20) - printstyled(lpad(cpu_alloc_str, alloc_align, " "), " | ", color=:white) - - cpu_rss_str = @sprintf("%5.2f", resp[6] / 2^20) - printstyled(lpad(cpu_rss_str, rss_align, " "), " |\n", color=:white) - finally - unlock(print_lock) - end -end -global print_testworker_started = (name, wrkr)->begin -end -function print_testworker_errored(name, wrkr) - lock(print_lock) - try - printstyled(name, color=:red) - printstyled(lpad("($wrkr)", name_align - textwidth(name) + 1, " "), " |", - " "^elapsed_align, " failed at $(now())\n", color=:red) - finally - unlock(print_lock) - end +custom_tests = Dict{String, Expr}() +for AT in (JLArray, Array), name in keys(TestSuite.tests) + custom_tests["$(AT)/$name"] = :(TestSuite.tests[$name]($AT)) end -# run tasks -t0 = now() -results = [] -all_tasks = Task[] -all_tests = copy(tests) -try - # Monitor stdin and kill this task on ^C - # but don't do this on Windows, because it may deadlock in the kernel - t = current_task() - running_tests = Dict{String, DateTime}() - if !Sys.iswindows() && isa(stdin, Base.TTY) - stdin_monitor = @async begin - term = REPL.Terminals.TTYTerminal("xterm", stdin, stdout, stderr) - try - REPL.Terminals.raw!(term, true) - while true - c = read(term, Char) - if c == '\x3' - Base.throwto(t, InterruptException()) - break - elseif c == '?' - println("Currently running: ") - tests = sort(collect(running_tests), by=x->x[2]) - foreach(tests) do (test, date) - println(test, " (running for ", round(now()-date, Minute), ")") - end - end - end - catch e - isa(e, InterruptException) || rethrow() - finally - REPL.Terminals.raw!(term, false) - end - end - end - @sync begin - function recycle_worker(p) - rmprocs(p, waitfor=30) - return nothing - end - - for p in workers() - @async begin - push!(all_tasks, current_task()) - while length(tests) > 0 - test = popfirst!(tests) - - # sometimes a worker failed, and we need to spawn a new one - if p === nothing - p = addworker(1)[1] - end - wrkr = p - - local resp - - # run the test - running_tests[test] = now() - try - resp = remotecall_fetch(runtests, wrkr, test_runners[test], test) - catch e - isa(e, InterruptException) && return - resp = Any[e] - end - delete!(running_tests, test) - push!(results, (test, resp)) - - # act on the results - if resp[1] isa Exception - print_testworker_errored(test, wrkr) - do_quickfail && Base.throwto(t, InterruptException()) - - # the worker encountered some failure, recycle it - # so future tests get a fresh environment - p = recycle_worker(p) - else - print_testworker_stats(test, wrkr, resp) - - cpu_rss = resp[6] - if haskey(ENV, "CI") && cpu_rss > 3*2^30 - # XXX: collecting garbage - # after each test, we are leaking CPU memory somewhere. - # this is a problem on CI, where2 we don't have much RAM. - # work around this by periodically recycling the worker. - p = recycle_worker(p) - end - end - end - - if p !== nothing - recycle_worker(p) - end - end - end - end -catch e - isa(e, InterruptException) || rethrow() - # If the test suite was merely interrupted, still print the - # summary, which can be useful to diagnose what's going on - foreach(task -> begin - istaskstarted(task) || return - istaskdone(task) && return - try - schedule(task, InterruptException(); error=true) - catch ex - @error "InterruptException" exception=ex,catch_backtrace() - end - end, all_tasks) - for t in all_tasks - # NOTE: we can't just wait, but need to discard the exception, - # because the throwto for --quickfail also kills the worker. - try - wait(t) - catch e - showerror(stderr, e) - end - end -finally - if @isdefined stdin_monitor - schedule(stdin_monitor, InterruptException(); error=true) +function testfilter(test) + if startswith(test, "testsuite") + return false end + return true end -t1 = now() -elapsed = canonicalize(Dates.CompoundPeriod(t1-t0)) -println("Testing finished in $elapsed") -# construct a testset to render the test results -o_ts = Test.DefaultTestSet("Overall") -function with_testset(f, testset) - @static if VERSION >= v"1.13.0-DEV.1044" - Test.@with_testset testset f() - else - Test.push_testset(testset) - try - f() - finally - Test.pop_testset() - end - end -end -with_testset(o_ts) do - completed_tests = Set{String}() - for (testname, (resp,)) in results - push!(completed_tests, testname) - if isa(resp, Test.DefaultTestSet) - with_testset(resp) do - Test.record(o_ts, resp) - end - elseif isa(resp, Tuple{Int,Int}) - fake = Test.DefaultTestSet(testname) - for i in 1:resp[1] - Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) - end - for i in 1:resp[2] - Test.record(fake, Test.Broken(:test, nothing)) - end - with_testset(fake) do - Test.record(o_ts, fake) - end - elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException) - println("Worker $(resp.pid) failed running test $(testname):") - Base.showerror(stdout, resp.captured) - println() - fake = Test.DefaultTestSet(testname) - for i in 1:resp.captured.ex.pass - Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) - end - for i in 1:resp.captured.ex.broken - Test.record(fake, Test.Broken(:test, nothing)) - end - for t in resp.captured.ex.errors_and_fails - Test.record(fake, t) - end - with_testset(fake) do - Test.record(o_ts, fake) - end - else - if !isa(resp, Exception) - resp = ErrorException(string("Unknown result type : ", typeof(resp))) - end - # If this test raised an exception that is not a remote testset exception, - # i.e. not a RemoteException capturing a TestSetException that means - # the test runner itself had some problem, so we may have hit a segfault, - # deserialization errors or something similar. Record this testset as Errored. - fake = Test.DefaultTestSet(testname) - Test.record(fake, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack([(exception=resp,backtrace=[])]), LineNumberNode(1))) - with_testset(fake) do - Test.record(o_ts, fake) - end - end - end - for test in tests - (test in completed_tests) && continue - fake = Test.DefaultTestSet(test) - Test.record(fake, Test.Error(:test_interrupted, test, nothing, Base.ExceptionStack([(exception="skipped",backtrace=[])]), LineNumberNode(1))) - with_testset(fake) do - Test.record(o_ts, fake) - end - end -end -println() -Test.print_test_results(o_ts, 1) -if (VERSION >= v"1.13.0-DEV.1037" && !Test.anynonpass(o_ts)) || - (VERSION < v"1.13.0-DEV.1037" && !o_ts.anynonpass) - println(" \033[32;1mSUCCESS\033[0m") -else - println(" \033[31;1mFAILURE\033[0m\n") - Test.print_test_errors(o_ts) - throw(Test.FallbackTestSetException("Test run finished with errors")) -end +runtests(ARGS; init_code, custom_tests, testfilter) \ No newline at end of file diff --git a/test/setup.jl b/test/setup.jl deleted file mode 100644 index aa1c12e5e..000000000 --- a/test/setup.jl +++ /dev/null @@ -1,85 +0,0 @@ -using Distributed, Test, JLArrays - -include("testsuite.jl") - -# Disable Float16-related tests until JuliaGPU/KernelAbstractions#600 is resolved -@static if isdefined(JLArrays.KernelAbstractions, :POCL) - TestSuite.supported_eltypes(::Type{<:JLArray}) = - setdiff(TestSuite.supported_eltypes(), [Float16, ComplexF16]) -end - -using Random - -if VERSION >= v"1.13.0-DEV.1044" -using Base.ScopedValues -end - -## entry point - -function runtests(f, name) - function inner() - # generate a temporary module to execute the tests in - mod_name = Symbol("Test", rand(1:100), "Main_", replace(name, '/' => '_')) - mod = @eval(Main, module $mod_name end) - @eval(mod, using Test, Random, JLArrays) - - let id = myid() - wait(@spawnat 1 print_testworker_started(name, id)) - end - - ex = quote - GC.gc(true) - Random.seed!(1) - JLArrays.allowscalar(false) - - @timed @testset $"$name" begin - $f() - end - end - data = @static if VERSION < v"1.13.0-DEV.1044" - Core.eval(mod, ex) - else - @with Test.TESTSET_PRINT_ENABLE => false Core.eval(mod, ex) - end - #data[1] is the testset - - # process results - cpu_rss = Sys.maxrss() - if VERSION >= v"1.11.0-DEV.1529" - tc = Test.get_test_counts(data[1]) - passes,fails,error,broken,c_passes,c_fails,c_errors,c_broken = - tc.passes, tc.fails, tc.errors, tc.broken, tc.cumulative_passes, - tc.cumulative_fails, tc.cumulative_errors, tc.cumulative_broken - else - passes,fails,errors,broken,c_passes,c_fails,c_errors,c_broken = - Test.get_test_counts(data[1]) - end - if data[1].anynonpass == false - data = ((passes+c_passes,broken+c_broken), - data[2], - data[3], - data[4], - data[5]) - end - res = vcat(collect(data), cpu_rss) - - GC.gc(true) - res - end - - @static if VERSION >= v"1.13.0-DEV.1044" - @with Test.TESTSET_PRINT_ENABLE=>false begin - inner() - end - else - old_print_setting = Test.TESTSET_PRINT_ENABLE[] - Test.TESTSET_PRINT_ENABLE[] = false - try - inner() - finally - Test.TESTSET_PRINT_ENABLE[] = old_print_setting - end - end -end - -nothing # File is loaded via a remotecall to "include". Ensure it returns "nothing". From 6c9dc96140fa46f5763a20b083304597d879f5b4 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 11 Oct 2025 14:48:21 +0200 Subject: [PATCH 2/3] Switch to symbols instead of using the types directly --- test/runtests.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index f674cd58b..6bd6343db 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,4 @@ using ParallelTestRunner -using JLArrays include("testsuite.jl") @@ -9,14 +8,14 @@ const init_code = quote include("testsuite.jl") # Disable Float16-related tests until JuliaGPU/KernelAbstractions#600 is resolved - @static if isdefined(JLArrays.KernelAbstractions, :POCL) + if isdefined(JLArrays.KernelAbstractions, :POCL) TestSuite.supported_eltypes(::Type{<:JLArray}) = setdiff(TestSuite.supported_eltypes(), [Float16, ComplexF16]) end end custom_tests = Dict{String, Expr}() -for AT in (JLArray, Array), name in keys(TestSuite.tests) +for AT in (:JLArray, :Array), name in keys(TestSuite.tests) custom_tests["$(AT)/$name"] = :(TestSuite.tests[$name]($AT)) end From 2957e3302772e6991f733b3d9f9982e91579a870 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 14 Oct 2025 15:30:11 +0200 Subject: [PATCH 3/3] Update to ParallelTestRunner@1.0 --- test/Project.toml | 2 +- test/runtests.jl | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/Project.toml b/test/Project.toml index 462c8e7b4..cdd776948 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -11,4 +11,4 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] -ParallelTestRunner = "0.1.2" +ParallelTestRunner = "1" diff --git a/test/runtests.jl b/test/runtests.jl index 6bd6343db..7778d1068 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,5 @@ -using ParallelTestRunner +using ParallelTestRunner: runtests +import GPUArrays include("testsuite.jl") @@ -19,11 +20,11 @@ for AT in (:JLArray, :Array), name in keys(TestSuite.tests) custom_tests["$(AT)/$name"] = :(TestSuite.tests[$name]($AT)) end -function testfilter(test) +function test_filter(test) if startswith(test, "testsuite") return false end return true end -runtests(ARGS; init_code, custom_tests, testfilter) \ No newline at end of file +runtests(GPUArrays, ARGS; init_code, custom_tests, test_filter)