diff --git a/test/runtests.jl b/test/runtests.jl index 66d6a0963..1efbbcc94 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -276,75 +276,149 @@ t1 = now() elapsed = canonicalize(Dates.CompoundPeriod(t1-t0)) println("Testing finished in $elapsed") -# construct a testset to render the test results -o_ts = Test.DefaultTestSet("Overall") -Test.push_testset(o_ts) -completed_tests = Set{String}() -for (testname, (resp,)) in results - push!(completed_tests, testname) - if isa(resp, Test.DefaultTestSet) - Test.push_testset(resp) - Test.record(o_ts, resp) - Test.pop_testset() - elseif isa(resp, Tuple{Int,Int}) - fake = Test.DefaultTestSet(testname) - for i in 1:resp[1] - Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) - end - for i in 1:resp[2] - Test.record(fake, Test.Broken(:test, nothing)) +@static if VERSION < v"1.13.0-DEV.1044" + # construct a testset to render the test results + o_ts = Test.DefaultTestSet("Overall") + Test.push_testset(o_ts) + completed_tests = Set{String}() + for (testname, (resp,)) in results + push!(completed_tests, testname) + if isa(resp, Test.DefaultTestSet) + Test.push_testset(resp) + Test.record(o_ts, resp) + Test.pop_testset() + elseif isa(resp, Tuple{Int,Int}) + fake = Test.DefaultTestSet(testname) + for i in 1:resp[1] + Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) + end + for i in 1:resp[2] + Test.record(fake, Test.Broken(:test, nothing)) + end + Test.push_testset(fake) + Test.record(o_ts, fake) + Test.pop_testset() + elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException) + println("Worker $(resp.pid) failed running test $(testname):") + Base.showerror(stdout, resp.captured) + println() + fake = Test.DefaultTestSet(testname) + for i in 1:resp.captured.ex.pass + Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) + end + for i in 1:resp.captured.ex.broken + Test.record(fake, Test.Broken(:test, nothing)) + end + for t in resp.captured.ex.errors_and_fails + Test.record(fake, t) + end + Test.push_testset(fake) + Test.record(o_ts, fake) + Test.pop_testset() + else + if !isa(resp, Exception) + resp = ErrorException(string("Unknown result type : ", typeof(resp))) + end + # If this test raised an exception that is not a remote testset exception, + # i.e. not a RemoteException capturing a TestSetException that means + # the test runner itself had some problem, so we may have hit a segfault, + # deserialization errors or something similar. Record this testset as Errored. + fake = Test.DefaultTestSet(testname) + Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1))) + Test.push_testset(fake) + Test.record(o_ts, fake) + Test.pop_testset() end + end + for test in all_tests + (test in completed_tests) && continue + fake = Test.DefaultTestSet(test) + Test.record(fake, Test.Error(:test_interrupted, test, nothing, + [("skipped", [])], LineNumberNode(1))) Test.push_testset(fake) Test.record(o_ts, fake) Test.pop_testset() - elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException) - println("Worker $(resp.pid) failed running test $(testname):") - Base.showerror(stdout, resp.captured) - println() - fake = Test.DefaultTestSet(testname) - for i in 1:resp.captured.ex.pass - Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) - end - for i in 1:resp.captured.ex.broken - Test.record(fake, Test.Broken(:test, nothing)) + end + println() + Test.print_test_results(o_ts, 1) + if !o_ts.anynonpass + println(" \033[32;1mSUCCESS\033[0m") + else + println(" \033[31;1mFAILURE\033[0m\n") + Test.print_test_errors(o_ts) + throw(Test.FallbackTestSetException("Test run finished with errors")) + end +else + # construct a testset to render the test results + o_ts = Test.DefaultTestSet("Overall") + Test.@with_testset o_ts begin + completed_tests = Set{String}() + for (testname, (resp,)) in results + push!(completed_tests, testname) + if isa(resp, Test.DefaultTestSet) + Test.@with_testset resp begin + Test.record(o_ts, resp) + end + elseif isa(resp, Tuple{Int,Int}) + fake = Test.DefaultTestSet(testname) + for i in 1:resp[1] + Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) + end + for i in 1:resp[2] + Test.record(fake, Test.Broken(:test, nothing)) + end + Test.@with_testset fake begin + Test.record(o_ts, fake) + end + elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException) + println("Worker $(resp.pid) failed running test $(testname):") + Base.showerror(stdout, resp.captured) + println() + fake = Test.DefaultTestSet(testname) + for i in 1:resp.captured.ex.pass + Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) + end + for i in 1:resp.captured.ex.broken + Test.record(fake, Test.Broken(:test, nothing)) + end + for t in resp.captured.ex.errors_and_fails + Test.record(fake, t) + end + Test.@with_testset fake begin + Test.record(o_ts, fake) + end + else + if !isa(resp, Exception) + resp = ErrorException(string("Unknown result type : ", typeof(resp))) + end + # If this test raised an exception that is not a remote testset exception, + # i.e. not a RemoteException capturing a TestSetException that means + # the test runner itself had some problem, so we may have hit a segfault, + # deserialization errors or something similar. Record this testset as Errored. + fake = Test.DefaultTestSet(testname) + Test.record(fake, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack([(exception=resp,backtrace=[])]), LineNumberNode(1))) + Test.@with_testset fake begin + Test.record(o_ts, fake) + end + end end - for t in resp.captured.ex.errors_and_fails - Test.record(fake, t) + end + for test in all_tests + (test in completed_tests) && continue + fake = Test.DefaultTestSet(test) + Test.record(fake, Test.Error(:test_interrupted, test, nothing, Base.ExceptionStack([(exception="skipped",backtrace=[])]), LineNumberNode(1))) + Test.@with_testset fake begin + Test.record(o_ts, fake) end - Test.push_testset(fake) - Test.record(o_ts, fake) - Test.pop_testset() + end + println() + Test.print_test_results(o_ts, 1) + if !o_ts.anynonpass + println(" \033[32;1mSUCCESS\033[0m") else - if !isa(resp, Exception) - resp = ErrorException(string("Unknown result type : ", typeof(resp))) - end - # If this test raised an exception that is not a remote testset exception, - # i.e. not a RemoteException capturing a TestSetException that means - # the test runner itself had some problem, so we may have hit a segfault, - # deserialization errors or something similar. Record this testset as Errored. - fake = Test.DefaultTestSet(testname) - Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1))) - Test.push_testset(fake) - Test.record(o_ts, fake) - Test.pop_testset() + println(" \033[31;1mFAILURE\033[0m\n") + Test.print_test_errors(o_ts) + throw(Test.FallbackTestSetException("Test run finished with errors")) end end -for test in all_tests - (test in completed_tests) && continue - fake = Test.DefaultTestSet(test) - Test.record(fake, Test.Error(:test_interrupted, test, nothing, - [("skipped", [])], LineNumberNode(1))) - Test.push_testset(fake) - Test.record(o_ts, fake) - Test.pop_testset() -end -println() -Test.print_test_results(o_ts, 1) -if !o_ts.anynonpass - println(" \033[32;1mSUCCESS\033[0m") -else - println(" \033[31;1mFAILURE\033[0m\n") - Test.print_test_errors(o_ts) - throw(Test.FallbackTestSetException("Test run finished with errors")) -end diff --git a/test/setup.jl b/test/setup.jl index 1e06e2f06..ea0945b28 100644 --- a/test/setup.jl +++ b/test/setup.jl @@ -9,7 +9,11 @@ using Random function runtests(f, name) old_print_setting = Test.TESTSET_PRINT_ENABLE[] - Test.TESTSET_PRINT_ENABLE[] = false + @static if VERSION < v"1.13.0-DEV.1044" + Test.TESTSET_PRINT_ENABLE[] = false + else + Test.TESTSET_PRINT_ENABLE[] => false + end try # generate a temporary module to execute the tests in @@ -56,7 +60,11 @@ function runtests(f, name) GC.gc(true) res finally - Test.TESTSET_PRINT_ENABLE[] = old_print_setting + @static if VERSION < v"1.13.0-DEV.1044" + Test.TESTSET_PRINT_ENABLE[] = old_print_setting + else + Test.TESTSET_PRINT_ENABLE[] => old_print_setting + end end end