Skip to content

Support @test_throws just checking the error message #41888

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
timholy merged 3 commits into from
Aug 17, 2021
Merged

Support @test_throws just checking the error message #41888

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ac4ec80
Support `@test_throws` just checking the error message
timholy Aug 14, 2021
016ddf9
Address review comments
timholy Aug 17, 2021
5147b46
Better NEWS and docstring
timholy Aug 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions NEWS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ New library functions
New library features
--------------------

* `@test_throws "some message" triggers_error()` can now be used to check whether the displayed error text
contains "some message" regardless of the specific exception type.
Regular expressions are also supported. ([#41888)

Standard library changes
------------------------
Expand Down
76 changes: 53 additions & 23 deletions stdlib/Test/src/Test.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,9 @@ struct Pass <: Result
data
value
source::Union{Nothing,LineNumberNode}
function Pass(test_type::Symbol, orig_expr, data, thrown, source=nothing)
return new(test_type, orig_expr, data, thrown isa String ? "String" : thrown, source)
message_only::Bool
function Pass(test_type::Symbol, orig_expr, data, thrown, source=nothing, message_only=false)
return new(test_type, orig_expr, data, thrown, source, message_only)
end
end

Expand All @@ -98,7 +99,11 @@ function Base.show(io::IO, t::Pass)
end
if t.test_type === :test_throws
# The correct type of exception was thrown
print(io, "\n Thrown: ", t.value isa String ? t.value : typeof(t.value))
if t.message_only
print(io, "\n Message: ", t.value)
else
print(io, "\n Thrown: ", typeof(t.value))
end
elseif t.test_type === :test && t.data !== nothing
# The test was an expression, so display the term-by-term
# evaluated version as well
Expand All @@ -118,12 +123,14 @@ struct Fail <: Result
data::Union{Nothing, String}
value::String
source::LineNumberNode
function Fail(test_type::Symbol, orig_expr, data, value, source::LineNumberNode)
message_only::Bool
function Fail(test_type::Symbol, orig_expr, data, value, source::LineNumberNode, message_only::Bool=false)
return new(test_type,
string(orig_expr),
data === nothing ? nothing : string(data),
string(isa(data, Type) ? typeof(value) : value),
source)
source,
message_only)
end
end

Expand All @@ -132,18 +139,24 @@ function Base.show(io::IO, t::Fail)
print(io, " at ")
printstyled(io, something(t.source.file, :none), ":", t.source.line, "\n"; bold=true, color=:default)
print(io, " Expression: ", t.orig_expr)
value, data = t.value, t.data
if t.test_type === :test_throws_wrong
# An exception was thrown, but it was of the wrong type
print(io, "\n Expected: ", t.data)
print(io, "\n Thrown: ", t.value)
if t.message_only
print(io, "\n Expected: ", data)
print(io, "\n Message: ", value)
else
print(io, "\n Expected: ", data)
print(io, "\n Thrown: ", value)
end
elseif t.test_type === :test_throws_nothing
# An exception was expected, but no exception was thrown
print(io, "\n Expected: ", t.data)
print(io, "\n Expected: ", data)
print(io, "\n No exception thrown")
elseif t.test_type === :test && t.data !== nothing
elseif t.test_type === :test && data !== nothing
# The test was an expression, so display the term-by-term
# evaluated version as well
print(io, "\n Evaluated: ", t.data)
print(io, "\n Evaluated: ", data)
end
end

Expand Down Expand Up @@ -238,6 +251,7 @@ function Serialization.serialize(s::Serialization.AbstractSerializer, t::Pass)
Serialization.serialize(s, t.data === nothing ? nothing : string(t.data))
Serialization.serialize(s, string(t.value))
Serialization.serialize(s, t.source === nothing ? nothing : t.source)
Serialization.serialize(s, t.message_only)
nothing
end

Expand Down Expand Up @@ -657,6 +671,7 @@ end

Tests that the expression `expr` throws `exception`.
The exception may specify either a type,
a string or regular expression occurring in the displayed error message,
or a value (which will be tested for equality by comparing fields).
Note that `@test_throws` does not support a trailing keyword form.

Expand All @@ -671,6 +686,11 @@ julia> @test_throws DimensionMismatch [1, 2, 3] + [1, 2]
Test Passed
Expression: [1, 2, 3] + [1, 2]
Thrown: DimensionMismatch

julia> @test_throws "Try sqrt(Complex" sqrt(-1)
Test Passed
Expression: sqrt(-1)
Message: "DomainError with -1.0:\\nsqrt will only return a complex result if called with a complex argument. Try sqrt(Complex(x))."
```
"""
macro test_throws(extype, ex)
Expand All @@ -697,13 +717,17 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype)
if isa(result, Threw)
# Check that the right type of exception was thrown
success = false
message_only = false
exc = result.exception
# NB: Throwing LoadError from macroexpands is deprecated, but in order to limit
# the breakage in package tests we add extra logic here.
from_macroexpand =
orig_expr isa Expr &&
orig_expr.head in (:call, :macrocall) &&
orig_expr.args[1] in MACROEXPAND_LIKE
if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc)
extype = extype.error # deprecated
end
if isa(extype, Type)
success =
if from_macroexpand && extype == LoadError && exc isa Exception
Expand All @@ -712,24 +736,30 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype)
else
isa(exc, extype)
end
else
if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc)
extype = extype.error # deprecated
end
if isa(exc, typeof(extype))
success = true
for fld in 1:nfields(extype)
if !isequal(getfield(extype, fld), getfield(exc, fld))
success = false
break
end
elseif isa(exc, typeof(extype))
success = true
for fld in 1:nfields(extype)
if !isequal(getfield(extype, fld), getfield(exc, fld))
success = false
break
end
end
elseif isa(extype, Exception)
else
message_only = true
exc = sprint(showerror, exc)
success = contains_warn(exc, extype)
exc = '"' * escape_string(exc) * '"'
if isa(extype, AbstractString)
extype = '"' * escape_string(extype) * '"'
elseif isa(extype, Function)
extype = "< match function >"
end
end
if success
testres = Pass(:test_throws, orig_expr, extype, exc, result.source)
testres = Pass(:test_throws, orig_expr, extype, exc, result.source, message_only)
else
testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source)
testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source, message_only)
end
else
testres = Fail(:test_throws_nothing, orig_expr, extype, nothing, result.source)
Expand Down
36 changes: 35 additions & 1 deletion stdlib/Test/test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,14 @@ end
"Thrown: ErrorException")
@test endswith(sprint(show, @test_throws ErrorException("test") error("test")),
"Thrown: ErrorException")
@test endswith(sprint(show, @test_throws "a test" error("a test")),
"Message: \"a test\"")
@test occursin("Message: \"DomainError",
sprint(show, @test_throws r"sqrt\([Cc]omplex" sqrt(-1)))
@test endswith(sprint(show, @test_throws str->occursin("a t", str) error("a test")),
"Message: \"a test\"")
@test endswith(sprint(show, @test_throws ["BoundsError", "access", "1-element", "at index [2]"] [1][2]),
"Message: \"BoundsError: attempt to access 1-element Vector{$Int} at index [2]\"")
end
# Test printing of Fail results
include("nothrow_testset.jl")
Expand Down Expand Up @@ -148,6 +156,11 @@ let fails = @testset NoThrowTestSet begin
@test contains(str1, str2)
# 22 - Fail - Type Comparison
@test typeof(1) <: typeof("julia")
# 23 - 26 - Fail - wrong message
@test_throws "A test" error("a test")
@test_throws r"sqrt\([Cc]omplx" sqrt(-1)
@test_throws str->occursin("a T", str) error("a test")
@test_throws ["BoundsError", "acess", "1-element", "at index [2]"] [1][2]
end
for fail in fails
@test fail isa Test.Fail
Expand Down Expand Up @@ -262,6 +275,27 @@ let fails = @testset NoThrowTestSet begin
@test occursin("Expression: typeof(1) <: typeof(\"julia\")", str)
@test occursin("Evaluated: $(typeof(1)) <: $(typeof("julia"))", str)
end

let str = sprint(show, fails[23])
@test occursin("Expected: \"A test\"", str)
@test occursin("Message: \"a test\"", str)
end

let str = sprint(show, fails[24])
@test occursin("Expected: r\"sqrt\\([Cc]omplx\"", str)
@test occursin(r"Message: .*Try sqrt\(Complex", str)
end

let str = sprint(show, fails[25])
@test occursin("Expected: < match function >", str)
@test occursin("Message: \"a test\"", str)
end

let str = sprint(show, fails[26])
@test occursin("Expected: [\"BoundsError\", \"acess\", \"1-element\", \"at index [2]\"]", str)
@test occursin(r"Message: \"BoundsError.* 1-element.*at index \[2\]", str)
end

end

let errors = @testset NoThrowTestSet begin
Expand Down Expand Up @@ -1202,4 +1236,4 @@ Test.finish(ts::PassInformationTestSet) = ts
@test ts.results[2].data == ErrorException
@test ts.results[2].value == ErrorException("Msg")
@test ts.results[2].source == LineNumberNode(test_throws_line_number, @__FILE__)
end
end