Skip to content

[reporting] Add summary message for failures/errors/skipped tests #2332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
daall merged 1 commit into from
Oct 14, 2020
Merged

[reporting] Add summary message for failures/errors/skipped tests #2332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 7 additions & 20 deletions test_reporting/junit_xml_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,10 @@
import defusedxml.ElementTree as ET


TEST_REPORT_CLIENT_VERSION = (1, 0, 0)
TEST_REPORT_CLIENT_VERSION = (1, 1, 0)

MAXIMUM_XML_SIZE = 20e7 # 20MB
MAXIMUM_SUMMARY_SIZE = 1024 # 1MB

# Fields found in the testsuite/root section of the JUnit XML file.
TESTSUITE_TAG = "testsuite"
Expand Down Expand Up @@ -264,25 +265,6 @@ def _validate_test_case(test_case):
f"\"{test_case.get('name', 'Name Not Found')}\""
)

# NOTE: "if failure" does not work with the ETree library.
failure = test_case.find("failure")
if failure is not None and failure.get("message") is None:
raise JUnitXMLValidationError(
f"no message found for failure in \"{test_case.get('name')}\""
)

error = test_case.find("error")
if error is not None and error.get("message") is None:
raise JUnitXMLValidationError(
f"no message found for error in \"{test_case.get('name')}\""
)

skipped = test_case.find("skipped")
if skipped is not None and skipped.get("message") is None:
raise JUnitXMLValidationError(
f"no message found for skip in \"{test_case.get('name')}\""
)

cases = root.findall(TESTCASE_TAG)

for test_case in cases:
Expand Down Expand Up @@ -373,13 +355,18 @@ def _parse_test_case(test_case):
# errored out during setup or teardown.
if failure is not None:
result["result"] = "failure"
summary = failure.get("message", "")
elif skipped is not None:
result["result"] = "skipped"
summary = skipped.get("message", "")
elif error is not None:
result["result"] = "error"
summary = error.get("message", "")
else:
result["result"] = "success"
summary = ""

result["summary"] = summary[:min(len(summary), MAXIMUM_SUMMARY_SIZE)]
result["error"] = error is not None

return feature, result
Expand Down
7 changes: 4 additions & 3 deletions test_reporting/tests/test_junit_xml_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
{
"classname": "acl.test_acl",
"result": "error",
"summary": "test machine broke",
"file": "acl/test_acl.py",
"line": "257",
"name": "test_acl",
Expand All @@ -58,6 +59,7 @@
"line": "369",
"name": "test_acl_2",
"result": "skipped",
"summary": "test machine skipped",
"time": "0.0",
"error": False
}
Expand All @@ -70,11 +72,13 @@
"name": "test_bgp_fact",
"time": "109.472",
"result": "success",
"summary": "",
"error": False
},
{
"classname": "bgp.test_bgp",
"result": "failure",
"summary": "test machine go brr",
"file": "bgp/test_bgp.py",
"line": "248",
"name": "test_bgp_speaker",
Expand Down Expand Up @@ -193,9 +197,6 @@ def test_invalid_junit_xml_metadata_errors(token, replacement, message):
[
("classname", "hehe", ".* not found in test case .*"),
('name="test_acl"', "", '.* not found in test case "Name Not Found"'),
('message="test machine go brr"', "", "no message found for failure .*"),
('message="test machine broke"', "", "no message found for error .*"),
('message="test machine skipped"', "", "no message found for skip .*"),
],
)
def test_invalid_junit_xml_test_case_errors(token, replacement, message):
Expand Down