fix finding of failed tests in output of PyTorch test step

easybuild/easyblocks/p/pytorch.py

@@ -42,6 +42,98 @@
 from easybuild.tools.systemtools import POWER, get_cpu_architecture
 
 
+def get_count_for_pattern(regex, text):
+    """Match the regexp containing a single group and return the integer value of the matched group.
+       Return zero if no or more than 1 match was found and warn for the latter case
+    """
+    match = re.findall(regex, text)
+    if len(match) == 1:
+        return int(match[0])
+    elif len(match) > 1:
+        # Shouldn't happen, but means something went wrong with the regular expressions.
+        # Throw warning, as the build might be fine, no need to error on this.
+        warn_msg = "Error in counting the number of test failures in the output of the PyTorch test suite.\n"
+        warn_msg += "Please check the EasyBuild log to verify the number of failures (if any) was acceptable."
+        print_warning(warn_msg)
+    return 0
+
+
+def extract_failed_tests_info(tests_out):
+    """
+    Extract information about failed tests from output produced by PyTorch tests
+    """
+
+    # Create clear summary report
+    failure_report = ""
+    failure_cnt = 0
+    error_cnt = 0
+    failed_test_suites = []
+
+    # Look for patterns like:
+    # Ran 219 tests in 67.325s
+    #
+    # FAILED (errors=10, skipped=190, expected failures=6)
+    # test_fx failed!
+    regex = (r"^Ran (?P<test_cnt>[0-9]+) tests.*$\n\n"
+             r"FAILED \((?P<failure_summary>.*)\)$\n"
+             r"(?:^(?:(?!failed!).)*$\n)*"
+             r"(?P<failed_test_suite_name>.*) failed!(?: Received signal: \w+)?\s*$")
+
+    for match in re.finditer(regex, tests_out, re.M):
+        # E.g. 'failures=3, errors=10, skipped=190, expected failures=6'
+        info = match.groupdict()
+        failure_summary = info['failure_summary']
+        test_cnt, test_suite = info['test_cnt'], info['failed_test_suite_name']
+        failure_report += test_suite + ' (' + test_cnt + " total tests, " + failure_summary + ')\n'
+        failure_cnt += get_count_for_pattern(r"(?<!expected )failures=([0-9]+)", failure_summary)
+        error_cnt += get_count_for_pattern(r"errors=([0-9]+)", failure_summary)
+        failed_test_suites.append(test_suite)
+
+    # Look for patterns like:
+    # ===================== 2 failed, 128 passed, 2 skipped, 2 warnings in 3.43s =====================
+    regex = r"^=+ (?P<failure_summary>.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P<failed_test_suite_name>.*) failed!$"
+
+    for match in re.finditer(regex, tests_out, re.M):
+        # E.g. '2 failed, 128 passed, 2 skipped, 2 warnings'
+        info = match.groupdict()
+        failure_summary, test_suite = info['failure_summary'], info['failed_test_suite_name']
+        failure_report += test_suite + ' ' + failure_summary + '\n'
+        failure_cnt += get_count_for_pattern(r"([0-9]+) failed", failure_summary)
+        error_cnt += get_count_for_pattern(r"([0-9]+) error", failure_summary)
+        failed_test_suites.append(test_suite)
+
+    # Look for patterns like:
+    # AssertionError: 4 unit test(s) failed:
+    # ...
+    # distributed/test_c10d_gloo failed!
+    regex = (r"^AssertionError: (?P<failed_test_cnt>[0-9]+) unit test\(s\) failed:$\n"
+             r"(?:^(?:(?!failed!).)*$\n)*"
+             r"(?P<failed_test_group_name>.*) failed!$")
+    for match in re.finditer(regex, tests_out, re.M):
+        info = match.groupdict()
+        test_group, failed_test_cnt = info['failed_test_group_name'], info['failed_test_cnt']
+        failure_report += test_group + ' (' + failed_test_cnt + " failed tests)\n"
+        failure_cnt += int(failed_test_cnt)
+        failed_test_suites.append(test_group)
+
+    # Look for patterns like:
+    # Running test_jit_cuda_fuser ... [2023-01-12 04:04:08.949222]
+    # ...
+    # AttributeError: 'NoneType' object has no attribute 'split'
+    # test_jit_cuda_fuser failed!
+    regex = (r"^Running (?P<test_name>test_[^\s]+) .*\n"
+             r"(?:^(?:(?!failed!).)*$\n)*"
+             r"\1 failed!$")
+    for match in re.finditer(regex, tests_out, re.M):
+        test_name = match.group('test_name')
+        failure_report += test_name + " (unknown failed test count)\n"
+        failure_cnt += 1
+        failed_test_suites.append(test_name)
+
+    return failure_report, failure_cnt, error_cnt, failed_test_suites
+
+
+
 class EB_PyTorch(PythonPackage):
     """Support for building/installing PyTorch."""
 
@@ -268,62 +360,7 @@ def test_step(self):
 
         tests_out, tests_ec = super(EB_PyTorch, self).test_step(return_output_ec=True)
 
-        def get_count_for_pattern(regex, text):
-            """Match the regexp containing a single group and return the integer value of the matched group.
-               Return zero if no or more than 1 match was found and warn for the latter case
-            """
-            match = re.findall(regex, text)
-            if len(match) == 1:
-                return int(match[0])
-            elif len(match) > 1:
-                # Shouldn't happen, but means something went wrong with the regular expressions.
-                # Throw warning, as the build might be fine, no need to error on this.
-                warn_msg = "Error in counting the number of test failures in the output of the PyTorch test suite.\n"
-                warn_msg += "Please check the EasyBuild log to verify the number of failures (if any) was acceptable."
-                print_warning(warn_msg)
-            return 0
-
-        # Create clear summary report
-        failure_report = ""
-        failure_cnt = 0
-        error_cnt = 0
-        failed_test_suites = []
-
-        # Grep for patterns like:
-        # Ran 219 tests in 67.325s
-        #
-        # FAILED (errors=10, skipped=190, expected failures=6)
-        # test_fx failed!
-        regex = (r"^Ran (?P<test_cnt>[0-9]+) tests.*$\n\n"
-                 r"FAILED \((?P<failure_summary>.*)\)$\n"
-                 r"(?:^(?:(?!failed!).)*$\n)*"
-                 r"(?P<failed_test_suite_name>.*) failed!(?: Received signal: \w+)?\s*$")
-
-        for m in re.finditer(regex, tests_out, re.M):
-            # E.g. 'failures=3, errors=10, skipped=190, expected failures=6'
-            failure_summary = m.group('failure_summary')
-            total, test_suite = m.group('test_cnt', 'failed_test_suite_name')
-            failure_report += "{test_suite} ({total} total tests, {failure_summary})\n".format(
-                    test_suite=test_suite, total=total, failure_summary=failure_summary
-                )
-            failure_cnt += get_count_for_pattern(r"(?<!expected )failures=([0-9]+)", failure_summary)
-            error_cnt += get_count_for_pattern(r"errors=([0-9]+)", failure_summary)
-            failed_test_suites.append(test_suite)
-
-        # Grep for patterns like:
-        # ===================== 2 failed, 128 passed, 2 skipped, 2 warnings in 3.43s =====================
-        regex = r"^=+ (?P<failure_summary>.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P<failed_test_suite_name>.*) failed!$"
-
-        for m in re.finditer(regex, tests_out, re.M):
-            # E.g. '2 failed, 128 passed, 2 skipped, 2 warnings'
-            failure_summary = m.group('failure_summary')
-            test_suite = m.group('failed_test_suite_name')
-            failure_report += "{test_suite} ({failure_summary})\n".format(
-                    test_suite=test_suite, failure_summary=failure_summary
-                )
-            failure_cnt += get_count_for_pattern(r"([0-9]+) failed", failure_summary)
-            error_cnt += get_count_for_pattern(r"([0-9]+) error", failure_summary)
-            failed_test_suites.append(test_suite)
+        failure_report, failure_cnt, error_cnt, failed_test_suites = extract_failed_tests_info(tests_out)
 
         # Make the names unique and sorted
         failed_test_suites = sorted(set(failed_test_suites))

test/easyblocks/easyblock_specific.py

@@ -40,6 +40,7 @@
 from easybuild.base.testing import TestCase
 from easybuild.easyblocks.generic.cmakemake import det_cmake_version
 from easybuild.easyblocks.generic.toolchain import Toolchain
+from easybuild.easyblocks import pytorch
 from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance
 from easybuild.framework.easyconfig.easyconfig import process_easyconfig
 from easybuild.tools import config
@@ -51,6 +52,72 @@
 from easybuild.tools.options import set_tmpdir
 from easybuild.tools.py2vs3 import StringIO
 
+PYTORCH_TESTS_OUTPUT = """
+...
+AssertionError: Expected zero exit code but got -6 for pid: 2006681
+
+----------------------------------------------------------------------
+Ran 2 tests in 6.576s
+
+FAILED (failures=2)
+distributed/fsdp/test_fsdp_input failed!
+Running distributed/fsdp/test_fsdp_multiple_forward ... [2023-01-12 05:46:45.746098]
+
+RuntimeError: Process 0 terminated or timed out after 610.0615825653076 seconds
+
+----------------------------------------------------------------------
+Ran 1 test in 610.744s
+
+FAILED (errors=1)
+Test exited with non-zero exitcode 1. Command to reproduce: /software/Python/3.9.6-GCCcore-11.2.0/bin/python distributed/test_c10d_gloo.py -v DistributedDataParallelTest.test_ddp_comm_hook_register_just_once
+
+RuntimeError: Process 0 terminated or timed out after 610.0726096630096 seconds
+
+----------------------------------------------------------------------
+Ran 1 test in 610.729s
+
+FAILED (errors=1)
+Test exited with non-zero exitcode 1. Command to reproduce: /software/Python/3.9.6-GCCcore-11.2.0/bin/python distributed/test_c10d_gloo.py -v DistributedDataParallelTest.test_ddp_invalid_comm_hook_init
+test_ddp_invalid_comm_hook_return_type (__main__.DistributedDataParallelTest)
+
+AssertionError: 4 unit test(s) failed:
+	DistributedDataParallelTest.test_ddp_comm_hook_register_just_once
+	DistributedDataParallelTest.test_ddp_invalid_comm_hook_init
+	ProcessGroupGlooTest.test_round_robin
+	ProcessGroupGlooTest.test_round_robin_create_destroy
+distributed/test_c10d_gloo failed!
+Running distributed/test_c10d_nccl ... [2023-01-12 07:43:41.085197]
+
+ValueError: For each axis slice, the sum of the observed frequencies must agree with the sum of the expected frequencies to a relative tolerance of 1e-08, but the percent differences are:
+4.535600093557479e-05
+
+----------------------------------------------------------------------
+Ran 216 tests in 22.396s
+
+FAILED (errors=4)
+distributions/test_distributions failed!
+
+...
+
+----------------------------------------------------------------------
+Ran 464 tests in 18.443s
+
+FAILED (failures=1, skipped=52, expected failures=1)
+test_autograd failed!
+Running test_binary_ufuncs ... [2023-01-12 09:02:45.049490]
+...
+
+Running test_jit_cuda_fuser ... [2023-01-12 04:04:08.949222]
+Executing ['/user/gent/400/vsc40023/eb_arcaninescratch/RHEL8/skylake-ib/software/Python/3.9.6-GCCcore-11.2.0/bin/python', 'test_jit_cuda_fuser.py', '-v'] ... [2023-01-12 04:04:08.949319]
+CUDA not available, skipping tests
+monkeytype is not installed. Skipping tests for Profile-Directed Typing
+Traceback (most recent call last):
+  File "/tmp/vsc40023/easybuild_build/PyTorch/1.11.0/foss-2021b/pytorch-v1.11.0/test/test_jit_cuda_fuser.py", line 25, in <module>
+    CUDA_MAJOR, CUDA_MINOR = (int(x) for x in torch.version.cuda.split('.'))
+AttributeError: 'NoneType' object has no attribute 'split'
+test_jit_cuda_fuser failed!
+"""  # noqa
+
 
 class EasyBlockSpecificTest(TestCase):
     """ Baseclass for easyblock testcases """
@@ -265,6 +332,30 @@ def test_det_cmake_version(self):
         """))
         self.assertEqual(det_cmake_version(), '1.2.3-rc4')
 
+    def test_pytorch_extract_failed_tests_info(self):
+        """
+        Test extract_failed_tests_info function from PyTorch easyblock.
+        """
+        res = pytorch.extract_failed_tests_info(PYTORCH_TESTS_OUTPUT)
+        self.assertEqual(len(res), 4)
+
+        expected_failure_report = '\n'.join([
+            "distributed/fsdp/test_fsdp_input (2 total tests, failures=2)",
+            "distributions/test_distributions (216 total tests, errors=4)",
+            "test_autograd (464 total tests, failures=1, skipped=52, expected failures=1)",
+            "distributed/test_c10d_gloo (4 failed tests)",
+            "test_jit_cuda_fuser (unknown failed test count)",
+        ])
+        self.assertEqual(res[0].strip(), expected_failure_report)
+        # test failures
+        self.assertEqual(res[1], 8)
+        # test errors
+        self.assertEqual(res[2], 4)
+
+        expected_failed_test_suites = ['distributed/fsdp/test_fsdp_input', 'distributions/test_distributions',
+                                       'test_autograd', 'distributed/test_c10d_gloo', 'test_jit_cuda_fuser']
+        self.assertEqual(res[3], expected_failed_test_suites)
+
 
 def suite():
     """Return all easyblock-specific tests."""