CI Build & Test: Show doctest failures, warnings as annotations in the 'Files changed' tab

.github/workflows/build.yml

@@ -185,14 +185,14 @@ jobs:
       - name: Test all files (sage -t --all --long)
         if: (success() || failure()) && steps.build.outcome == 'success'
         run: |
-          ../sage -python -m pip install coverage
-          ../sage -python -m coverage run ./bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303
-        working-directory: ./worktree-image/src
+          ./sage -python -m pip install coverage
+          ./sage -python -m coverage run ./src/bin/sage-runtests --all --long -p2 --format github --random-seed=286735480429121101562228604801325644303
+        working-directory: ./worktree-image
 
       - name: Prepare coverage results
         if: (success() || failure()) && steps.build.outcome == 'success'
         run: |
-          ./venv/bin/python3 -m coverage combine src/.coverage/
+          ./venv/bin/python3 -m coverage combine .coverage/
           ./venv/bin/python3 -m coverage xml
           mkdir -p coverage-report
           mv coverage.xml coverage-report/

src/bin/sage-runtests

@@ -34,6 +34,8 @@ if __name__ == "__main__":
     what.add_argument("--installed", action="store_true", default=False, help="test all installed modules of the Sage library")
     parser.add_argument("--logfile", type=argparse.FileType('a'), metavar="FILE", help="log all output to FILE")
 
+    parser.add_argument("--format", choices=["sage", "github"], default="sage",
+                        help="set format of error messages and warnings")
     parser.add_argument("-l", "--long", action="store_true", default=False, help="include lines with the phrase 'long time'")
     parser.add_argument("-s", "--short", dest="target_walltime", nargs='?',
                         type=int, default=-1, const=300, metavar="SECONDS",
@@ -141,7 +143,8 @@ if __name__ == "__main__":
             in_filenames = True
             new_arguments.append('--')
         new_arguments.append(arg)
-        afterlog = bool(arg == '--logfile')
+        afterlog = arg in ['--logfile', '--stats_path', '--stats-path',
+                           '--baseline_stats_path', '--baseline-stats-path']
 
     args = parser.parse_args(new_arguments)
 

src/sage/doctest/control.py

@@ -139,6 +139,7 @@ def __init__(self, **kwds):
         self.show_skipped = False
         self.target_walltime = -1
         self.baseline_stats_path = None
+        self.format = "sage"
 
         # sage-runtests contains more optional tags. Technically, adding
         # auto_optional_tags here is redundant, since that is added
@@ -1097,6 +1098,35 @@ def sort_key(source):
                 return -self.stats.get(basename, default).get('walltime', 0), basename
             self.sources = sorted(self.sources, key=sort_key)
 
+    def source_baseline(self, source):
+        r"""
+        Return the ``baseline_stats`` value of ``source``.
+
+        INPUT:
+
+        - ``source`` -- a :class:`DocTestSource` instance
+
+        OUTPUT:
+
+        A dictionary.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults, DocTestController
+            sage: from sage.env import SAGE_SRC
+            sage: import os
+            sage: filename = os.path.join(SAGE_SRC,'sage','doctest','util.py')
+            sage: DD = DocTestDefaults()
+            sage: DC = DocTestController(DD, [filename])
+            sage: DC.expand_files_into_sources()
+            sage: DC.source_baseline(DC.sources[0])
+            {}
+        """
+        if self.baseline_stats:
+            basename = source.basename
+            return self.baseline_stats.get(basename, {})
+        return {}
+
     def run_doctests(self):
         """
         Actually runs the doctests.
@@ -1142,6 +1172,8 @@ def run_doctests(self):
             iterations = ", ".join(iterations)
             if iterations:
                 iterations = " (%s)" % (iterations)
+            if self.baseline_stats:
+                self.log(f"Using --baseline-stats-path={self.options.baseline_stats_path}")
             self.log("Doctesting %s%s%s." % (filestr, threads, iterations))
             self.reporter = DocTestReporter(self)
             self.dispatcher = DocTestDispatcher(self)

src/sage/doctest/forker.py

@@ -513,7 +513,7 @@ def __init__(self, *args, **kwds):
 
         - ``stdout`` -- an open file to restore for debugging
 
-        - ``checker`` -- None, or an instance of
+        - ``checker`` -- ``None``, or an instance of
           :class:`doctest.OutputChecker`
 
         - ``verbose`` -- boolean, determines whether verbose printing
@@ -522,6 +522,8 @@ def __init__(self, *args, **kwds):
         - ``optionflags`` -- Controls the comparison with the expected
           output.  See :mod:`testmod` for more information.
 
+        - ``baseline`` -- dictionary, the ``baseline_stats`` value
+
         EXAMPLES::
 
             sage: from sage.doctest.parsing import SageOutputChecker
@@ -535,6 +537,7 @@ def __init__(self, *args, **kwds):
         O = kwds.pop('outtmpfile', None)
         self.msgfile = kwds.pop('msgfile', None)
         self.options = kwds.pop('sage_options')
+        self.baseline = kwds.pop('baseline', {})
         doctest.DocTestRunner.__init__(self, *args, **kwds)
         self._fakeout = SageSpoofInOut(O)
         if self.msgfile is None:
@@ -1232,16 +1235,41 @@ def _failure_header(self, test, example, message='Failed example:'):
         """
         out = [self.DIVIDER]
         with OriginalSource(example):
-            if test.filename:
-                if test.lineno is not None and example.lineno is not None:
-                    lineno = test.lineno + example.lineno + 1
+            if self.options.format == 'sage':
+                if test.filename:
+                    if test.lineno is not None and example.lineno is not None:
+                        lineno = test.lineno + example.lineno + 1
+                    else:
+                        lineno = '?'
+                    out.append('File "%s", line %s, in %s' %
+                               (test.filename, lineno, test.name))
+                else:
+                    out.append('Line %s, in %s' % (example.lineno + 1, test.name))
+                out.append(message)
+            elif self.options.format == 'github':
+                # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#using-workflow-commands-to-access-toolkit-functions
+                if message.startswith('Warning: '):
+                    command = f'::warning title={message}'
+                    message = message[len('Warning: '):]
+                elif self.baseline.get('failed', False):
+                    command = f'::notice title={message}'
+                    message += ' [failed in baseline]'
+                else:
+                    command = f'::error title={message}'
+                if extra := getattr(example, 'extra', None):
+                    message += f': {extra}'
+                if test.filename:
+                    command += f',file={test.filename}'
+                    if test.lineno is not None and example.lineno is not None:
+                        lineno = test.lineno + example.lineno + 1
+                        command += f',line={lineno}'
+                    lineno = None
                 else:
-                    lineno = '?'
-                out.append('File "%s", line %s, in %s' %
-                           (test.filename, lineno, test.name))
+                    command += f',line={example.lineno + 1}'
+                command += f'::{message}'
+                out.append(command)
             else:
-                out.append('Line %s, in %s' % (example.lineno + 1, test.name))
-            out.append(message)
+                raise ValueError(f'unknown format option: {self.options.format}')
             source = example.source
             out.append(doctest._indent(source))
             return '\n'.join(out)
@@ -1414,6 +1442,7 @@ def report_failure(self, out, test, example, got, globs):
         """
         if not self.options.initial or self.no_failure_yet:
             self.no_failure_yet = False
+            example.extra = f'Got: {got!r}'
             returnval = doctest.DocTestRunner.report_failure(self, out, test, example, got)
             if self.options.debug:
                 self._fakeout.stop_spoofing()
@@ -1564,6 +1593,8 @@ def report_unexpected_exception(self, out, test, example, exc_info):
         """
         if not self.options.initial or self.no_failure_yet:
             self.no_failure_yet = False
+
+            example.extra = "Exception raised: " + repr("".join(traceback.format_exception(*exc_info)))
             returnval = doctest.DocTestRunner.report_unexpected_exception(self, out, test, example, exc_info)
             if self.options.debug:
                 self._fakeout.stop_spoofing()
@@ -1721,12 +1752,14 @@ def serial_dispatch(self):
         """
         for source in self.controller.sources:
             heading = self.controller.reporter.report_head(source)
+            baseline = self.controller.source_baseline(source)
             if not self.controller.options.only_errors:
                 self.controller.log(heading)
 
             with tempfile.TemporaryFile() as outtmpfile:
                 result = DocTestTask(source)(self.controller.options,
-                                             outtmpfile, self.controller.logger)
+                                             outtmpfile, self.controller.logger,
+                                             baseline=baseline)
                 outtmpfile.seek(0)
                 output = bytes_to_str(outtmpfile.read())
 
@@ -1984,9 +2017,10 @@ def sel_exit():
                             # Start a new worker.
                             import copy
                             worker_options = copy.copy(opt)
+                            baseline = self.controller.source_baseline(source)
                             if target_endtime is not None:
                                 worker_options.target_walltime = (target_endtime - now) / (max(1, pending_tests / opt.nthreads))
-                            w = DocTestWorker(source, options=worker_options, funclist=[sel_exit])
+                            w = DocTestWorker(source, options=worker_options, funclist=[sel_exit], baseline=baseline)
                             heading = self.controller.reporter.report_head(w.source)
                             if not self.controller.options.only_errors:
                                 w.messages = heading + "\n"
@@ -2127,6 +2161,8 @@ class should be accessed by the child process.
     - ``funclist`` -- a list of callables to be called at the start of
       the child process.
 
+    - ``baseline`` -- dictionary, the ``baseline_stats`` value
+
     EXAMPLES::
 
         sage: from sage.doctest.forker import DocTestWorker, DocTestTask
@@ -2146,7 +2182,7 @@ class should be accessed by the child process.
         sage: reporter.report(FDS, False, W.exitcode, result, "")
             [... tests, ... s]
     """
-    def __init__(self, source, options, funclist=[]):
+    def __init__(self, source, options, funclist=[], baseline=None):
         """
         Initialization.
 
@@ -2170,6 +2206,7 @@ def __init__(self, source, options, funclist=[]):
         self.source = source
         self.options = options
         self.funclist = funclist
+        self.baseline = baseline
 
         # Open pipe for messages. These are raw file descriptors,
         # not Python file objects!
@@ -2241,7 +2278,8 @@ def run(self):
         os.close(self.rmessages)
         msgpipe = os.fdopen(self.wmessages, "w")
         try:
-            task(self.options, self.outtmpfile, msgpipe, self.result_queue)
+            task(self.options, self.outtmpfile, msgpipe, self.result_queue,
+                 baseline=self.baseline)
         finally:
             msgpipe.close()
             self.outtmpfile.close()
@@ -2507,7 +2545,8 @@ def __init__(self, source):
         """
         self.source = source
 
-    def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None):
+    def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None, *,
+                 baseline=None):
         """
         Calling the task does the actual work of running the doctests.
 
@@ -2524,6 +2563,8 @@ def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None):
         - ``result_queue`` -- an instance of :class:`multiprocessing.Queue`
           to store the doctest result. For testing, this can also be None.
 
+        - ``baseline`` -- a dictionary, the ``baseline_stats`` value.
+
         OUTPUT:
 
         - ``(doctests, result_dict)`` where ``doctests`` is the number of
@@ -2559,7 +2600,8 @@ def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None):
                 outtmpfile=outtmpfile,
                 msgfile=msgfile,
                 sage_options=options,
-                optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS)
+                optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,
+                baseline=baseline)
             runner.basename = self.source.basename
             runner.filename = self.source.path
             N = options.file_iterations