gh-38619: Re-add py3.9 support to sage_autodoc.py
    
This PR corrects the fact the sage_autodoc.py changes needed for sphinx
8 dropped python 3.9 support. This is a consequence of the fact that
sphinx 8 has dropped python 3.9.
Since we still want to support python 3.9, some support has to be re-
introduced.
See #38549 (comment)
and after.

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->

- [x] The title is concise and informative.
- [x] The description explains in detail what this PR is about.
- [x] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation and checked the documentation
preview.
    
URL: #38619
Reported by: François Bissey
Reviewer(s): Kwankyu Lee

Author: Release Manager

src/sage_docbuild/ext/sage_autodoc.py

@@ -33,12 +33,15 @@
 - Kwankyu Lee (2024-02-14): rebased on Sphinx 7.2.6
 
 - François Bissey (2024-08-24): rebased on Sphinx 8.0.2
+
+- François Bissey (2024-09-10): Tweaks to support python 3.9 (and older sphinx) as well
 """
 
 from __future__ import annotations
 
 import functools
 import operator
+import sys
 import re
 from inspect import Parameter, Signature
 from typing import TYPE_CHECKING, Any, ClassVar, NewType, TypeVar
@@ -670,7 +673,7 @@ def add_content(self, more_content: StringList | None) -> None:
 
         # add additional content (e.g. from document), if present
         if more_content:
-            for line, src in zip(more_content.data, more_content.items, strict=True):
+            for line, src in zip(more_content.data, more_content.items):
                 self.add_line(line, src[0], src[1])
 
     def get_object_members(self, want_all: bool) -> tuple[bool, list[ObjectMember]]:
@@ -1041,7 +1044,7 @@ def add_content(self, more_content: StringList | None) -> None:
         super().add_content(None)
         self.indent = old_indent
         if more_content:
-            for line, src in zip(more_content.data, more_content.items, strict=True):
+            for line, src in zip(more_content.data, more_content.items):
                 self.add_line(line, src[0], src[1])
 
     @classmethod
@@ -1576,8 +1579,14 @@ def __init__(self, *args: Any) -> None:
     def can_document_member(
         cls: type[Documenter], member: Any, membername: str, isattr: bool, parent: Any,
     ) -> bool:
-        return isinstance(member, type) or (
-            isattr and isinstance(member, NewType | TypeVar))
+        # support both sphinx 8 and py3.9/older sphinx
+        try:
+            result_bool = isinstance(member, type) or (
+                isattr and isinstance(member, NewType | TypeVar))
+        except:
+            result_bool = isinstance(member, type) or (
+                isattr and (inspect.isNewType(member) or isinstance(member, TypeVar)))
+        return result_bool
 
     def import_object(self, raiseerror: bool = False) -> bool:
         ret = super().import_object(raiseerror)
@@ -1650,7 +1659,12 @@ def import_object(self, raiseerror: bool = False) -> bool:
             # -------------------------------------------------------------------
             else:
                 self.doc_as_attr = True
-            if isinstance(self.object, NewType | TypeVar):
+            # support both sphinx 8 and py3.9/older sphinx
+            try:
+                test_bool = isinstance(self.object, NewType | TypeVar)
+            except:
+                test_bool = inspect.isNewType(self.object) or isinstance(self.object, TypeVar)
+            if test_bool:
                 modname = getattr(self.object, '__module__', self.modname)
                 if modname != self.modname and self.modname.startswith(modname):
                     bases = self.modname[len(modname):].strip('.').split('.')
@@ -1659,7 +1673,12 @@ def import_object(self, raiseerror: bool = False) -> bool:
         return ret
 
     def _get_signature(self) -> tuple[Any | None, str | None, Signature | None]:
-        if isinstance(self.object, NewType | TypeVar):
+        # support both sphinx 8 and py3.9/older sphinx
+        try:
+            test_bool = isinstance(self.object, NewType | TypeVar)
+        except:
+            test_bool = inspect.isNewType(self.object) or isinstance(self.object, TypeVar)
+        if test_bool:
             # Suppress signature
             return None, None, None
 
@@ -1844,14 +1863,24 @@ def add_directive_header(self, sig: str) -> None:
             self.directivetype = 'attribute'
         super().add_directive_header(sig)
 
-        if isinstance(self.object, NewType | TypeVar):
+        # support both sphinx 8 and py3.9/older sphinx
+        try:
+            test_bool = isinstance(self.object, NewType | TypeVar)
+        except:
+            test_bool = inspect.isNewType(self.object) or isinstance(self.object, TypeVar)
+        if test_bool:
             return
 
         if self.analyzer and '.'.join(self.objpath) in self.analyzer.finals:
             self.add_line('   :final:', sourcename)
 
         canonical_fullname = self.get_canonical_fullname()
-        if (not self.doc_as_attr and not isinstance(self.object, NewType)
+        # support both sphinx 8 and py3.9/older sphinx
+        try:
+            newtype_test = isinstance(self.object, NewType)
+        except:
+            newtype_test = inspect.isNewType(self.object)
+        if (not self.doc_as_attr and not newtype_test
                 and canonical_fullname and self.fullname != canonical_fullname):
             self.add_line('   :canonical: %s' % canonical_fullname, sourcename)
 
@@ -1903,6 +1932,28 @@ def get_doc(self) -> list[list[str]] | None:
         if isinstance(self.object, TypeVar):
             if self.object.__doc__ == TypeVar.__doc__:
                 return []
+        # ------------------------------------------------------------------
+        # This section is kept for compatibility with python 3.9
+        # see https://github.com/sagemath/sage/pull/38549#issuecomment-2327790930
+        if sys.version_info[:2] < (3, 10):
+            if inspect.isNewType(self.object) or isinstance(self.object, TypeVar):
+                parts = self.modname.strip('.').split('.')
+                orig_objpath = self.objpath
+                for i in range(len(parts)):
+                    new_modname = '.'.join(parts[:len(parts) - i])
+                    new_objpath = parts[len(parts) - i:] + orig_objpath
+                    try:
+                        analyzer = ModuleAnalyzer.for_module(new_modname)
+                        analyzer.analyze()
+                        key = ('', new_objpath[-1])
+                        comment = list(analyzer.attr_docs.get(key, []))
+                        if comment:
+                            self.objpath = new_objpath
+                            self.modname = new_modname
+                            return [comment]
+                    except PycodeError:
+                        pass
+        # ------------------------------------------------------------------
         if self.doc_as_attr:
             # Don't show the docstring of the class when it is an alias.
             if self.get_variable_comment():
@@ -1966,7 +2017,12 @@ def get_variable_comment(self) -> list[str] | None:
             return None
 
     def add_content(self, more_content: StringList | None) -> None:
-        if isinstance(self.object, NewType):
+        # support both sphinx 8 and py3.9/older sphinx
+        try:
+            newtype_test = isinstance(self.object, NewType)
+        except:
+            newtype_test = inspect.isNewType(self.object)
+        if newtype_test:
             if self.config.autodoc_typehints_format == "short":
                 supertype = restify(self.object.__supertype__, "smart")
             else: