gh-123881: make compiler add the .generic_base base class without constructing AST nodes

Doc/whatsnew/3.14.rst

@@ -108,10 +108,8 @@ ast
   (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.)
 
 * Add support for :func:`copy.replace` for AST nodes.
-
   (Contributed by Bénédikt Tran in :gh:`121141`.)
 
-
 ctypes
 ------
 
@@ -401,6 +399,10 @@ typing
 * Remove :class:`!typing.ByteString`. It had previously raised a
   :exc:`DeprecationWarning` since Python 3.12.
 
+* The ``.generic_base`` base class of generic classes is now added by
+  the parser rather than the compiler. This means that it is visible in
+  the ``AST``. (Contributed by Irit Katriel in :gh:`123881`.)
+
 urllib
 ------
 

Grammar/python.gram

@@ -254,10 +254,7 @@ class_def[stmt_ty]:
 class_def_raw[stmt_ty]:
     | invalid_class_def_raw
     | 'class' a=NAME t=[type_params] b=['(' z=[arguments] ')' { z }] ':' c=block {
-        _PyAST_ClassDef(a->v.Name.id,
-                     (b) ? ((expr_ty) b)->v.Call.args : NULL,
-                     (b) ? ((expr_ty) b)->v.Call.keywords : NULL,
-                     c, NULL, t, EXTRA) }
+        _PyPegen_class_def(a, b, c, t, EXTRA) }
 
 # Function definitions
 # --------------------

Include/internal/pycore_compile.h

@@ -136,7 +136,6 @@ int _PyCompile_IsNestedScope(struct _PyCompiler *c);
 int _PyCompile_IsInInlinedComp(struct _PyCompiler *c);
 int _PyCompile_ScopeType(struct _PyCompiler *c);
 int _PyCompile_OptimizationLevel(struct _PyCompiler *c);
-PyArena *_PyCompile_Arena(struct _PyCompiler *c);
 int _PyCompile_LookupArg(struct _PyCompiler *c, PyCodeObject *co, PyObject *name);
 PyObject *_PyCompile_Qualname(struct _PyCompiler *c);
 _PyCompile_CodeUnitMetadata *_PyCompile_Metadata(struct _PyCompiler *c);

Lib/ast.py

@@ -997,6 +997,8 @@ def visit_ClassDef(self, node):
         with self.delimit_if("(", ")", condition = node.bases or node.keywords):
             comma = False
             for e in node.bases:
+                if isinstance(e, Name) and e.id == ".generic_base":
+                    continue
                 if comma:
                     self.write(", ")
                 else:

Lib/test/test_ast/snippets.py

@@ -503,11 +503,11 @@ def main():
 ('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), None), ('TypeVarTuple', (1, 15, 1, 18), 'Ts', None), ('ParamSpec', (1, 20, 1, 23), 'P', None)], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []),
 ('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 22, 1, 25), 'Ts', None), ('ParamSpec', (1, 27, 1, 30), 'P', None)], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []),
 ('Module', [('TypeAlias', (1, 0, 1, 48), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 17), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Constant', (1, 16, 1, 17), 1, None)), ('TypeVarTuple', (1, 19, 1, 26), 'Ts', ('Constant', (1, 25, 1, 26), 2, None)), ('ParamSpec', (1, 28, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))], ('Tuple', (1, 38, 1, 48), [('Name', (1, 39, 1, 40), 'T', ('Load',)), ('Name', (1, 42, 1, 44), 'Ts', ('Load',)), ('Name', (1, 46, 1, 47), 'P', ('Load',))], ('Load',)))], []),
-('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []),
-('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []),
-('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []),
-('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []),
-('Module', [('ClassDef', (1, 0, 1, 43), 'X', [], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []),
+('Module', [('ClassDef', (1, 0, 1, 16), 'X', [('Name', (1, 0, 1, 16), '.generic_base', ('Load',))], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []),
+('Module', [('ClassDef', (1, 0, 1, 26), 'X', [('Name', (1, 0, 1, 26), '.generic_base', ('Load',))], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []),
+('Module', [('ClassDef', (1, 0, 1, 31), 'X', [('Name', (1, 0, 1, 31), '.generic_base', ('Load',))], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []),
+('Module', [('ClassDef', (1, 0, 1, 38), 'X', [('Name', (1, 0, 1, 38), '.generic_base', ('Load',))], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []),
+('Module', [('ClassDef', (1, 0, 1, 43), 'X', [('Name', (1, 0, 1, 43), '.generic_base', ('Load',))], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []),
 ('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None)])], []),
 ('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts', None), ('ParamSpec', (1, 14, 1, 17), 'P', None)])], []),
 ('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), None), ('TypeVarTuple', (1, 14, 1, 17), 'Ts', None), ('ParamSpec', (1, 19, 1, 22), 'P', None)])], []),

Lib/test/test_ast/test_ast.py

@@ -3290,13 +3290,15 @@ def test_folding_type_param_in_class_def(self):
             optimized_target = self.wrap_statement(
                 ast.ClassDef(
                     name='foo',
+                    bases=[ast.Name(id='.generic_base', ctx=ast.Load())],
                     body=[ast.Pass()],
                     type_params=[type_param(name=name, default_value=ast.Constant(2))]
                 )
             )
             non_optimized_target = self.wrap_statement(
                 ast.ClassDef(
                     name='foo',
+                    bases=[ast.Name(id='.generic_base', ctx=ast.Load())],
                     body=[ast.Pass()],
                     type_params=[type_param(name=name, default_value=unoptimized_binop)]
                 )

Misc/NEWS.d/next/Core and Builtins/2024-09-09-22-05-07.gh-issue-123881.Lz6-Lt.rst

@@ -0,0 +1,3 @@
+The ``.generic_base`` base class of generic classes is now added by the
+parser rather than the compiler. This means that it is visible in the
+``AST``.

Parser/action_helpers.c

@@ -746,6 +746,46 @@ _PyPegen_function_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty f
         function_def->end_lineno, function_def->end_col_offset, p->arena);
 }
 
+/* Construct a ClassDef */
+stmt_ty
+_PyPegen_class_def(expr_ty a, arguments_ty b, asdl_stmt_seq *c, asdl_type_param_seq *t,
+                   int lineno, int col_offset, int end_lineno, int end_col_offset,
+                   PyArena *arena)
+{
+    identifier name = a->v.Name.id;
+    asdl_expr_seq *bases = (b) ? ((expr_ty) b)->v.Call.args : NULL;
+    asdl_keyword_seq *keywords = (b) ? ((expr_ty) b)->v.Call.keywords : NULL;
+    asdl_stmt_seq *body = c;
+    asdl_expr_seq *decorator_list = NULL;
+    asdl_type_param_seq *type_params = t;
+
+    int is_generic = asdl_seq_LEN(type_params) > 0;
+    if (is_generic) {
+        /* Add */
+        asdl_expr_seq *orig_bases = bases;
+        Py_ssize_t orig_len = asdl_seq_LEN(orig_bases);
+        bases = _Py_asdl_expr_seq_new(orig_len + 1, arena);
+        if (bases == NULL) {
+            return NULL;
+        }
+        for (Py_ssize_t i = 0; i < orig_len; i++) {
+            asdl_seq_SET(bases, i, asdl_seq_GET(orig_bases, i));
+        }
+        _Py_DECLARE_STR(generic_base, ".generic_base");
+        expr_ty name_node = _PyAST_Name(
+            &_Py_STR(generic_base), Load,
+            lineno, col_offset, end_lineno, end_col_offset, arena
+        );
+        if (name_node == NULL) {
+            return NULL;
+        }
+        asdl_seq_SET(bases, orig_len, name_node);
+    }
+
+    return _PyAST_ClassDef(name, bases, keywords, body, decorator_list, type_params,
+                           lineno, col_offset, end_lineno, end_col_offset, arena);
+}
+
 /* Construct a ClassDef equivalent to class_def, but with decorators */
 stmt_ty
 _PyPegen_class_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty class_def)

Parser/pegen.h

@@ -322,6 +322,8 @@ expr_ty _PyPegen_formatted_value(Parser *, expr_ty, Token *, ResultTokenWithMeta
                                  int, int, int, int, PyArena *);
 AugOperator *_PyPegen_augoperator(Parser*, operator_ty type);
 stmt_ty _PyPegen_function_def_decorators(Parser *, asdl_expr_seq *, stmt_ty);
+stmt_ty _PyPegen_class_def(expr_ty, arguments_ty, asdl_stmt_seq *,
+                           asdl_type_param_seq *, int, int, int, int, PyArena *);
 stmt_ty _PyPegen_class_def_decorators(Parser *, asdl_expr_seq *, stmt_ty);
 KeywordOrStarred *_PyPegen_keyword_or_starred(Parser *, void *, int);
 asdl_expr_seq *_PyPegen_seq_extract_starred_exprs(Parser *, asdl_seq *);

Python/codegen.c

@@ -76,7 +76,6 @@ typedef struct _PyCompiler compiler;
 #define SCOPE_TYPE(C) _PyCompile_ScopeType(C)
 #define QUALNAME(C) _PyCompile_Qualname(C)
 #define METADATA(C) _PyCompile_Metadata(C)
-#define ARENA(C) _PyCompile_Arena(C)
 
 typedef _PyInstruction instruction;
 typedef _PyInstructionSequence instr_sequence;
@@ -1549,27 +1548,8 @@ codegen_class(compiler *c, stmt_ty s)
         ADDOP_I_IN_SCOPE(c, loc, CALL_INTRINSIC_1, INTRINSIC_SUBSCRIPT_GENERIC);
         RETURN_IF_ERROR_IN_SCOPE(c, codegen_nameop(c, loc, &_Py_STR(generic_base), Store));
 
-        Py_ssize_t original_len = asdl_seq_LEN(s->v.ClassDef.bases);
-        asdl_expr_seq *bases = _Py_asdl_expr_seq_new(
-            original_len + 1, ARENA(c));
-        if (bases == NULL) {
-            _PyCompile_ExitScope(c);
-            return ERROR;
-        }
-        for (Py_ssize_t i = 0; i < original_len; i++) {
-            asdl_seq_SET(bases, i, asdl_seq_GET(s->v.ClassDef.bases, i));
-        }
-        expr_ty name_node = _PyAST_Name(
-            &_Py_STR(generic_base), Load,
-            loc.lineno, loc.col_offset, loc.end_lineno, loc.end_col_offset, ARENA(c)
-        );
-        if (name_node == NULL) {
-            _PyCompile_ExitScope(c);
-            return ERROR;
-        }
-        asdl_seq_SET(bases, original_len, name_node);
         RETURN_IF_ERROR_IN_SCOPE(c, codegen_call_helper(c, loc, 2,
-                                                        bases,
+                                                        s->v.ClassDef.bases,
                                                         s->v.ClassDef.keywords));
 
         PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0);

Python/compile.c

@@ -1244,12 +1244,6 @@ _PyCompile_Metadata(compiler *c)
     return &c->u->u_metadata;
 }
 
-PyArena *
-_PyCompile_Arena(compiler *c)
-{
-    return c->c_arena;
-}
-
 #ifndef NDEBUG
 int
 _PyCompile_IsTopLevelAwait(compiler *c)