WIP bpo-44800: Rename _PyInterpreterFrame to _Py_framedata

Include/cpython/ceval.h

@@ -19,7 +19,7 @@ PyAPI_FUNC(PyObject *) _PyEval_GetBuiltinId(_Py_Identifier *);
    flag was set, else return 0. */
 PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
 
-PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _interpreter_frame *f, int exc);
+PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyExecFrame *xf, int exc);
 
 PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds);
 PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void);

Include/cpython/frameobject.h

@@ -4,10 +4,26 @@
 #  error "this header file must not be included directly"
 #endif
 
+/* Starting in CPython 3.11, CPython separates the frame state between the
+ * full "introspection frames" exposed by the Python and C runtime state
+ * introspection APIs, and internal lighter weight "execution frames", which
+ * are simple C structures owned by either the interpreter eval loop (while
+ * executing ordinary functions), by a generator or coroutine object (for
+ * frames that are able to be suspended), or by their corresponding
+ * introspection frame (if an instrospection API has been invoked and the
+ * introspection frame created).
+ *
+ * This split storage eliminates a lot of allocation and deallocation of full
+ * Python objects during code execution, providing a significant speed gain
+ * over the previous approach of using full Python objects for both introspection
+ * and code execution.
+ */
+// Declaration of _PyExecFrame is in cpython/pystate.h for use in PyThreadState
+
 struct _frame {
     PyObject_HEAD
     struct _frame *f_back;      /* previous frame, or NULL */
-    struct _interpreter_frame *f_frame; /* points to the frame data */
+    _PyExecFrame *f_xframe;     /* points to the frame runtime data */
     PyObject *f_trace;          /* Trace function */
     int f_lineno;               /* Current line number. Only valid if non-zero */
     char f_trace_lines;         /* Emit per-line trace events? */
@@ -24,11 +40,6 @@ PyAPI_DATA(PyTypeObject) PyFrame_Type;
 PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
                                         PyObject *, PyObject *);
 
-/* only internal use */
-PyFrameObject*
-_PyFrame_New_NoTrack(struct _interpreter_frame *, int);
-
-
 /* The rest of the interface is specific for frame objects */
 
 /* Conversions between "fast locals" and locals in dictionary */

Include/cpython/pystate.h

@@ -69,6 +69,12 @@ typedef struct _stack_chunk {
     PyObject * data[1]; /* Variable sized */
 } _PyStackChunk;
 
+
+// Declared here so the thread state can use it without creating an include loop
+// See cpython/frameobject.h for an explanation of the type
+// See internal/pycore_xframe.h for the struct definition
+typedef struct _Py_execution_frame _PyExecFrame;
+
 // The PyThreadState typedef is in Include/pystate.h.
 struct _ts {
     /* See Python/ceval.c for comments explaining most fields */
@@ -77,8 +83,8 @@ struct _ts {
     struct _ts *next;
     PyInterpreterState *interp;
 
-    /* Borrowed reference to the current frame (it can be NULL) */
-    struct _interpreter_frame *frame;
+    /* Borrowed reference to the current execution frame (it can be NULL) */
+    _PyExecFrame *xframe;
     int recursion_depth;
     int recursion_headroom; /* Allow 50 more calls to handle any errors. */
     int stackcheck_counter;
@@ -223,7 +229,7 @@ PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
 
 /* Frame evaluation API */
 
-typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _interpreter_frame *, int);
+typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyExecFrame *, int);
 
 PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc(
     PyInterpreterState *interp);

Include/genobject.h

@@ -15,8 +15,8 @@ extern "C" {
    and coroutine objects. */
 #define _PyGenObject_HEAD(prefix)                                           \
     PyObject_HEAD                                                           \
-    /* Note: gi_frame can be NULL if the generator is "finished" */         \
-    struct _interpreter_frame *prefix##_xframe;                             \
+    /* Note: gi_xframe can be NULL if the generator is "finished" */        \
+    _PyExecFrame *prefix##_xframe;                                          \
     /* The code object backing the generator */                             \
     PyCodeObject *prefix##_code;                                            \
     /* List of weak reference. */                                           \

Include/internal/pycore_ceval.h

@@ -41,9 +41,9 @@ extern PyObject *_PyEval_BuiltinsFromGlobals(
 
 
 static inline PyObject*
-_PyEval_EvalFrame(PyThreadState *tstate, struct _interpreter_frame *frame, int throwflag)
+_PyEval_EvalFrame(PyThreadState *tstate, _PyExecFrame *xframe, int throwflag)
 {
-    return tstate->interp->eval_frame(tstate, frame, throwflag);
+    return tstate->interp->eval_frame(tstate, xframe, throwflag);
 }
 
 extern PyObject *
@@ -107,9 +107,9 @@ static inline void _Py_LeaveRecursiveCall_inline(void)  {
 
 #define Py_LeaveRecursiveCall() _Py_LeaveRecursiveCall_inline()
 
-struct _interpreter_frame *_PyEval_GetFrame(void);
+_PyExecFrame *_PyEval_GetExecFrame(void);
 
-PyObject *_Py_MakeCoro(PyFrameConstructor *, struct _interpreter_frame *);
+PyObject *_Py_MakeCoro(PyFrameConstructor *, _PyExecFrame *);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_xframe.h

@@ -0,0 +1,133 @@
+#ifndef Py_INTERNAL_XFRAME_H
+#define Py_INTERNAL_XFRAME_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Internal-use-only introspection frame constructor */
+PyFrameObject*
+_PyFrame_New_NoTrack(_PyExecFrame *, int);
+
+/* These values are chosen so that the inline functions below all
+ * compare f_state to zero.
+ */
+enum _framestate {
+    FRAME_CREATED = -2,
+    FRAME_SUSPENDED = -1,
+    FRAME_EXECUTING = 0,
+    FRAME_RETURNED = 1,
+    FRAME_UNWINDING = 2,
+    FRAME_RAISED = 3,
+    FRAME_CLEARED = 4
+};
+
+typedef signed char PyFrameState;
+
+// The _PyExecFrame typedef is in Include/pyframeobject.h
+struct _Py_execution_frame {
+    PyObject *xf_globals;
+    PyObject *xf_builtins;
+    PyObject *xf_locals;
+    PyCodeObject *xf_code;
+    PyFrameObject *xf_frame_obj; // Introspection frame (if created)
+    /* Borrowed reference to a generator, or NULL */
+    PyObject *xf_generator;
+    _PyExecFrame *xf_previous;
+    int xf_lasti;       /* Last instruction if called */
+    int xf_stackdepth;  /* Depth of value stack */
+    int xf_nlocalsplus;
+    PyFrameState xf_state;       /* What state the frame is in */
+    PyObject *xf_stack[1];
+};
+
+static inline int _PyExecFrame_IsRunnable(_PyExecFrame *xf) {
+    return xf->xf_state < FRAME_EXECUTING;
+}
+
+static inline int _PyExecFrame_IsExecuting(_PyExecFrame *xf) {
+    return xf->xf_state == FRAME_EXECUTING;
+}
+
+static inline int _PyExecFrame_HasCompleted(_PyExecFrame *xf) {
+    return xf->xf_state > FRAME_EXECUTING;
+}
+
+#define FRAME_SPECIALS_SIZE ((sizeof(_PyExecFrame)-1)/sizeof(PyObject *))
+
+_PyExecFrame *
+_PyExecFrame_HeapAlloc(PyFrameConstructor *con, PyObject *locals);
+
+static inline void
+_PyExecFrame_InitializeSpecials(
+    _PyExecFrame *xframe, PyFrameConstructor *con,
+    PyObject *locals, int nlocalsplus)
+{
+    xframe->xf_code = (PyCodeObject *)Py_NewRef(con->fc_code);
+    xframe->xf_builtins = Py_NewRef(con->fc_builtins);
+    xframe->xf_globals = Py_NewRef(con->fc_globals);
+    xframe->xf_locals = Py_XNewRef(locals);
+    xframe->xf_nlocalsplus = nlocalsplus;
+    xframe->xf_stackdepth = 0;
+    xframe->xf_frame_obj = NULL;
+    xframe->xf_generator = NULL;
+    xframe->xf_lasti = -1;
+    xframe->xf_state = FRAME_CREATED;
+}
+
+/* Gets the pointer to the locals array
+ * that precedes this frame.
+ */
+static inline PyObject**
+_PyExecFrame_GetLocalsArray(_PyExecFrame *xframe)
+{
+    return ((PyObject **)xframe) - xframe->xf_nlocalsplus;
+}
+
+/* For use by _PyExecFrame_GetFrameObject
+  Do not call directly. */
+PyFrameObject *
+_PyExecFrame_MakeAndSetFrameObject(_PyExecFrame *xframe);
+
+/* Gets the PyFrameObject for this frame, lazily
+ * creating it if necessary.
+ * Returns a borrowed referennce */
+static inline PyFrameObject *
+_PyExecFrame_GetFrameObject(_PyExecFrame *xframe)
+{
+    PyFrameObject *res = xframe->xf_frame_obj;
+    if (res != NULL) {
+        return res;
+    }
+    return _PyExecFrame_MakeAndSetFrameObject(xframe);
+}
+
+/* Clears all references in the frame.
+ * If take is non-zero, then the execution frame
+ * may be transfered to the frame object it references
+ * instead of being cleared. Either way
+ * the caller no longer owns the references
+ * in the frame.
+ * take should  be set to 1 for heap allocated
+ * frames like the ones in generators and coroutines.
+ */
+int
+_PyExecFrame_Clear(_PyExecFrame *xframe, int take);
+
+int
+_PyExecFrame_Traverse(_PyExecFrame *xframe, visitproc visit, void *arg);
+
+int
+_PyExecFrame_FastToLocalsWithError(_PyExecFrame *frame);
+
+void
+_PyExecFrame_LocalsToFast(_PyExecFrame *frame, int clear);
+
+_PyExecFrame *_PyThreadState_PushExecFrame(
+    PyThreadState *tstate, PyFrameConstructor *con, PyObject *locals);
+
+void _PyThreadState_PopExecFrame(PyThreadState *tstate, _PyExecFrame *frame);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_XFRAME_H */

Misc/NEWS.d/next/Core and Builtins/2021-08-01-18-18-51.bpo-44800.TCsfH3.rst

@@ -0,0 +1,4 @@
+Refactored internal APIs for the new lazy introspection frame creation to
+consistently distinguish between the classic introspection frames (full
+Python objects) and the new lighter weight internal execution frames (C
+structs with no intrinsic instance lifecycle management)