Preliminary PEP 573 (module state access) implementation

Doc/c-api/structures.rst

@@ -147,23 +147,56 @@ Implementing functions and methods
    value of the function as exposed in Python.  The function must return a new
    reference.
 
+   The function signature is::
+
+      PyObject *PyCFunction(PyObject *self,
+                            PyObject *const *args);
 
 .. c:type:: PyCFunctionWithKeywords
 
    Type of the functions used to implement Python callables in C
    with signature :const:`METH_VARARGS | METH_KEYWORDS`.
+   The function signature is::
+
+      PyObject *PyCFunctionWithKeywords(PyObject *self,
+                                        PyObject *const *args,
+                                        PyObject *kwargs);
 
 
 .. c:type:: _PyCFunctionFast
 
    Type of the functions used to implement Python callables in C
    with signature :const:`METH_FASTCALL`.
+   The function signature is::
 
+      PyObject *_PyCFunctionFast(PyObject *self,
+                                 PyObject *const *args,
+                                 Py_ssize_t nargs);
 
 .. c:type:: _PyCFunctionFastWithKeywords
 
    Type of the functions used to implement Python callables in C
    with signature :const:`METH_FASTCALL | METH_KEYWORDS`.
+   The function signature is::
+
+      PyObject *_PyCFunctionFastWithKeywords(PyObject *self,
+                                             PyObject *const *args,
+                                             Py_ssize_t nargs,
+                                             PyObject *kwnames);
+
+.. c:type:: PyCMethod
+
+   Type of the functions used to implement Python callables in C
+   with signature :const:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS`.
+   The function signature is::
+
+      PyObject *PyCMethod(PyObject *self,
+                          PyTypeObject *defining_class,
+                          PyObject *const *args,
+                          Py_ssize_t nargs,
+                          PyObject *kwnames)
+
+   .. versionadded:: 3.9
 
 
 .. c:type:: PyMethodDef
@@ -197,9 +230,7 @@ The :attr:`ml_flags` field is a bitfield which can include the following flags.
 The individual flags indicate either a calling convention or a binding
 convention.
 
-There are four basic calling conventions for positional arguments
-and two of them can be combined with :const:`METH_KEYWORDS` to support
-also keyword arguments.  So there are a total of 6 calling conventions:
+There are these calling conventions:
 
 .. data:: METH_VARARGS
 
@@ -250,6 +281,19 @@ also keyword arguments.  So there are a total of 6 calling conventions:
    .. versionadded:: 3.7
 
 
+.. data:: METH_METHOD | METH_FASTCALL | METH_KEYWORDS
+
+   Extension of :const:`METH_FASTCALL | METH_KEYWORDS` supporting the *defining
+   class*, that is, the class that contains the method in question.
+   The defining class might be a superclass of ``Py_TYPE(self)``.
+
+   The method needs to be of type :c:type:`PyCMethod`, the same as for
+   ``METH_FASTCALL | METH_KEYWORDS`` with ``defining_class`` argument added after
+   ``self``.
+
+   .. versionadded:: 3.9
+
+
 .. data:: METH_NOARGS
 
    Methods without parameters don't need to check whether arguments are given if

Doc/c-api/type.rst

@@ -109,14 +109,38 @@ Type Objects
 
    .. versionadded:: 3.4
 
+.. c:function:: void* PyType_GetModule(PyTypeObject *type)
+
+   Return the module object associated with the given type when the type was
+   created using :c:func:`PyType_FromModuleAndSpec`.
+
+   If no module is associated with the given type, sets :py:class:`TypeError`
+   and returns ``NULL``.
+
+   .. versionadded:: 3.9
+
+.. c:function:: void* PyType_GetModuleState(PyTypeObject *type)
+
+   Return the state of the module object associated with the given type.
+   This is a shortcut for calling :c:func:`PyModule_GetState()` on the result
+   of :c:func:`PyType_GetModule`.
+
+   If no module is associated with the given type, sets :py:class:`TypeError`
+   and returns ``NULL``.
+
+   If the *type* has an associated module but its state is ``NULL``,
+   returns ``NULL`` without setting an exception.
+
+   .. versionadded:: 3.9
+
 
 Creating Heap-Allocated Types
 .............................
 
 The following functions and structs are used to create
 :ref:`heap types <heap-types>`.
 
-.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
+.. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
 
    Creates and returns a heap type object from the *spec*
    (:const:`Py_TPFLAGS_HEAPTYPE`).
@@ -127,8 +151,18 @@ The following functions and structs are used to create
    If *bases* is ``NULL``, the *Py_tp_base* slot is used instead.
    If that also is ``NULL``, the new type derives from :class:`object`.
 
+   The *module* must be a module object or ``NULL``.
+   If not ``NULL``, the module is associated with the new type and can later be
+   retreived with :c:func:`PyType_GetModule`.
+
    This function calls :c:func:`PyType_Ready` on the new type.
 
+   .. versionadded:: 3.9
+
+.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
+
+   Equivalent to ``PyType_FromModuleAndSpec(NULL, spec, NULL)``.
+
    .. versionadded:: 3.3
 
 .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)

Include/cpython/methodobject.h

@@ -0,0 +1,30 @@
+#ifndef Py_CPYTHON_METHODOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+/* Macros for direct access to these values. Type checks are *not*
+   done, so use with care. */
+#define PyCFunction_GET_FUNCTION(func) \
+        (((PyCFunctionObject *)func) -> m_ml -> ml_meth)
+#define PyCFunction_GET_SELF(func) \
+        (((PyCFunctionObject *)func) -> m_ml -> ml_flags & METH_STATIC ? \
+         NULL : ((PyCFunctionObject *)func) -> m_self)
+#define PyCFunction_GET_FLAGS(func) \
+        (((PyCFunctionObject *)func) -> m_ml -> ml_flags)
+#define PyCFunction_GET_CLASS(func) \
+    (((PyCFunctionObject *)func) -> m_ml -> ml_flags & METH_METHOD ? \
+         ((PyCMethodObject *)func) -> mm_class : NULL)
+
+typedef struct {
+    PyObject_VAR_HEAD
+    PyMethodDef *m_ml; /* Description of the C function to call */
+    PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
+    PyObject    *m_module; /* The __module__ attribute, can be anything */
+    PyObject    *m_weakreflist; /* List of weak references */
+    vectorcallfunc vectorcall;
+} PyCFunctionObject;
+
+typedef struct {
+    PyCFunctionObject func;
+    PyTypeObject *mm_class; /* Class that defines this method */
+} PyCMethodObject;

Include/cpython/object.h

@@ -289,6 +289,7 @@ typedef struct _heaptypeobject {
     PyBufferProcs as_buffer;
     PyObject *ht_name, *ht_slots, *ht_qualname;
     struct _dictkeysobject *ht_cached_keys;
+    PyObject *ht_module;
     /* here are optional user slots, followed by the members. */
 } PyHeapTypeObject;
 

Include/methodobject.h

@@ -22,21 +22,13 @@ typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
 typedef PyObject *(*_PyCFunctionFastWithKeywords) (PyObject *,
                                                    PyObject *const *, Py_ssize_t,
                                                    PyObject *);
+typedef PyObject *(*PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *,
+                               size_t, PyObject *);
+
 PyAPI_FUNC(PyCFunction) PyCFunction_GetFunction(PyObject *);
 PyAPI_FUNC(PyObject *) PyCFunction_GetSelf(PyObject *);
 PyAPI_FUNC(int) PyCFunction_GetFlags(PyObject *);
 
-/* Macros for direct access to these values. Type checks are *not*
-   done, so use with care. */
-#ifndef Py_LIMITED_API
-#define PyCFunction_GET_FUNCTION(func) \
-        (((PyCFunctionObject *)func) -> m_ml -> ml_meth)
-#define PyCFunction_GET_SELF(func) \
-        (((PyCFunctionObject *)func) -> m_ml -> ml_flags & METH_STATIC ? \
-         NULL : ((PyCFunctionObject *)func) -> m_self)
-#define PyCFunction_GET_FLAGS(func) \
-        (((PyCFunctionObject *)func) -> m_ml -> ml_flags)
-#endif
 Py_DEPRECATED(3.9) PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *);
 
 struct PyMethodDef {
@@ -52,6 +44,13 @@ typedef struct PyMethodDef PyMethodDef;
 PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *,
                                          PyObject *);
 
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000
+#define PyCFunction_NewEx(ML, SELF, MOD) PyCMethod_New((ML), (SELF), (MOD), NULL)
+PyAPI_FUNC(PyObject *) PyCMethod_New(PyMethodDef *, PyObject *,
+                                     PyObject *, PyTypeObject *);
+#endif
+
+
 /* Flag passed to newmethodobject */
 /* #define METH_OLDARGS  0x0000   -- unsupported now */
 #define METH_VARARGS  0x0001
@@ -84,15 +83,24 @@ PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *,
 #define METH_STACKLESS 0x0000
 #endif
 
+/* METH_METHOD means the function stores an
+ * additional reference to the class that defines it;
+ * both self and class are passed to it.
+ * It uses PyCMethodObject instead of PyCFunctionObject.
+ * May not be combined with METH_NOARGS, METH_O, METH_CLASS or METH_STATIC.
+ */
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000
+#define METH_METHOD 0x0200
+#endif
+
+
 #ifndef Py_LIMITED_API
-typedef struct {
-    PyObject_HEAD
-    PyMethodDef *m_ml; /* Description of the C function to call */
-    PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
-    PyObject    *m_module; /* The __module__ attribute, can be anything */
-    PyObject    *m_weakreflist; /* List of weak references */
-    vectorcallfunc vectorcall;
-} PyCFunctionObject;
+
+#define Py_CPYTHON_METHODOBJECT_H
+#include  "cpython/methodobject.h"
+#undef Py_CPYTHON_METHODOBJECT_H
+
 #endif
 
 #ifdef __cplusplus

Include/object.h

@@ -213,6 +213,11 @@ PyAPI_FUNC(PyObject*) PyType_FromSpecWithBases(PyType_Spec*, PyObject*);
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03040000
 PyAPI_FUNC(void*) PyType_GetSlot(PyTypeObject*, int);
 #endif
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000
+PyAPI_FUNC(PyObject*) PyType_FromModuleAndSpec(PyObject *, PyType_Spec *, PyObject *);
+PyAPI_FUNC(PyObject *) PyType_GetModule(struct _typeobject *);
+PyAPI_FUNC(void *) PyType_GetModuleState(struct _typeobject *);
+#endif
 
 /* Generic type check */
 PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *);

Lib/test/test_importlib/extension/test_loader.py

@@ -267,6 +267,66 @@ def test_nonascii(self):
                 self.assertEqual(module.__name__, name)
                 self.assertEqual(module.__doc__, "Module named in %s" % lang)
 
+    def test_subclass_get_module(self):
+        testmultiphase = self.load_module_by_name("_testmultiphase_meth_state_access")
+
+        class StateAccessType_Subclass(testmultiphase.StateAccessType):
+            pass
+
+        ex = StateAccessType_Subclass()
+
+        self.assertIs(ex.get_defining_module(), testmultiphase)
+
+    def test_subclass_get_module_with_super(self):
+        testmultiphase = self.load_module_by_name("_testmultiphase_meth_state_access")
+
+        class StateAccessType_Subclass(testmultiphase.StateAccessType):
+            def get_defining_module(self):
+                return super().get_defining_module()
+
+        ex = StateAccessType_Subclass()
+
+        self.assertIs(ex.get_defining_module(), testmultiphase)
+
+    def test_state_counter(self):
+        testmultiphase = self.load_module_by_name("_testmultiphase_meth_state_access")
+
+        a = testmultiphase.StateAccessType()
+        b = testmultiphase.StateAccessType()
+
+        self.assertEqual(a.get_count(), b.get_count())
+        self.assertEqual(a.get_count(), 0)
+
+        a.increment_count()
+        self.assertEqual(a.get_count(), b.get_count())
+        self.assertEqual(a.get_count(), 1)
+
+        b.increment_count()
+        self.assertEqual(a.get_count(), b.get_count())
+        self.assertEqual(a.get_count(), 2)
+
+        a.decrement_count()
+        self.assertEqual(a.get_count(), b.get_count())
+        self.assertEqual(a.get_count(), 1)
+
+        b.decrement_count()
+        self.assertEqual(a.get_count(), b.get_count())
+        self.assertEqual(a.get_count(), 0)
+
+        a.decrement_count(2)
+        self.assertEqual(a.get_count(), b.get_count())
+        self.assertEqual(a.get_count(), -2)
+
+        a.decrement_count(3, twice=True)
+        self.assertEqual(a.get_count(), b.get_count())
+        self.assertEqual(a.get_count(), -8)
+
+        with self.assertRaises(TypeError):
+            a.decrement_count(thrice=3)
+
+        with self.assertRaises(TypeError):
+            a.decrement_count(1, 2, 3)
+
 
 (Frozen_MultiPhaseExtensionModuleTests,
  Source_MultiPhaseExtensionModuleTests

Lib/test/test_sys.py

@@ -1117,7 +1117,7 @@ def test_objecttypes(self):
         # buffer
         # XXX
         # builtin_function_or_method
-        check(len, size('5P'))
+        check(len, vsize('5P'))
         # bytearray
         samples = [b'', b'u'*100000]
         for sample in samples:
@@ -1322,7 +1322,7 @@ def delx(self): del self.__x
                   '3P'                  # PyMappingMethods
                   '10P'                 # PySequenceMethods
                   '2P'                  # PyBufferProcs
-                  '4P')
+                  '5P')
         class newstyleclass(object): pass
         # Separate block for PyDictKeysObject with 8 keys and 5 entries
         check(newstyleclass, s + calcsize("2nP2n0P") + 8 + 5*calcsize("n2P"))

Misc/NEWS.d/next/C API/2020-01-22-12-38-59.bpo-38787.HUH6hd.rst

@@ -0,0 +1,2 @@
+Module C state is now accessible from C-defined heap type methods. (PEP-573)
+Patch by Marcel Plch and Petr Viktorin.