Fix a long-standing bug in the handling of Python multiple inheritance (#30056)

* Snapshot of pybind#4762 applied to pywrapcc

* Universal `bases.size() != vhs.size()` (not as `assert()`)

* Revert "Universal `bases.size() != vhs.size()` (not as `assert()`)"

This reverts commit 4c2407d17e982e1512f43ad89bf8752c0d2c7fe0.

* Streamline implementation and avoid unsafe `reinterpret_cast<instance *>()` introduced with PR pybind#2152

The `reinterpret_cast<instance *>(self)` is unsafe if `__new__` is mocked,
which was actually found in the wild: the mock returned `None` for `self`.
This was inconsequential because `inst` is currently cast straight back to
`PyObject *` to compute `all_type_info()`, which is empty if `self` is not
a pybind11 `instance`, and then `inst` is never dereferenced. However, the
unsafe detour through `instance *` is easily avoided and the updated
implementation is less prone to accidents while debugging or refactoring.

* Fix actual undefined behavior exposed by previous changes.

It turns out the previous commit message is incorrect, the `inst` pointer is actually dereferenced, in the `value_and_holder` ctor here:

https://github.com/pybind/pybind11/blob/f3e0602802c7840992c97f4960515777cad6a5c7/include/pybind11/detail/type_caster_base.h#L262-L263

```
259     // Main constructor for a found value/holder:
260     value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index)
261         : inst{i}, index{index}, type{type},
262           vh{inst->simple_layout ? inst->simple_value_holder
263                                  : &inst->nonsimple.values_and_holders[vpos]} {}
```

* Add test_mock_new()

Author: rwgk

include/pybind11/detail/class.h

@@ -189,12 +189,10 @@ extern "C" inline PyObject *pybind11_meta_call(PyObject *type, PyObject *args, P
         return nullptr;
     }
 
-    // This must be a pybind11 instance
-    auto *instance = reinterpret_cast<detail::instance *>(self);
-
     // Ensure that the base __init__ function(s) were called
-    for (const auto &vh : values_and_holders(instance)) {
-        if (!vh.holder_constructed()) {
+    values_and_holders vhs(self);
+    for (const auto &vh : vhs) {
+        if (!vh.holder_constructed() && !vhs.is_redundant_value_and_holder(vh)) {
             PyErr_Format(PyExc_TypeError,
                          "%.200s.__init__() must be called when overriding __init__",
                          get_fully_qualified_tp_name(vh.type->type).c_str());

include/pybind11/detail/type_caster_base.h

@@ -102,8 +102,22 @@ class loader_life_support {
 inline std::pair<decltype(internals::registered_types_py)::iterator, bool>
 all_type_info_get_cache(PyTypeObject *type);
 
+// Band-aid workaround to fix a subtle but serious bug in a minimalistic fashion. See PR #4762.
+inline void all_type_info_add_base_most_derived_first(std::vector<type_info *> &bases,
+                                                      type_info *addl_base) {
+    for (auto it = bases.begin(); it != bases.end(); it++) {
+        type_info *existing_base = *it;
+        if (PyType_IsSubtype(addl_base->type, existing_base->type) != 0) {
+            bases.insert(it, addl_base);
+            return;
+        }
+    }
+    bases.push_back(addl_base);
+}
+
 // Populates a just-created cache entry.
 PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector<type_info *> &bases) {
+    assert(bases.empty());
     std::vector<PyTypeObject *> check;
     for (handle parent : reinterpret_borrow<tuple>(t->tp_bases)) {
         check.push_back((PyTypeObject *) parent.ptr());
@@ -136,7 +150,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector<type_
                     }
                 }
                 if (!found) {
-                    bases.push_back(tinfo);
+                    all_type_info_add_base_most_derived_first(bases, tinfo);
                 }
             }
         } else if (type->tp_bases) {
@@ -322,18 +336,29 @@ struct values_and_holders {
     explicit values_and_holders(instance *inst)
         : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {}
 
+    explicit values_and_holders(PyObject *obj)
+        : inst{nullptr}, tinfo(all_type_info(Py_TYPE(obj))) {
+        if (!tinfo.empty()) {
+            inst = reinterpret_cast<instance *>(obj);
+        }
+    }
+
     struct iterator {
     private:
         instance *inst = nullptr;
         const type_vec *types = nullptr;
         value_and_holder curr;
         friend struct values_and_holders;
-        iterator(instance *inst, const type_vec *tinfo)
-            : inst{inst}, types{tinfo},
-              curr(inst /* instance */,
-                   types->empty() ? nullptr : (*types)[0] /* type info */,
-                   0, /* vpos: (non-simple types only): the first vptr comes first */
-                   0 /* index */) {}
+        iterator(instance *inst, const type_vec *tinfo) : inst{inst}, types{tinfo} {
+            if (inst != nullptr) {
+                assert(!types->empty());
+                curr = value_and_holder(
+                    inst /* instance */,
+                    (*types)[0] /* type info */,
+                    0, /* vpos: (non-simple types only): the first vptr comes first */
+                    0 /* index */);
+            }
+        }
         // Past-the-end iterator:
         explicit iterator(size_t end) : curr(end) {}
 
@@ -364,6 +389,16 @@ struct values_and_holders {
     }
 
     size_t size() { return tinfo.size(); }
+
+    // Band-aid workaround to fix a subtle but serious bug in a minimalistic fashion. See PR #4762.
+    bool is_redundant_value_and_holder(const value_and_holder &vh) {
+        for (size_t i = 0; i < vh.index; i++) {
+            if (PyType_IsSubtype(tinfo[i]->type, tinfo[vh.index]->type) != 0) {
+                return true;
+            }
+        }
+        return false;
+    }
 };
 
 /**

tests/CMakeLists.txt

@@ -166,6 +166,7 @@ set(PYBIND11_TEST_FILES
     test_opaque_types
     test_operator_overloading
     test_pickling
+    test_python_multiple_inheritance
     test_pytypes
     test_return_value_policy_override
     test_return_value_policy_pack

tests/test_class.py

@@ -1,3 +1,5 @@
+from unittest import mock
+
 import pytest
 
 import env
@@ -203,6 +205,18 @@ def __init__(self):
     assert msg(exc_info.value) == expected
 
 
+@pytest.mark.parametrize(
+    "mock_return_value", [None, (1, 2, 3), m.Pet("Polly", "parrot"), m.Dog("Molly")]
+)
+def test_mock_new(mock_return_value):
+    with mock.patch.object(
+        m.Pet, "__new__", return_value=mock_return_value
+    ) as mock_new:
+        obj = m.Pet("Noname", "Nospecies")
+    assert obj is mock_return_value
+    mock_new.assert_called_once_with(m.Pet, "Noname", "Nospecies")
+
+
 def test_automatic_upcasting():
     assert type(m.return_class_1()).__name__ == "DerivedClass1"
     assert type(m.return_class_2()).__name__ == "DerivedClass2"

tests/test_python_multiple_inheritance.cpp

@@ -0,0 +1,45 @@
+#include "pybind11_tests.h"
+
+namespace test_python_multiple_inheritance {
+
+// Copied from:
+// https://github.com/google/clif/blob/5718e4d0807fd3b6a8187dde140069120b81ecef/clif/testing/python_multiple_inheritance.h
+
+struct CppBase {
+    explicit CppBase(int value) : base_value(value) {}
+    int get_base_value() const { return base_value; }
+    void reset_base_value(int new_value) { base_value = new_value; }
+
+private:
+    int base_value;
+};
+
+struct CppDrvd : CppBase {
+    explicit CppDrvd(int value) : CppBase(value), drvd_value(value * 3) {}
+    int get_drvd_value() const { return drvd_value; }
+    void reset_drvd_value(int new_value) { drvd_value = new_value; }
+
+    int get_base_value_from_drvd() const { return get_base_value(); }
+    void reset_base_value_from_drvd(int new_value) { reset_base_value(new_value); }
+
+private:
+    int drvd_value;
+};
+
+} // namespace test_python_multiple_inheritance
+
+TEST_SUBMODULE(python_multiple_inheritance, m) {
+    using namespace test_python_multiple_inheritance;
+
+    py::class_<CppBase>(m, "CppBase")
+        .def(py::init<int>())
+        .def("get_base_value", &CppBase::get_base_value)
+        .def("reset_base_value", &CppBase::reset_base_value);
+
+    py::class_<CppDrvd, CppBase>(m, "CppDrvd")
+        .def(py::init<int>())
+        .def("get_drvd_value", &CppDrvd::get_drvd_value)
+        .def("reset_drvd_value", &CppDrvd::reset_drvd_value)
+        .def("get_base_value_from_drvd", &CppDrvd::get_base_value_from_drvd)
+        .def("reset_base_value_from_drvd", &CppDrvd::reset_base_value_from_drvd);
+}

tests/test_python_multiple_inheritance.py

@@ -0,0 +1,35 @@
+# Adapted from:
+# https://github.com/google/clif/blob/5718e4d0807fd3b6a8187dde140069120b81ecef/clif/testing/python/python_multiple_inheritance_test.py
+
+from pybind11_tests import python_multiple_inheritance as m
+
+
+class PC(m.CppBase):
+    pass
+
+
+class PPCC(PC, m.CppDrvd):
+    pass
+
+
+def test_PC():
+    d = PC(11)
+    assert d.get_base_value() == 11
+    d.reset_base_value(13)
+    assert d.get_base_value() == 13
+
+
+def test_PPCC():
+    d = PPCC(11)
+    assert d.get_drvd_value() == 33
+    d.reset_drvd_value(55)
+    assert d.get_drvd_value() == 55
+
+    assert d.get_base_value() == 11
+    assert d.get_base_value_from_drvd() == 11
+    d.reset_base_value(20)
+    assert d.get_base_value() == 20
+    assert d.get_base_value_from_drvd() == 20
+    d.reset_base_value_from_drvd(30)
+    assert d.get_base_value() == 30
+    assert d.get_base_value_from_drvd() == 30