Add py::potentially_slicing_weak_ptr(handle) function

docs/advanced/classes.rst

@@ -428,6 +428,51 @@ Python side by allowing the Python function to return ``None`` or an ``int``:
         return false;  // Alternatively return MyClass::myMethod(value);
     }
 
+Avoiding Inheritance Slicing and ``std::weak_ptr`` surprises
+------------------------------------------------------------
+
+When working with classes that use virtual functions and are subclassed
+in Python, special care must be taken when converting Python objects to
+``std::shared_ptr<T>``. Depending on whether the class uses a plain
+``std::shared_ptr`` holder or ``py::smart_holder``, the resulting
+``shared_ptr`` may either allow inheritance slicing or lead to potentially
+surprising behavior when constructing ``std::weak_ptr`` instances.
+
+This section explains how ``std::shared_ptr`` and ``py::smart_holder`` manage
+object lifetimes differently, how these differences affect trampoline-derived
+objects, and what options are available to achieve the situation-specific
+desired behavior.
+
+When using ``std::shared_ptr`` as the holder type, converting a Python object
+to a ``std::shared_ptr<T>`` (e.g., ``obj.cast<std::shared_ptr<T>>()``, or simply
+passing the Python object as an argument to a ``.def()``-ed function) returns
+a ``shared_ptr`` that shares ownership with the original ``class_`` holder,
+usually preserving object lifetime. However, for Python classes that derive from
+a trampoline, if the Python object is destroyed, only the base C++ object may
+remain alive, leading to inheritance slicing
+(see `#1333 <https://github.com/pybind/pybind11/issues/1333>`_).
+
+In contrast, with ``py::smart_holder``, converting a Python object to
+a ``std::shared_ptr<T>`` returns a new ``shared_ptr`` with an independent
+control block that keeps the derived Python object alive. This avoids
+inheritance slicing but can lead to unintended behavior when creating
+``std::weak_ptr`` instances
+(see `#5623 <https://github.com/pybind/pybind11/issues/5623>`_).
+
+If it is necessary to obtain a ``std::weak_ptr`` that shares the control block
+with the ``smart_holder``—at the cost of reintroducing potential inheritance
+slicing—you can use ``py::potentially_slicing_weak_ptr<T>(obj)``.
+
+When precise lifetime management of derived Python objects is important,
+using a Python-side ``weakref`` is the most reliable approach, as it avoids
+both inheritance slicing and unintended interactions with ``std::weak_ptr``
+semantics in C++.
+
+.. seealso::
+
+    * :func:`potentially_slicing_weak_ptr` C++ documentation
+    * :file:`tests/test_potentially_slicing_weak_ptr.cpp`
+
 
 .. _custom_constructors:
 

include/pybind11/cast.h

@@ -983,6 +983,17 @@ struct copyable_holder_caster<
         return shared_ptr_storage;
     }
 
+    std::weak_ptr<type> potentially_slicing_weak_ptr() {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+            // Reusing shared_ptr code to minimize code complexity.
+            shared_ptr_storage
+                = sh_load_helper.load_as_shared_ptr(value,
+                                                    /*responsible_parent=*/nullptr,
+                                                    /*force_potentially_slicing_shared_ptr=*/true);
+        }
+        return shared_ptr_storage;
+    }
+
     static handle
     cast(const std::shared_ptr<type> &src, return_value_policy policy, handle parent) {
         const auto *ptr = src.get();
@@ -1077,6 +1088,54 @@ struct copyable_holder_caster<
 template <typename T>
 class type_caster<std::shared_ptr<T>> : public copyable_holder_caster<T, std::shared_ptr<T>> {};
 
+PYBIND11_NAMESPACE_END(detail)
+
+/// Return a std::shared_ptr with the SAME CONTROL BLOCK as the std::shared_ptr owned by the
+/// class_ holder. For class_-wrapped types with trampolines, the returned std::shared_ptr
+/// does NOT keep any derived Python objects alive (see issue #1333).
+///
+/// For class_-wrapped types using std::shared_ptr as the holder, the following expressions
+/// produce equivalent results (see tests/test_potentially_slicing_weak_ptr.cpp,py):
+///
+///     - obj.cast<std::shared_ptr<T>>()
+///     - py::potentially_slicing_weak_ptr<T>(obj).lock()
+///
+/// For class_-wrapped types with trampolines and using py::smart_holder, obj.cast<>()
+/// produces a std::shared_ptr that keeps any derived Python objects alive for its own lifetime,
+/// but this is achieved by introducing a std::shared_ptr control block that is independent of
+/// the one owned by the py::smart_holder. This can lead to surprising std::weak_ptr behavior
+/// (see issue #5623). An easy solution is to use py::potentially_slicing_weak_ptr<>(obj),
+/// as exercised in tests/test_potentially_slicing_weak_ptr.cpp,py (look for
+/// "set_wp_potentially_slicing"). Note, however, that this reintroduces the inheritance
+/// slicing issue (see issue #1333). The ideal — but usually more involved — solution is to use
+/// a Python weakref to the derived Python object, instead of a C++ base-class std::weak_ptr.
+///
+/// It is not possible (at least no known approach exists at the time of this writing) to
+/// simultaneously achieve both desirable properties:
+///
+///     - the same std::shared_ptr control block as the class_ holder
+///     - automatic lifetime extension of any derived Python objects
+///
+/// The reason is that this would introduce a reference cycle that cannot be garbage collected:
+///
+///     - the derived Python object owns the class_ holder
+///     - the class_ holder owns the std::shared_ptr
+///     - the std::shared_ptr would own a reference to the derived Python object,
+///       completing the cycle
+template <typename T>
+std::weak_ptr<T> potentially_slicing_weak_ptr(handle obj) {
+    detail::make_caster<std::shared_ptr<T>> caster;
+    if (caster.load(obj, /*convert=*/true)) {
+        return caster.potentially_slicing_weak_ptr();
+    }
+    const char *obj_type_name = detail::obj_class_name(obj.ptr());
+    throw type_error("\"" + std::string(obj_type_name)
+                     + "\" object is not convertible to std::weak_ptr<T> (with T = " + type_id<T>()
+                     + ")");
+}
+
+PYBIND11_NAMESPACE_BEGIN(detail)
+
 // SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to unique_ptr specialization.
 /// Type caster for holder types like std::unique_ptr.
 /// Please consider the SFINAE hook an implementation detail, as explained

include/pybind11/detail/type_caster_base.h

@@ -759,7 +759,11 @@ struct load_helper : value_and_holder_helper {
     }
 
     std::shared_ptr<T> load_as_shared_ptr(void *void_raw_ptr,
-                                          handle responsible_parent = nullptr) const {
+                                          handle responsible_parent = nullptr,
+                                          // to support py::potentially_slicing_weak_ptr
+                                          // with minimal added code complexity:
+                                          bool force_potentially_slicing_shared_ptr
+                                          = false) const {
         if (!have_holder()) {
             return nullptr;
         }
@@ -774,7 +778,7 @@ struct load_helper : value_and_holder_helper {
             throw std::runtime_error("Non-owning holder (load_as_shared_ptr).");
         }
         auto *type_raw_ptr = static_cast<T *>(void_raw_ptr);
-        if (python_instance_is_alias) {
+        if (python_instance_is_alias && !force_potentially_slicing_shared_ptr) {
             auto *vptr_gd_ptr = std::get_deleter<memory::guarded_delete>(hld.vptr);
             if (vptr_gd_ptr != nullptr) {
                 std::shared_ptr<void> released_ptr = vptr_gd_ptr->released_ptr.lock();

tests/CMakeLists.txt

@@ -161,6 +161,7 @@ set(PYBIND11_TEST_FILES
     test_opaque_types
     test_operator_overloading
     test_pickling
+    test_potentially_slicing_weak_ptr
     test_python_multiple_inheritance
     test_pytypes
     test_sequences_and_iterators

tests/test_potentially_slicing_weak_ptr.cpp

@@ -0,0 +1,168 @@
+// Copyright (c) 2025 The pybind Community.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#include "pybind11_tests.h"
+
+#include <memory>
+
+namespace pybind11_tests {
+namespace potentially_slicing_weak_ptr {
+
+template <int> // Using int as a trick to easily generate multiple types.
+struct VirtBase {
+    virtual ~VirtBase() = default;
+    virtual int get_code() { return 100; }
+};
+
+using VirtBaseSH = VirtBase<0>; // for testing with py::smart_holder
+using VirtBaseSP = VirtBase<1>; // for testing with std::shared_ptr as holder
+
+// Similar to trampoline_self_life_support
+struct trampoline_is_alive_simple {
+    std::uint64_t magic_token = 197001010000u;
+
+    trampoline_is_alive_simple() = default;
+
+    ~trampoline_is_alive_simple() { magic_token = 20380118191407u; }
+
+    trampoline_is_alive_simple(const trampoline_is_alive_simple &other) = default;
+    trampoline_is_alive_simple(trampoline_is_alive_simple &&other) noexcept
+        : magic_token(other.magic_token) {
+        other.magic_token = 20380118191407u;
+    }
+
+    trampoline_is_alive_simple &operator=(const trampoline_is_alive_simple &) = delete;
+    trampoline_is_alive_simple &operator=(trampoline_is_alive_simple &&) = delete;
+};
+
+template <typename VB>
+const char *determine_trampoline_state(const std::shared_ptr<VB> &sp) {
+    if (!sp) {
+        return "sp nullptr";
+    }
+    auto *tias = dynamic_cast<trampoline_is_alive_simple *>(sp.get());
+    if (!tias) {
+        return "dynamic_cast failed";
+    }
+    if (tias->magic_token == 197001010000u) {
+        return "trampoline alive";
+    }
+    if (tias->magic_token == 20380118191407u) {
+        return "trampoline DEAD";
+    }
+    return "UNDEFINED BEHAVIOR";
+}
+
+struct PyVirtBaseSH : VirtBaseSH, py::trampoline_self_life_support, trampoline_is_alive_simple {
+    using VirtBaseSH::VirtBaseSH;
+    int get_code() override { PYBIND11_OVERRIDE(int, VirtBaseSH, get_code); }
+};
+
+struct PyVirtBaseSP : VirtBaseSP, trampoline_is_alive_simple { // self-life-support not available
+    using VirtBaseSP::VirtBaseSP;
+    int get_code() override { PYBIND11_OVERRIDE(int, VirtBaseSP, get_code); }
+};
+
+template <typename VB>
+std::shared_ptr<VB> rtrn_obj_cast_shared_ptr(py::handle obj) {
+    return obj.cast<std::shared_ptr<VB>>();
+}
+
+// There is no type_caster<std::weak_ptr<VB>>, and to minimize code complexity
+// we do not want to add one, therefore we have to return a shared_ptr here.
+template <typename VB>
+std::shared_ptr<VB> rtrn_potentially_slicing_shared_ptr(py::handle obj) {
+    return py::potentially_slicing_weak_ptr<VB>(obj).lock();
+}
+
+template <typename VB>
+struct SpOwner {
+    void set_sp(const std::shared_ptr<VB> &sp_) { sp = sp_; }
+
+    int get_code() const {
+        if (!sp) {
+            return -888;
+        }
+        return sp->get_code();
+    }
+
+    const char *get_trampoline_state() const { return determine_trampoline_state(sp); }
+
+private:
+    std::shared_ptr<VB> sp;
+};
+
+template <typename VB>
+struct WpOwner {
+    void set_wp(const std::weak_ptr<VB> &wp_) { wp = wp_; }
+
+    int get_code() const {
+        auto sp = wp.lock();
+        if (!sp) {
+            return -999;
+        }
+        return sp->get_code();
+    }
+
+    const char *get_trampoline_state() const { return determine_trampoline_state(wp.lock()); }
+
+private:
+    std::weak_ptr<VB> wp;
+};
+
+template <typename VB>
+void wrap(py::module_ &m,
+          const char *roc_pyname,
+          const char *rps_pyname,
+          const char *spo_pyname,
+          const char *wpo_pyname) {
+    m.def(roc_pyname, rtrn_obj_cast_shared_ptr<VB>);
+    m.def(rps_pyname, rtrn_potentially_slicing_shared_ptr<VB>);
+
+    py::classh<SpOwner<VB>>(m, spo_pyname)
+        .def(py::init<>())
+        .def("set_sp", &SpOwner<VB>::set_sp)
+        .def("get_code", &SpOwner<VB>::get_code)
+        .def("get_trampoline_state", &SpOwner<VB>::get_trampoline_state);
+
+    py::classh<WpOwner<VB>>(m, wpo_pyname)
+        .def(py::init<>())
+        .def("set_wp",
+             [](WpOwner<VB> &self, py::handle obj) {
+                 self.set_wp(obj.cast<std::shared_ptr<VB>>());
+             })
+        .def("set_wp_potentially_slicing",
+             [](WpOwner<VB> &self, py::handle obj) {
+                 self.set_wp(py::potentially_slicing_weak_ptr<VB>(obj));
+             })
+        .def("get_code", &WpOwner<VB>::get_code)
+        .def("get_trampoline_state", &WpOwner<VB>::get_trampoline_state);
+}
+
+} // namespace potentially_slicing_weak_ptr
+} // namespace pybind11_tests
+
+using namespace pybind11_tests::potentially_slicing_weak_ptr;
+
+TEST_SUBMODULE(potentially_slicing_weak_ptr, m) {
+    py::classh<VirtBaseSH, PyVirtBaseSH>(m, "VirtBaseSH")
+        .def(py::init<>())
+        .def("get_code", &VirtBaseSH::get_code);
+
+    py::class_<VirtBaseSP, std::shared_ptr<VirtBaseSP>, PyVirtBaseSP>(m, "VirtBaseSP")
+        .def(py::init<>())
+        .def("get_code", &VirtBaseSP::get_code);
+
+    wrap<VirtBaseSH>(m,
+                     "SH_rtrn_obj_cast_shared_ptr",
+                     "SH_rtrn_potentially_slicing_shared_ptr",
+                     "SH_SpOwner",
+                     "SH_WpOwner");
+
+    wrap<VirtBaseSP>(m,
+                     "SP_rtrn_obj_cast_shared_ptr",
+                     "SP_rtrn_potentially_slicing_shared_ptr",
+                     "SP_SpOwner",
+                     "SP_WpOwner");
+}