Merge branch 'master' into prepv300/review

rwgk · rwgk · commit 1e21e37f9ddf · 2025-05-18T21:38:16.000-07:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -34,7 +34,9 @@ jobs:
         python:
         - '3.8'
         - '3.13'
+        - '3.13t'
         - '3.14'
+        - '3.14t'
         - 'pypy-3.10'
         - 'pypy-3.11'
         - 'graalpy-24.2'
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -130,28 +130,23 @@ if(PYBIND11_MASTER_PROJECT)
     set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}")
   endif()
 
-  if(NOT DEFINED Python3_EXECUTABLE
-     AND NOT DEFINED Python_EXECUTABLE
-     AND NOT DEFINED Python_ROOT_DIR
-     AND NOT DEFINED ENV{VIRTUALENV}
-     AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.venv")
-    message(STATUS "Autodetecting Python in virtual environment")
-    set(Python_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/.venv")
-  endif()
-
   # This is a shortcut that is primarily for the venv cmake preset,
   # but can be used to quickly setup tests manually, too
   set(PYBIND11_CREATE_WITH_UV
       ""
-      CACHE STRING "Create a virtualenv in Python_ROOT_DIR with uv if it doesn't exist")
+      CACHE STRING "Create a virtualenv if it doesn't exist")
 
   if(NOT PYBIND11_CREATE_WITH_UV STREQUAL "")
-    if(NOT DEFINED Python_ROOT_DIR)
-      message(FATAL_ERROR "Python_ROOT_DIR must be defined to use PYBIND11_CREATE_WITH_UV")
-    endif()
+    set(Python_ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/.venv")
     if(EXISTS "${Python_ROOT_DIR}")
-      message(STATUS "Using existing venv at ${Python_ROOT_DIR}, remove to recreate")
-    else()
+      if(EXISTS "${CMAKE_BINARY_DIR}/CMakeCache.txt")
+        message(STATUS "Using existing venv at ${Python_ROOT_DIR}, remove or --fresh to recreate")
+      else()
+        # --fresh used to remove the cache
+        file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/.venv")
+      endif()
+    endif()
+    if(NOT EXISTS "${Python_ROOT_DIR}")
       find_program(UV uv REQUIRED)
       # CMake 3.19+ would be able to use COMMAND_ERROR_IS_FATAL
       message(
@@ -172,8 +167,16 @@ if(PYBIND11_MASTER_PROJECT)
         message(FATAL_ERROR "uv pip install failed with '${_pip_result}'")
       endif()
     endif()
+  else()
+    if(NOT DEFINED Python3_EXECUTABLE
+       AND NOT DEFINED Python_EXECUTABLE
+       AND NOT DEFINED Python_ROOT_DIR
+       AND NOT DEFINED ENV{VIRTUALENV}
+       AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.venv")
+      message(STATUS "Autodetecting Python in virtual environment")
+      set(Python_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/.venv")
+    endif()
   endif()
-
 endif()
 
 # NB: when adding a header don't forget to also add it to setup.py
diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst
@@ -1384,13 +1384,21 @@ You can do that using ``py::custom_type_setup``:
            auto *type = &heap_type->ht_type;
            type->tp_flags |= Py_TPFLAGS_HAVE_GC;
            type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) {
-               auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
-               Py_VISIT(self.value.ptr());
+               // https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse
+   #if PY_VERSION_HEX >= 0x03090000
+               Py_VISIT(Py_TYPE(self_base));
+   #endif
+               if (py::detail::is_holder_constructed(self_base)) {
+                   auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
+                   Py_VISIT(self.value.ptr());
+               }
                return 0;
            };
            type->tp_clear = [](PyObject *self_base) {
-               auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
-               self.value = py::none();
+               if (py::detail::is_holder_constructed(self_base)) {
+                   auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
+                   self.value = py::none();
+               }
                return 0;
            };
        }));
diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst
@@ -212,6 +212,11 @@ naturally:
         assert(locals["message"].cast<std::string>() == "1 + 2 = 3");
     }
 
+``PYBIND11_EMBEDDED_MODULE`` also accepts
+:func:`py::mod_gil_not_used()`,
+:func:`py::multiple_interpreters::per_interpreter_gil()`, and
+:func:`py::multiple_interpreters::shared_gil()` tags just like ``PYBIND11_MODULE``.
+See :ref:`misc_subinterp` and :ref:`misc_free_threading` for more information.
 
 Interpreter lifetime
 ====================
diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst
@@ -155,6 +155,8 @@ following checklist.
   within pybind11 that will throw exceptions on certain GIL handling errors
   (reference counting operations).
 
+.. _misc_free_threading:
+
 Free-threading support
 ==================================================================
 
@@ -178,6 +180,8 @@ your code is thread safe.  Modules must still be built against the Python free-t
 enable free-threading, even if they specify this tag.  Adding this tag does not break
 compatibility with non-free-threaded Python.
 
+.. _misc_subinterp:
+
 Sub-interpreter support
 ==================================================================
 
diff --git a/include/pybind11/detail/value_and_holder.h b/include/pybind11/detail/value_and_holder.h
@@ -74,5 +74,17 @@ struct value_and_holder {
     }
 };
 
+// This is a semi-public API to check if the corresponding instance has been constructed with a
+// holder. That is, if the instance has been constructed with a holder, the `__init__` method is
+// called and the C++ object is valid. Otherwise, the C++ object might only be allocated, but not
+// initialized. This will lead to **SEGMENTATION FAULTS** if the C++ object is used in any way.
+// Example usage: https://pybind11.readthedocs.io/en/stable/advanced/classes.html#custom-type-setup
+//                for `tp_traverse` and `tp_clear` implementations.
+// WARNING: The caller is responsible for ensuring that the `reinterpret_cast` is valid.
+inline bool is_holder_constructed(PyObject *obj) {
+    auto *const instance = reinterpret_cast<pybind11::detail::instance *>(obj);
+    return instance->get_value_and_holder().holder_constructed();
+}
+
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)
diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h
@@ -38,24 +38,43 @@
             });
         }
  \endrst */
-#define PYBIND11_EMBEDDED_MODULE(name, variable)                                                  \
+PYBIND11_WARNING_PUSH
+PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments")
+#define PYBIND11_EMBEDDED_MODULE(name, variable, ...)                                             \
     static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name);           \
+    static ::pybind11::module_::slots_array PYBIND11_CONCAT(pybind11_module_slots_, name);        \
+    static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *);                                 \
     static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &);                     \
     static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() {                            \
-        auto m = ::pybind11::module_::create_extension_module(                                    \
-            PYBIND11_TOSTRING(name), nullptr, &PYBIND11_CONCAT(pybind11_module_def_, name));      \
+        static auto result = []() {                                                               \
+            auto &slots = PYBIND11_CONCAT(pybind11_module_slots_, name);                          \
+            slots[0] = {Py_mod_exec,                                                              \
+                        reinterpret_cast<void *>(&PYBIND11_CONCAT(pybind11_exec_, name))};        \
+            slots[1] = {0, nullptr};                                                              \
+            return ::pybind11::module_::initialize_multiphase_module_def(                         \
+                PYBIND11_TOSTRING(name),                                                          \
+                nullptr,                                                                          \
+                &PYBIND11_CONCAT(pybind11_module_def_, name),                                     \
+                slots,                                                                            \
+                ##__VA_ARGS__);                                                                   \
+        }();                                                                                      \
+        return result.ptr();                                                                      \
+    }                                                                                             \
+    PYBIND11_EMBEDDED_MODULE_IMPL(name)                                                           \
+    ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)(                  \
+        PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name));                     \
+    int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject * pm) {                                    \
         try {                                                                                     \
+            auto m = pybind11::reinterpret_borrow<::pybind11::module_>(pm);                       \
             PYBIND11_CONCAT(pybind11_init_, name)(m);                                             \
-            return m.ptr();                                                                       \
+            return 0;                                                                             \
         }                                                                                         \
         PYBIND11_CATCH_INIT_EXCEPTIONS                                                            \
-        return nullptr;                                                                           \
+        return -1;                                                                                \
     }                                                                                             \
-    PYBIND11_EMBEDDED_MODULE_IMPL(name)                                                           \
-    ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)(                  \
-        PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name));                     \
     void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_                                \
                                                & variable) // NOLINT(bugprone-macro-parentheses)
+PYBIND11_WARNING_POP
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h
@@ -1484,7 +1484,7 @@ class module_ : public object {
     using slots_array = std::array<PyModuleDef_Slot, 4>;
 
     /** \rst
-        Initialized a module def for use with multi-phase module initialization.
+        Initialize a module def for use with multi-phase module initialization.
 
         ``def`` should point to a statically allocated module_def.
         ``slots`` must already contain a Py_mod_exec or Py_mod_create slot and will be filled with
diff --git a/tests/test_custom_type_setup.cpp b/tests/test_custom_type_setup.cpp
@@ -26,13 +26,21 @@ TEST_SUBMODULE(custom_type_setup, m) {
             auto *type = &heap_type->ht_type;
             type->tp_flags |= Py_TPFLAGS_HAVE_GC;
             type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) {
-                auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
-                Py_VISIT(self.value.ptr());
+        // https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse
+#if PY_VERSION_HEX >= 0x03090000
+                Py_VISIT(Py_TYPE(self_base));
+#endif
+                if (py::detail::is_holder_constructed(self_base)) {
+                    auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
+                    Py_VISIT(self.value.ptr());
+                }
                 return 0;
             };
             type->tp_clear = [](PyObject *self_base) {
-                auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
-                self.value = py::none();
+                if (py::detail::is_holder_constructed(self_base)) {
+                    auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
+                    self.value = py::none();
+                }
                 return 0;
             };
         }));
diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp
@@ -55,7 +55,7 @@ class test_override_cache_helper_trampoline : public test_override_cache_helper
     int func() override { PYBIND11_OVERRIDE(int, test_override_cache_helper, func); }
 };
 
-PYBIND11_EMBEDDED_MODULE(widget_module, m) {
+PYBIND11_EMBEDDED_MODULE(widget_module, m, py::multiple_interpreters::per_interpreter_gil()) {
     py::class_<Widget, PyWidget>(m, "Widget")
         .def(py::init<std::string>())
         .def_property_readonly("the_message", &Widget::the_message);
@@ -336,6 +336,7 @@ TEST_CASE("Restart the interpreter") {
     REQUIRE(py_widget.attr("the_message").cast<std::string>() == "Hello after restart");
 }
 
+#if defined(PYBIND11_SUBINTERPRETER_SUPPORT)
 TEST_CASE("Subinterpreter") {
     py::module_::import("external_module"); // in the main interpreter
 
@@ -347,6 +348,10 @@ TEST_CASE("Subinterpreter") {
 
         REQUIRE(m.attr("add")(1, 2).cast<int>() == 3);
     }
+
+    auto main_int
+        = py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>();
+
     REQUIRE(has_state_dict_internals_obj());
     REQUIRE(has_pybind11_internals_static());
 
@@ -359,7 +364,6 @@ TEST_CASE("Subinterpreter") {
     // Subinterpreters get their own copy of builtins.
     REQUIRE_FALSE(has_state_dict_internals_obj());
 
-#if defined(PYBIND11_SUBINTERPRETER_SUPPORT) && PY_VERSION_HEX >= 0x030C0000
     // internals hasn't been populated yet, but will be different for the subinterpreter
     REQUIRE_FALSE(has_pybind11_internals_static());
 
@@ -369,14 +373,12 @@ TEST_CASE("Subinterpreter") {
     py::detail::get_internals();
     REQUIRE(has_pybind11_internals_static());
     REQUIRE(get_details_as_uintptr() == ext_int);
-#else
-    // This static is still defined
-    REQUIRE(has_pybind11_internals_static());
-#endif
+    REQUIRE(main_int != ext_int);
 
     // Modules tags should be gone.
     REQUIRE_FALSE(py::hasattr(py::module_::import("__main__"), "tag"));
     {
+        REQUIRE_NOTHROW(py::module_::import("widget_module"));
         auto m = py::module_::import("widget_module");
         REQUIRE_FALSE(py::hasattr(m, "extension_module_tag"));
 
@@ -397,7 +399,6 @@ TEST_CASE("Subinterpreter") {
     REQUIRE(has_state_dict_internals_obj());
 }
 
-#if defined(PYBIND11_SUBINTERPRETER_SUPPORT)
 TEST_CASE("Multiple Subinterpreters") {
     // Make sure the module is in the main interpreter and save its pointer
     auto *main_ext = py::module_::import("external_module").ptr();
@@ -512,10 +513,11 @@ TEST_CASE("Per-Subinterpreter GIL") {
 
         // we have switched to the new interpreter and released the main gil
 
-        // widget_module did not provide the mod_per_interpreter_gil tag, so it cannot be imported
+        // trampoline_module did not provide the per_interpreter_gil tag, so it cannot be
+        // imported
         bool caught = false;
         try {
-            py::module_::import("widget_module");
+            py::module_::import("trampoline_module");
         } catch (pybind11::error_already_set &pe) {
             T_REQUIRE(pe.matches(PyExc_ImportError));
             std::string msg(pe.what());
@@ -525,6 +527,9 @@ TEST_CASE("Per-Subinterpreter GIL") {
         }
         T_REQUIRE(caught);
 
+        // widget_module did provide the per_interpreter_gil tag, so it this does not throw
+        py::module_::import("widget_module");
+
         T_REQUIRE(!py::hasattr(py::module_::import("external_module"), "multi_interp"));
         py::module_::import("external_module").attr("multi_interp") = std::to_string(num);
 
@@ -547,8 +552,8 @@ TEST_CASE("Per-Subinterpreter GIL") {
 
         Py_EndInterpreter(sub);
 
-        PyThreadState_Swap(
-            main_tstate); // switch back so the scoped_acquire can release the GIL properly
+        // switch back so the scoped_acquire can release the GIL properly
+        PyThreadState_Swap(main_tstate);
     };
 
     std::thread t1(thread_main, 1);
@@ -622,12 +627,26 @@ TEST_CASE("Threads") {
 
     {
         py::gil_scoped_release gil_release{};
+#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX < 0x030E0000
+        std::mutex mutex;
+#endif
 
         auto threads = std::vector<std::thread>();
         for (auto i = 0; i < num_threads; ++i) {
             threads.emplace_back([&]() {
                 py::gil_scoped_acquire gil{};
+#ifdef Py_GIL_DISABLED
+#    if PY_VERSION_HEX < 0x030E0000
+                std::lock_guard<std::mutex> lock(mutex);
+                locals["count"] = locals["count"].cast<int>() + 1;
+#    else
+                Py_BEGIN_CRITICAL_SECTION(locals.ptr());
                 locals["count"] = locals["count"].cast<int>() + 1;
+                Py_END_CRITICAL_SECTION();
+#    endif
+#else
+                locals["count"] = locals["count"].cast<int>() + 1;
+#endif
             });
         }
 
