feat: enable conversions between native Python enum types and C++ enums

CMakeLists.txt

@@ -134,6 +134,7 @@ set(PYBIND11_HEADERS
     include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h
     include/pybind11/detail/init.h
     include/pybind11/detail/internals.h
+    include/pybind11/detail/native_enum_data.h
     include/pybind11/detail/struct_smart_holder.h
     include/pybind11/detail/type_caster_base.h
     include/pybind11/detail/typeid.h
@@ -160,6 +161,7 @@ set(PYBIND11_HEADERS
     include/pybind11/gil_safe_call_once.h
     include/pybind11/iostream.h
     include/pybind11/functional.h
+    include/pybind11/native_enum.h
     include/pybind11/numpy.h
     include/pybind11/operators.h
     include/pybind11/pybind11.h

docs/classes.rst

@@ -459,8 +459,8 @@ you can use ``py::detail::overload_cast_impl`` with an additional set of parenth
     other using the ``.def(py::init<...>())`` syntax. The existing machinery
     for specifying keyword and default arguments also works.
 
-Enumerations and internal types
-===============================
+Enumerations
+============
 
 Let's now suppose that the example class contains internal types like enumerations, e.g.:
 
@@ -494,69 +494,37 @@ The binding code for this example looks as follows:
         .def_readwrite("type", &Pet::type)
         .def_readwrite("attr", &Pet::attr);
 
-    py::enum_<Pet::Kind>(pet, "Kind")
+    py::native_enum<Pet::Kind>(pet, "Kind")
         .value("Dog", Pet::Kind::Dog)
         .value("Cat", Pet::Kind::Cat)
-        .export_values();
+        .export_values()
+        .finalize();
 
     py::class_<Pet::Attributes>(pet, "Attributes")
         .def(py::init<>())
         .def_readwrite("age", &Pet::Attributes::age);
 
 
-To ensure that the nested types ``Kind`` and ``Attributes`` are created within the scope of ``Pet``, the
-``pet`` ``py::class_`` instance must be supplied to the :class:`enum_` and ``py::class_``
-constructor. The :func:`enum_::export_values` function exports the enum entries
-into the parent scope, which should be skipped for newer C++11-style strongly
-typed enums.
-
-.. code-block:: pycon
-
-    >>> p = Pet("Lucy", Pet.Cat)
-    >>> p.type
-    Kind.Cat
-    >>> int(p.type)
-    1L
-
-The entries defined by the enumeration type are exposed in the ``__members__`` property:
-
-.. code-block:: pycon
-
-    >>> Pet.Kind.__members__
-    {'Dog': Kind.Dog, 'Cat': Kind.Cat}
-
-The ``name`` property returns the name of the enum value as a unicode string.
-
-.. note::
-
-    It is also possible to use ``str(enum)``, however these accomplish different
-    goals. The following shows how these two approaches differ.
-
-    .. code-block:: pycon
-
-        >>> p = Pet("Lucy", Pet.Cat)
-        >>> pet_type = p.type
-        >>> pet_type
-        Pet.Cat
-        >>> str(pet_type)
-        'Pet.Cat'
-        >>> pet_type.name
-        'Cat'
-
-.. note::
-
-    When the special tag ``py::arithmetic()`` is specified to the ``enum_``
-    constructor, pybind11 creates an enumeration that also supports rudimentary
-    arithmetic and bit-level operations like comparisons, and, or, xor, negation,
-    etc.
-
-    .. code-block:: cpp
-
-        py::enum_<Pet::Kind>(pet, "Kind", py::arithmetic())
-           ...
-
-    By default, these are omitted to conserve space.
-
-.. warning::
-
-    Contrary to Python customs, enum values from the wrappers should not be compared using ``is``, but with ``==`` (see `#1177 <https://github.com/pybind/pybind11/issues/1177>`_ for background).
+To ensure that the nested types ``Kind`` and ``Attributes`` are created
+within the scope of ``Pet``, the ``pet`` ``py::class_`` instance must be
+supplied to the ``py::native_enum`` and ``py::class_`` constructors. The
+``.export_values()`` function is available for exporting the enum entries
+into the parent scope, if desired.
+
+The ``.finalize()`` call above is needed because Python's native enums
+cannot be built incrementally, but all name/value pairs need to be passed at
+once. To achieve this, ``py::native_enum`` acts as a buffer to collect the
+name/value pairs. The ``.finalize()`` call uses the accumulated name/value
+pairs to build the arguments for constructing a native Python enum type.
+
+``py::native_enum`` was introduced with pybind11 release v3.0.0. It binds C++
+enum types to Python's native enum.Enum, making them PEP 435 compatible. This
+is the recommended way to bind C++ enums. The older ``py::enum_`` is
+not PEP 435 compatible
+(see `issue #2332 <https://github.com/pybind/pybind11/issues/2332>`_)
+but remains supported indefinitely for backward compatibility.
+New bindings should prefer ``py::native_enum``.
+
+For details about the deprecated ``py::enum_``, please refer to
+:file:`tests/test_enum.cpp` and
+:file:`tests/test_enum.py`.

include/pybind11/cast.h

@@ -12,6 +12,7 @@
 
 #include "detail/common.h"
 #include "detail/descr.h"
+#include "detail/native_enum_data.h"
 #include "detail/type_caster_base.h"
 #include "detail/typeid.h"
 #include "pytypes.h"
@@ -53,6 +54,104 @@ cast_op(make_caster<T> &&caster) {
     return std::move(caster).operator result_t();
 }
 
+template <typename EnumType>
+class type_caster_enum_type {
+private:
+    using Underlying = typename std::underlying_type<EnumType>::type;
+
+public:
+    static constexpr auto name = const_name<EnumType>();
+
+    template <typename SrcType>
+    static handle cast(SrcType &&src, return_value_policy, handle parent) {
+        handle native_enum
+            = global_internals_native_enum_type_map_get_item(std::type_index(typeid(EnumType)));
+        if (native_enum) {
+            return native_enum(static_cast<Underlying>(src)).release();
+        }
+        return type_caster_base<EnumType>::cast(
+            std::forward<SrcType>(src),
+            // Fixes https://github.com/pybind/pybind11/pull/3643#issuecomment-1022987818:
+            return_value_policy::copy,
+            parent);
+    }
+
+    bool load(handle src, bool convert) {
+        handle native_enum
+            = global_internals_native_enum_type_map_get_item(std::type_index(typeid(EnumType)));
+        if (native_enum) {
+            if (!isinstance(src, native_enum)) {
+                return false;
+            }
+            type_caster<Underlying> underlying_caster;
+            if (!underlying_caster.load(src.attr("value"), convert)) {
+                pybind11_fail("native_enum internal consistency failure.");
+            }
+            value = static_cast<EnumType>(static_cast<Underlying>(underlying_caster));
+            return true;
+        }
+        if (!pybind11_enum_) {
+            pybind11_enum_.reset(new type_caster_base<EnumType>());
+        }
+        return pybind11_enum_->load(src, convert);
+    }
+
+    template <typename T>
+    using cast_op_type = detail::cast_op_type<T>;
+
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator EnumType *() {
+        if (!pybind11_enum_) {
+            return &value;
+        }
+        return pybind11_enum_->operator EnumType *();
+    }
+
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator EnumType &() {
+        if (!pybind11_enum_) {
+            return value;
+        }
+        return pybind11_enum_->operator EnumType &();
+    }
+
+private:
+    std::unique_ptr<type_caster_base<EnumType>> pybind11_enum_;
+    EnumType value;
+};
+
+template <typename EnumType, typename SFINAE = void>
+struct type_caster_enum_type_enabled : std::true_type {};
+
+template <typename T>
+struct type_uses_type_caster_enum_type {
+    static constexpr bool value
+        = std::is_enum<T>::value && type_caster_enum_type_enabled<T>::value;
+};
+
+template <typename EnumType>
+class type_caster<EnumType, detail::enable_if_t<type_uses_type_caster_enum_type<EnumType>::value>>
+    : public type_caster_enum_type<EnumType> {};
+
+template <typename T, detail::enable_if_t<std::is_enum<T>::value, int> = 0>
+bool isinstance_native_enum_impl(handle obj, const std::type_info &tp) {
+    handle native_enum = global_internals_native_enum_type_map_get_item(tp);
+    if (!native_enum) {
+        return false;
+    }
+    return isinstance(obj, native_enum);
+}
+
+template <typename T, detail::enable_if_t<!std::is_enum<T>::value, int> = 0>
+bool isinstance_native_enum_impl(handle, const std::type_info &) {
+    return false;
+}
+
+template <typename T>
+bool isinstance_native_enum(handle obj, const std::type_info &tp) {
+    return isinstance_native_enum_impl<intrinsic_t<T>>(obj, tp);
+}
+
 template <typename type>
 class type_caster<std::reference_wrapper<type>> {
 private:
@@ -1468,8 +1567,17 @@ template <typename T,
           = 0>
 T cast(const handle &handle) {
     using namespace detail;
-    static_assert(!cast_is_temporary_value_reference<T>::value,
+    constexpr bool is_enum_cast = type_uses_type_caster_enum_type<intrinsic_t<T>>::value;
+    static_assert(!cast_is_temporary_value_reference<T>::value || is_enum_cast,
                   "Unable to cast type to reference: value is local to type caster");
+#ifndef NDEBUG
+    if (is_enum_cast && cast_is_temporary_value_reference<T>::value) {
+        if (detail::global_internals_native_enum_type_map_contains(
+                std::type_index(typeid(intrinsic_t<T>)))) {
+            pybind11_fail("Unable to cast native enum type to reference");
+        }
+    }
+#endif
     return cast_op<T>(load_type<T>(handle));
 }
 

include/pybind11/detail/class.h

@@ -716,15 +716,7 @@ inline PyObject *make_new_python_type(const type_record &rec) {
             PyUnicode_FromFormat("%U.%U", rec.scope.attr("__qualname__").ptr(), name.ptr()));
     }
 
-    object module_;
-    if (rec.scope) {
-        if (hasattr(rec.scope, "__module__")) {
-            module_ = rec.scope.attr("__module__");
-        } else if (hasattr(rec.scope, "__name__")) {
-            module_ = rec.scope.attr("__name__");
-        }
-    }
-
+    object module_ = get_module_name_if_available(rec.scope);
     const auto *full_name = c_str(
 #if !defined(PYPY_VERSION)
         module_ ? str(module_).cast<std::string>() + "." + rec.name :

include/pybind11/detail/internals.h

@@ -37,11 +37,11 @@
 /// further ABI-incompatible changes may be made before the ABI is officially
 /// changed to the new version.
 #ifndef PYBIND11_INTERNALS_VERSION
-#    define PYBIND11_INTERNALS_VERSION 7
+#    define PYBIND11_INTERNALS_VERSION 8
 #endif
 
-#if PYBIND11_INTERNALS_VERSION < 7
-#    error "PYBIND11_INTERNALS_VERSION 7 is the minimum for all platforms for pybind11v3."
+#if PYBIND11_INTERNALS_VERSION < 8
+#    error "PYBIND11_INTERNALS_VERSION 8 is the minimum for all platforms for pybind11v3."
 #endif
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
@@ -194,6 +194,8 @@ struct internals {
     // We want unique addresses since we use pointer equality to compare function records
     std::string function_record_capsule_name = internals_function_record_capsule_name;
 
+    type_map<PyObject *> native_enum_type_map;
+
     internals() = default;
     internals(const internals &other) = delete;
     internals &operator=(const internals &other) = delete;