From fbf100cad8dc98b38b00ee3f650e950a22235c65 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 27 Sep 2022 19:06:09 +0000 Subject: [PATCH 001/141] First draft of Eigen::Tensor support --- include/pybind11/eigen.h | 347 +++++++++++++++++++++++++++++++++++++++ tests/test_eigen.cpp | 73 ++++++++ tests/test_eigen.py | 54 +++++- 3 files changed, 473 insertions(+), 1 deletion(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 8316252294..ac7020281f 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -34,6 +34,7 @@ #include #include +#include #if defined(_MSC_VER) # pragma warning(pop) @@ -641,6 +642,350 @@ struct type_caster::value>> { using cast_op_type = Type; }; +template +struct eigen_to_numpy {}; + +template<> +struct eigen_to_numpy { + static const int flag = array::f_style; +}; + +template<> +struct eigen_to_numpy { + static const int flag = array::c_style; +}; + +template +struct get_eigen_data {}; + +template +struct get_eigen_data::value>> { + template + static const T* get_data(const D& d) { + return d.data(); + } +}; + +template +struct get_eigen_data::value>> { + template + static T* get_data(D& d) { + return d.mutable_data(); + } +}; + +template +struct eigen_helper { +}; + +template +struct eigen_helper> { + typedef Eigen::Tensor Type; + typedef Eigen::Tensor ConstType; + + typedef E Element; + static const int N = dim; + static const int Options = O; + + typedef void ValidType; + + static std::array get_shape(const Type& f) { + return f.dimensions(); + } + + static bool is_correct_shape(const std::array& /*shape*/) { + return true; + } +}; + +template +struct eigen_helper, O>> { + typedef Eigen::TensorFixedSize, O> Type; + typedef Eigen::TensorFixedSize, O> ConstType; + + typedef E Element; + static const int N = Eigen::Sizes::count; + static const int Options = O; + typedef void ValidType; + + static std::array get_shape(const Type& /*f*/) { + return get_shape(); + } + + static std::array get_shape() { + return {Indices...}; + } + + static bool is_correct_shape(const std::array& shape) { + return get_shape() == shape; + } +}; + +template +struct type_caster::ValidType> { + using H = eigen_helper; + PYBIND11_TYPE_CASTER(Type, const_name("eigen::Tensor")); + + bool load(handle src, bool /*convert*/) { + array_t::flag> a(reinterpret_borrow(src)); + + if (a.ndim() != H::N) { + return false; + } + + std::array shape; + std::copy(a.shape(), a.shape() + H::N, shape.begin()); + + if (!H::is_correct_shape(shape)) { + return false; + } + + + value = Eigen::TensorMap(a.data(), shape); + + return true; + } + + static handle cast(Type &&src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::reference || policy == return_value_policy::reference_internal) { + pybind11_fail("Cannot use a reference return value policy for an rvalue"); + } + return cast_impl(&src, return_value_policy::move, parent); + } + + static handle cast(const Type &&src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::reference || policy == return_value_policy::reference_internal) { + pybind11_fail("Cannot use a reference return value policy for an rvalue"); + } + return cast_impl(&src, return_value_policy::move, parent); + } + + static handle cast(Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + + static handle cast(const Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + + static handle cast(Type *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + static handle cast(const Type *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + template + static handle cast_impl(C *src, return_value_policy policy, handle parent) { + bool dec_parent; + bool writeable; + switch (policy) { + case return_value_policy::move: + if (std::is_const::value) { + pybind11_fail("Cannot move from a constant reference"); + } + { + Type* copy = new Type(std::move(*src)); + src = copy; + } + parent = capsule(src, [](void* ptr) { + delete (Type*) ptr; + }).release(); + dec_parent = true; + writeable = true; + break; + + case return_value_policy::take_ownership: + if (std::is_const::value) { + pybind11_fail("Cannot take ownership of a const reference"); + } + parent = capsule(src, [](void* ptr) {delete (Type*) ptr;}).release(); + dec_parent = true; + writeable = true; + break; + + case return_value_policy::copy: + parent = {}; + dec_parent = false; + writeable = true; + break; + + case return_value_policy::reference: + parent = none().release(); + dec_parent = true; + writeable = !std::is_const::value; + break; + + case return_value_policy::reference_internal: + // Default should do the right thing + dec_parent = false; + writeable = !std::is_const::value; + break; + + default: + pybind11_fail("pybind11 bug in eigen.h, please file a bug report"); + } + + handle result = array_t::flag>( + H::get_shape(*src), + src->data(), + parent).release(); + + if (!writeable) { + array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + if (dec_parent) { + parent.dec_ref(); + } + + return result; + } +}; + +template +struct type_caster, typename eigen_helper::ValidType> { + using H = eigen_helper; + + bool load(handle src, bool /*convert*/) { + // Note that we have a lot more checks here as we want to make sure to avoid copies + array a = reinterpret_borrow(src); + if ((a.flags() & eigen_to_numpy::flag) == 0) { + return false; + } + + if (!a.dtype().is(dtype::of())) { + return false; + } + + if (a.ndim() != H::N) { + return false; + } + + std::array shape; + std::copy(a.shape(), a.shape() + H::N, shape.begin()); + + if (!H::is_correct_shape(shape)) { + return false; + } + + value = std::make_unique>(reinterpret_cast(a.mutable_data()), shape); + + return true; + } + + static handle cast(Eigen::TensorMap &&src, return_value_policy policy, handle parent) { + return cast_impl(&src, policy, parent); + } + + static handle cast(const Eigen::TensorMap &&src, return_value_policy policy, handle parent) { + return cast_impl(&src, policy, parent); + } + + static handle cast(Eigen::TensorMap &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + + static handle cast(const Eigen::TensorMap &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + + static handle cast(Eigen::TensorMap *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + static handle cast(const Eigen::TensorMap *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + template + static handle cast_impl(C *src, return_value_policy policy, handle parent) { + bool dec_parent; + bool writeable; + switch (policy) { + case return_value_policy::reference: + parent = none().release(); + dec_parent = true; + writeable = !std::is_const::value; + break; + + case return_value_policy::reference_internal: + // Default should do the right thing + dec_parent = false; + writeable = !std::is_const::value; + break; + + default: + // move, take_ownership don't make any sense for a ref/map: + pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either reference or reference_internal"); + } + + handle result = array_t::flag>( + H::get_shape(*src), + src->data(), + parent).release(); + + if (dec_parent) { + parent.dec_ref(); + } + + if (!writeable) { + array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + return result; + } + + protected: + // TODO: Move to std::optional once std::optional has more support + std::unique_ptr> value; + public: + static constexpr auto name = const_name("Eigen::TensorMap"); + operator Eigen::TensorMap *() { return value.get(); } /* NOLINT(bugprone-macro-parentheses) */ \ + operator Eigen::TensorMap &() { return *value; } /* NOLINT(bugprone-macro-parentheses) */ \ + operator Eigen::TensorMap &&() && { return std::move(*value); } /* NOLINT(bugprone-macro-parentheses) */ \ + + template \ + using cast_op_type = ::pybind11::detail::movable_cast_op_type; +}; + template struct type_caster::value>> { using Scalar = typename Type::Scalar; @@ -709,5 +1054,7 @@ struct type_caster::value>> { + npy_format_descriptor::name + const_name("]")); }; + + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/test_eigen.cpp b/tests/test_eigen.cpp index b0c7bdb48a..8aca00315c 100644 --- a/tests/test_eigen.cpp +++ b/tests/test_eigen.cpp @@ -32,6 +32,13 @@ void reset_ref(M &x) { } } +template +void reset_tensor(M& x) { + for (int i = 0; i< x.size(); i++) { + x(i) = i; + } +} + // Returns a static, column-major matrix Eigen::MatrixXd &get_cm() { static Eigen::MatrixXd *x; @@ -50,6 +57,33 @@ MatrixXdR &get_rm() { } return *x; } + +Eigen::Tensor& get_tensor() { + static Eigen::Tensor *x; + + if (!x) { + x = new Eigen::Tensor(3, 1, 2); + reset_tensor(*x); + } + + return *x; +} + +Eigen::TensorFixedSize>& get_fixed_tensor() { + static Eigen::TensorFixedSize> *x; + + if (!x) { + x = new Eigen::TensorFixedSize>(); + reset_tensor(*x); + } + + return *x; +} + +const Eigen::Tensor& get_const_tensor() { + return get_tensor(); +} + // Resets the values of the static matrices returned by get_cm()/get_rm() void reset_refs() { reset_ref(get_cm()); @@ -427,4 +461,43 @@ TEST_SUBMODULE(eigen, m) { py::module_::import("numpy").attr("ones")(10); return v[0](5); }); + + m.def("copy_fixed_global_tensor", [](){ + return get_fixed_tensor(); + }); + + m.def("copy_global_tensor", [](){ + return get_tensor(); + }); + + m.def("copy_const_global_tensor", [](){ + return get_const_tensor(); + }); + + m.def("reference_global_tensor", [](){ + return &get_tensor(); + }, py::return_value_policy::reference); + + m.def("reference_const_global_tensor", [](){ + return &get_const_tensor(); + }, py::return_value_policy::reference); + + m.def("reference_view_of_global_tensor", []() { + return Eigen::TensorMap>(get_tensor()); + }, py::return_value_policy::reference); + + m.def("reference_view_of_fixed_global_tensor", []() { + return Eigen::TensorMap>>(get_fixed_tensor()); + }, py::return_value_policy::reference); + + m.def("round_trip_tensor", [](py::array_t foo) { + auto blah = foo.cast>(); + return blah; + }); + + m.def("round_trip_view_tensor", [](py::array_t foo) { + auto view = foo.cast>>(); + return view; + }, py::return_value_policy::reference); + } diff --git a/tests/test_eigen.py b/tests/test_eigen.py index 713432a815..509a4274a3 100644 --- a/tests/test_eigen.py +++ b/tests/test_eigen.py @@ -20,7 +20,6 @@ def assert_equal_ref(mat): np.testing.assert_array_equal(mat, ref) - def assert_sparse_equal_ref(sparse_mat): assert_equal_ref(sparse_mat.toarray()) @@ -782,3 +781,56 @@ def test_custom_operator_new(): o = m.CustomOperatorNew() np.testing.assert_allclose(o.a, 0.0) np.testing.assert_allclose(o.b.diagonal(), 1.0) + +tensor_ref = np.array( + [ + [[0, 3]], + [[1, 4]], + [[2, 5]], + ] +) + +def assert_equal_tensor_ref(mat, writeable=True, modified=0): + assert mat.flags.writeable == writeable + + if modified != 0: + tensor_ref[0, 0, 0] = modified + + np.testing.assert_array_equal(mat, tensor_ref) + + tensor_ref[0, 0, 0] = 0 + +def test_convert_tensor_to_py(): + assert_equal_tensor_ref(m.copy_global_tensor()) + assert_equal_tensor_ref(m.copy_fixed_global_tensor()) + assert_equal_tensor_ref(m.copy_const_global_tensor()) + + assert_equal_tensor_ref(m.reference_global_tensor()) + assert_equal_tensor_ref(m.reference_view_of_global_tensor()) + assert_equal_tensor_ref(m.reference_view_of_fixed_global_tensor()) + assert_equal_tensor_ref(m.reference_const_global_tensor(), writeable=False) + +def test_references_actually_refer(): + a = m.reference_global_tensor() + a[0, 0,0] = 100 + assert_equal_tensor_ref(m.copy_const_global_tensor(), modified = 100) + a[0, 0,0] = 0 + assert_equal_tensor_ref(m.copy_const_global_tensor()) + + a = m.reference_view_of_global_tensor() + a[0, 0,0] = 100 + assert_equal_tensor_ref(m.copy_const_global_tensor(), modified = 100) + a[0, 0,0] = 0 + assert_equal_tensor_ref(m.copy_const_global_tensor()) + +def test_round_trip(): + assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) + +def test_round_trip_references_actually_refer(): + # Need to create a copy that matches the type on the C side + copy = np.array(tensor_ref, dtype=np.float64, order='F') + a = m.round_trip_view_tensor(copy) + a[0, 0,0] = 100 + assert_equal_tensor_ref(copy, modified = 100) + a[0, 0,0] = 0 + assert_equal_tensor_ref(copy) \ No newline at end of file From 5a6fd069d2ab0cce95e0fb673938806186350e86 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 19:15:45 +0000 Subject: [PATCH 002/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/eigen.h | 125 +++++++++++++++++++-------------------- tests/test_eigen.cpp | 67 ++++++++++----------- tests/test_eigen.py | 35 ++++++----- 3 files changed, 116 insertions(+), 111 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index ac7020281f..1ea382f64f 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -645,12 +645,12 @@ struct type_caster::value>> { template struct eigen_to_numpy {}; -template<> +template <> struct eigen_to_numpy { static const int flag = array::f_style; }; -template<> +template <> struct eigen_to_numpy { static const int flag = array::c_style; }; @@ -658,27 +658,26 @@ struct eigen_to_numpy { template struct get_eigen_data {}; -template +template struct get_eigen_data::value>> { - template - static const T* get_data(const D& d) { + template + static const T *get_data(const D &d) { return d.data(); } }; -template +template struct get_eigen_data::value>> { - template - static T* get_data(D& d) { + template + static T *get_data(D &d) { return d.mutable_data(); } }; template -struct eigen_helper { -}; +struct eigen_helper {}; -template +template struct eigen_helper> { typedef Eigen::Tensor Type; typedef Eigen::Tensor ConstType; @@ -689,16 +688,12 @@ struct eigen_helper> { typedef void ValidType; - static std::array get_shape(const Type& f) { - return f.dimensions(); - } + static std::array get_shape(const Type &f) { return f.dimensions(); } - static bool is_correct_shape(const std::array& /*shape*/) { - return true; - } + static bool is_correct_shape(const std::array & /*shape*/) { return true; } }; -template +template struct eigen_helper, O>> { typedef Eigen::TensorFixedSize, O> Type; typedef Eigen::TensorFixedSize, O> ConstType; @@ -708,26 +703,23 @@ struct eigen_helper, O>> { static const int Options = O; typedef void ValidType; - static std::array get_shape(const Type& /*f*/) { - return get_shape(); - } + static std::array get_shape(const Type & /*f*/) { return get_shape(); } - static std::array get_shape() { - return {Indices...}; - } + static std::array get_shape() { return {Indices...}; } - static bool is_correct_shape(const std::array& shape) { + static bool is_correct_shape(const std::array &shape) { return get_shape() == shape; } }; -template +template struct type_caster::ValidType> { using H = eigen_helper; PYBIND11_TYPE_CASTER(Type, const_name("eigen::Tensor")); bool load(handle src, bool /*convert*/) { - array_t::flag> a(reinterpret_borrow(src)); + array_t::flag> a( + reinterpret_borrow(src)); if (a.ndim() != H::N) { return false; @@ -740,21 +732,22 @@ struct type_caster::ValidType> { return false; } - value = Eigen::TensorMap(a.data(), shape); return true; } static handle cast(Type &&src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::reference || policy == return_value_policy::reference_internal) { + if (policy == return_value_policy::reference + || policy == return_value_policy::reference_internal) { pybind11_fail("Cannot use a reference return value policy for an rvalue"); } return cast_impl(&src, return_value_policy::move, parent); } static handle cast(const Type &&src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::reference || policy == return_value_policy::reference_internal) { + if (policy == return_value_policy::reference + || policy == return_value_policy::reference_internal) { pybind11_fail("Cannot use a reference return value policy for an rvalue"); } return cast_impl(&src, return_value_policy::move, parent); @@ -794,7 +787,7 @@ struct type_caster::ValidType> { return cast_impl(src, policy, parent); } - template + template static handle cast_impl(C *src, return_value_policy policy, handle parent) { bool dec_parent; bool writeable; @@ -804,12 +797,10 @@ struct type_caster::ValidType> { pybind11_fail("Cannot move from a constant reference"); } { - Type* copy = new Type(std::move(*src)); + Type *copy = new Type(std::move(*src)); src = copy; } - parent = capsule(src, [](void* ptr) { - delete (Type*) ptr; - }).release(); + parent = capsule(src, [](void *ptr) { delete (Type *) ptr; }).release(); dec_parent = true; writeable = true; break; @@ -818,7 +809,7 @@ struct type_caster::ValidType> { if (std::is_const::value) { pybind11_fail("Cannot take ownership of a const reference"); } - parent = capsule(src, [](void* ptr) {delete (Type*) ptr;}).release(); + parent = capsule(src, [](void *ptr) { delete (Type *) ptr; }).release(); dec_parent = true; writeable = true; break; @@ -846,10 +837,9 @@ struct type_caster::ValidType> { } handle result = array_t::flag>( - H::get_shape(*src), - src->data(), - parent).release(); - + H::get_shape(*src), src->data(), parent) + .release(); + if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; } @@ -862,7 +852,7 @@ struct type_caster::ValidType> { } }; -template +template struct type_caster, typename eigen_helper::ValidType> { using H = eigen_helper; @@ -888,7 +878,8 @@ struct type_caster, typename eigen_helper::ValidTyp return false; } - value = std::make_unique>(reinterpret_cast(a.mutable_data()), shape); + value = std::make_unique>( + reinterpret_cast(a.mutable_data()), shape); return true; } @@ -897,7 +888,8 @@ struct type_caster, typename eigen_helper::ValidTyp return cast_impl(&src, policy, parent); } - static handle cast(const Eigen::TensorMap &&src, return_value_policy policy, handle parent) { + static handle + cast(const Eigen::TensorMap &&src, return_value_policy policy, handle parent) { return cast_impl(&src, policy, parent); } @@ -909,7 +901,8 @@ struct type_caster, typename eigen_helper::ValidTyp return cast_impl(&src, policy, parent); } - static handle cast(const Eigen::TensorMap &src, return_value_policy policy, handle parent) { + static handle + cast(const Eigen::TensorMap &src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) { policy = return_value_policy::copy; @@ -926,7 +919,8 @@ struct type_caster, typename eigen_helper::ValidTyp return cast_impl(src, policy, parent); } - static handle cast(const Eigen::TensorMap *src, return_value_policy policy, handle parent) { + static handle + cast(const Eigen::TensorMap *src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic) { policy = return_value_policy::take_ownership; } else if (policy == return_value_policy::automatic_reference) { @@ -935,7 +929,7 @@ struct type_caster, typename eigen_helper::ValidTyp return cast_impl(src, policy, parent); } - template + template static handle cast_impl(C *src, return_value_policy policy, handle parent) { bool dec_parent; bool writeable; @@ -954,13 +948,13 @@ struct type_caster, typename eigen_helper::ValidTyp default: // move, take_ownership don't make any sense for a ref/map: - pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either reference or reference_internal"); + pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either " + "reference or reference_internal"); } handle result = array_t::flag>( - H::get_shape(*src), - src->data(), - parent).release(); + H::get_shape(*src), src->data(), parent) + .release(); if (dec_parent) { parent.dec_ref(); @@ -972,18 +966,23 @@ struct type_caster, typename eigen_helper::ValidTyp return result; } - - protected: - // TODO: Move to std::optional once std::optional has more support - std::unique_ptr> value; - public: - static constexpr auto name = const_name("Eigen::TensorMap"); - operator Eigen::TensorMap *() { return value.get(); } /* NOLINT(bugprone-macro-parentheses) */ \ - operator Eigen::TensorMap &() { return *value; } /* NOLINT(bugprone-macro-parentheses) */ \ - operator Eigen::TensorMap &&() && { return std::move(*value); } /* NOLINT(bugprone-macro-parentheses) */ \ - - template \ - using cast_op_type = ::pybind11::detail::movable_cast_op_type; + +protected: + // TODO: Move to std::optional once std::optional has more support + std::unique_ptr> value; + +public: + static constexpr auto name = const_name("Eigen::TensorMap"); + operator Eigen::TensorMap *() { + return value.get(); + } /* NOLINT(bugprone-macro-parentheses) */ + operator Eigen::TensorMap &() { return *value; } /* NOLINT(bugprone-macro-parentheses) */ + operator Eigen::TensorMap &&() && { + return std::move(*value); + } /* NOLINT(bugprone-macro-parentheses) */ + + template + using cast_op_type = ::pybind11::detail::movable_cast_op_type; }; template @@ -1054,7 +1053,5 @@ struct type_caster::value>> { + npy_format_descriptor::name + const_name("]")); }; - - PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/test_eigen.cpp b/tests/test_eigen.cpp index 8aca00315c..ef14c0fa4e 100644 --- a/tests/test_eigen.cpp +++ b/tests/test_eigen.cpp @@ -33,8 +33,8 @@ void reset_ref(M &x) { } template -void reset_tensor(M& x) { - for (int i = 0; i< x.size(); i++) { +void reset_tensor(M &x) { + for (int i = 0; i < x.size(); i++) { x(i) = i; } } @@ -58,7 +58,7 @@ MatrixXdR &get_rm() { return *x; } -Eigen::Tensor& get_tensor() { +Eigen::Tensor &get_tensor() { static Eigen::Tensor *x; if (!x) { @@ -69,7 +69,7 @@ Eigen::Tensor& get_tensor() { return *x; } -Eigen::TensorFixedSize>& get_fixed_tensor() { +Eigen::TensorFixedSize> &get_fixed_tensor() { static Eigen::TensorFixedSize> *x; if (!x) { @@ -80,9 +80,7 @@ Eigen::TensorFixedSize>& get_fixed_tensor() { return *x; } -const Eigen::Tensor& get_const_tensor() { - return get_tensor(); -} +const Eigen::Tensor &get_const_tensor() { return get_tensor(); } // Resets the values of the static matrices returned by get_cm()/get_rm() void reset_refs() { @@ -462,42 +460,45 @@ TEST_SUBMODULE(eigen, m) { return v[0](5); }); - m.def("copy_fixed_global_tensor", [](){ - return get_fixed_tensor(); - }); + m.def("copy_fixed_global_tensor", []() { return get_fixed_tensor(); }); - m.def("copy_global_tensor", [](){ - return get_tensor(); - }); + m.def("copy_global_tensor", []() { return get_tensor(); }); - m.def("copy_const_global_tensor", [](){ - return get_const_tensor(); - }); + m.def("copy_const_global_tensor", []() { return get_const_tensor(); }); - m.def("reference_global_tensor", [](){ - return &get_tensor(); - }, py::return_value_policy::reference); + m.def( + "reference_global_tensor", + []() { return &get_tensor(); }, + py::return_value_policy::reference); - m.def("reference_const_global_tensor", [](){ - return &get_const_tensor(); - }, py::return_value_policy::reference); + m.def( + "reference_const_global_tensor", + []() { return &get_const_tensor(); }, + py::return_value_policy::reference); - m.def("reference_view_of_global_tensor", []() { - return Eigen::TensorMap>(get_tensor()); - }, py::return_value_policy::reference); + m.def( + "reference_view_of_global_tensor", + []() { return Eigen::TensorMap>(get_tensor()); }, + py::return_value_policy::reference); - m.def("reference_view_of_fixed_global_tensor", []() { - return Eigen::TensorMap>>(get_fixed_tensor()); - }, py::return_value_policy::reference); + m.def( + "reference_view_of_fixed_global_tensor", + []() { + return Eigen::TensorMap>>( + get_fixed_tensor()); + }, + py::return_value_policy::reference); m.def("round_trip_tensor", [](py::array_t foo) { auto blah = foo.cast>(); return blah; }); - m.def("round_trip_view_tensor", [](py::array_t foo) { - auto view = foo.cast>>(); - return view; - }, py::return_value_policy::reference); - + m.def( + "round_trip_view_tensor", + [](py::array_t foo) { + auto view = foo.cast>>(); + return view; + }, + py::return_value_policy::reference); } diff --git a/tests/test_eigen.py b/tests/test_eigen.py index 509a4274a3..4bdd5f8f27 100644 --- a/tests/test_eigen.py +++ b/tests/test_eigen.py @@ -20,6 +20,7 @@ def assert_equal_ref(mat): np.testing.assert_array_equal(mat, ref) + def assert_sparse_equal_ref(sparse_mat): assert_equal_ref(sparse_mat.toarray()) @@ -782,6 +783,7 @@ def test_custom_operator_new(): np.testing.assert_allclose(o.a, 0.0) np.testing.assert_allclose(o.b.diagonal(), 1.0) + tensor_ref = np.array( [ [[0, 3]], @@ -790,16 +792,18 @@ def test_custom_operator_new(): ] ) + def assert_equal_tensor_ref(mat, writeable=True, modified=0): assert mat.flags.writeable == writeable - + if modified != 0: tensor_ref[0, 0, 0] = modified - + np.testing.assert_array_equal(mat, tensor_ref) - + tensor_ref[0, 0, 0] = 0 + def test_convert_tensor_to_py(): assert_equal_tensor_ref(m.copy_global_tensor()) assert_equal_tensor_ref(m.copy_fixed_global_tensor()) @@ -810,27 +814,30 @@ def test_convert_tensor_to_py(): assert_equal_tensor_ref(m.reference_view_of_fixed_global_tensor()) assert_equal_tensor_ref(m.reference_const_global_tensor(), writeable=False) + def test_references_actually_refer(): a = m.reference_global_tensor() - a[0, 0,0] = 100 - assert_equal_tensor_ref(m.copy_const_global_tensor(), modified = 100) - a[0, 0,0] = 0 + a[0, 0, 0] = 100 + assert_equal_tensor_ref(m.copy_const_global_tensor(), modified=100) + a[0, 0, 0] = 0 assert_equal_tensor_ref(m.copy_const_global_tensor()) a = m.reference_view_of_global_tensor() - a[0, 0,0] = 100 - assert_equal_tensor_ref(m.copy_const_global_tensor(), modified = 100) - a[0, 0,0] = 0 + a[0, 0, 0] = 100 + assert_equal_tensor_ref(m.copy_const_global_tensor(), modified=100) + a[0, 0, 0] = 0 assert_equal_tensor_ref(m.copy_const_global_tensor()) + def test_round_trip(): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) + def test_round_trip_references_actually_refer(): # Need to create a copy that matches the type on the C side - copy = np.array(tensor_ref, dtype=np.float64, order='F') + copy = np.array(tensor_ref, dtype=np.float64, order="F") a = m.round_trip_view_tensor(copy) - a[0, 0,0] = 100 - assert_equal_tensor_ref(copy, modified = 100) - a[0, 0,0] = 0 - assert_equal_tensor_ref(copy) \ No newline at end of file + a[0, 0, 0] = 100 + assert_equal_tensor_ref(copy, modified=100) + a[0, 0, 0] = 0 + assert_equal_tensor_ref(copy) From a6a4bcd856895195ab3b0af3e92369e0fb03673c Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 27 Sep 2022 19:32:47 +0000 Subject: [PATCH 003/141] Fix build errors --- include/pybind11/eigen.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 1ea382f64f..bb08901073 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -659,7 +659,7 @@ template struct get_eigen_data {}; template -struct get_eigen_data::value>> { +struct get_eigen_data::value>> { template static const T *get_data(const D &d) { return d.data(); @@ -667,7 +667,7 @@ struct get_eigen_data::value>> { }; template -struct get_eigen_data::value>> { +struct get_eigen_data::value>> { template static T *get_data(D &d) { return d.mutable_data(); @@ -878,8 +878,8 @@ struct type_caster, typename eigen_helper::ValidTyp return false; } - value = std::make_unique>( - reinterpret_cast(a.mutable_data()), shape); + value.reset(new Eigen::TensorMap( + reinterpret_cast(a.mutable_data()), shape)); return true; } From 5996f801cf711ce1c51ca8460aba5aefc0720103 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 27 Sep 2022 20:13:06 +0000 Subject: [PATCH 004/141] Weird allocator stuff? --- include/pybind11/eigen.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index bb08901073..15291df3de 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -797,10 +797,17 @@ struct type_caster::ValidType> { pybind11_fail("Cannot move from a constant reference"); } { - Type *copy = new Type(std::move(*src)); + Eigen::aligned_allocator allocator; + Type* copy = ::new (allocator.allocate(1)) Type(std::move(*src)); src = copy; } - parent = capsule(src, [](void *ptr) { delete (Type *) ptr; }).release(); + + parent = capsule(src, [](void *ptr) { + Eigen::aligned_allocator allocator; + Type* copy = (Type*) ptr; + copy->~Type(); + allocator.deallocate(copy, 1); + }).release(); dec_parent = true; writeable = true; break; From 25a4d7eff161d4f00d15cb94c727a4b141be1646 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 20:18:58 +0000 Subject: [PATCH 005/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/eigen.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 15291df3de..eb46af5a9b 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -798,16 +798,16 @@ struct type_caster::ValidType> { } { Eigen::aligned_allocator allocator; - Type* copy = ::new (allocator.allocate(1)) Type(std::move(*src)); + Type *copy = ::new (allocator.allocate(1)) Type(std::move(*src)); src = copy; } parent = capsule(src, [](void *ptr) { - Eigen::aligned_allocator allocator; - Type* copy = (Type*) ptr; - copy->~Type(); - allocator.deallocate(copy, 1); - }).release(); + Eigen::aligned_allocator allocator; + Type *copy = (Type *) ptr; + copy->~Type(); + allocator.deallocate(copy, 1); + }).release(); dec_parent = true; writeable = true; break; From 21c240bd473f8615be9566ed42a924d279ffca1d Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 27 Sep 2022 20:41:07 +0000 Subject: [PATCH 006/141] Remove unused + additional allocator junk --- include/pybind11/eigen.h | 22 ++-------------------- tests/test_eigen.cpp | 3 ++- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index eb46af5a9b..be167e8b24 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -655,25 +655,6 @@ struct eigen_to_numpy { static const int flag = array::c_style; }; -template -struct get_eigen_data {}; - -template -struct get_eigen_data::value>> { - template - static const T *get_data(const D &d) { - return d.data(); - } -}; - -template -struct get_eigen_data::value>> { - template - static T *get_data(D &d) { - return d.mutable_data(); - } -}; - template struct eigen_helper {}; @@ -701,11 +682,12 @@ struct eigen_helper, O>> { typedef E Element; static const int N = Eigen::Sizes::count; static const int Options = O; + typedef void ValidType; static std::array get_shape(const Type & /*f*/) { return get_shape(); } - static std::array get_shape() { return {Indices...}; } + static std::array get_shape() { return {{Indices...}}; } static bool is_correct_shape(const std::array &shape) { return get_shape() == shape; diff --git a/tests/test_eigen.cpp b/tests/test_eigen.cpp index ef14c0fa4e..ed6d7f3215 100644 --- a/tests/test_eigen.cpp +++ b/tests/test_eigen.cpp @@ -73,7 +73,8 @@ Eigen::TensorFixedSize> &get_fixed_tensor() { static Eigen::TensorFixedSize> *x; if (!x) { - x = new Eigen::TensorFixedSize>(); + Eigen::aligned_allocator>> allocator; + x = new (allocator.allocate(1)) Eigen::TensorFixedSize>(); reset_tensor(*x); } From 7d49a312aa5e21da58529b472029353b98fc4658 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 27 Sep 2022 20:49:04 +0000 Subject: [PATCH 007/141] Disable warning --- include/pybind11/eigen.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index be167e8b24..22c1d05361 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -24,6 +24,7 @@ // it is probably best to keep this around indefinitely. #if defined(_MSC_VER) # pragma warning(push) +# pragma warning(disable : 4554) // Tensor.h warning # pragma warning(disable : 4127) // C4127: conditional expression is constant # pragma warning(disable : 5054) // https://github.com/pybind/pybind11/pull/3741 // C5054: operator '&': deprecated between enumerations of different types From f4292b56ee8ac3bcd00497ee9a6fcd5ac6960717 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 27 Sep 2022 20:56:30 +0000 Subject: [PATCH 008/141] Use constexpr --- include/pybind11/eigen.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 22c1d05361..086ed9e242 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -648,12 +648,12 @@ struct eigen_to_numpy {}; template <> struct eigen_to_numpy { - static const int flag = array::f_style; + static constexpr int flag = array::f_style; }; template <> struct eigen_to_numpy { - static const int flag = array::c_style; + static constexpr int flag = array::c_style; }; template @@ -665,14 +665,14 @@ struct eigen_helper> { typedef Eigen::Tensor ConstType; typedef E Element; - static const int N = dim; - static const int Options = O; + static constexpr int N = dim; + static constexpr int Options = O; typedef void ValidType; static std::array get_shape(const Type &f) { return f.dimensions(); } - static bool is_correct_shape(const std::array & /*shape*/) { return true; } + static constexpr bool is_correct_shape(const std::array & /*shape*/) { return true; } }; template @@ -681,14 +681,14 @@ struct eigen_helper, O>> { typedef Eigen::TensorFixedSize, O> ConstType; typedef E Element; - static const int N = Eigen::Sizes::count; - static const int Options = O; + static constexpr int N = Eigen::Sizes::count; + static constexpr int Options = O; typedef void ValidType; static std::array get_shape(const Type & /*f*/) { return get_shape(); } - static std::array get_shape() { return {{Indices...}}; } + static constexpr std::array get_shape() { return {{Indices...}}; } static bool is_correct_shape(const std::array &shape) { return get_shape() == shape; From 68f06e71c788d26bb8f62575d14d2d6d9bc181c3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 21:04:05 +0000 Subject: [PATCH 009/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/eigen.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 086ed9e242..89dc4bff8e 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -672,7 +672,9 @@ struct eigen_helper> { static std::array get_shape(const Type &f) { return f.dimensions(); } - static constexpr bool is_correct_shape(const std::array & /*shape*/) { return true; } + static constexpr bool is_correct_shape(const std::array & /*shape*/) { + return true; + } }; template From a77b38ca1b6a7c472bd269636f0eb9b82700d1f3 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 27 Sep 2022 21:13:35 +0000 Subject: [PATCH 010/141] clang tidy fixes --- include/pybind11/eigen.h | 33 +++++++++++++++++---------------- tests/test_eigen.cpp | 4 ++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 89dc4bff8e..07ae34a081 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -661,14 +661,15 @@ struct eigen_helper {}; template struct eigen_helper> { - typedef Eigen::Tensor Type; - typedef Eigen::Tensor ConstType; + using Type = Eigen::Tensor; + using ConstType = Eigen::Tensor; + + using Element = E; - typedef E Element; static constexpr int N = dim; static constexpr int Options = O; - typedef void ValidType; + using ValidType = void; static std::array get_shape(const Type &f) { return f.dimensions(); } @@ -679,14 +680,14 @@ struct eigen_helper> { template struct eigen_helper, O>> { - typedef Eigen::TensorFixedSize, O> Type; - typedef Eigen::TensorFixedSize, O> ConstType; + using Type = Eigen::TensorFixedSize, O>; + using ConstType = Eigen::TensorFixedSize, O>; - typedef E Element; + using Element = E; static constexpr int N = Eigen::Sizes::count; static constexpr int Options = O; - typedef void ValidType; + using ValidType = void; static std::array get_shape(const Type & /*f*/) { return get_shape(); } @@ -774,8 +775,8 @@ struct type_caster::ValidType> { template static handle cast_impl(C *src, return_value_policy policy, handle parent) { - bool dec_parent; - bool writeable; + bool dec_parent = false; + bool writeable = false; switch (policy) { case return_value_policy::move: if (std::is_const::value) { @@ -850,7 +851,7 @@ struct type_caster, typename eigen_helper::ValidTyp bool load(handle src, bool /*convert*/) { // Note that we have a lot more checks here as we want to make sure to avoid copies - array a = reinterpret_borrow(src); + auto a = reinterpret_borrow(src); if ((a.flags() & eigen_to_numpy::flag) == 0) { return false; } @@ -923,8 +924,8 @@ struct type_caster, typename eigen_helper::ValidTyp template static handle cast_impl(C *src, return_value_policy policy, handle parent) { - bool dec_parent; - bool writeable; + bool dec_parent = false; + bool writeable = false; switch (policy) { case return_value_policy::reference: parent = none().release(); @@ -965,11 +966,11 @@ struct type_caster, typename eigen_helper::ValidTyp public: static constexpr auto name = const_name("Eigen::TensorMap"); - operator Eigen::TensorMap *() { + explicit operator Eigen::TensorMap *() { return value.get(); } /* NOLINT(bugprone-macro-parentheses) */ - operator Eigen::TensorMap &() { return *value; } /* NOLINT(bugprone-macro-parentheses) */ - operator Eigen::TensorMap &&() && { + explicit operator Eigen::TensorMap &() { return *value; } /* NOLINT(bugprone-macro-parentheses) */ + explicit operator Eigen::TensorMap &&() && { return std::move(*value); } /* NOLINT(bugprone-macro-parentheses) */ diff --git a/tests/test_eigen.cpp b/tests/test_eigen.cpp index ed6d7f3215..9014d72b90 100644 --- a/tests/test_eigen.cpp +++ b/tests/test_eigen.cpp @@ -490,14 +490,14 @@ TEST_SUBMODULE(eigen, m) { }, py::return_value_policy::reference); - m.def("round_trip_tensor", [](py::array_t foo) { + m.def("round_trip_tensor", [](const py::array_t& foo) { auto blah = foo.cast>(); return blah; }); m.def( "round_trip_view_tensor", - [](py::array_t foo) { + [](const py::array_t& foo) { auto view = foo.cast>>(); return view; }, From 24e91209317dc4c3ec36e48d92da7a7dd0637d8c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 21:19:25 +0000 Subject: [PATCH 011/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/eigen.h | 6 ++++-- tests/test_eigen.cpp | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 07ae34a081..69751d1c40 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -968,8 +968,10 @@ struct type_caster, typename eigen_helper::ValidTyp static constexpr auto name = const_name("Eigen::TensorMap"); explicit operator Eigen::TensorMap *() { return value.get(); - } /* NOLINT(bugprone-macro-parentheses) */ - explicit operator Eigen::TensorMap &() { return *value; } /* NOLINT(bugprone-macro-parentheses) */ + } /* NOLINT(bugprone-macro-parentheses) */ + explicit operator Eigen::TensorMap &() { + return *value; + } /* NOLINT(bugprone-macro-parentheses) */ explicit operator Eigen::TensorMap &&() && { return std::move(*value); } /* NOLINT(bugprone-macro-parentheses) */ diff --git a/tests/test_eigen.cpp b/tests/test_eigen.cpp index 9014d72b90..3b71f1e2ab 100644 --- a/tests/test_eigen.cpp +++ b/tests/test_eigen.cpp @@ -490,14 +490,14 @@ TEST_SUBMODULE(eigen, m) { }, py::return_value_policy::reference); - m.def("round_trip_tensor", [](const py::array_t& foo) { + m.def("round_trip_tensor", [](const py::array_t &foo) { auto blah = foo.cast>(); return blah; }); m.def( "round_trip_view_tensor", - [](const py::array_t& foo) { + [](const py::array_t &foo) { auto view = foo.cast>>(); return view; }, From ca2303cdf0a29b23750b61a89d2582a68fa70ad6 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 28 Sep 2022 17:31:38 +0000 Subject: [PATCH 012/141] Resolve comments --- include/pybind11/eigen.h | 147 +++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 82 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 69751d1c40..ccfbf6c867 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -33,6 +33,7 @@ # pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif +#include #include #include #include @@ -643,82 +644,79 @@ struct type_caster::value>> { using cast_op_type = Type; }; -template -struct eigen_to_numpy {}; - -template <> -struct eigen_to_numpy { - static constexpr int flag = array::f_style; -}; - -template <> -struct eigen_to_numpy { - static constexpr int flag = array::c_style; -}; - -template -struct eigen_helper {}; +template +constexpr int compute_array_flag_from_tensor() { + static_assert(((int)T::Layout == (int)Eigen::RowMajor) || ((int)T::Layout == (int)Eigen::ColMajor), "Layout must be row or column major"); + return ((int)T::Layout == (int)Eigen::RowMajor) ? array::c_style : array::f_style; +} -template -struct eigen_helper> { - using Type = Eigen::Tensor; - using ConstType = Eigen::Tensor; - using Element = E; - static constexpr int N = dim; - static constexpr int Options = O; +template +struct eigen_tensor_helper {}; +template +struct eigen_tensor_helper> { + using T = Eigen::Tensor; using ValidType = void; - static std::array get_shape(const Type &f) { return f.dimensions(); } + static std::array get_shape(const T &f) { return f.dimensions(); } - static constexpr bool is_correct_shape(const std::array & /*shape*/) { + static constexpr bool is_correct_shape(const std::array & /*shape*/) { return true; } -}; -template -struct eigen_helper, O>> { - using Type = Eigen::TensorFixedSize, O>; - using ConstType = Eigen::TensorFixedSize, O>; + template + static constexpr auto get_dimensions_descriptor_helper(index_sequence) { + return concat(const_name(((void) Is, "?"))...); + } - using Element = E; - static constexpr int N = Eigen::Sizes::count; - static constexpr int Options = O; + static constexpr auto dimensions_descriptor = get_dimensions_descriptor_helper(make_index_sequence()); +}; +template +struct eigen_tensor_helper, Options_, IndexType>> { + using T = Eigen::TensorFixedSize, Options_, IndexType>; using ValidType = void; - static std::array get_shape(const Type & /*f*/) { return get_shape(); } + static constexpr std::array get_shape(const T & /*f*/) { return get_shape(); } - static constexpr std::array get_shape() { return {{Indices...}}; } + static constexpr std::array get_shape() { return {{Indices...}}; } - static bool is_correct_shape(const std::array &shape) { + static bool is_correct_shape(const std::array &shape) { return get_shape() == shape; } + + static constexpr auto dimensions_descriptor = concat(const_name()...); }; +template +constexpr auto get_tensor_descriptor() { + return const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + + eigen_tensor_helper::dimensions_descriptor + const_name("], flags.writeable, ") + const_name<(int)T::Layout == (int)Eigen::RowMajor>("flags.c_contiguous", "flags.f_contiguous"); +} + template -struct type_caster::ValidType> { - using H = eigen_helper; - PYBIND11_TYPE_CASTER(Type, const_name("eigen::Tensor")); +struct type_caster::ValidType> { + using H = eigen_tensor_helper; + PYBIND11_TYPE_CASTER(Type, get_tensor_descriptor()); bool load(handle src, bool /*convert*/) { - array_t::flag> a( + array_t()> a( reinterpret_borrow(src)); - if (a.ndim() != H::N) { + if (a.ndim() != Type::NumIndices) { return false; } - std::array shape; - std::copy(a.shape(), a.shape() + H::N, shape.begin()); + std::array shape; + std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); if (!H::is_correct_shape(shape)) { return false; } - value = Eigen::TensorMap(a.data(), shape); + value = Eigen::TensorMap(const_cast(a.data()), shape); return true; } @@ -775,7 +773,7 @@ struct type_caster::ValidType> { template static handle cast_impl(C *src, return_value_policy policy, handle parent) { - bool dec_parent = false; + object parent_object; bool writeable = false; switch (policy) { case return_value_policy::move: @@ -788,13 +786,12 @@ struct type_caster::ValidType> { src = copy; } - parent = capsule(src, [](void *ptr) { + parent_object = capsule(src, [](void *ptr) { Eigen::aligned_allocator allocator; Type *copy = (Type *) ptr; copy->~Type(); allocator.deallocate(copy, 1); - }).release(); - dec_parent = true; + }); writeable = true; break; @@ -802,26 +799,23 @@ struct type_caster::ValidType> { if (std::is_const::value) { pybind11_fail("Cannot take ownership of a const reference"); } - parent = capsule(src, [](void *ptr) { delete (Type *) ptr; }).release(); - dec_parent = true; + parent_object = capsule(src, [](void *ptr) { delete (Type *) ptr; }); writeable = true; break; case return_value_policy::copy: - parent = {}; - dec_parent = false; + parent_object = {}; writeable = true; break; case return_value_policy::reference: - parent = none().release(); - dec_parent = true; + parent_object = none(); writeable = !std::is_const::value; break; case return_value_policy::reference_internal: // Default should do the right thing - dec_parent = false; + parent_object = reinterpret_borrow(parent); writeable = !std::is_const::value; break; @@ -829,50 +823,46 @@ struct type_caster::ValidType> { pybind11_fail("pybind11 bug in eigen.h, please file a bug report"); } - handle result = array_t::flag>( - H::get_shape(*src), src->data(), parent) + handle result = array_t()>( + H::get_shape(*src), src->data(), parent_object) .release(); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; } - if (dec_parent) { - parent.dec_ref(); - } - return result; } }; template -struct type_caster, typename eigen_helper::ValidType> { - using H = eigen_helper; +struct type_caster, typename eigen_tensor_helper::ValidType> { + using H = eigen_tensor_helper; bool load(handle src, bool /*convert*/) { // Note that we have a lot more checks here as we want to make sure to avoid copies auto a = reinterpret_borrow(src); - if ((a.flags() & eigen_to_numpy::flag) == 0) { + if ((a.flags() & compute_array_flag_from_tensor()) == 0) { return false; } - if (!a.dtype().is(dtype::of())) { + if (!a.dtype().is(dtype::of())) { return false; } - if (a.ndim() != H::N) { + if (a.ndim() != Type::NumIndices) { return false; } - std::array shape; - std::copy(a.shape(), a.shape() + H::N, shape.begin()); + std::array shape; + std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); if (!H::is_correct_shape(shape)) { return false; } value.reset(new Eigen::TensorMap( - reinterpret_cast(a.mutable_data()), shape)); + reinterpret_cast(a.mutable_data()), shape)); return true; } @@ -924,19 +914,16 @@ struct type_caster, typename eigen_helper::ValidTyp template static handle cast_impl(C *src, return_value_policy policy, handle parent) { - bool dec_parent = false; - bool writeable = false; + object parent_object; + bool writeable = !std::is_const::value; switch (policy) { case return_value_policy::reference: - parent = none().release(); - dec_parent = true; - writeable = !std::is_const::value; + parent_object = none(); break; case return_value_policy::reference_internal: // Default should do the right thing - dec_parent = false; - writeable = !std::is_const::value; + parent_object = reinterpret_borrow(parent); break; default: @@ -945,14 +932,10 @@ struct type_caster, typename eigen_helper::ValidTyp "reference or reference_internal"); } - handle result = array_t::flag>( - H::get_shape(*src), src->data(), parent) + handle result = array_t()>( + H::get_shape(*src), src->data(), parent_object) .release(); - if (dec_parent) { - parent.dec_ref(); - } - if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; } @@ -965,7 +948,7 @@ struct type_caster, typename eigen_helper::ValidTyp std::unique_ptr> value; public: - static constexpr auto name = const_name("Eigen::TensorMap"); + static constexpr auto name = get_tensor_descriptor(); explicit operator Eigen::TensorMap *() { return value.get(); } /* NOLINT(bugprone-macro-parentheses) */ From 3a978dc6841e272220a473f9d324c9f8005e6360 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Sep 2022 17:37:25 +0000 Subject: [PATCH 013/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/eigen.h | 57 ++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index ccfbf6c867..efd5d52789 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -34,6 +34,7 @@ #endif #include + #include #include #include @@ -644,44 +645,53 @@ struct type_caster::value>> { using cast_op_type = Type; }; -template +template constexpr int compute_array_flag_from_tensor() { - static_assert(((int)T::Layout == (int)Eigen::RowMajor) || ((int)T::Layout == (int)Eigen::ColMajor), "Layout must be row or column major"); - return ((int)T::Layout == (int)Eigen::RowMajor) ? array::c_style : array::f_style; + static_assert(((int) T::Layout == (int) Eigen::RowMajor) + || ((int) T::Layout == (int) Eigen::ColMajor), + "Layout must be row or column major"); + return ((int) T::Layout == (int) Eigen::RowMajor) ? array::c_style : array::f_style; } - - template struct eigen_tensor_helper {}; -template +template struct eigen_tensor_helper> { using T = Eigen::Tensor; using ValidType = void; - static std::array get_shape(const T &f) { return f.dimensions(); } + static std::array get_shape(const T &f) { + return f.dimensions(); + } - static constexpr bool is_correct_shape(const std::array & /*shape*/) { + static constexpr bool + is_correct_shape(const std::array & /*shape*/) { return true; } - template + template static constexpr auto get_dimensions_descriptor_helper(index_sequence) { return concat(const_name(((void) Is, "?"))...); } - static constexpr auto dimensions_descriptor = get_dimensions_descriptor_helper(make_index_sequence()); + static constexpr auto dimensions_descriptor + = get_dimensions_descriptor_helper(make_index_sequence()); }; -template -struct eigen_tensor_helper, Options_, IndexType>> { +template +struct eigen_tensor_helper< + Eigen::TensorFixedSize, Options_, IndexType>> { using T = Eigen::TensorFixedSize, Options_, IndexType>; using ValidType = void; - static constexpr std::array get_shape(const T & /*f*/) { return get_shape(); } + static constexpr std::array get_shape(const T & /*f*/) { + return get_shape(); + } - static constexpr std::array get_shape() { return {{Indices...}}; } + static constexpr std::array get_shape() { + return {{Indices...}}; + } static bool is_correct_shape(const std::array &shape) { return get_shape() == shape; @@ -690,10 +700,13 @@ struct eigen_tensor_helper()...); }; -template +template constexpr auto get_tensor_descriptor() { - return const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") - + eigen_tensor_helper::dimensions_descriptor + const_name("], flags.writeable, ") + const_name<(int)T::Layout == (int)Eigen::RowMajor>("flags.c_contiguous", "flags.f_contiguous"); + return const_name("numpy.ndarray[") + npy_format_descriptor::name + + const_name("[") + eigen_tensor_helper::dimensions_descriptor + + const_name("], flags.writeable, ") + + const_name<(int) T::Layout == (int) Eigen::RowMajor>("flags.c_contiguous", + "flags.f_contiguous"); } template @@ -716,7 +729,7 @@ struct type_caster::ValidType> { return false; } - value = Eigen::TensorMap(const_cast(a.data()), shape); + value = Eigen::TensorMap(const_cast(a.data()), shape); return true; } @@ -787,10 +800,10 @@ struct type_caster::ValidType> { } parent_object = capsule(src, [](void *ptr) { - Eigen::aligned_allocator allocator; - Type *copy = (Type *) ptr; - copy->~Type(); - allocator.deallocate(copy, 1); + Eigen::aligned_allocator allocator; + Type *copy = (Type *) ptr; + copy->~Type(); + allocator.deallocate(copy, 1); }); writeable = true; break; From 38fbc37bc9f146b0a1487c07d8ea61713687d8ac Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 28 Sep 2022 17:42:39 +0000 Subject: [PATCH 014/141] Remove auto constexpr function --- include/pybind11/eigen.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index efd5d52789..8368973547 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -701,18 +701,19 @@ struct eigen_tensor_helper< }; template -constexpr auto get_tensor_descriptor() { - return const_name("numpy.ndarray[") + npy_format_descriptor::name +struct get_tensor_descriptor { + static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("], flags.writeable, ") + const_name<(int) T::Layout == (int) Eigen::RowMajor>("flags.c_contiguous", "flags.f_contiguous"); -} + +}; template struct type_caster::ValidType> { using H = eigen_tensor_helper; - PYBIND11_TYPE_CASTER(Type, get_tensor_descriptor()); + PYBIND11_TYPE_CASTER(Type, get_tensor_descriptor::value); bool load(handle src, bool /*convert*/) { array_t()> a( @@ -961,7 +962,7 @@ struct type_caster, typename eigen_tensor_helper::V std::unique_ptr> value; public: - static constexpr auto name = get_tensor_descriptor(); + static constexpr auto name = get_tensor_descriptor::value; explicit operator Eigen::TensorMap *() { return value.get(); } /* NOLINT(bugprone-macro-parentheses) */ From bd6a679fc1cca64b3a5b0bf7774f95fbc96a089b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Sep 2022 17:50:07 +0000 Subject: [PATCH 015/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/eigen.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 8368973547..4d819cc1e3 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -702,12 +702,12 @@ struct eigen_tensor_helper< template struct get_tensor_descriptor { - static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") + eigen_tensor_helper::dimensions_descriptor - + const_name("], flags.writeable, ") - + const_name<(int) T::Layout == (int) Eigen::RowMajor>("flags.c_contiguous", - "flags.f_contiguous"); - + static constexpr auto value + = const_name("numpy.ndarray[") + npy_format_descriptor::name + + const_name("[") + eigen_tensor_helper::dimensions_descriptor + + const_name("], flags.writeable, ") + + const_name<(int) T::Layout == (int) Eigen::RowMajor>("flags.c_contiguous", + "flags.f_contiguous"); }; template From 8d64196d49e7a4f55edfc1d68636194bc0890889 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 28 Sep 2022 17:57:59 +0000 Subject: [PATCH 016/141] Try again for older C++ --- include/pybind11/eigen.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 4d819cc1e3..7d43c4a627 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -670,13 +670,16 @@ struct eigen_tensor_helper + struct helper {}; + template - static constexpr auto get_dimensions_descriptor_helper(index_sequence) { - return concat(const_name(((void) Is, "?"))...); - } + struct helper> { + static constexpr auto value = concat(const_name(((void) Is, "?"))...); + }; static constexpr auto dimensions_descriptor - = get_dimensions_descriptor_helper(make_index_sequence()); + = helper())>::value; }; template From 09557c5f772a49b29982fb7bfc07a72a3c675ed6 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 28 Sep 2022 18:03:28 +0000 Subject: [PATCH 017/141] Oops forgot constexpr --- include/pybind11/eigen.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 7d43c4a627..a6d3a52211 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -932,7 +932,7 @@ struct type_caster, typename eigen_tensor_helper::V template static handle cast_impl(C *src, return_value_policy policy, handle parent) { object parent_object; - bool writeable = !std::is_const::value; + constexpr bool writeable = !std::is_const::value; switch (policy) { case return_value_policy::reference: parent_object = none(); From 0a60475d28e7f892889076d93fbdbdfdc9903b2c Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 28 Sep 2022 20:36:30 +0000 Subject: [PATCH 018/141] Move to new files as suggested --- include/pybind11/detail/eigen_matrix.h | 708 +++++++++++ include/pybind11/detail/eigen_tensor.h | 383 ++++++ include/pybind11/eigen.h | 1039 +---------------- tests/CMakeLists.txt | 8 +- .../{test_eigen.cpp => test_eigen_matrix.cpp} | 77 +- tests/{test_eigen.py => test_eigen_matrix.py} | 63 +- tests/test_eigen_tensor.cpp | 88 ++ tests/test_eigen_tensor.py | 64 + 8 files changed, 1255 insertions(+), 1175 deletions(-) create mode 100644 include/pybind11/detail/eigen_matrix.h create mode 100644 include/pybind11/detail/eigen_tensor.h rename tests/{test_eigen.cpp => test_eigen_matrix.cpp} (89%) rename tests/{test_eigen.py => test_eigen_matrix.py} (94%) create mode 100644 tests/test_eigen_tensor.cpp create mode 100644 tests/test_eigen_tensor.py diff --git a/include/pybind11/detail/eigen_matrix.h b/include/pybind11/detail/eigen_matrix.h new file mode 100644 index 0000000000..c4445ccb52 --- /dev/null +++ b/include/pybind11/detail/eigen_matrix.h @@ -0,0 +1,708 @@ + +/* + pybind11/eigen.h: Transparent conversion for dense and sparse Eigen matrices + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +/* HINT: To suppress warnings originating from the Eigen headers, use -isystem. + See also: + https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir + https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler +*/ + +#include "../numpy.h" + +// The C4127 suppression was introduced for Eigen 3.4.0. In theory we could +// make it version specific, or even remove it later, but considering that +// 1. C4127 is generally far more distracting than useful for modern template code, and +// 2. we definitely want to ignore any MSVC warnings originating from Eigen code, +// it is probably best to keep this around indefinitely. +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4127) // C4127: conditional expression is constant +# pragma warning(disable : 5054) // https://github.com/pybind/pybind11/pull/3741 +// C5054: operator '&': deprecated between enumerations of different types +#elif defined(__MINGW32__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + +#include +#include + +#if defined(_MSC_VER) +# pragma warning(pop) +#elif defined(__MINGW32__) +# pragma GCC diagnostic pop +#endif + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +// Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: +using EigenDStride = Eigen::Stride; +template +using EigenDRef = Eigen::Ref; +template +using EigenDMap = Eigen::Map; + +PYBIND11_NAMESPACE_BEGIN(detail) + +#if EIGEN_VERSION_AT_LEAST(3, 3, 0) +using EigenIndex = Eigen::Index; +template +using EigenMapSparseMatrix = Eigen::Map>; +#else +using EigenIndex = EIGEN_DEFAULT_DENSE_INDEX_TYPE; +template +using EigenMapSparseMatrix = Eigen::MappedSparseMatrix; +#endif + +// Matches Eigen::Map, Eigen::Ref, blocks, etc: +template +using is_eigen_dense_map = all_of, + std::is_base_of, T>>; +template +using is_eigen_mutable_map = std::is_base_of, T>; +template +using is_eigen_dense_plain + = all_of>, is_template_base_of>; +template +using is_eigen_sparse = is_template_base_of; +// Test for objects inheriting from EigenBase that aren't captured by the above. This +// basically covers anything that can be assigned to a dense matrix but that don't have a typical +// matrix data layout that can be copied from their .data(). For example, DiagonalMatrix and +// SelfAdjointView fall into this category. +template +using is_eigen_other + = all_of, + negation, is_eigen_dense_plain, is_eigen_sparse>>>; + +// Captures numpy/eigen conformability status (returned by EigenProps::conformable()): +template +struct EigenConformable { + bool conformable = false; + EigenIndex rows = 0, cols = 0; + EigenDStride stride{0, 0}; // Only valid if negativestrides is false! + bool negativestrides = false; // If true, do not use stride! + + // NOLINTNEXTLINE(google-explicit-constructor) + EigenConformable(bool fits = false) : conformable{fits} {} + // Matrix type: + EigenConformable(EigenIndex r, EigenIndex c, EigenIndex rstride, EigenIndex cstride) + : conformable{true}, rows{r}, cols{c}, + // TODO: when Eigen bug #747 is fixed, remove the tests for non-negativity. + // http://eigen.tuxfamily.org/bz/show_bug.cgi?id=747 + stride{EigenRowMajor ? (rstride > 0 ? rstride : 0) + : (cstride > 0 ? cstride : 0) /* outer stride */, + EigenRowMajor ? (cstride > 0 ? cstride : 0) + : (rstride > 0 ? rstride : 0) /* inner stride */}, + negativestrides{rstride < 0 || cstride < 0} {} + // Vector type: + EigenConformable(EigenIndex r, EigenIndex c, EigenIndex stride) + : EigenConformable(r, c, r == 1 ? c * stride : stride, c == 1 ? r : r * stride) {} + + template + bool stride_compatible() const { + // To have compatible strides, we need (on both dimensions) one of fully dynamic strides, + // matching strides, or a dimension size of 1 (in which case the stride value is + // irrelevant). Alternatively, if any dimension size is 0, the strides are not relevant + // (and numpy ≥ 1.23 sets the strides to 0 in that case, so we need to check explicitly). + if (negativestrides) { + return false; + } + if (rows == 0 || cols == 0) { + return true; + } + return (props::inner_stride == Eigen::Dynamic || props::inner_stride == stride.inner() + || (EigenRowMajor ? cols : rows) == 1) + && (props::outer_stride == Eigen::Dynamic || props::outer_stride == stride.outer() + || (EigenRowMajor ? rows : cols) == 1); + } + // NOLINTNEXTLINE(google-explicit-constructor) + operator bool() const { return conformable; } +}; + +template +struct eigen_extract_stride { + using type = Type; +}; +template +struct eigen_extract_stride> { + using type = StrideType; +}; +template +struct eigen_extract_stride> { + using type = StrideType; +}; + +// Helper struct for extracting information from an Eigen type +template +struct EigenProps { + using Type = Type_; + using Scalar = typename Type::Scalar; + using StrideType = typename eigen_extract_stride::type; + static constexpr EigenIndex rows = Type::RowsAtCompileTime, cols = Type::ColsAtCompileTime, + size = Type::SizeAtCompileTime; + static constexpr bool row_major = Type::IsRowMajor, + vector + = Type::IsVectorAtCompileTime, // At least one dimension has fixed size 1 + fixed_rows = rows != Eigen::Dynamic, fixed_cols = cols != Eigen::Dynamic, + fixed = size != Eigen::Dynamic, // Fully-fixed size + dynamic = !fixed_rows && !fixed_cols; // Fully-dynamic size + + template + using if_zero = std::integral_constant; + static constexpr EigenIndex inner_stride + = if_zero::value, + outer_stride = if_zero < StrideType::OuterStrideAtCompileTime, + vector ? size + : row_major ? cols + : rows > ::value; + static constexpr bool dynamic_stride + = inner_stride == Eigen::Dynamic && outer_stride == Eigen::Dynamic; + static constexpr bool requires_row_major + = !dynamic_stride && !vector && (row_major ? inner_stride : outer_stride) == 1; + static constexpr bool requires_col_major + = !dynamic_stride && !vector && (row_major ? outer_stride : inner_stride) == 1; + + // Takes an input array and determines whether we can make it fit into the Eigen type. If + // the array is a vector, we attempt to fit it into either an Eigen 1xN or Nx1 vector + // (preferring the latter if it will fit in either, i.e. for a fully dynamic matrix type). + static EigenConformable conformable(const array &a) { + const auto dims = a.ndim(); + if (dims < 1 || dims > 2) { + return false; + } + + if (dims == 2) { // Matrix type: require exact match (or dynamic) + + EigenIndex np_rows = a.shape(0), np_cols = a.shape(1), + np_rstride = a.strides(0) / static_cast(sizeof(Scalar)), + np_cstride = a.strides(1) / static_cast(sizeof(Scalar)); + if ((PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && np_rows != rows) + || (PYBIND11_SILENCE_MSVC_C4127(fixed_cols) && np_cols != cols)) { + return false; + } + + return {np_rows, np_cols, np_rstride, np_cstride}; + } + + // Otherwise we're storing an n-vector. Only one of the strides will be used, but + // whichever is used, we want the (single) numpy stride value. + const EigenIndex n = a.shape(0), + stride = a.strides(0) / static_cast(sizeof(Scalar)); + + if (vector) { // Eigen type is a compile-time vector + if (PYBIND11_SILENCE_MSVC_C4127(fixed) && size != n) { + return false; // Vector size mismatch + } + return {rows == 1 ? 1 : n, cols == 1 ? 1 : n, stride}; + } + if (fixed) { + // The type has a fixed size, but is not a vector: abort + return false; + } + if (fixed_cols) { + // Since this isn't a vector, cols must be != 1. We allow this only if it exactly + // equals the number of elements (rows is Dynamic, and so 1 row is allowed). + if (cols != n) { + return false; + } + return {1, n, stride}; + } // Otherwise it's either fully dynamic, or column dynamic; both become a column vector + if (PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && rows != n) { + return false; + } + return {n, 1, stride}; + } + + static constexpr bool show_writeable + = is_eigen_dense_map::value && is_eigen_mutable_map::value; + static constexpr bool show_order = is_eigen_dense_map::value; + static constexpr bool show_c_contiguous = show_order && requires_row_major; + static constexpr bool show_f_contiguous + = !show_c_contiguous && show_order && requires_col_major; + + static constexpr auto descriptor + = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + + const_name(const_name<(size_t) rows>(), const_name("m")) + const_name(", ") + + const_name(const_name<(size_t) cols>(), const_name("n")) + const_name("]") + + + // For a reference type (e.g. Ref) we have other constraints that might need to + // be satisfied: writeable=True (for a mutable reference), and, depending on the map's + // stride options, possibly f_contiguous or c_contiguous. We include them in the + // descriptor output to provide some hint as to why a TypeError is occurring (otherwise + // it can be confusing to see that a function accepts a 'numpy.ndarray[float64[3,2]]' and + // an error message that you *gave* a numpy.ndarray of the right type and dimensions. + const_name(", flags.writeable", "") + + const_name(", flags.c_contiguous", "") + + const_name(", flags.f_contiguous", "") + const_name("]"); +}; + +// Casts an Eigen type to numpy array. If given a base, the numpy array references the src data, +// otherwise it'll make a copy. writeable lets you turn off the writeable flag for the array. +template +handle +eigen_array_cast(typename props::Type const &src, handle base = handle(), bool writeable = true) { + constexpr ssize_t elem_size = sizeof(typename props::Scalar); + array a; + if (props::vector) { + a = array({src.size()}, {elem_size * src.innerStride()}, src.data(), base); + } else { + a = array({src.rows(), src.cols()}, + {elem_size * src.rowStride(), elem_size * src.colStride()}, + src.data(), + base); + } + + if (!writeable) { + array_proxy(a.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + return a.release(); +} + +// Takes an lvalue ref to some Eigen type and a (python) base object, creating a numpy array that +// reference the Eigen object's data with `base` as the python-registered base class (if omitted, +// the base will be set to None, and lifetime management is up to the caller). The numpy array is +// non-writeable if the given type is const. +template +handle eigen_ref_array(Type &src, handle parent = none()) { + // none here is to get past array's should-we-copy detection, which currently always + // copies when there is no base. Setting the base to None should be harmless. + return eigen_array_cast(src, parent, !std::is_const::value); +} + +// Takes a pointer to some dense, plain Eigen type, builds a capsule around it, then returns a +// numpy array that references the encapsulated data with a python-side reference to the capsule to +// tie its destruction to that of any dependent python objects. Const-ness is determined by +// whether or not the Type of the pointer given is const. +template ::value>> +handle eigen_encapsulate(Type *src) { + capsule base(src, [](void *o) { delete static_cast(o); }); + return eigen_ref_array(*src, base); +} + +// Type caster for regular, dense matrix types (e.g. MatrixXd), but not maps/refs/etc. of dense +// types. +template +struct type_caster::value>> { + using Scalar = typename Type::Scalar; + using props = EigenProps; + + bool load(handle src, bool convert) { + // If we're in no-convert mode, only load if given an array of the correct type + if (!convert && !isinstance>(src)) { + return false; + } + + // Coerce into an array, but don't do type conversion yet; the copy below handles it. + auto buf = array::ensure(src); + + if (!buf) { + return false; + } + + auto dims = buf.ndim(); + if (dims < 1 || dims > 2) { + return false; + } + + auto fits = props::conformable(buf); + if (!fits) { + return false; + } + + // Allocate the new type, then build a numpy reference into it + value = Type(fits.rows, fits.cols); + auto ref = reinterpret_steal(eigen_ref_array(value)); + if (dims == 1) { + ref = ref.squeeze(); + } else if (ref.ndim() == 1) { + buf = buf.squeeze(); + } + + int result = detail::npy_api::get().PyArray_CopyInto_(ref.ptr(), buf.ptr()); + + if (result < 0) { // Copy failed! + PyErr_Clear(); + return false; + } + + return true; + } + +private: + // Cast implementation + template + static handle cast_impl(CType *src, return_value_policy policy, handle parent) { + switch (policy) { + case return_value_policy::take_ownership: + case return_value_policy::automatic: + return eigen_encapsulate(src); + case return_value_policy::move: + return eigen_encapsulate(new CType(std::move(*src))); + case return_value_policy::copy: + return eigen_array_cast(*src); + case return_value_policy::reference: + case return_value_policy::automatic_reference: + return eigen_ref_array(*src); + case return_value_policy::reference_internal: + return eigen_ref_array(*src, parent); + default: + throw cast_error("unhandled return_value_policy: should not happen!"); + }; + } + +public: + // Normal returned non-reference, non-const value: + static handle cast(Type &&src, return_value_policy /* policy */, handle parent) { + return cast_impl(&src, return_value_policy::move, parent); + } + // If you return a non-reference const, we mark the numpy array readonly: + static handle cast(const Type &&src, return_value_policy /* policy */, handle parent) { + return cast_impl(&src, return_value_policy::move, parent); + } + // lvalue reference return; default (automatic) becomes copy + static handle cast(Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + // const lvalue reference return; default (automatic) becomes copy + static handle cast(const Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + // non-const pointer return + static handle cast(Type *src, return_value_policy policy, handle parent) { + return cast_impl(src, policy, parent); + } + // const pointer return + static handle cast(const Type *src, return_value_policy policy, handle parent) { + return cast_impl(src, policy, parent); + } + + static constexpr auto name = props::descriptor; + + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type *() { return &value; } + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type &() { return value; } + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type &&() && { return std::move(value); } + template + using cast_op_type = movable_cast_op_type; + +private: + Type value; +}; + +// Base class for casting reference/map/block/etc. objects back to python. +template +struct eigen_map_caster { +private: + using props = EigenProps; + +public: + // Directly referencing a ref/map's data is a bit dangerous (whatever the map/ref points to has + // to stay around), but we'll allow it under the assumption that you know what you're doing + // (and have an appropriate keep_alive in place). We return a numpy array pointing directly at + // the ref's data (The numpy array ends up read-only if the ref was to a const matrix type.) + // Note that this means you need to ensure you don't destroy the object in some other way (e.g. + // with an appropriate keep_alive, or with a reference to a statically allocated matrix). + static handle cast(const MapType &src, return_value_policy policy, handle parent) { + switch (policy) { + case return_value_policy::copy: + return eigen_array_cast(src); + case return_value_policy::reference_internal: + return eigen_array_cast(src, parent, is_eigen_mutable_map::value); + case return_value_policy::reference: + case return_value_policy::automatic: + case return_value_policy::automatic_reference: + return eigen_array_cast(src, none(), is_eigen_mutable_map::value); + default: + // move, take_ownership don't make any sense for a ref/map: + pybind11_fail("Invalid return_value_policy for Eigen Map/Ref/Block type"); + } + } + + static constexpr auto name = props::descriptor; + + // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return + // types but not bound arguments). We still provide them (with an explicitly delete) so that + // you end up here if you try anyway. + bool load(handle, bool) = delete; + operator MapType() = delete; + template + using cast_op_type = MapType; +}; + +// We can return any map-like object (but can only load Refs, specialized next): +template +struct type_caster::value>> : eigen_map_caster {}; + +// Loader for Ref<...> arguments. See the documentation for info on how to make this work without +// copying (it requires some extra effort in many cases). +template +struct type_caster< + Eigen::Ref, + enable_if_t>::value>> + : public eigen_map_caster> { +private: + using Type = Eigen::Ref; + using props = EigenProps; + using Scalar = typename props::Scalar; + using MapType = Eigen::Map; + using Array + = array_t; + static constexpr bool need_writeable = is_eigen_mutable_map::value; + // Delay construction (these have no default constructor) + std::unique_ptr map; + std::unique_ptr ref; + // Our array. When possible, this is just a numpy array pointing to the source data, but + // sometimes we can't avoid copying (e.g. input is not a numpy array at all, has an + // incompatible layout, or is an array of a type that needs to be converted). Using a numpy + // temporary (rather than an Eigen temporary) saves an extra copy when we need both type + // conversion and storage order conversion. (Note that we refuse to use this temporary copy + // when loading an argument for a Ref with M non-const, i.e. a read-write reference). + Array copy_or_ref; + +public: + bool load(handle src, bool convert) { + // First check whether what we have is already an array of the right type. If not, we + // can't avoid a copy (because the copy is also going to do type conversion). + bool need_copy = !isinstance(src); + + EigenConformable fits; + if (!need_copy) { + // We don't need a converting copy, but we also need to check whether the strides are + // compatible with the Ref's stride requirements + auto aref = reinterpret_borrow(src); + + if (aref && (!need_writeable || aref.writeable())) { + fits = props::conformable(aref); + if (!fits) { + return false; // Incompatible dimensions + } + if (!fits.template stride_compatible()) { + need_copy = true; + } else { + copy_or_ref = std::move(aref); + } + } else { + need_copy = true; + } + } + + if (need_copy) { + // We need to copy: If we need a mutable reference, or we're not supposed to convert + // (either because we're in the no-convert overload pass, or because we're explicitly + // instructed not to copy (via `py::arg().noconvert()`) we have to fail loading. + if (!convert || need_writeable) { + return false; + } + + Array copy = Array::ensure(src); + if (!copy) { + return false; + } + fits = props::conformable(copy); + if (!fits || !fits.template stride_compatible()) { + return false; + } + copy_or_ref = std::move(copy); + loader_life_support::add_patient(copy_or_ref); + } + + ref.reset(); + map.reset(new MapType(data(copy_or_ref), + fits.rows, + fits.cols, + make_stride(fits.stride.outer(), fits.stride.inner()))); + ref.reset(new Type(*map)); + + return true; + } + + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type *() { return ref.get(); } + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type &() { return *ref; } + template + using cast_op_type = pybind11::detail::cast_op_type<_T>; + +private: + template ::value, int> = 0> + Scalar *data(Array &a) { + return a.mutable_data(); + } + + template ::value, int> = 0> + const Scalar *data(Array &a) { + return a.data(); + } + + // Attempt to figure out a constructor of `Stride` that will work. + // If both strides are fixed, use a default constructor: + template + using stride_ctor_default = bool_constant::value>; + // Otherwise, if there is a two-index constructor, assume it is (outer,inner) like + // Eigen::Stride, and use it: + template + using stride_ctor_dual + = bool_constant::value + && std::is_constructible::value>; + // Otherwise, if there is a one-index constructor, and just one of the strides is dynamic, use + // it (passing whichever stride is dynamic). + template + using stride_ctor_outer + = bool_constant, stride_ctor_dual>::value + && S::OuterStrideAtCompileTime == Eigen::Dynamic + && S::InnerStrideAtCompileTime != Eigen::Dynamic + && std::is_constructible::value>; + template + using stride_ctor_inner + = bool_constant, stride_ctor_dual>::value + && S::InnerStrideAtCompileTime == Eigen::Dynamic + && S::OuterStrideAtCompileTime != Eigen::Dynamic + && std::is_constructible::value>; + + template ::value, int> = 0> + static S make_stride(EigenIndex, EigenIndex) { + return S(); + } + template ::value, int> = 0> + static S make_stride(EigenIndex outer, EigenIndex inner) { + return S(outer, inner); + } + template ::value, int> = 0> + static S make_stride(EigenIndex outer, EigenIndex) { + return S(outer); + } + template ::value, int> = 0> + static S make_stride(EigenIndex, EigenIndex inner) { + return S(inner); + } +}; + +// type_caster for special matrix types (e.g. DiagonalMatrix), which are EigenBase, but not +// EigenDense (i.e. they don't have a data(), at least not with the usual matrix layout). +// load() is not supported, but we can cast them into the python domain by first copying to a +// regular Eigen::Matrix, then casting that. +template +struct type_caster::value>> { +protected: + using Matrix + = Eigen::Matrix; + using props = EigenProps; + +public: + static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { + handle h = eigen_encapsulate(new Matrix(src)); + return h; + } + static handle cast(const Type *src, return_value_policy policy, handle parent) { + return cast(*src, policy, parent); + } + + static constexpr auto name = props::descriptor; + + // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return + // types but not bound arguments). We still provide them (with an explicitly delete) so that + // you end up here if you try anyway. + bool load(handle, bool) = delete; + operator Type() = delete; + template + using cast_op_type = Type; +}; + +template +struct type_caster::value>> { + using Scalar = typename Type::Scalar; + using StorageIndex = remove_reference_t().outerIndexPtr())>; + using Index = typename Type::Index; + static constexpr bool rowMajor = Type::IsRowMajor; + + bool load(handle src, bool) { + if (!src) { + return false; + } + + auto obj = reinterpret_borrow(src); + object sparse_module = module_::import("scipy.sparse"); + object matrix_type = sparse_module.attr(rowMajor ? "csr_matrix" : "csc_matrix"); + + if (!type::handle_of(obj).is(matrix_type)) { + try { + obj = matrix_type(obj); + } catch (const error_already_set &) { + return false; + } + } + + auto values = array_t((object) obj.attr("data")); + auto innerIndices = array_t((object) obj.attr("indices")); + auto outerIndices = array_t((object) obj.attr("indptr")); + auto shape = pybind11::tuple((pybind11::object) obj.attr("shape")); + auto nnz = obj.attr("nnz").cast(); + + if (!values || !innerIndices || !outerIndices) { + return false; + } + + value = EigenMapSparseMatrix(shape[0].cast(), + shape[1].cast(), + std::move(nnz), + outerIndices.mutable_data(), + innerIndices.mutable_data(), + values.mutable_data()); + + return true; + } + + static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { + const_cast(src).makeCompressed(); + + object matrix_type + = module_::import("scipy.sparse").attr(rowMajor ? "csr_matrix" : "csc_matrix"); + + array data(src.nonZeros(), src.valuePtr()); + array outerIndices((rowMajor ? src.rows() : src.cols()) + 1, src.outerIndexPtr()); + array innerIndices(src.nonZeros(), src.innerIndexPtr()); + + return matrix_type(pybind11::make_tuple( + std::move(data), std::move(innerIndices), std::move(outerIndices)), + pybind11::make_tuple(src.rows(), src.cols())) + .release(); + } + + PYBIND11_TYPE_CASTER(Type, + const_name<(Type::IsRowMajor) != 0>("scipy.sparse.csr_matrix[", + "scipy.sparse.csc_matrix[") + + npy_format_descriptor::name + const_name("]")); +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) \ No newline at end of file diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h new file mode 100644 index 0000000000..c3042101ce --- /dev/null +++ b/include/pybind11/detail/eigen_tensor.h @@ -0,0 +1,383 @@ + +/* + pybind11/eigen_tensor.h: Transparent conversion for Eigen tensors + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +/* HINT: To suppress warnings originating from the Eigen headers, use -isystem. + See also: + https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir + https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler +*/ + +#include "../numpy.h" + +// The C4127 suppression was introduced for Eigen 3.4.0. In theory we could +// make it version specific, or even remove it later, but considering that +// 1. C4127 is generally far more distracting than useful for modern template code, and +// 2. we definitely want to ignore any MSVC warnings originating from Eigen code, +// it is probably best to keep this around indefinitely. +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4554) // Tensor.h warning +// C5054: operator '&': deprecated between enumerations of different types +#elif defined(__MINGW32__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + +#include + +#if defined(_MSC_VER) +# pragma warning(pop) +#elif defined(__MINGW32__) +# pragma GCC diagnostic pop +#endif + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +constexpr int compute_array_flag_from_tensor() { + static_assert(((int) T::Layout == (int) Eigen::RowMajor) + || ((int) T::Layout == (int) Eigen::ColMajor), + "Layout must be row or column major"); + return ((int) T::Layout == (int) Eigen::RowMajor) ? array::c_style : array::f_style; +} + +template +struct eigen_tensor_helper {}; + +template +struct eigen_tensor_helper> { + using T = Eigen::Tensor; + using ValidType = void; + + static std::array get_shape(const T &f) { + return f.dimensions(); + } + + static constexpr bool + is_correct_shape(const std::array & /*shape*/) { + return true; + } + + template + struct helper {}; + + template + struct helper> { + static constexpr auto value = concat(const_name(((void) Is, "?"))...); + }; + + static constexpr auto dimensions_descriptor + = helper())>::value; +}; + +template +struct eigen_tensor_helper< + Eigen::TensorFixedSize, Options_, IndexType>> { + using T = Eigen::TensorFixedSize, Options_, IndexType>; + using ValidType = void; + + static constexpr std::array get_shape(const T & /*f*/) { + return get_shape(); + } + + static constexpr std::array get_shape() { + return {{Indices...}}; + } + + static bool is_correct_shape(const std::array &shape) { + return get_shape() == shape; + } + + static constexpr auto dimensions_descriptor = concat(const_name()...); +}; + +template +struct get_tensor_descriptor { + static constexpr auto value + = const_name("numpy.ndarray[") + npy_format_descriptor::name + + const_name("[") + eigen_tensor_helper::dimensions_descriptor + + const_name("], flags.writeable, ") + + const_name<(int) T::Layout == (int) Eigen::RowMajor>("flags.c_contiguous", + "flags.f_contiguous"); +}; + +template +struct type_caster::ValidType> { + using H = eigen_tensor_helper; + PYBIND11_TYPE_CASTER(Type, get_tensor_descriptor::value); + + bool load(handle src, bool /*convert*/) { + array_t()> a( + reinterpret_borrow(src)); + + if (a.ndim() != Type::NumIndices) { + return false; + } + + std::array shape; + std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); + + if (!H::is_correct_shape(shape)) { + return false; + } + + value = Eigen::TensorMap(const_cast(a.data()), shape); + + return true; + } + + static handle cast(Type &&src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::reference + || policy == return_value_policy::reference_internal) { + pybind11_fail("Cannot use a reference return value policy for an rvalue"); + } + return cast_impl(&src, return_value_policy::move, parent); + } + + static handle cast(const Type &&src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::reference + || policy == return_value_policy::reference_internal) { + pybind11_fail("Cannot use a reference return value policy for an rvalue"); + } + return cast_impl(&src, return_value_policy::move, parent); + } + + static handle cast(Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + + static handle cast(const Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + + static handle cast(Type *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + static handle cast(const Type *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + template + static handle cast_impl(C *src, return_value_policy policy, handle parent) { + object parent_object; + bool writeable = false; + switch (policy) { + case return_value_policy::move: + if (std::is_const::value) { + pybind11_fail("Cannot move from a constant reference"); + } + { + Eigen::aligned_allocator allocator; + Type *copy = ::new (allocator.allocate(1)) Type(std::move(*src)); + src = copy; + } + + parent_object = capsule(src, [](void *ptr) { + Eigen::aligned_allocator allocator; + Type *copy = (Type *) ptr; + copy->~Type(); + allocator.deallocate(copy, 1); + }); + writeable = true; + break; + + case return_value_policy::take_ownership: + if (std::is_const::value) { + pybind11_fail("Cannot take ownership of a const reference"); + } + parent_object = capsule(src, [](void *ptr) { delete (Type *) ptr; }); + writeable = true; + break; + + case return_value_policy::copy: + parent_object = {}; + writeable = true; + break; + + case return_value_policy::reference: + parent_object = none(); + writeable = !std::is_const::value; + break; + + case return_value_policy::reference_internal: + // Default should do the right thing + parent_object = reinterpret_borrow(parent); + writeable = !std::is_const::value; + break; + + default: + pybind11_fail("pybind11 bug in eigen.h, please file a bug report"); + } + + handle result = array_t()>( + H::get_shape(*src), src->data(), parent_object) + .release(); + + if (!writeable) { + array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + return result; + } +}; + +template +struct type_caster, typename eigen_tensor_helper::ValidType> { + using H = eigen_tensor_helper; + + bool load(handle src, bool /*convert*/) { + // Note that we have a lot more checks here as we want to make sure to avoid copies + auto a = reinterpret_borrow(src); + if ((a.flags() & compute_array_flag_from_tensor()) == 0) { + return false; + } + + if (!a.dtype().is(dtype::of())) { + return false; + } + + if (a.ndim() != Type::NumIndices) { + return false; + } + + std::array shape; + std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); + + if (!H::is_correct_shape(shape)) { + return false; + } + + value.reset(new Eigen::TensorMap( + reinterpret_cast(a.mutable_data()), shape)); + + return true; + } + + static handle cast(Eigen::TensorMap &&src, return_value_policy policy, handle parent) { + return cast_impl(&src, policy, parent); + } + + static handle + cast(const Eigen::TensorMap &&src, return_value_policy policy, handle parent) { + return cast_impl(&src, policy, parent); + } + + static handle cast(Eigen::TensorMap &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + + static handle + cast(const Eigen::TensorMap &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + + static handle cast(Eigen::TensorMap *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + static handle + cast(const Eigen::TensorMap *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + template + static handle cast_impl(C *src, return_value_policy policy, handle parent) { + object parent_object; + constexpr bool writeable = !std::is_const::value; + switch (policy) { + case return_value_policy::reference: + parent_object = none(); + break; + + case return_value_policy::reference_internal: + // Default should do the right thing + parent_object = reinterpret_borrow(parent); + break; + + default: + // move, take_ownership don't make any sense for a ref/map: + pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either " + "reference or reference_internal"); + } + + handle result = array_t()>( + H::get_shape(*src), src->data(), parent_object) + .release(); + + if (!writeable) { + array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + return result; + } + +protected: + // TODO: Move to std::optional once std::optional has more support + std::unique_ptr> value; + +public: + static constexpr auto name = get_tensor_descriptor::value; + explicit operator Eigen::TensorMap *() { + return value.get(); + } /* NOLINT(bugprone-macro-parentheses) */ + explicit operator Eigen::TensorMap &() { + return *value; + } /* NOLINT(bugprone-macro-parentheses) */ + explicit operator Eigen::TensorMap &&() && { + return std::move(*value); + } /* NOLINT(bugprone-macro-parentheses) */ + + template + using cast_op_type = ::pybind11::detail::movable_cast_op_type; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) \ No newline at end of file diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index a6d3a52211..69fb528f99 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -9,1044 +9,11 @@ #pragma once -/* HINT: To suppress warnings originating from the Eigen headers, use -isystem. - See also: - https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir - https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler -*/ - -#include "numpy.h" - -// The C4127 suppression was introduced for Eigen 3.4.0. In theory we could -// make it version specific, or even remove it later, but considering that -// 1. C4127 is generally far more distracting than useful for modern template code, and -// 2. we definitely want to ignore any MSVC warnings originating from Eigen code, -// it is probably best to keep this around indefinitely. -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4554) // Tensor.h warning -# pragma warning(disable : 4127) // C4127: conditional expression is constant -# pragma warning(disable : 5054) // https://github.com/pybind/pybind11/pull/3741 -// C5054: operator '&': deprecated between enumerations of different types -#elif defined(__MINGW32__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - -#include - -#include -#include -#include - -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__MINGW32__) -# pragma GCC diagnostic pop -#endif +#include "detail/eigen_matrix.h" +#include "detail/eigen_tensor.h" // Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit // move constructors that break things. We could detect this an explicitly copy, but an extra copy // of matrices seems highly undesirable. static_assert(EIGEN_VERSION_AT_LEAST(3, 2, 7), - "Eigen support in pybind11 requires Eigen >= 3.2.7"); - -PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) - -// Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: -using EigenDStride = Eigen::Stride; -template -using EigenDRef = Eigen::Ref; -template -using EigenDMap = Eigen::Map; - -PYBIND11_NAMESPACE_BEGIN(detail) - -#if EIGEN_VERSION_AT_LEAST(3, 3, 0) -using EigenIndex = Eigen::Index; -template -using EigenMapSparseMatrix = Eigen::Map>; -#else -using EigenIndex = EIGEN_DEFAULT_DENSE_INDEX_TYPE; -template -using EigenMapSparseMatrix = Eigen::MappedSparseMatrix; -#endif - -// Matches Eigen::Map, Eigen::Ref, blocks, etc: -template -using is_eigen_dense_map = all_of, - std::is_base_of, T>>; -template -using is_eigen_mutable_map = std::is_base_of, T>; -template -using is_eigen_dense_plain - = all_of>, is_template_base_of>; -template -using is_eigen_sparse = is_template_base_of; -// Test for objects inheriting from EigenBase that aren't captured by the above. This -// basically covers anything that can be assigned to a dense matrix but that don't have a typical -// matrix data layout that can be copied from their .data(). For example, DiagonalMatrix and -// SelfAdjointView fall into this category. -template -using is_eigen_other - = all_of, - negation, is_eigen_dense_plain, is_eigen_sparse>>>; - -// Captures numpy/eigen conformability status (returned by EigenProps::conformable()): -template -struct EigenConformable { - bool conformable = false; - EigenIndex rows = 0, cols = 0; - EigenDStride stride{0, 0}; // Only valid if negativestrides is false! - bool negativestrides = false; // If true, do not use stride! - - // NOLINTNEXTLINE(google-explicit-constructor) - EigenConformable(bool fits = false) : conformable{fits} {} - // Matrix type: - EigenConformable(EigenIndex r, EigenIndex c, EigenIndex rstride, EigenIndex cstride) - : conformable{true}, rows{r}, cols{c}, - // TODO: when Eigen bug #747 is fixed, remove the tests for non-negativity. - // http://eigen.tuxfamily.org/bz/show_bug.cgi?id=747 - stride{EigenRowMajor ? (rstride > 0 ? rstride : 0) - : (cstride > 0 ? cstride : 0) /* outer stride */, - EigenRowMajor ? (cstride > 0 ? cstride : 0) - : (rstride > 0 ? rstride : 0) /* inner stride */}, - negativestrides{rstride < 0 || cstride < 0} {} - // Vector type: - EigenConformable(EigenIndex r, EigenIndex c, EigenIndex stride) - : EigenConformable(r, c, r == 1 ? c * stride : stride, c == 1 ? r : r * stride) {} - - template - bool stride_compatible() const { - // To have compatible strides, we need (on both dimensions) one of fully dynamic strides, - // matching strides, or a dimension size of 1 (in which case the stride value is - // irrelevant). Alternatively, if any dimension size is 0, the strides are not relevant - // (and numpy ≥ 1.23 sets the strides to 0 in that case, so we need to check explicitly). - if (negativestrides) { - return false; - } - if (rows == 0 || cols == 0) { - return true; - } - return (props::inner_stride == Eigen::Dynamic || props::inner_stride == stride.inner() - || (EigenRowMajor ? cols : rows) == 1) - && (props::outer_stride == Eigen::Dynamic || props::outer_stride == stride.outer() - || (EigenRowMajor ? rows : cols) == 1); - } - // NOLINTNEXTLINE(google-explicit-constructor) - operator bool() const { return conformable; } -}; - -template -struct eigen_extract_stride { - using type = Type; -}; -template -struct eigen_extract_stride> { - using type = StrideType; -}; -template -struct eigen_extract_stride> { - using type = StrideType; -}; - -// Helper struct for extracting information from an Eigen type -template -struct EigenProps { - using Type = Type_; - using Scalar = typename Type::Scalar; - using StrideType = typename eigen_extract_stride::type; - static constexpr EigenIndex rows = Type::RowsAtCompileTime, cols = Type::ColsAtCompileTime, - size = Type::SizeAtCompileTime; - static constexpr bool row_major = Type::IsRowMajor, - vector - = Type::IsVectorAtCompileTime, // At least one dimension has fixed size 1 - fixed_rows = rows != Eigen::Dynamic, fixed_cols = cols != Eigen::Dynamic, - fixed = size != Eigen::Dynamic, // Fully-fixed size - dynamic = !fixed_rows && !fixed_cols; // Fully-dynamic size - - template - using if_zero = std::integral_constant; - static constexpr EigenIndex inner_stride - = if_zero::value, - outer_stride = if_zero < StrideType::OuterStrideAtCompileTime, - vector ? size - : row_major ? cols - : rows > ::value; - static constexpr bool dynamic_stride - = inner_stride == Eigen::Dynamic && outer_stride == Eigen::Dynamic; - static constexpr bool requires_row_major - = !dynamic_stride && !vector && (row_major ? inner_stride : outer_stride) == 1; - static constexpr bool requires_col_major - = !dynamic_stride && !vector && (row_major ? outer_stride : inner_stride) == 1; - - // Takes an input array and determines whether we can make it fit into the Eigen type. If - // the array is a vector, we attempt to fit it into either an Eigen 1xN or Nx1 vector - // (preferring the latter if it will fit in either, i.e. for a fully dynamic matrix type). - static EigenConformable conformable(const array &a) { - const auto dims = a.ndim(); - if (dims < 1 || dims > 2) { - return false; - } - - if (dims == 2) { // Matrix type: require exact match (or dynamic) - - EigenIndex np_rows = a.shape(0), np_cols = a.shape(1), - np_rstride = a.strides(0) / static_cast(sizeof(Scalar)), - np_cstride = a.strides(1) / static_cast(sizeof(Scalar)); - if ((PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && np_rows != rows) - || (PYBIND11_SILENCE_MSVC_C4127(fixed_cols) && np_cols != cols)) { - return false; - } - - return {np_rows, np_cols, np_rstride, np_cstride}; - } - - // Otherwise we're storing an n-vector. Only one of the strides will be used, but - // whichever is used, we want the (single) numpy stride value. - const EigenIndex n = a.shape(0), - stride = a.strides(0) / static_cast(sizeof(Scalar)); - - if (vector) { // Eigen type is a compile-time vector - if (PYBIND11_SILENCE_MSVC_C4127(fixed) && size != n) { - return false; // Vector size mismatch - } - return {rows == 1 ? 1 : n, cols == 1 ? 1 : n, stride}; - } - if (fixed) { - // The type has a fixed size, but is not a vector: abort - return false; - } - if (fixed_cols) { - // Since this isn't a vector, cols must be != 1. We allow this only if it exactly - // equals the number of elements (rows is Dynamic, and so 1 row is allowed). - if (cols != n) { - return false; - } - return {1, n, stride}; - } // Otherwise it's either fully dynamic, or column dynamic; both become a column vector - if (PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && rows != n) { - return false; - } - return {n, 1, stride}; - } - - static constexpr bool show_writeable - = is_eigen_dense_map::value && is_eigen_mutable_map::value; - static constexpr bool show_order = is_eigen_dense_map::value; - static constexpr bool show_c_contiguous = show_order && requires_row_major; - static constexpr bool show_f_contiguous - = !show_c_contiguous && show_order && requires_col_major; - - static constexpr auto descriptor - = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") - + const_name(const_name<(size_t) rows>(), const_name("m")) + const_name(", ") - + const_name(const_name<(size_t) cols>(), const_name("n")) + const_name("]") - + - // For a reference type (e.g. Ref) we have other constraints that might need to - // be satisfied: writeable=True (for a mutable reference), and, depending on the map's - // stride options, possibly f_contiguous or c_contiguous. We include them in the - // descriptor output to provide some hint as to why a TypeError is occurring (otherwise - // it can be confusing to see that a function accepts a 'numpy.ndarray[float64[3,2]]' and - // an error message that you *gave* a numpy.ndarray of the right type and dimensions. - const_name(", flags.writeable", "") - + const_name(", flags.c_contiguous", "") - + const_name(", flags.f_contiguous", "") + const_name("]"); -}; - -// Casts an Eigen type to numpy array. If given a base, the numpy array references the src data, -// otherwise it'll make a copy. writeable lets you turn off the writeable flag for the array. -template -handle -eigen_array_cast(typename props::Type const &src, handle base = handle(), bool writeable = true) { - constexpr ssize_t elem_size = sizeof(typename props::Scalar); - array a; - if (props::vector) { - a = array({src.size()}, {elem_size * src.innerStride()}, src.data(), base); - } else { - a = array({src.rows(), src.cols()}, - {elem_size * src.rowStride(), elem_size * src.colStride()}, - src.data(), - base); - } - - if (!writeable) { - array_proxy(a.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; - } - - return a.release(); -} - -// Takes an lvalue ref to some Eigen type and a (python) base object, creating a numpy array that -// reference the Eigen object's data with `base` as the python-registered base class (if omitted, -// the base will be set to None, and lifetime management is up to the caller). The numpy array is -// non-writeable if the given type is const. -template -handle eigen_ref_array(Type &src, handle parent = none()) { - // none here is to get past array's should-we-copy detection, which currently always - // copies when there is no base. Setting the base to None should be harmless. - return eigen_array_cast(src, parent, !std::is_const::value); -} - -// Takes a pointer to some dense, plain Eigen type, builds a capsule around it, then returns a -// numpy array that references the encapsulated data with a python-side reference to the capsule to -// tie its destruction to that of any dependent python objects. Const-ness is determined by -// whether or not the Type of the pointer given is const. -template ::value>> -handle eigen_encapsulate(Type *src) { - capsule base(src, [](void *o) { delete static_cast(o); }); - return eigen_ref_array(*src, base); -} - -// Type caster for regular, dense matrix types (e.g. MatrixXd), but not maps/refs/etc. of dense -// types. -template -struct type_caster::value>> { - using Scalar = typename Type::Scalar; - using props = EigenProps; - - bool load(handle src, bool convert) { - // If we're in no-convert mode, only load if given an array of the correct type - if (!convert && !isinstance>(src)) { - return false; - } - - // Coerce into an array, but don't do type conversion yet; the copy below handles it. - auto buf = array::ensure(src); - - if (!buf) { - return false; - } - - auto dims = buf.ndim(); - if (dims < 1 || dims > 2) { - return false; - } - - auto fits = props::conformable(buf); - if (!fits) { - return false; - } - - // Allocate the new type, then build a numpy reference into it - value = Type(fits.rows, fits.cols); - auto ref = reinterpret_steal(eigen_ref_array(value)); - if (dims == 1) { - ref = ref.squeeze(); - } else if (ref.ndim() == 1) { - buf = buf.squeeze(); - } - - int result = detail::npy_api::get().PyArray_CopyInto_(ref.ptr(), buf.ptr()); - - if (result < 0) { // Copy failed! - PyErr_Clear(); - return false; - } - - return true; - } - -private: - // Cast implementation - template - static handle cast_impl(CType *src, return_value_policy policy, handle parent) { - switch (policy) { - case return_value_policy::take_ownership: - case return_value_policy::automatic: - return eigen_encapsulate(src); - case return_value_policy::move: - return eigen_encapsulate(new CType(std::move(*src))); - case return_value_policy::copy: - return eigen_array_cast(*src); - case return_value_policy::reference: - case return_value_policy::automatic_reference: - return eigen_ref_array(*src); - case return_value_policy::reference_internal: - return eigen_ref_array(*src, parent); - default: - throw cast_error("unhandled return_value_policy: should not happen!"); - }; - } - -public: - // Normal returned non-reference, non-const value: - static handle cast(Type &&src, return_value_policy /* policy */, handle parent) { - return cast_impl(&src, return_value_policy::move, parent); - } - // If you return a non-reference const, we mark the numpy array readonly: - static handle cast(const Type &&src, return_value_policy /* policy */, handle parent) { - return cast_impl(&src, return_value_policy::move, parent); - } - // lvalue reference return; default (automatic) becomes copy - static handle cast(Type &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic - || policy == return_value_policy::automatic_reference) { - policy = return_value_policy::copy; - } - return cast_impl(&src, policy, parent); - } - // const lvalue reference return; default (automatic) becomes copy - static handle cast(const Type &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic - || policy == return_value_policy::automatic_reference) { - policy = return_value_policy::copy; - } - return cast(&src, policy, parent); - } - // non-const pointer return - static handle cast(Type *src, return_value_policy policy, handle parent) { - return cast_impl(src, policy, parent); - } - // const pointer return - static handle cast(const Type *src, return_value_policy policy, handle parent) { - return cast_impl(src, policy, parent); - } - - static constexpr auto name = props::descriptor; - - // NOLINTNEXTLINE(google-explicit-constructor) - operator Type *() { return &value; } - // NOLINTNEXTLINE(google-explicit-constructor) - operator Type &() { return value; } - // NOLINTNEXTLINE(google-explicit-constructor) - operator Type &&() && { return std::move(value); } - template - using cast_op_type = movable_cast_op_type; - -private: - Type value; -}; - -// Base class for casting reference/map/block/etc. objects back to python. -template -struct eigen_map_caster { -private: - using props = EigenProps; - -public: - // Directly referencing a ref/map's data is a bit dangerous (whatever the map/ref points to has - // to stay around), but we'll allow it under the assumption that you know what you're doing - // (and have an appropriate keep_alive in place). We return a numpy array pointing directly at - // the ref's data (The numpy array ends up read-only if the ref was to a const matrix type.) - // Note that this means you need to ensure you don't destroy the object in some other way (e.g. - // with an appropriate keep_alive, or with a reference to a statically allocated matrix). - static handle cast(const MapType &src, return_value_policy policy, handle parent) { - switch (policy) { - case return_value_policy::copy: - return eigen_array_cast(src); - case return_value_policy::reference_internal: - return eigen_array_cast(src, parent, is_eigen_mutable_map::value); - case return_value_policy::reference: - case return_value_policy::automatic: - case return_value_policy::automatic_reference: - return eigen_array_cast(src, none(), is_eigen_mutable_map::value); - default: - // move, take_ownership don't make any sense for a ref/map: - pybind11_fail("Invalid return_value_policy for Eigen Map/Ref/Block type"); - } - } - - static constexpr auto name = props::descriptor; - - // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return - // types but not bound arguments). We still provide them (with an explicitly delete) so that - // you end up here if you try anyway. - bool load(handle, bool) = delete; - operator MapType() = delete; - template - using cast_op_type = MapType; -}; - -// We can return any map-like object (but can only load Refs, specialized next): -template -struct type_caster::value>> : eigen_map_caster {}; - -// Loader for Ref<...> arguments. See the documentation for info on how to make this work without -// copying (it requires some extra effort in many cases). -template -struct type_caster< - Eigen::Ref, - enable_if_t>::value>> - : public eigen_map_caster> { -private: - using Type = Eigen::Ref; - using props = EigenProps; - using Scalar = typename props::Scalar; - using MapType = Eigen::Map; - using Array - = array_t; - static constexpr bool need_writeable = is_eigen_mutable_map::value; - // Delay construction (these have no default constructor) - std::unique_ptr map; - std::unique_ptr ref; - // Our array. When possible, this is just a numpy array pointing to the source data, but - // sometimes we can't avoid copying (e.g. input is not a numpy array at all, has an - // incompatible layout, or is an array of a type that needs to be converted). Using a numpy - // temporary (rather than an Eigen temporary) saves an extra copy when we need both type - // conversion and storage order conversion. (Note that we refuse to use this temporary copy - // when loading an argument for a Ref with M non-const, i.e. a read-write reference). - Array copy_or_ref; - -public: - bool load(handle src, bool convert) { - // First check whether what we have is already an array of the right type. If not, we - // can't avoid a copy (because the copy is also going to do type conversion). - bool need_copy = !isinstance(src); - - EigenConformable fits; - if (!need_copy) { - // We don't need a converting copy, but we also need to check whether the strides are - // compatible with the Ref's stride requirements - auto aref = reinterpret_borrow(src); - - if (aref && (!need_writeable || aref.writeable())) { - fits = props::conformable(aref); - if (!fits) { - return false; // Incompatible dimensions - } - if (!fits.template stride_compatible()) { - need_copy = true; - } else { - copy_or_ref = std::move(aref); - } - } else { - need_copy = true; - } - } - - if (need_copy) { - // We need to copy: If we need a mutable reference, or we're not supposed to convert - // (either because we're in the no-convert overload pass, or because we're explicitly - // instructed not to copy (via `py::arg().noconvert()`) we have to fail loading. - if (!convert || need_writeable) { - return false; - } - - Array copy = Array::ensure(src); - if (!copy) { - return false; - } - fits = props::conformable(copy); - if (!fits || !fits.template stride_compatible()) { - return false; - } - copy_or_ref = std::move(copy); - loader_life_support::add_patient(copy_or_ref); - } - - ref.reset(); - map.reset(new MapType(data(copy_or_ref), - fits.rows, - fits.cols, - make_stride(fits.stride.outer(), fits.stride.inner()))); - ref.reset(new Type(*map)); - - return true; - } - - // NOLINTNEXTLINE(google-explicit-constructor) - operator Type *() { return ref.get(); } - // NOLINTNEXTLINE(google-explicit-constructor) - operator Type &() { return *ref; } - template - using cast_op_type = pybind11::detail::cast_op_type<_T>; - -private: - template ::value, int> = 0> - Scalar *data(Array &a) { - return a.mutable_data(); - } - - template ::value, int> = 0> - const Scalar *data(Array &a) { - return a.data(); - } - - // Attempt to figure out a constructor of `Stride` that will work. - // If both strides are fixed, use a default constructor: - template - using stride_ctor_default = bool_constant::value>; - // Otherwise, if there is a two-index constructor, assume it is (outer,inner) like - // Eigen::Stride, and use it: - template - using stride_ctor_dual - = bool_constant::value - && std::is_constructible::value>; - // Otherwise, if there is a one-index constructor, and just one of the strides is dynamic, use - // it (passing whichever stride is dynamic). - template - using stride_ctor_outer - = bool_constant, stride_ctor_dual>::value - && S::OuterStrideAtCompileTime == Eigen::Dynamic - && S::InnerStrideAtCompileTime != Eigen::Dynamic - && std::is_constructible::value>; - template - using stride_ctor_inner - = bool_constant, stride_ctor_dual>::value - && S::InnerStrideAtCompileTime == Eigen::Dynamic - && S::OuterStrideAtCompileTime != Eigen::Dynamic - && std::is_constructible::value>; - - template ::value, int> = 0> - static S make_stride(EigenIndex, EigenIndex) { - return S(); - } - template ::value, int> = 0> - static S make_stride(EigenIndex outer, EigenIndex inner) { - return S(outer, inner); - } - template ::value, int> = 0> - static S make_stride(EigenIndex outer, EigenIndex) { - return S(outer); - } - template ::value, int> = 0> - static S make_stride(EigenIndex, EigenIndex inner) { - return S(inner); - } -}; - -// type_caster for special matrix types (e.g. DiagonalMatrix), which are EigenBase, but not -// EigenDense (i.e. they don't have a data(), at least not with the usual matrix layout). -// load() is not supported, but we can cast them into the python domain by first copying to a -// regular Eigen::Matrix, then casting that. -template -struct type_caster::value>> { -protected: - using Matrix - = Eigen::Matrix; - using props = EigenProps; - -public: - static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { - handle h = eigen_encapsulate(new Matrix(src)); - return h; - } - static handle cast(const Type *src, return_value_policy policy, handle parent) { - return cast(*src, policy, parent); - } - - static constexpr auto name = props::descriptor; - - // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return - // types but not bound arguments). We still provide them (with an explicitly delete) so that - // you end up here if you try anyway. - bool load(handle, bool) = delete; - operator Type() = delete; - template - using cast_op_type = Type; -}; - -template -constexpr int compute_array_flag_from_tensor() { - static_assert(((int) T::Layout == (int) Eigen::RowMajor) - || ((int) T::Layout == (int) Eigen::ColMajor), - "Layout must be row or column major"); - return ((int) T::Layout == (int) Eigen::RowMajor) ? array::c_style : array::f_style; -} - -template -struct eigen_tensor_helper {}; - -template -struct eigen_tensor_helper> { - using T = Eigen::Tensor; - using ValidType = void; - - static std::array get_shape(const T &f) { - return f.dimensions(); - } - - static constexpr bool - is_correct_shape(const std::array & /*shape*/) { - return true; - } - - template - struct helper {}; - - template - struct helper> { - static constexpr auto value = concat(const_name(((void) Is, "?"))...); - }; - - static constexpr auto dimensions_descriptor - = helper())>::value; -}; - -template -struct eigen_tensor_helper< - Eigen::TensorFixedSize, Options_, IndexType>> { - using T = Eigen::TensorFixedSize, Options_, IndexType>; - using ValidType = void; - - static constexpr std::array get_shape(const T & /*f*/) { - return get_shape(); - } - - static constexpr std::array get_shape() { - return {{Indices...}}; - } - - static bool is_correct_shape(const std::array &shape) { - return get_shape() == shape; - } - - static constexpr auto dimensions_descriptor = concat(const_name()...); -}; - -template -struct get_tensor_descriptor { - static constexpr auto value - = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") + eigen_tensor_helper::dimensions_descriptor - + const_name("], flags.writeable, ") - + const_name<(int) T::Layout == (int) Eigen::RowMajor>("flags.c_contiguous", - "flags.f_contiguous"); -}; - -template -struct type_caster::ValidType> { - using H = eigen_tensor_helper; - PYBIND11_TYPE_CASTER(Type, get_tensor_descriptor::value); - - bool load(handle src, bool /*convert*/) { - array_t()> a( - reinterpret_borrow(src)); - - if (a.ndim() != Type::NumIndices) { - return false; - } - - std::array shape; - std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); - - if (!H::is_correct_shape(shape)) { - return false; - } - - value = Eigen::TensorMap(const_cast(a.data()), shape); - - return true; - } - - static handle cast(Type &&src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::reference - || policy == return_value_policy::reference_internal) { - pybind11_fail("Cannot use a reference return value policy for an rvalue"); - } - return cast_impl(&src, return_value_policy::move, parent); - } - - static handle cast(const Type &&src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::reference - || policy == return_value_policy::reference_internal) { - pybind11_fail("Cannot use a reference return value policy for an rvalue"); - } - return cast_impl(&src, return_value_policy::move, parent); - } - - static handle cast(Type &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic - || policy == return_value_policy::automatic_reference) { - policy = return_value_policy::copy; - } - return cast_impl(&src, policy, parent); - } - - static handle cast(const Type &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic - || policy == return_value_policy::automatic_reference) { - policy = return_value_policy::copy; - } - return cast(&src, policy, parent); - } - - static handle cast(Type *src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic) { - policy = return_value_policy::take_ownership; - } else if (policy == return_value_policy::automatic_reference) { - policy = return_value_policy::reference; - } - return cast_impl(src, policy, parent); - } - - static handle cast(const Type *src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic) { - policy = return_value_policy::take_ownership; - } else if (policy == return_value_policy::automatic_reference) { - policy = return_value_policy::reference; - } - return cast_impl(src, policy, parent); - } - - template - static handle cast_impl(C *src, return_value_policy policy, handle parent) { - object parent_object; - bool writeable = false; - switch (policy) { - case return_value_policy::move: - if (std::is_const::value) { - pybind11_fail("Cannot move from a constant reference"); - } - { - Eigen::aligned_allocator allocator; - Type *copy = ::new (allocator.allocate(1)) Type(std::move(*src)); - src = copy; - } - - parent_object = capsule(src, [](void *ptr) { - Eigen::aligned_allocator allocator; - Type *copy = (Type *) ptr; - copy->~Type(); - allocator.deallocate(copy, 1); - }); - writeable = true; - break; - - case return_value_policy::take_ownership: - if (std::is_const::value) { - pybind11_fail("Cannot take ownership of a const reference"); - } - parent_object = capsule(src, [](void *ptr) { delete (Type *) ptr; }); - writeable = true; - break; - - case return_value_policy::copy: - parent_object = {}; - writeable = true; - break; - - case return_value_policy::reference: - parent_object = none(); - writeable = !std::is_const::value; - break; - - case return_value_policy::reference_internal: - // Default should do the right thing - parent_object = reinterpret_borrow(parent); - writeable = !std::is_const::value; - break; - - default: - pybind11_fail("pybind11 bug in eigen.h, please file a bug report"); - } - - handle result = array_t()>( - H::get_shape(*src), src->data(), parent_object) - .release(); - - if (!writeable) { - array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; - } - - return result; - } -}; - -template -struct type_caster, typename eigen_tensor_helper::ValidType> { - using H = eigen_tensor_helper; - - bool load(handle src, bool /*convert*/) { - // Note that we have a lot more checks here as we want to make sure to avoid copies - auto a = reinterpret_borrow(src); - if ((a.flags() & compute_array_flag_from_tensor()) == 0) { - return false; - } - - if (!a.dtype().is(dtype::of())) { - return false; - } - - if (a.ndim() != Type::NumIndices) { - return false; - } - - std::array shape; - std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); - - if (!H::is_correct_shape(shape)) { - return false; - } - - value.reset(new Eigen::TensorMap( - reinterpret_cast(a.mutable_data()), shape)); - - return true; - } - - static handle cast(Eigen::TensorMap &&src, return_value_policy policy, handle parent) { - return cast_impl(&src, policy, parent); - } - - static handle - cast(const Eigen::TensorMap &&src, return_value_policy policy, handle parent) { - return cast_impl(&src, policy, parent); - } - - static handle cast(Eigen::TensorMap &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic - || policy == return_value_policy::automatic_reference) { - policy = return_value_policy::copy; - } - return cast_impl(&src, policy, parent); - } - - static handle - cast(const Eigen::TensorMap &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic - || policy == return_value_policy::automatic_reference) { - policy = return_value_policy::copy; - } - return cast(&src, policy, parent); - } - - static handle cast(Eigen::TensorMap *src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic) { - policy = return_value_policy::take_ownership; - } else if (policy == return_value_policy::automatic_reference) { - policy = return_value_policy::reference; - } - return cast_impl(src, policy, parent); - } - - static handle - cast(const Eigen::TensorMap *src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic) { - policy = return_value_policy::take_ownership; - } else if (policy == return_value_policy::automatic_reference) { - policy = return_value_policy::reference; - } - return cast_impl(src, policy, parent); - } - - template - static handle cast_impl(C *src, return_value_policy policy, handle parent) { - object parent_object; - constexpr bool writeable = !std::is_const::value; - switch (policy) { - case return_value_policy::reference: - parent_object = none(); - break; - - case return_value_policy::reference_internal: - // Default should do the right thing - parent_object = reinterpret_borrow(parent); - break; - - default: - // move, take_ownership don't make any sense for a ref/map: - pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either " - "reference or reference_internal"); - } - - handle result = array_t()>( - H::get_shape(*src), src->data(), parent_object) - .release(); - - if (!writeable) { - array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; - } - - return result; - } - -protected: - // TODO: Move to std::optional once std::optional has more support - std::unique_ptr> value; - -public: - static constexpr auto name = get_tensor_descriptor::value; - explicit operator Eigen::TensorMap *() { - return value.get(); - } /* NOLINT(bugprone-macro-parentheses) */ - explicit operator Eigen::TensorMap &() { - return *value; - } /* NOLINT(bugprone-macro-parentheses) */ - explicit operator Eigen::TensorMap &&() && { - return std::move(*value); - } /* NOLINT(bugprone-macro-parentheses) */ - - template - using cast_op_type = ::pybind11::detail::movable_cast_op_type; -}; - -template -struct type_caster::value>> { - using Scalar = typename Type::Scalar; - using StorageIndex = remove_reference_t().outerIndexPtr())>; - using Index = typename Type::Index; - static constexpr bool rowMajor = Type::IsRowMajor; - - bool load(handle src, bool) { - if (!src) { - return false; - } - - auto obj = reinterpret_borrow(src); - object sparse_module = module_::import("scipy.sparse"); - object matrix_type = sparse_module.attr(rowMajor ? "csr_matrix" : "csc_matrix"); - - if (!type::handle_of(obj).is(matrix_type)) { - try { - obj = matrix_type(obj); - } catch (const error_already_set &) { - return false; - } - } - - auto values = array_t((object) obj.attr("data")); - auto innerIndices = array_t((object) obj.attr("indices")); - auto outerIndices = array_t((object) obj.attr("indptr")); - auto shape = pybind11::tuple((pybind11::object) obj.attr("shape")); - auto nnz = obj.attr("nnz").cast(); - - if (!values || !innerIndices || !outerIndices) { - return false; - } - - value = EigenMapSparseMatrix(shape[0].cast(), - shape[1].cast(), - std::move(nnz), - outerIndices.mutable_data(), - innerIndices.mutable_data(), - values.mutable_data()); - - return true; - } - - static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { - const_cast(src).makeCompressed(); - - object matrix_type - = module_::import("scipy.sparse").attr(rowMajor ? "csr_matrix" : "csc_matrix"); - - array data(src.nonZeros(), src.valuePtr()); - array outerIndices((rowMajor ? src.rows() : src.cols()) + 1, src.outerIndexPtr()); - array innerIndices(src.nonZeros(), src.innerIndexPtr()); - - return matrix_type(pybind11::make_tuple( - std::move(data), std::move(innerIndices), std::move(outerIndices)), - pybind11::make_tuple(src.rows(), src.cols())) - .release(); - } - - PYBIND11_TYPE_CASTER(Type, - const_name<(Type::IsRowMajor) != 0>("scipy.sparse.csr_matrix[", - "scipy.sparse.csc_matrix[") - + npy_format_descriptor::name + const_name("]")); -}; - -PYBIND11_NAMESPACE_END(detail) -PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) + "Eigen support in pybind11 requires Eigen >= 3.2.7"); \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7296cd1b81..9adac392e0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -128,7 +128,8 @@ set(PYBIND11_TEST_FILES test_custom_type_casters test_custom_type_setup test_docstring_options - test_eigen + test_eigen_tensor + test_eigen_matrix test_enum test_eval test_exceptions @@ -233,7 +234,10 @@ list(GET PYBIND11_EIGEN_VERSION_AND_HASH 1 PYBIND11_EIGEN_VERSION_HASH) # Check if Eigen is available; if not, remove from PYBIND11_TEST_FILES (but # keep it in PYBIND11_PYTEST_FILES, so that we get the "eigen is not installed" # skip message). -list(FIND PYBIND11_TEST_FILES test_eigen.cpp PYBIND11_TEST_FILES_EIGEN_I) +list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) +if(PYBIND11_TEST_FILES_EIGEN_I EQUAL -1) + list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) +endif() if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) # Try loading via newer Eigen's Eigen3Config first (bypassing tools/FindEigen3.cmake). # Eigen 3.3.1+ exports a cmake 3.0+ target for handling dependency requirements, but also diff --git a/tests/test_eigen.cpp b/tests/test_eigen_matrix.cpp similarity index 89% rename from tests/test_eigen.cpp rename to tests/test_eigen_matrix.cpp index 3b71f1e2ab..b98eee492e 100644 --- a/tests/test_eigen.cpp +++ b/tests/test_eigen_matrix.cpp @@ -32,13 +32,6 @@ void reset_ref(M &x) { } } -template -void reset_tensor(M &x) { - for (int i = 0; i < x.size(); i++) { - x(i) = i; - } -} - // Returns a static, column-major matrix Eigen::MatrixXd &get_cm() { static Eigen::MatrixXd *x; @@ -57,32 +50,6 @@ MatrixXdR &get_rm() { } return *x; } - -Eigen::Tensor &get_tensor() { - static Eigen::Tensor *x; - - if (!x) { - x = new Eigen::Tensor(3, 1, 2); - reset_tensor(*x); - } - - return *x; -} - -Eigen::TensorFixedSize> &get_fixed_tensor() { - static Eigen::TensorFixedSize> *x; - - if (!x) { - Eigen::aligned_allocator>> allocator; - x = new (allocator.allocate(1)) Eigen::TensorFixedSize>(); - reset_tensor(*x); - } - - return *x; -} - -const Eigen::Tensor &get_const_tensor() { return get_tensor(); } - // Resets the values of the static matrices returned by get_cm()/get_rm() void reset_refs() { reset_ref(get_cm()); @@ -114,7 +81,7 @@ struct CustomOperatorNew { EIGEN_MAKE_ALIGNED_OPERATOR_NEW; }; -TEST_SUBMODULE(eigen, m) { +TEST_SUBMODULE(eigen_matrix, m) { using FixedMatrixR = Eigen::Matrix; using FixedMatrixC = Eigen::Matrix; using DenseMatrixR = Eigen::Matrix; @@ -460,46 +427,4 @@ TEST_SUBMODULE(eigen, m) { py::module_::import("numpy").attr("ones")(10); return v[0](5); }); - - m.def("copy_fixed_global_tensor", []() { return get_fixed_tensor(); }); - - m.def("copy_global_tensor", []() { return get_tensor(); }); - - m.def("copy_const_global_tensor", []() { return get_const_tensor(); }); - - m.def( - "reference_global_tensor", - []() { return &get_tensor(); }, - py::return_value_policy::reference); - - m.def( - "reference_const_global_tensor", - []() { return &get_const_tensor(); }, - py::return_value_policy::reference); - - m.def( - "reference_view_of_global_tensor", - []() { return Eigen::TensorMap>(get_tensor()); }, - py::return_value_policy::reference); - - m.def( - "reference_view_of_fixed_global_tensor", - []() { - return Eigen::TensorMap>>( - get_fixed_tensor()); - }, - py::return_value_policy::reference); - - m.def("round_trip_tensor", [](const py::array_t &foo) { - auto blah = foo.cast>(); - return blah; - }); - - m.def( - "round_trip_view_tensor", - [](const py::array_t &foo) { - auto view = foo.cast>>(); - return view; - }, - py::return_value_policy::reference); } diff --git a/tests/test_eigen.py b/tests/test_eigen_matrix.py similarity index 94% rename from tests/test_eigen.py rename to tests/test_eigen_matrix.py index 4bdd5f8f27..2da5a6ff52 100644 --- a/tests/test_eigen.py +++ b/tests/test_eigen_matrix.py @@ -3,7 +3,7 @@ from pybind11_tests import ConstructorStats np = pytest.importorskip("numpy") -m = pytest.importorskip("pybind11_tests.eigen") +m = pytest.importorskip("pybind11_tests.eigen_matrix") ref = np.array( @@ -781,63 +781,4 @@ def test_custom_operator_new(): o = m.CustomOperatorNew() np.testing.assert_allclose(o.a, 0.0) - np.testing.assert_allclose(o.b.diagonal(), 1.0) - - -tensor_ref = np.array( - [ - [[0, 3]], - [[1, 4]], - [[2, 5]], - ] -) - - -def assert_equal_tensor_ref(mat, writeable=True, modified=0): - assert mat.flags.writeable == writeable - - if modified != 0: - tensor_ref[0, 0, 0] = modified - - np.testing.assert_array_equal(mat, tensor_ref) - - tensor_ref[0, 0, 0] = 0 - - -def test_convert_tensor_to_py(): - assert_equal_tensor_ref(m.copy_global_tensor()) - assert_equal_tensor_ref(m.copy_fixed_global_tensor()) - assert_equal_tensor_ref(m.copy_const_global_tensor()) - - assert_equal_tensor_ref(m.reference_global_tensor()) - assert_equal_tensor_ref(m.reference_view_of_global_tensor()) - assert_equal_tensor_ref(m.reference_view_of_fixed_global_tensor()) - assert_equal_tensor_ref(m.reference_const_global_tensor(), writeable=False) - - -def test_references_actually_refer(): - a = m.reference_global_tensor() - a[0, 0, 0] = 100 - assert_equal_tensor_ref(m.copy_const_global_tensor(), modified=100) - a[0, 0, 0] = 0 - assert_equal_tensor_ref(m.copy_const_global_tensor()) - - a = m.reference_view_of_global_tensor() - a[0, 0, 0] = 100 - assert_equal_tensor_ref(m.copy_const_global_tensor(), modified=100) - a[0, 0, 0] = 0 - assert_equal_tensor_ref(m.copy_const_global_tensor()) - - -def test_round_trip(): - assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) - - -def test_round_trip_references_actually_refer(): - # Need to create a copy that matches the type on the C side - copy = np.array(tensor_ref, dtype=np.float64, order="F") - a = m.round_trip_view_tensor(copy) - a[0, 0, 0] = 100 - assert_equal_tensor_ref(copy, modified=100) - a[0, 0, 0] = 0 - assert_equal_tensor_ref(copy) + np.testing.assert_allclose(o.b.diagonal(), 1.0) \ No newline at end of file diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp new file mode 100644 index 0000000000..c3f174c3ad --- /dev/null +++ b/tests/test_eigen_tensor.cpp @@ -0,0 +1,88 @@ +/* + tests/eigen_tensor.cpp -- automatic conversion of Eigen Tensor + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include + +#include "pybind11_tests.h" + +template +void reset_tensor(M &x) { + for (int i = 0; i < x.size(); i++) { + x(i) = i; + } +} + +Eigen::Tensor &get_tensor() { + static Eigen::Tensor *x; + + if (!x) { + x = new Eigen::Tensor(3, 1, 2); + reset_tensor(*x); + } + + return *x; +} + +Eigen::TensorFixedSize> &get_fixed_tensor() { + static Eigen::TensorFixedSize> *x; + + if (!x) { + Eigen::aligned_allocator>> allocator; + x = new (allocator.allocate(1)) Eigen::TensorFixedSize>(); + reset_tensor(*x); + } + + return *x; +} + +const Eigen::Tensor &get_const_tensor() { return get_tensor(); } + +TEST_SUBMODULE(eigen_tensor, m) { + m.def("copy_fixed_global_tensor", []() { return get_fixed_tensor(); }); + + m.def("copy_global_tensor", []() { return get_tensor(); }); + + m.def("copy_const_global_tensor", []() { return get_const_tensor(); }); + + m.def( + "reference_global_tensor", + []() { return &get_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_const_global_tensor", + []() { return &get_const_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_global_tensor", + []() { return Eigen::TensorMap>(get_tensor()); }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_fixed_global_tensor", + []() { + return Eigen::TensorMap>>( + get_fixed_tensor()); + }, + py::return_value_policy::reference); + + m.def("round_trip_tensor", [](const py::array_t &foo) { + auto blah = foo.cast>(); + return blah; + }); + + m.def( + "round_trip_view_tensor", + [](const py::array_t &foo) { + auto view = foo.cast>>(); + return view; + }, + py::return_value_policy::reference); +} diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py new file mode 100644 index 0000000000..932104b3d7 --- /dev/null +++ b/tests/test_eigen_tensor.py @@ -0,0 +1,64 @@ +import pytest + +from pybind11_tests import ConstructorStats + +np = pytest.importorskip("numpy") +m = pytest.importorskip("pybind11_tests.eigen_tensor") + +tensor_ref = np.array( + [ + [[0, 3]], + [[1, 4]], + [[2, 5]], + ] +) + + +def assert_equal_tensor_ref(mat, writeable=True, modified=0): + assert mat.flags.writeable == writeable + + if modified != 0: + tensor_ref[0, 0, 0] = modified + + np.testing.assert_array_equal(mat, tensor_ref) + + tensor_ref[0, 0, 0] = 0 + + +def test_convert_tensor_to_py(): + assert_equal_tensor_ref(m.copy_global_tensor()) + assert_equal_tensor_ref(m.copy_fixed_global_tensor()) + assert_equal_tensor_ref(m.copy_const_global_tensor()) + + assert_equal_tensor_ref(m.reference_global_tensor()) + assert_equal_tensor_ref(m.reference_view_of_global_tensor()) + assert_equal_tensor_ref(m.reference_view_of_fixed_global_tensor()) + assert_equal_tensor_ref(m.reference_const_global_tensor(), writeable=False) + + +def test_references_actually_refer(): + a = m.reference_global_tensor() + a[0, 0, 0] = 100 + assert_equal_tensor_ref(m.copy_const_global_tensor(), modified=100) + a[0, 0, 0] = 0 + assert_equal_tensor_ref(m.copy_const_global_tensor()) + + a = m.reference_view_of_global_tensor() + a[0, 0, 0] = 100 + assert_equal_tensor_ref(m.copy_const_global_tensor(), modified=100) + a[0, 0, 0] = 0 + assert_equal_tensor_ref(m.copy_const_global_tensor()) + + +def test_round_trip(): + assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) + + +def test_round_trip_references_actually_refer(): + # Need to create a copy that matches the type on the C side + copy = np.array(tensor_ref, dtype=np.float64, order="F") + a = m.round_trip_view_tensor(copy) + a[0, 0, 0] = 100 + assert_equal_tensor_ref(copy, modified=100) + a[0, 0, 0] = 0 + assert_equal_tensor_ref(copy) From 11aac3a568a69115ff7a0180387a4d60a9db7e05 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Sep 2022 20:47:48 +0000 Subject: [PATCH 019/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/detail/eigen_matrix.h | 2 +- include/pybind11/detail/eigen_tensor.h | 2 +- include/pybind11/eigen.h | 2 +- tests/test_eigen_matrix.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/eigen_matrix.h b/include/pybind11/detail/eigen_matrix.h index c4445ccb52..274604f7c4 100644 --- a/include/pybind11/detail/eigen_matrix.h +++ b/include/pybind11/detail/eigen_matrix.h @@ -705,4 +705,4 @@ struct type_caster::value>> { }; PYBIND11_NAMESPACE_END(detail) -PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) \ No newline at end of file +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index c3042101ce..c021d24584 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -380,4 +380,4 @@ struct type_caster, typename eigen_tensor_helper::V }; PYBIND11_NAMESPACE_END(detail) -PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) \ No newline at end of file +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 69fb528f99..60377b1075 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -16,4 +16,4 @@ // move constructors that break things. We could detect this an explicitly copy, but an extra copy // of matrices seems highly undesirable. static_assert(EIGEN_VERSION_AT_LEAST(3, 2, 7), - "Eigen support in pybind11 requires Eigen >= 3.2.7"); \ No newline at end of file + "Eigen support in pybind11 requires Eigen >= 3.2.7"); diff --git a/tests/test_eigen_matrix.py b/tests/test_eigen_matrix.py index 2da5a6ff52..4407fa6aee 100644 --- a/tests/test_eigen_matrix.py +++ b/tests/test_eigen_matrix.py @@ -781,4 +781,4 @@ def test_custom_operator_new(): o = m.CustomOperatorNew() np.testing.assert_allclose(o.a, 0.0) - np.testing.assert_allclose(o.b.diagonal(), 1.0) \ No newline at end of file + np.testing.assert_allclose(o.b.diagonal(), 1.0) From fb53ffd46bf0e1d47ead757d1e6c8ef80a164e79 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 28 Sep 2022 20:52:23 +0000 Subject: [PATCH 020/141] Fix weird tests --- tests/CMakeLists.txt | 10 +++++++++- tests/extra_python_package/test_files.py | 2 ++ tests/test_eigen_tensor.py | 2 -- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9adac392e0..403f9a0aba 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -293,7 +293,15 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) endif() message(STATUS "Building tests with Eigen v${EIGEN3_VERSION}") else() - list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() + + list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() message( STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN=ON on CMake 3.11+ to download") endif() diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 8e1ddd8508..e8bac488af 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -49,6 +49,8 @@ "include/pybind11/detail/class.h", "include/pybind11/detail/common.h", "include/pybind11/detail/descr.h", + "include/pybind11/detail/eigen_tensor.h", + "include/pybind11/detail/eigen_matrix.h", "include/pybind11/detail/init.h", "include/pybind11/detail/internals.h", "include/pybind11/detail/type_caster_base.h", diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 932104b3d7..e278ffd009 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -1,7 +1,5 @@ import pytest -from pybind11_tests import ConstructorStats - np = pytest.importorskip("numpy") m = pytest.importorskip("pybind11_tests.eigen_tensor") From 03a0595b70b225ec63f1dfb42ea47f2eef047ef2 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 30 Sep 2022 17:14:31 +0000 Subject: [PATCH 021/141] Fix nits --- include/pybind11/detail/eigen_tensor.h | 37 +++++++++++++++----------- tests/test_eigen_tensor.cpp | 8 +++--- tests/test_eigen_tensor.py | 1 + 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index c021d24584..71c19a5710 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -26,7 +26,7 @@ #if defined(_MSC_VER) # pragma warning(push) # pragma warning(disable : 4554) // Tensor.h warning -// C5054: operator '&': deprecated between enumerations of different types +# pragma warning(disable : 4127) // Tensor.h warning #elif defined(__MINGW32__) # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wmaybe-uninitialized" @@ -46,10 +46,10 @@ PYBIND11_NAMESPACE_BEGIN(detail) template constexpr int compute_array_flag_from_tensor() { - static_assert(((int) T::Layout == (int) Eigen::RowMajor) - || ((int) T::Layout == (int) Eigen::ColMajor), + static_assert((static_cast(T::Layout) == static_cast(Eigen::RowMajor)) + || (static_cast(T::Layout) == static_cast(Eigen::ColMajor)), "Layout must be row or column major"); - return ((int) T::Layout == (int) Eigen::RowMajor) ? array::c_style : array::f_style; + return (static_cast(T::Layout) == static_cast(Eigen::RowMajor)) ? array::c_style : array::f_style; } template @@ -74,7 +74,8 @@ struct eigen_tensor_helper struct helper> { - static constexpr auto value = concat(const_name(((void) Is, "?"))...); + // Hack to work around gcc 4.8 bugs. Feel free to remove when we drop gcc 4.8 support. + static constexpr descr value = concat(const_name(((void) Is, "?"))...); }; static constexpr auto dimensions_descriptor @@ -231,6 +232,9 @@ struct type_caster::ValidType> { case return_value_policy::reference_internal: // Default should do the right thing + if (!parent) { + pybind11_fail("Cannot use reference internal when there is no parent"); + } parent_object = reinterpret_borrow(parent); writeable = !std::is_const::value; break; @@ -239,15 +243,14 @@ struct type_caster::ValidType> { pybind11_fail("pybind11 bug in eigen.h, please file a bug report"); } - handle result = array_t()>( - H::get_shape(*src), src->data(), parent_object) - .release(); + object result = array_t()>( + H::get_shape(*src), src->data(), parent_object); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; } - return result; + return result.release(); } }; @@ -339,6 +342,9 @@ struct type_caster, typename eigen_tensor_helper::V case return_value_policy::reference_internal: // Default should do the right thing + if (!parent) { + pybind11_fail("Cannot use reference internal when there is no parent"); + } parent_object = reinterpret_borrow(parent); break; @@ -348,15 +354,14 @@ struct type_caster, typename eigen_tensor_helper::V "reference or reference_internal"); } - handle result = array_t()>( - H::get_shape(*src), src->data(), parent_object) - .release(); + object result = array_t()>( + H::get_shape(*src), src->data(), parent_object); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; } - return result; + return result.release(); } protected: @@ -367,13 +372,13 @@ struct type_caster, typename eigen_tensor_helper::V static constexpr auto name = get_tensor_descriptor::value; explicit operator Eigen::TensorMap *() { return value.get(); - } /* NOLINT(bugprone-macro-parentheses) */ + } explicit operator Eigen::TensorMap &() { return *value; - } /* NOLINT(bugprone-macro-parentheses) */ + } explicit operator Eigen::TensorMap &&() && { return std::move(*value); - } /* NOLINT(bugprone-macro-parentheses) */ + } template using cast_op_type = ::pybind11::detail::movable_cast_op_type; diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index c3f174c3ad..16ee8fa07a 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -73,15 +73,13 @@ TEST_SUBMODULE(eigen_tensor, m) { }, py::return_value_policy::reference); - m.def("round_trip_tensor", [](const py::array_t &foo) { - auto blah = foo.cast>(); - return blah; + m.def("round_trip_tensor", [](const Eigen::Tensor &tensor) { + return tensor; }); m.def( "round_trip_view_tensor", - [](const py::array_t &foo) { - auto view = foo.cast>>(); + [](Eigen::TensorMap> view) { return view; }, py::return_value_policy::reference); diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index e278ffd009..178df4f1e4 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -1,4 +1,5 @@ import pytest +from inspect import signature np = pytest.importorskip("numpy") m = pytest.importorskip("pybind11_tests.eigen_tensor") From 0b0fa220934077504d5734cb54d628068530c6d2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Sep 2022 17:21:06 +0000 Subject: [PATCH 022/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/detail/eigen_tensor.h | 22 +++++++++------------- tests/test_eigen_tensor.cpp | 8 ++------ tests/test_eigen_tensor.py | 3 ++- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 71c19a5710..ac4f49e011 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -49,7 +49,8 @@ constexpr int compute_array_flag_from_tensor() { static_assert((static_cast(T::Layout) == static_cast(Eigen::RowMajor)) || (static_cast(T::Layout) == static_cast(Eigen::ColMajor)), "Layout must be row or column major"); - return (static_cast(T::Layout) == static_cast(Eigen::RowMajor)) ? array::c_style : array::f_style; + return (static_cast(T::Layout) == static_cast(Eigen::RowMajor)) ? array::c_style + : array::f_style; } template @@ -75,7 +76,8 @@ struct eigen_tensor_helper struct helper> { // Hack to work around gcc 4.8 bugs. Feel free to remove when we drop gcc 4.8 support. - static constexpr descr value = concat(const_name(((void) Is, "?"))...); + static constexpr descr value + = concat(const_name(((void) Is, "?"))...); }; static constexpr auto dimensions_descriptor @@ -244,7 +246,7 @@ struct type_caster::ValidType> { } object result = array_t()>( - H::get_shape(*src), src->data(), parent_object); + H::get_shape(*src), src->data(), parent_object); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; @@ -355,7 +357,7 @@ struct type_caster, typename eigen_tensor_helper::V } object result = array_t()>( - H::get_shape(*src), src->data(), parent_object); + H::get_shape(*src), src->data(), parent_object); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; @@ -370,15 +372,9 @@ struct type_caster, typename eigen_tensor_helper::V public: static constexpr auto name = get_tensor_descriptor::value; - explicit operator Eigen::TensorMap *() { - return value.get(); - } - explicit operator Eigen::TensorMap &() { - return *value; - } - explicit operator Eigen::TensorMap &&() && { - return std::move(*value); - } + explicit operator Eigen::TensorMap *() { return value.get(); } + explicit operator Eigen::TensorMap &() { return *value; } + explicit operator Eigen::TensorMap &&() && { return std::move(*value); } template using cast_op_type = ::pybind11::detail::movable_cast_op_type; diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 16ee8fa07a..92f19c14a1 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -73,14 +73,10 @@ TEST_SUBMODULE(eigen_tensor, m) { }, py::return_value_policy::reference); - m.def("round_trip_tensor", [](const Eigen::Tensor &tensor) { - return tensor; - }); + m.def("round_trip_tensor", [](const Eigen::Tensor &tensor) { return tensor; }); m.def( "round_trip_view_tensor", - [](Eigen::TensorMap> view) { - return view; - }, + [](Eigen::TensorMap> view) { return view; }, py::return_value_policy::reference); } diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 178df4f1e4..54ade5d959 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -1,6 +1,7 @@ -import pytest from inspect import signature +import pytest + np = pytest.importorskip("numpy") m = pytest.importorskip("pybind11_tests.eigen_tensor") From 5a7b9c5d7539d93fd2488eb21c25dc42b2009afd Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 30 Sep 2022 17:19:14 +0000 Subject: [PATCH 023/141] Oops, forgot import --- tests/test_eigen_tensor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 54ade5d959..e278ffd009 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -1,5 +1,3 @@ -from inspect import signature - import pytest np = pytest.importorskip("numpy") From 27377d9f2d9336165fc42c62fd296b262c73ad84 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 30 Sep 2022 22:29:31 +0000 Subject: [PATCH 024/141] Fix clang 3.6 bug --- include/pybind11/detail/eigen_tensor.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index ac4f49e011..613ed04d6c 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -223,7 +223,10 @@ struct type_caster::ValidType> { break; case return_value_policy::copy: - parent_object = {}; + { + // Clang 3.6 / 3.7 has a very confusing bug that seems to be fixed by adding this scope + parent_object = {}; + } writeable = true; break; From a4a96f336f3d1d7dc729c9be5d82ecf234718508 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Sep 2022 22:35:20 +0000 Subject: [PATCH 025/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/detail/eigen_tensor.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 613ed04d6c..3e47109d8a 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -222,11 +222,11 @@ struct type_caster::ValidType> { writeable = true; break; - case return_value_policy::copy: - { - // Clang 3.6 / 3.7 has a very confusing bug that seems to be fixed by adding this scope - parent_object = {}; - } + case return_value_policy::copy: { + // Clang 3.6 / 3.7 has a very confusing bug that seems to be fixed by adding this + // scope + parent_object = {}; + } writeable = true; break; From 06a816c08d6070d8db259e6d3f7eab0ad90d6f0c Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 30 Sep 2022 23:38:31 +0000 Subject: [PATCH 026/141] More comprehensive test suite --- tests/test_eigen_tensor.cpp | 37 ++++++++++++++++++++++++++++++------- tests/test_eigen_tensor.py | 34 +++++++++++++++++++++------------- 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 92f19c14a1..1b57d3c45e 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -44,29 +44,52 @@ Eigen::TensorFixedSize> &get_fixed_tensor() { const Eigen::Tensor &get_const_tensor() { return get_tensor(); } TEST_SUBMODULE(eigen_tensor, m) { - m.def("copy_fixed_global_tensor", []() { return get_fixed_tensor(); }); + m.def("copy_fixed_tensor", []() { return &get_fixed_tensor(); }, + py::return_value_policy::copy + ); - m.def("copy_global_tensor", []() { return get_tensor(); }); + m.def("copy_tensor", []() { return &get_tensor(); }, + py::return_value_policy::copy + ); - m.def("copy_const_global_tensor", []() { return get_const_tensor(); }); + m.def("copy_const_tensor", []() { return &get_const_tensor(); }, + py::return_value_policy::copy + ); + + m.def("move_fixed_tensor", []() { return get_fixed_tensor(); }); + + m.def("move_tensor", []() { return get_tensor(); }); + + m.def("take_fixed_tensor", []() { return new Eigen::TensorFixedSize>(get_fixed_tensor()); }, + py::return_value_policy::take_ownership + ); + + m.def("take_tensor", []() { return new Eigen::Tensor(get_tensor()); }, + py::return_value_policy::take_ownership + ); + + m.def( + "reference_tensor", + []() { return &get_tensor(); }, + py::return_value_policy::reference); m.def( - "reference_global_tensor", + "reference_fixed_tensor", []() { return &get_tensor(); }, py::return_value_policy::reference); m.def( - "reference_const_global_tensor", + "reference_const_tensor", []() { return &get_const_tensor(); }, py::return_value_policy::reference); m.def( - "reference_view_of_global_tensor", + "reference_view_of_tensor", []() { return Eigen::TensorMap>(get_tensor()); }, py::return_value_policy::reference); m.def( - "reference_view_of_fixed_global_tensor", + "reference_view_of_fixed_tensor", []() { return Eigen::TensorMap>>( get_fixed_tensor()); diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index e278ffd009..5dd22dcbac 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -24,28 +24,36 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=0): def test_convert_tensor_to_py(): - assert_equal_tensor_ref(m.copy_global_tensor()) - assert_equal_tensor_ref(m.copy_fixed_global_tensor()) - assert_equal_tensor_ref(m.copy_const_global_tensor()) + assert_equal_tensor_ref(m.copy_tensor()) + assert_equal_tensor_ref(m.copy_fixed_tensor()) + assert_equal_tensor_ref(m.copy_const_tensor()) - assert_equal_tensor_ref(m.reference_global_tensor()) - assert_equal_tensor_ref(m.reference_view_of_global_tensor()) - assert_equal_tensor_ref(m.reference_view_of_fixed_global_tensor()) - assert_equal_tensor_ref(m.reference_const_global_tensor(), writeable=False) + assert_equal_tensor_ref(m.move_tensor()) + assert_equal_tensor_ref(m.move_fixed_tensor()) + + assert_equal_tensor_ref(m.take_tensor()) + assert_equal_tensor_ref(m.take_fixed_tensor()) + + assert_equal_tensor_ref(m.reference_tensor()) + assert_equal_tensor_ref(m.reference_fixed_tensor()) + + assert_equal_tensor_ref(m.reference_view_of_tensor()) + assert_equal_tensor_ref(m.reference_view_of_fixed_tensor()) + assert_equal_tensor_ref(m.reference_const_tensor(), writeable=False) def test_references_actually_refer(): - a = m.reference_global_tensor() + a = m.reference_tensor() a[0, 0, 0] = 100 - assert_equal_tensor_ref(m.copy_const_global_tensor(), modified=100) + assert_equal_tensor_ref(m.copy_const_tensor(), modified=100) a[0, 0, 0] = 0 - assert_equal_tensor_ref(m.copy_const_global_tensor()) + assert_equal_tensor_ref(m.copy_const_tensor()) - a = m.reference_view_of_global_tensor() + a = m.reference_view_of_tensor() a[0, 0, 0] = 100 - assert_equal_tensor_ref(m.copy_const_global_tensor(), modified=100) + assert_equal_tensor_ref(m.copy_const_tensor(), modified=100) a[0, 0, 0] = 0 - assert_equal_tensor_ref(m.copy_const_global_tensor()) + assert_equal_tensor_ref(m.copy_const_tensor()) def test_round_trip(): From d03a0319ed13d6bb1991e5ddaafb062401a82d80 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Sep 2022 23:44:28 +0000 Subject: [PATCH 027/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_eigen_tensor.cpp | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 1b57d3c45e..6073749225 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -44,34 +44,33 @@ Eigen::TensorFixedSize> &get_fixed_tensor() { const Eigen::Tensor &get_const_tensor() { return get_tensor(); } TEST_SUBMODULE(eigen_tensor, m) { - m.def("copy_fixed_tensor", []() { return &get_fixed_tensor(); }, - py::return_value_policy::copy - ); + m.def( + "copy_fixed_tensor", []() { return &get_fixed_tensor(); }, py::return_value_policy::copy); - m.def("copy_tensor", []() { return &get_tensor(); }, - py::return_value_policy::copy - ); + m.def( + "copy_tensor", []() { return &get_tensor(); }, py::return_value_policy::copy); - m.def("copy_const_tensor", []() { return &get_const_tensor(); }, - py::return_value_policy::copy - ); + m.def( + "copy_const_tensor", []() { return &get_const_tensor(); }, py::return_value_policy::copy); m.def("move_fixed_tensor", []() { return get_fixed_tensor(); }); m.def("move_tensor", []() { return get_tensor(); }); - m.def("take_fixed_tensor", []() { return new Eigen::TensorFixedSize>(get_fixed_tensor()); }, - py::return_value_policy::take_ownership - ); + m.def( + "take_fixed_tensor", + []() { + return new Eigen::TensorFixedSize>(get_fixed_tensor()); + }, + py::return_value_policy::take_ownership); - m.def("take_tensor", []() { return new Eigen::Tensor(get_tensor()); }, - py::return_value_policy::take_ownership - ); + m.def( + "take_tensor", + []() { return new Eigen::Tensor(get_tensor()); }, + py::return_value_policy::take_ownership); m.def( - "reference_tensor", - []() { return &get_tensor(); }, - py::return_value_policy::reference); + "reference_tensor", []() { return &get_tensor(); }, py::return_value_policy::reference); m.def( "reference_fixed_tensor", From 069f3fc2540ced4183e269d9d7997c9dfc195caa Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 1 Oct 2022 00:28:08 +0000 Subject: [PATCH 028/141] Refactor allocators to make things more clear --- include/pybind11/detail/eigen_tensor.h | 38 +++++++++++++++++++------- tests/test_eigen_tensor.cpp | 12 ++++---- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 3e47109d8a..0db3f4595d 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -82,6 +82,15 @@ struct eigen_tensor_helper())>::value; + + template + static T* alloc(Args&&... args) { + return new T(std::forward(args)...); + } + + static void free(T* tensor) { + delete tensor; + } }; template @@ -103,6 +112,18 @@ struct eigen_tensor_helper< } static constexpr auto dimensions_descriptor = concat(const_name()...); + + template + static T* alloc(Args&&... args) { + Eigen::aligned_allocator allocator; + return ::new (allocator.allocate(1)) T(std::forward(args)...); + } + + static void free(T* tensor) { + Eigen::aligned_allocator allocator; + tensor->~T(); + allocator.deallocate(tensor, 1); + } }; template @@ -199,17 +220,11 @@ struct type_caster::ValidType> { if (std::is_const::value) { pybind11_fail("Cannot move from a constant reference"); } - { - Eigen::aligned_allocator allocator; - Type *copy = ::new (allocator.allocate(1)) Type(std::move(*src)); - src = copy; - } + + src = H::alloc(std::move(*src)); parent_object = capsule(src, [](void *ptr) { - Eigen::aligned_allocator allocator; - Type *copy = (Type *) ptr; - copy->~Type(); - allocator.deallocate(copy, 1); + H::free(reinterpret_cast(ptr)); }); writeable = true; break; @@ -218,7 +233,10 @@ struct type_caster::ValidType> { if (std::is_const::value) { pybind11_fail("Cannot take ownership of a const reference"); } - parent_object = capsule(src, [](void *ptr) { delete (Type *) ptr; }); + + parent_object = capsule(src, [](void *ptr) { + H::free(reinterpret_cast(ptr)); + }); writeable = true; break; diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 6073749225..c8a03cb052 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -57,12 +57,12 @@ TEST_SUBMODULE(eigen_tensor, m) { m.def("move_tensor", []() { return get_tensor(); }); - m.def( - "take_fixed_tensor", - []() { - return new Eigen::TensorFixedSize>(get_fixed_tensor()); - }, - py::return_value_policy::take_ownership); + m.def("take_fixed_tensor", []() { + Eigen::aligned_allocator>> allocator; + return new (allocator.allocate(1)) Eigen::TensorFixedSize>(get_fixed_tensor()); + }, + py::return_value_policy::take_ownership + ); m.def( "take_tensor", From ea47cd153bb6b71a863d437e4bdc38b22dbdb906 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 1 Oct 2022 00:34:42 +0000 Subject: [PATCH 029/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/detail/eigen_tensor.h | 24 ++++++++++-------------- tests/test_eigen_tensor.cpp | 15 +++++++++------ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 0db3f4595d..eb9f4efece 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -83,14 +83,12 @@ struct eigen_tensor_helper())>::value; - template - static T* alloc(Args&&... args) { + template + static T *alloc(Args &&...args) { return new T(std::forward(args)...); } - static void free(T* tensor) { - delete tensor; - } + static void free(T *tensor) { delete tensor; } }; template @@ -113,13 +111,13 @@ struct eigen_tensor_helper< static constexpr auto dimensions_descriptor = concat(const_name()...); - template - static T* alloc(Args&&... args) { + template + static T *alloc(Args &&...args) { Eigen::aligned_allocator allocator; return ::new (allocator.allocate(1)) T(std::forward(args)...); } - static void free(T* tensor) { + static void free(T *tensor) { Eigen::aligned_allocator allocator; tensor->~T(); allocator.deallocate(tensor, 1); @@ -223,9 +221,8 @@ struct type_caster::ValidType> { src = H::alloc(std::move(*src)); - parent_object = capsule(src, [](void *ptr) { - H::free(reinterpret_cast(ptr)); - }); + parent_object + = capsule(src, [](void *ptr) { H::free(reinterpret_cast(ptr)); }); writeable = true; break; @@ -234,9 +231,8 @@ struct type_caster::ValidType> { pybind11_fail("Cannot take ownership of a const reference"); } - parent_object = capsule(src, [](void *ptr) { - H::free(reinterpret_cast(ptr)); - }); + parent_object + = capsule(src, [](void *ptr) { H::free(reinterpret_cast(ptr)); }); writeable = true; break; diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index c8a03cb052..20270697f5 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -57,12 +57,15 @@ TEST_SUBMODULE(eigen_tensor, m) { m.def("move_tensor", []() { return get_tensor(); }); - m.def("take_fixed_tensor", []() { - Eigen::aligned_allocator>> allocator; - return new (allocator.allocate(1)) Eigen::TensorFixedSize>(get_fixed_tensor()); - }, - py::return_value_policy::take_ownership - ); + m.def( + "take_fixed_tensor", + []() { + Eigen::aligned_allocator>> + allocator; + return new (allocator.allocate(1)) + Eigen::TensorFixedSize>(get_fixed_tensor()); + }, + py::return_value_policy::take_ownership); m.def( "take_tensor", From c343f4d5e12a3e05c588b9f2f0bebcb7b0277788 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 1 Oct 2022 01:30:50 +0000 Subject: [PATCH 030/141] Switch to std::copy --- include/pybind11/detail/eigen_tensor.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index eb9f4efece..97c9e968d1 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -62,7 +62,10 @@ struct eigen_tensor_helper get_shape(const T &f) { - return f.dimensions(); + std::array result; + const auto& dims = f.dimensions(); + std::copy(std::begin(dims), std::end(dims), std::begin(result)); + return result; } static constexpr bool From 25fc7a84acdd8162ea37e807271d43083377edb2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 1 Oct 2022 01:36:44 +0000 Subject: [PATCH 031/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/detail/eigen_tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 97c9e968d1..b6ec8918dd 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -63,7 +63,7 @@ struct eigen_tensor_helper get_shape(const T &f) { std::array result; - const auto& dims = f.dimensions(); + const auto &dims = f.dimensions(); std::copy(std::begin(dims), std::end(dims), std::begin(result)); return result; } From 22c554d6b0b917547233327b61f0707009708e2f Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 1 Oct 2022 02:28:10 +0000 Subject: [PATCH 032/141] Switch to DSizes instead of array --- include/pybind11/detail/eigen_tensor.h | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index b6ec8918dd..6e89657c7d 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -61,15 +61,12 @@ struct eigen_tensor_helper; using ValidType = void; - static std::array get_shape(const T &f) { - std::array result; - const auto &dims = f.dimensions(); - std::copy(std::begin(dims), std::end(dims), std::begin(result)); - return result; + static Eigen::DSizes get_shape(const T &f) { + return f.dimensions(); } static constexpr bool - is_correct_shape(const std::array & /*shape*/) { + is_correct_shape(const Eigen::DSizes & /*shape*/) { return true; } @@ -100,15 +97,15 @@ struct eigen_tensor_helper< using T = Eigen::TensorFixedSize, Options_, IndexType>; using ValidType = void; - static constexpr std::array get_shape(const T & /*f*/) { + static constexpr Eigen::DSizes get_shape(const T & /*f*/) { return get_shape(); } - static constexpr std::array get_shape() { - return {{Indices...}}; + static constexpr Eigen::DSizes get_shape() { + return Eigen::DSizes(Indices...); } - static bool is_correct_shape(const std::array &shape) { + static bool is_correct_shape(const Eigen::DSizes &shape) { return get_shape() == shape; } @@ -150,7 +147,7 @@ struct type_caster::ValidType> { return false; } - std::array shape; + Eigen::DSizes shape; std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); if (!H::is_correct_shape(shape)) { @@ -295,7 +292,7 @@ struct type_caster, typename eigen_tensor_helper::V return false; } - std::array shape; + Eigen::DSizes shape; std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); if (!H::is_correct_shape(shape)) { From 0a7163741d15129cf4fd0662462f511113ce7028 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 1 Oct 2022 02:34:21 +0000 Subject: [PATCH 033/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/detail/eigen_tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 6e89657c7d..ba6cac949d 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -147,7 +147,7 @@ struct type_caster::ValidType> { return false; } - Eigen::DSizes shape; + Eigen::DSizes shape; std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); if (!H::is_correct_shape(shape)) { From 8d1c6d08fbe1136770a4feb41c5613f706c6946e Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 3 Oct 2022 16:46:01 +0000 Subject: [PATCH 034/141] Address feedback --- include/pybind11/detail/eigen_tensor.h | 90 +++++++++++++------------- tests/test_eigen_tensor.cpp | 18 +++--- tests/test_eigen_tensor.py | 39 ++++++----- 3 files changed, 75 insertions(+), 72 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index ba6cac949d..678ee2b4a2 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -2,8 +2,6 @@ /* pybind11/eigen_tensor.h: Transparent conversion for Eigen tensors - Copyright (c) 2016 Wenzel Jakob - All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. */ @@ -58,15 +56,15 @@ struct eigen_tensor_helper {}; template struct eigen_tensor_helper> { - using T = Eigen::Tensor; + using Type = Eigen::Tensor; using ValidType = void; - static Eigen::DSizes get_shape(const T &f) { + static Eigen::DSizes get_shape(const Type &f) { return f.dimensions(); } static constexpr bool - is_correct_shape(const Eigen::DSizes & /*shape*/) { + is_correct_shape(const Eigen::DSizes & /*shape*/) { return true; } @@ -81,80 +79,80 @@ struct eigen_tensor_helper())>::value; + = helper())>::value; template - static T *alloc(Args &&...args) { - return new T(std::forward(args)...); + static Type *alloc(Args &&...args) { + return new Type(std::forward(args)...); } - static void free(T *tensor) { delete tensor; } + static void free(Type *tensor) { delete tensor; } }; template struct eigen_tensor_helper< Eigen::TensorFixedSize, Options_, IndexType>> { - using T = Eigen::TensorFixedSize, Options_, IndexType>; + using Type = Eigen::TensorFixedSize, Options_, IndexType>; using ValidType = void; - static constexpr Eigen::DSizes get_shape(const T & /*f*/) { + static constexpr Eigen::DSizes get_shape(const Type & /*f*/) { return get_shape(); } - static constexpr Eigen::DSizes get_shape() { - return Eigen::DSizes(Indices...); + static constexpr Eigen::DSizes get_shape() { + return Eigen::DSizes(Indices...); } - static bool is_correct_shape(const Eigen::DSizes &shape) { + static bool is_correct_shape(const Eigen::DSizes &shape) { return get_shape() == shape; } static constexpr auto dimensions_descriptor = concat(const_name()...); template - static T *alloc(Args &&...args) { - Eigen::aligned_allocator allocator; - return ::new (allocator.allocate(1)) T(std::forward(args)...); + static Type *alloc(Args &&...args) { + Eigen::aligned_allocator allocator; + return ::new (allocator.allocate(1)) Type(std::forward(args)...); } - static void free(T *tensor) { - Eigen::aligned_allocator allocator; - tensor->~T(); + static void free(Type *tensor) { + Eigen::aligned_allocator allocator; + tensor->~Type(); allocator.deallocate(tensor, 1); } }; -template +template struct get_tensor_descriptor { static constexpr auto value - = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") + eigen_tensor_helper::dimensions_descriptor + = const_name("numpy.ndarray[") + npy_format_descriptor::name + + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("], flags.writeable, ") - + const_name<(int) T::Layout == (int) Eigen::RowMajor>("flags.c_contiguous", + + const_name<(int) Type::Layout == (int) Eigen::RowMajor>("flags.c_contiguous", "flags.f_contiguous"); }; template struct type_caster::ValidType> { - using H = eigen_tensor_helper; + using Helper = eigen_tensor_helper; PYBIND11_TYPE_CASTER(Type, get_tensor_descriptor::value); bool load(handle src, bool /*convert*/) { - array_t()> a( + array_t()> arr( reinterpret_borrow(src)); - if (a.ndim() != Type::NumIndices) { + if (arr.ndim() != Type::NumIndices) { return false; } - Eigen::DSizes shape; - std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); + Eigen::DSizes shape; + std::copy(arr.shape(), arr.shape() + Type::NumIndices, shape.begin()); - if (!H::is_correct_shape(shape)) { + if (!Helper::is_correct_shape(shape)) { return false; } - value = Eigen::TensorMap(const_cast(a.data()), shape); + value = Eigen::TensorMap(const_cast(arr.data()), shape); return true; } @@ -219,10 +217,10 @@ struct type_caster::ValidType> { pybind11_fail("Cannot move from a constant reference"); } - src = H::alloc(std::move(*src)); + src = Helper::alloc(std::move(*src)); parent_object - = capsule(src, [](void *ptr) { H::free(reinterpret_cast(ptr)); }); + = capsule(src, [](void *ptr) { Helper::free(reinterpret_cast(ptr)); }); writeable = true; break; @@ -232,7 +230,7 @@ struct type_caster::ValidType> { } parent_object - = capsule(src, [](void *ptr) { H::free(reinterpret_cast(ptr)); }); + = capsule(src, [](void *ptr) { Helper::free(reinterpret_cast(ptr)); }); writeable = true; break; @@ -262,8 +260,8 @@ struct type_caster::ValidType> { pybind11_fail("pybind11 bug in eigen.h, please file a bug report"); } - object result = array_t()>( - H::get_shape(*src), src->data(), parent_object); + auto result = array_t()>( + Helper::get_shape(*src), src->data(), parent_object); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; @@ -275,32 +273,32 @@ struct type_caster::ValidType> { template struct type_caster, typename eigen_tensor_helper::ValidType> { - using H = eigen_tensor_helper; + using Helper = eigen_tensor_helper; bool load(handle src, bool /*convert*/) { // Note that we have a lot more checks here as we want to make sure to avoid copies - auto a = reinterpret_borrow(src); - if ((a.flags() & compute_array_flag_from_tensor()) == 0) { + auto arr = reinterpret_borrow(src); + if ((arr.flags() & compute_array_flag_from_tensor()) == 0) { return false; } - if (!a.dtype().is(dtype::of())) { + if (!arr.dtype().is(dtype::of())) { return false; } - if (a.ndim() != Type::NumIndices) { + if (arr.ndim() != Type::NumIndices) { return false; } Eigen::DSizes shape; - std::copy(a.shape(), a.shape() + Type::NumIndices, shape.begin()); + std::copy(arr.shape(), arr.shape() + Type::NumIndices, shape.begin()); - if (!H::is_correct_shape(shape)) { + if (!Helper::is_correct_shape(shape)) { return false; } value.reset(new Eigen::TensorMap( - reinterpret_cast(a.mutable_data()), shape)); + reinterpret_cast(arr.mutable_data()), shape)); return true; } @@ -373,8 +371,8 @@ struct type_caster, typename eigen_tensor_helper::V "reference or reference_internal"); } - object result = array_t()>( - H::get_shape(*src), src->data(), parent_object); + auto result = array_t()>( + Helper::get_shape(*src), src->data(), parent_object); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 20270697f5..933d74c6c8 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -1,8 +1,6 @@ /* tests/eigen_tensor.cpp -- automatic conversion of Eigen Tensor - Copyright (c) 2016 Wenzel Jakob - All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. */ @@ -22,19 +20,19 @@ Eigen::Tensor &get_tensor() { static Eigen::Tensor *x; if (!x) { - x = new Eigen::Tensor(3, 1, 2); + x = new Eigen::Tensor(3, 5, 2); reset_tensor(*x); } return *x; } -Eigen::TensorFixedSize> &get_fixed_tensor() { - static Eigen::TensorFixedSize> *x; +Eigen::TensorFixedSize> &get_fixed_tensor() { + static Eigen::TensorFixedSize> *x; if (!x) { - Eigen::aligned_allocator>> allocator; - x = new (allocator.allocate(1)) Eigen::TensorFixedSize>(); + Eigen::aligned_allocator>> allocator; + x = new (allocator.allocate(1)) Eigen::TensorFixedSize>(); reset_tensor(*x); } @@ -60,10 +58,10 @@ TEST_SUBMODULE(eigen_tensor, m) { m.def( "take_fixed_tensor", []() { - Eigen::aligned_allocator>> + Eigen::aligned_allocator>> allocator; return new (allocator.allocate(1)) - Eigen::TensorFixedSize>(get_fixed_tensor()); + Eigen::TensorFixedSize>(get_fixed_tensor()); }, py::return_value_policy::take_ownership); @@ -93,7 +91,7 @@ TEST_SUBMODULE(eigen_tensor, m) { m.def( "reference_view_of_fixed_tensor", []() { - return Eigen::TensorMap>>( + return Eigen::TensorMap>>( get_fixed_tensor()); }, py::return_value_policy::reference); diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 5dd22dcbac..73cf97cff0 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -4,24 +4,22 @@ m = pytest.importorskip("pybind11_tests.eigen_tensor") tensor_ref = np.array( - [ - [[0, 3]], - [[1, 4]], - [[2, 5]], - ] + [[[0.0, 15.0], [3.0, 18.0], [6.0, 21.0], [9.0, 24.0], [12.0, 27.0]], + [[1.0, 16.0], [4.0, 19.0], [7.0, 22.0], [10.0, 25.0], [13.0, 28.0]], + [[2.0, 17.0], [5.0, 20.0], [8.0, 23.0], [11.0, 26.0], [14.0, 29.0]]], ) +indices = (2, 3, 1) + def assert_equal_tensor_ref(mat, writeable=True, modified=0): assert mat.flags.writeable == writeable + copy = np.array(tensor_ref) if modified != 0: - tensor_ref[0, 0, 0] = modified - - np.testing.assert_array_equal(mat, tensor_ref) - - tensor_ref[0, 0, 0] = 0 + copy[indices] = modified + np.testing.assert_array_equal(mat, copy) def test_convert_tensor_to_py(): assert_equal_tensor_ref(m.copy_tensor()) @@ -44,15 +42,16 @@ def test_convert_tensor_to_py(): def test_references_actually_refer(): a = m.reference_tensor() - a[0, 0, 0] = 100 + temp = a[indices] + a[indices] = 100 assert_equal_tensor_ref(m.copy_const_tensor(), modified=100) - a[0, 0, 0] = 0 + a[indices] = temp assert_equal_tensor_ref(m.copy_const_tensor()) a = m.reference_view_of_tensor() - a[0, 0, 0] = 100 + a[indices] = 100 assert_equal_tensor_ref(m.copy_const_tensor(), modified=100) - a[0, 0, 0] = 0 + a[indices] = temp assert_equal_tensor_ref(m.copy_const_tensor()) @@ -64,7 +63,15 @@ def test_round_trip_references_actually_refer(): # Need to create a copy that matches the type on the C side copy = np.array(tensor_ref, dtype=np.float64, order="F") a = m.round_trip_view_tensor(copy) - a[0, 0, 0] = 100 + temp = a[indices] + a[indices] = 100 assert_equal_tensor_ref(copy, modified=100) - a[0, 0, 0] = 0 + a[indices] = temp assert_equal_tensor_ref(copy) + +def test_doc_string(doc): + assert doc(m.copy_tensor) == 'copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous' + assert doc(m.copy_fixed_tensor) == 'copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2], flags.writeable, flags.f_contiguous' + + # Ideally this type signature wouldn't have flags.writeable, but I don't see a way to avoid it due to names being set at compile time ... + assert doc(m.reference_const_tensor) == 'reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous' \ No newline at end of file From 98b2e126e7464dffb8a874405bba28ee15c8d945 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 16:52:58 +0000 Subject: [PATCH 035/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/detail/eigen_tensor.h | 10 ++++++---- tests/test_eigen_tensor.py | 25 +++++++++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 678ee2b4a2..78302757b4 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -95,7 +95,8 @@ struct eigen_tensor_helper< using Type = Eigen::TensorFixedSize, Options_, IndexType>; using ValidType = void; - static constexpr Eigen::DSizes get_shape(const Type & /*f*/) { + static constexpr Eigen::DSizes + get_shape(const Type & /*f*/) { return get_shape(); } @@ -103,7 +104,8 @@ struct eigen_tensor_helper< return Eigen::DSizes(Indices...); } - static bool is_correct_shape(const Eigen::DSizes &shape) { + static bool + is_correct_shape(const Eigen::DSizes &shape) { return get_shape() == shape; } @@ -129,7 +131,7 @@ struct get_tensor_descriptor { + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("], flags.writeable, ") + const_name<(int) Type::Layout == (int) Eigen::RowMajor>("flags.c_contiguous", - "flags.f_contiguous"); + "flags.f_contiguous"); }; template @@ -145,7 +147,7 @@ struct type_caster::ValidType> { return false; } - Eigen::DSizes shape; + Eigen::DSizes shape; std::copy(arr.shape(), arr.shape() + Type::NumIndices, shape.begin()); if (!Helper::is_correct_shape(shape)) { diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 73cf97cff0..9abca313a5 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -4,9 +4,11 @@ m = pytest.importorskip("pybind11_tests.eigen_tensor") tensor_ref = np.array( - [[[0.0, 15.0], [3.0, 18.0], [6.0, 21.0], [9.0, 24.0], [12.0, 27.0]], - [[1.0, 16.0], [4.0, 19.0], [7.0, 22.0], [10.0, 25.0], [13.0, 28.0]], - [[2.0, 17.0], [5.0, 20.0], [8.0, 23.0], [11.0, 26.0], [14.0, 29.0]]], + [ + [[0.0, 15.0], [3.0, 18.0], [6.0, 21.0], [9.0, 24.0], [12.0, 27.0]], + [[1.0, 16.0], [4.0, 19.0], [7.0, 22.0], [10.0, 25.0], [13.0, 28.0]], + [[2.0, 17.0], [5.0, 20.0], [8.0, 23.0], [11.0, 26.0], [14.0, 29.0]], + ], ) indices = (2, 3, 1) @@ -21,6 +23,7 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=0): np.testing.assert_array_equal(mat, copy) + def test_convert_tensor_to_py(): assert_equal_tensor_ref(m.copy_tensor()) assert_equal_tensor_ref(m.copy_fixed_tensor()) @@ -69,9 +72,19 @@ def test_round_trip_references_actually_refer(): a[indices] = temp assert_equal_tensor_ref(copy) + def test_doc_string(doc): - assert doc(m.copy_tensor) == 'copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous' - assert doc(m.copy_fixed_tensor) == 'copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2], flags.writeable, flags.f_contiguous' + assert ( + doc(m.copy_tensor) + == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous" + ) + assert ( + doc(m.copy_fixed_tensor) + == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2], flags.writeable, flags.f_contiguous" + ) # Ideally this type signature wouldn't have flags.writeable, but I don't see a way to avoid it due to names being set at compile time ... - assert doc(m.reference_const_tensor) == 'reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous' \ No newline at end of file + assert ( + doc(m.reference_const_tensor) + == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous" + ) From cc59682d1761fc1db3087b7fbd988a63f118c438 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 3 Oct 2022 16:51:47 +0000 Subject: [PATCH 036/141] Fix python + dummy c++ change to trigger build --- tests/test_eigen_tensor.cpp | 6 ++---- tests/test_eigen_tensor.py | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 933d74c6c8..10f3f87fea 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -58,10 +58,8 @@ TEST_SUBMODULE(eigen_tensor, m) { m.def( "take_fixed_tensor", []() { - Eigen::aligned_allocator>> - allocator; - return new (allocator.allocate(1)) - Eigen::TensorFixedSize>(get_fixed_tensor()); + Eigen::aligned_allocator>> allocator; + return new (allocator.allocate(1)) Eigen::TensorFixedSize>(get_fixed_tensor()); }, py::return_value_policy::take_ownership); diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 9abca313a5..33b1c8c451 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -83,7 +83,8 @@ def test_doc_string(doc): == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2], flags.writeable, flags.f_contiguous" ) - # Ideally this type signature wouldn't have flags.writeable, but I don't see a way to avoid it due to names being set at compile time ... + # Ideally this type signature wouldn't have flags.writeable, but + # I don't see a way to avoid it due to names being set at compile time ... assert ( doc(m.reference_const_tensor) == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous" From fda4a0001d7baaf142cedd725457158e9bd8a6c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 16:57:37 +0000 Subject: [PATCH 037/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_eigen_tensor.cpp | 6 ++++-- tests/test_eigen_tensor.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 10f3f87fea..933d74c6c8 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -58,8 +58,10 @@ TEST_SUBMODULE(eigen_tensor, m) { m.def( "take_fixed_tensor", []() { - Eigen::aligned_allocator>> allocator; - return new (allocator.allocate(1)) Eigen::TensorFixedSize>(get_fixed_tensor()); + Eigen::aligned_allocator>> + allocator; + return new (allocator.allocate(1)) + Eigen::TensorFixedSize>(get_fixed_tensor()); }, py::return_value_policy::take_ownership); diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 33b1c8c451..a033fdddb9 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -83,7 +83,7 @@ def test_doc_string(doc): == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2], flags.writeable, flags.f_contiguous" ) - # Ideally this type signature wouldn't have flags.writeable, but + # Ideally this type signature wouldn't have flags.writeable, but # I don't see a way to avoid it due to names being set at compile time ... assert ( doc(m.reference_const_tensor) From 9153db5f1b1d6d5824de5ccce9af24375e50d584 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 3 Oct 2022 19:45:51 +0000 Subject: [PATCH 038/141] Alignment --- include/pybind11/detail/eigen_tensor.h | 42 +++++++++++++++++--------- tests/test_eigen_tensor.cpp | 5 +++ tests/test_eigen_tensor.py | 4 +++ 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 78302757b4..76e57a7659 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -42,6 +42,11 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +bool is_tensor_aligned(const void* data) { + return (std::size_t(data) % EIGEN_DEFAULT_ALIGN_BYTES) == 0; +} + + template constexpr int compute_array_flag_from_tensor() { static_assert((static_cast(T::Layout) == static_cast(Eigen::RowMajor)) @@ -154,7 +159,11 @@ struct type_caster::ValidType> { return false; } - value = Eigen::TensorMap(const_cast(arr.data()), shape); + if (is_tensor_aligned(arr.data())) { + value = Eigen::TensorMap(const_cast(arr.data()), shape); + } else { + value = Eigen::TensorMap(const_cast(arr.data()), shape); + } return true; } @@ -273,8 +282,9 @@ struct type_caster::ValidType> { } }; -template -struct type_caster, typename eigen_tensor_helper::ValidType> { +template +struct type_caster, typename eigen_tensor_helper::ValidType> { + using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper; bool load(handle src, bool /*convert*/) { @@ -291,6 +301,10 @@ struct type_caster, typename eigen_tensor_helper::V if (arr.ndim() != Type::NumIndices) { return false; } + + if ((Options & Eigen::Aligned) != 0 && !is_tensor_aligned(arr.data())) { + return false; + } Eigen::DSizes shape; std::copy(arr.shape(), arr.shape() + Type::NumIndices, shape.begin()); @@ -299,22 +313,22 @@ struct type_caster, typename eigen_tensor_helper::V return false; } - value.reset(new Eigen::TensorMap( + value.reset(new MapType( reinterpret_cast(arr.mutable_data()), shape)); return true; } - static handle cast(Eigen::TensorMap &&src, return_value_policy policy, handle parent) { + static handle cast(MapType &&src, return_value_policy policy, handle parent) { return cast_impl(&src, policy, parent); } static handle - cast(const Eigen::TensorMap &&src, return_value_policy policy, handle parent) { + cast(const MapType &&src, return_value_policy policy, handle parent) { return cast_impl(&src, policy, parent); } - static handle cast(Eigen::TensorMap &src, return_value_policy policy, handle parent) { + static handle cast(MapType &src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) { policy = return_value_policy::copy; @@ -323,7 +337,7 @@ struct type_caster, typename eigen_tensor_helper::V } static handle - cast(const Eigen::TensorMap &src, return_value_policy policy, handle parent) { + cast(const MapType &src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) { policy = return_value_policy::copy; @@ -331,7 +345,7 @@ struct type_caster, typename eigen_tensor_helper::V return cast(&src, policy, parent); } - static handle cast(Eigen::TensorMap *src, return_value_policy policy, handle parent) { + static handle cast(MapType *src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic) { policy = return_value_policy::take_ownership; } else if (policy == return_value_policy::automatic_reference) { @@ -341,7 +355,7 @@ struct type_caster, typename eigen_tensor_helper::V } static handle - cast(const Eigen::TensorMap *src, return_value_policy policy, handle parent) { + cast(const MapType *src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic) { policy = return_value_policy::take_ownership; } else if (policy == return_value_policy::automatic_reference) { @@ -385,13 +399,13 @@ struct type_caster, typename eigen_tensor_helper::V protected: // TODO: Move to std::optional once std::optional has more support - std::unique_ptr> value; + std::unique_ptr value; public: static constexpr auto name = get_tensor_descriptor::value; - explicit operator Eigen::TensorMap *() { return value.get(); } - explicit operator Eigen::TensorMap &() { return *value; } - explicit operator Eigen::TensorMap &&() && { return std::move(*value); } + explicit operator MapType *() { return value.get(); } + explicit operator MapType &() { return *value; } + explicit operator MapType &&() && { return std::move(*value); } template using cast_op_type = ::pybind11::detail::movable_cast_op_type; diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 933d74c6c8..446d5ce10c 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -102,4 +102,9 @@ TEST_SUBMODULE(eigen_tensor, m) { "round_trip_view_tensor", [](Eigen::TensorMap> view) { return view; }, py::return_value_policy::reference); + + m.def( + "round_trip_aligned_view_tensor", + [](Eigen::TensorMap, Eigen::Aligned> view) { return view; }, + py::return_value_policy::reference); } diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index a033fdddb9..2b69bbba32 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -61,6 +61,10 @@ def test_references_actually_refer(): def test_round_trip(): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) +def test_aligned_view(): + copy = np.array(tensor_ref, dtype=np.float64, order="F") + assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) + assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(copy)) def test_round_trip_references_actually_refer(): # Need to create a copy that matches the type on the C side From 44681e002d05aec1249003d4e0cf9fa1e8e6ccd9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 19:53:44 +0000 Subject: [PATCH 039/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/detail/eigen_tensor.h | 38 ++++++++++++-------------- tests/test_eigen_tensor.py | 2 ++ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 76e57a7659..d4283b128b 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -42,11 +42,10 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) -bool is_tensor_aligned(const void* data) { - return (std::size_t(data) % EIGEN_DEFAULT_ALIGN_BYTES) == 0; +bool is_tensor_aligned(const void *data) { + return (std::size_t(data) % EIGEN_DEFAULT_ALIGN_BYTES) == 0; } - template constexpr int compute_array_flag_from_tensor() { static_assert((static_cast(T::Layout) == static_cast(Eigen::RowMajor)) @@ -159,11 +158,12 @@ struct type_caster::ValidType> { return false; } - if (is_tensor_aligned(arr.data())) { - value = Eigen::TensorMap(const_cast(arr.data()), shape); - } else { + if (is_tensor_aligned(arr.data())) { + value = Eigen::TensorMap( + const_cast(arr.data()), shape); + } else { value = Eigen::TensorMap(const_cast(arr.data()), shape); - } + } return true; } @@ -283,7 +283,8 @@ struct type_caster::ValidType> { }; template -struct type_caster, typename eigen_tensor_helper::ValidType> { +struct type_caster, + typename eigen_tensor_helper::ValidType> { using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper; @@ -301,10 +302,10 @@ struct type_caster, typename eigen_tensor_helper if (arr.ndim() != Type::NumIndices) { return false; } - - if ((Options & Eigen::Aligned) != 0 && !is_tensor_aligned(arr.data())) { + + if ((Options & Eigen::Aligned) != 0 && !is_tensor_aligned(arr.data())) { return false; - } + } Eigen::DSizes shape; std::copy(arr.shape(), arr.shape() + Type::NumIndices, shape.begin()); @@ -313,8 +314,8 @@ struct type_caster, typename eigen_tensor_helper return false; } - value.reset(new MapType( - reinterpret_cast(arr.mutable_data()), shape)); + value.reset( + new MapType(reinterpret_cast(arr.mutable_data()), shape)); return true; } @@ -323,8 +324,7 @@ struct type_caster, typename eigen_tensor_helper return cast_impl(&src, policy, parent); } - static handle - cast(const MapType &&src, return_value_policy policy, handle parent) { + static handle cast(const MapType &&src, return_value_policy policy, handle parent) { return cast_impl(&src, policy, parent); } @@ -336,8 +336,7 @@ struct type_caster, typename eigen_tensor_helper return cast_impl(&src, policy, parent); } - static handle - cast(const MapType &src, return_value_policy policy, handle parent) { + static handle cast(const MapType &src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) { policy = return_value_policy::copy; @@ -354,8 +353,7 @@ struct type_caster, typename eigen_tensor_helper return cast_impl(src, policy, parent); } - static handle - cast(const MapType *src, return_value_policy policy, handle parent) { + static handle cast(const MapType *src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic) { policy = return_value_policy::take_ownership; } else if (policy == return_value_policy::automatic_reference) { @@ -403,7 +401,7 @@ struct type_caster, typename eigen_tensor_helper public: static constexpr auto name = get_tensor_descriptor::value; - explicit operator MapType *() { return value.get(); } + explicit operator MapType *() { return value.get(); } explicit operator MapType &() { return *value; } explicit operator MapType &&() && { return std::move(*value); } diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 2b69bbba32..cec81cb372 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -61,11 +61,13 @@ def test_references_actually_refer(): def test_round_trip(): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) + def test_aligned_view(): copy = np.array(tensor_ref, dtype=np.float64, order="F") assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(copy)) + def test_round_trip_references_actually_refer(): # Need to create a copy that matches the type on the C side copy = np.array(tensor_ref, dtype=np.float64, order="F") From 6c3bfa4e59371853f43106cc60fd49449c94fe42 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 3 Oct 2022 19:48:36 +0000 Subject: [PATCH 040/141] Add include guard --- include/pybind11/eigen.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 60377b1075..151b78e602 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -10,7 +10,10 @@ #pragma once #include "detail/eigen_matrix.h" + +#if EIGEN_VERSION_AT_LEAST(3, 3, 0) #include "detail/eigen_tensor.h" +#endif // Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit // move constructors that break things. We could detect this an explicitly copy, but an extra copy From e01bbd8bd15f84fa2ec7dff2f6aa9dd37fb24eb2 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 3 Oct 2022 19:55:52 +0000 Subject: [PATCH 041/141] Forgot inline --- include/pybind11/detail/eigen_tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index d4283b128b..797c338581 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -42,7 +42,7 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) -bool is_tensor_aligned(const void *data) { +inline bool is_tensor_aligned(const void *data) { return (std::size_t(data) % EIGEN_DEFAULT_ALIGN_BYTES) == 0; } From 8ef248d0f5da87c64c20e15065341c3ea1019402 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 3 Oct 2022 20:43:52 +0000 Subject: [PATCH 042/141] Fix compiler warning --- include/pybind11/detail/eigen_tensor.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 797c338581..6414160de0 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -303,7 +303,9 @@ struct type_caster, return false; } - if ((Options & Eigen::Aligned) != 0 && !is_tensor_aligned(arr.data())) { + // Use temporary to avoid MSVC warning ... + bool is_aligned = (Options & Eigen::Aligned) != 0; + if (is_aligned && !is_tensor_aligned(arr.data())) { return false; } From ad8e9879171edaae716cd99ec722fb0254a697f5 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 3 Oct 2022 21:02:00 +0000 Subject: [PATCH 043/141] Remove bad test --- include/pybind11/detail/eigen_tensor.h | 3 +-- tests/test_eigen_tensor.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 6414160de0..720f8932d0 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -283,8 +283,7 @@ struct type_caster::ValidType> { }; template -struct type_caster, - typename eigen_tensor_helper::ValidType> { +struct type_caster, typename eigen_tensor_helper::ValidType> { using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper; diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index cec81cb372..93016db72a 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -63,9 +63,7 @@ def test_round_trip(): def test_aligned_view(): - copy = np.array(tensor_ref, dtype=np.float64, order="F") assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) - assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(copy)) def test_round_trip_references_actually_refer(): From 6bf2ffbd70c8cec40f0a36353a2b24a6a9d8e3a3 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 3 Oct 2022 23:08:45 +0000 Subject: [PATCH 044/141] Better type signatures --- include/pybind11/detail/eigen_tensor.h | 4 ++-- tests/test_eigen_tensor.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 720f8932d0..0db2e6a822 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -134,8 +134,8 @@ struct get_tensor_descriptor { = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("], flags.writeable, ") - + const_name<(int) Type::Layout == (int) Eigen::RowMajor>("flags.c_contiguous", - "flags.f_contiguous"); + + const_name<(int) Type::Layout == (int) Eigen::RowMajor>("flags.c_contiguous]", + "flags.f_contiguous]"); }; template diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 93016db72a..6ff04aa448 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -80,16 +80,16 @@ def test_round_trip_references_actually_refer(): def test_doc_string(doc): assert ( doc(m.copy_tensor) - == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous" + == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]" ) assert ( doc(m.copy_fixed_tensor) - == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2], flags.writeable, flags.f_contiguous" + == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2], flags.writeable, flags.f_contiguous]" ) # Ideally this type signature wouldn't have flags.writeable, but # I don't see a way to avoid it due to names being set at compile time ... assert ( doc(m.reference_const_tensor) - == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous" + == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]" ) From ba79142042cdc20dcffda0d7e114faf4d07a2675 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 4 Oct 2022 18:28:59 +0000 Subject: [PATCH 045/141] Add guards to make compiler requirements more explicit --- include/pybind11/detail/eigen_tensor.h | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index 0db2e6a822..b27021d064 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -77,9 +77,18 @@ struct eigen_tensor_helper struct helper> { - // Hack to work around gcc 4.8 bugs. Feel free to remove when we drop gcc 4.8 support. + #if defined(__GNUC__) && __GNUC__ <= 4 + + // Hack to work around gcc 4.8 bugs. static constexpr descr value = concat(const_name(((void) Is, "?"))...); + + #else + + static constexpr auto value + = concat(const_name(((void) Is, "?"))...); + + #endif }; static constexpr auto dimensions_descriptor @@ -245,11 +254,16 @@ struct type_caster::ValidType> { writeable = true; break; - case return_value_policy::copy: { - // Clang 3.6 / 3.7 has a very confusing bug that seems to be fixed by adding this - // scope + case return_value_policy::copy: + #if defined(__clang_major__) && __clang_major__ <= 3 + // Hack to work around clang bugs + { parent_object = {}; } + #else + parent_object = {}; + #endif + writeable = true; break; From 880ef5df9bbbf9e03d3e4fddb07914a3baa0a272 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Oct 2022 18:36:12 +0000 Subject: [PATCH 046/141] style: pre-commit fixes --- include/pybind11/detail/eigen_tensor.h | 30 ++++++++++++-------------- include/pybind11/eigen.h | 2 +- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index b27021d064..a10a4463fd 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -77,18 +77,17 @@ struct eigen_tensor_helper struct helper> { - #if defined(__GNUC__) && __GNUC__ <= 4 +#if defined(__GNUC__) && __GNUC__ <= 4 - // Hack to work around gcc 4.8 bugs. + // Hack to work around gcc 4.8 bugs. static constexpr descr value = concat(const_name(((void) Is, "?"))...); - #else +#else - static constexpr auto value - = concat(const_name(((void) Is, "?"))...); - - #endif + static constexpr auto value = concat(const_name(((void) Is, "?"))...); + +#endif }; static constexpr auto dimensions_descriptor @@ -254,15 +253,13 @@ struct type_caster::ValidType> { writeable = true; break; - case return_value_policy::copy: - #if defined(__clang_major__) && __clang_major__ <= 3 - // Hack to work around clang bugs - { + case return_value_policy::copy: +#if defined(__clang_major__) && __clang_major__ <= 3 + // Hack to work around clang bugs + { parent_object = {}; } +#else parent_object = {}; - } - #else - parent_object = {}; - #endif +#endif writeable = true; break; @@ -297,7 +294,8 @@ struct type_caster::ValidType> { }; template -struct type_caster, typename eigen_tensor_helper::ValidType> { +struct type_caster, + typename eigen_tensor_helper::ValidType> { using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper; diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 151b78e602..f9bc568cbd 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -12,7 +12,7 @@ #include "detail/eigen_matrix.h" #if EIGEN_VERSION_AT_LEAST(3, 3, 0) -#include "detail/eigen_tensor.h" +# include "detail/eigen_tensor.h" #endif // Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit From afcdf3e4005887e3f2518f78b2b5685c4fff5cc0 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 4 Oct 2022 21:12:11 +0000 Subject: [PATCH 047/141] Force rerun of tests due to flake --- include/pybind11/eigen.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index f9bc568cbd..a30572be71 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -11,8 +11,8 @@ #include "detail/eigen_matrix.h" -#if EIGEN_VERSION_AT_LEAST(3, 3, 0) -# include "detail/eigen_tensor.h" +#if EIGEN_VERSION_AT_LEAST(3, 3, 0 ) +#include "detail/eigen_tensor.h" #endif // Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit From 73ec5a5f6b6927c678e7ef345ff7b3cb132b1a7e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Oct 2022 21:18:37 +0000 Subject: [PATCH 048/141] style: pre-commit fixes --- include/pybind11/eigen.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index a30572be71..f9bc568cbd 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -11,8 +11,8 @@ #include "detail/eigen_matrix.h" -#if EIGEN_VERSION_AT_LEAST(3, 3, 0 ) -#include "detail/eigen_tensor.h" +#if EIGEN_VERSION_AT_LEAST(3, 3, 0) +# include "detail/eigen_tensor.h" #endif // Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit From f54847824f51418ee8f71bf5ac31487a794d014b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 4 Oct 2022 15:07:44 -0700 Subject: [PATCH 049/141] Keep pragmas & all related comments together, add PLEASE KEEP IN SYNC --- include/pybind11/detail/eigen_matrix.h | 6 +++--- include/pybind11/detail/eigen_tensor.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/pybind11/detail/eigen_matrix.h b/include/pybind11/detail/eigen_matrix.h index 274604f7c4..ad375f902f 100644 --- a/include/pybind11/detail/eigen_matrix.h +++ b/include/pybind11/detail/eigen_matrix.h @@ -10,14 +10,14 @@ #pragma once +#include "../numpy.h" + +// Similar to comments & pragma block in eigen_tensor.h. PLEASE KEEP IN SYNC. /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. See also: https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler */ - -#include "../numpy.h" - // The C4127 suppression was introduced for Eigen 3.4.0. In theory we could // make it version specific, or even remove it later, but considering that // 1. C4127 is generally far more distracting than useful for modern template code, and diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/detail/eigen_tensor.h index a10a4463fd..0e19d8549c 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/detail/eigen_tensor.h @@ -8,14 +8,14 @@ #pragma once +#include "../numpy.h" + +// Similar to comments & pragma block in eigen_matrix.h. PLEASE KEEP IN SYNC. /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. See also: https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler */ - -#include "../numpy.h" - // The C4127 suppression was introduced for Eigen 3.4.0. In theory we could // make it version specific, or even remove it later, but considering that // 1. C4127 is generally far more distracting than useful for modern template code, and From 8829b4b1dce6680cfda92200d1eac7a385f3c58e Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 5 Oct 2022 01:02:55 +0000 Subject: [PATCH 050/141] Move headers out of detail --- include/pybind11/eigen.h | 12 +++--------- include/pybind11/{detail => }/eigen_matrix.h | 8 +++++++- include/pybind11/{detail => }/eigen_tensor.h | 6 +++++- tests/test_eigen_matrix.cpp | 2 +- tests/test_eigen_tensor.cpp | 2 +- 5 files changed, 17 insertions(+), 13 deletions(-) rename include/pybind11/{detail => }/eigen_matrix.h (98%) rename include/pybind11/{detail => }/eigen_tensor.h (99%) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index f9bc568cbd..f9e6dc7d89 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -9,14 +9,8 @@ #pragma once -#include "detail/eigen_matrix.h" +#include "eigen_matrix.h" #if EIGEN_VERSION_AT_LEAST(3, 3, 0) -# include "detail/eigen_tensor.h" -#endif - -// Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit -// move constructors that break things. We could detect this an explicitly copy, but an extra copy -// of matrices seems highly undesirable. -static_assert(EIGEN_VERSION_AT_LEAST(3, 2, 7), - "Eigen support in pybind11 requires Eigen >= 3.2.7"); +# include "eigen_tensor.h" +#endif \ No newline at end of file diff --git a/include/pybind11/detail/eigen_matrix.h b/include/pybind11/eigen_matrix.h similarity index 98% rename from include/pybind11/detail/eigen_matrix.h rename to include/pybind11/eigen_matrix.h index ad375f902f..348641fc43 100644 --- a/include/pybind11/detail/eigen_matrix.h +++ b/include/pybind11/eigen_matrix.h @@ -10,7 +10,7 @@ #pragma once -#include "../numpy.h" +#include "numpy.h" // Similar to comments & pragma block in eigen_tensor.h. PLEASE KEEP IN SYNC. /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. @@ -42,6 +42,12 @@ # pragma GCC diagnostic pop #endif +// Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit +// move constructors that break things. We could detect this an explicitly copy, but an extra copy +// of matrices seems highly undesirable. +static_assert(EIGEN_VERSION_AT_LEAST(3, 2, 7), + "Eigen matrix support in pybind11 requires Eigen >= 3.2.7"); + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) // Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: diff --git a/include/pybind11/detail/eigen_tensor.h b/include/pybind11/eigen_tensor.h similarity index 99% rename from include/pybind11/detail/eigen_tensor.h rename to include/pybind11/eigen_tensor.h index 0e19d8549c..f23a05efd4 100644 --- a/include/pybind11/detail/eigen_tensor.h +++ b/include/pybind11/eigen_tensor.h @@ -8,7 +8,7 @@ #pragma once -#include "../numpy.h" +#include "numpy.h" // Similar to comments & pragma block in eigen_matrix.h. PLEASE KEEP IN SYNC. /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. @@ -16,6 +16,7 @@ https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler */ + // The C4127 suppression was introduced for Eigen 3.4.0. In theory we could // make it version specific, or even remove it later, but considering that // 1. C4127 is generally far more distracting than useful for modern template code, and @@ -38,6 +39,9 @@ # pragma GCC diagnostic pop #endif +static_assert(EIGEN_VERSION_AT_LEAST(3, 3, 0), + "Eigen Tensor support in pybind11 requires Eigen >= 3.3.0"); + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) diff --git a/tests/test_eigen_matrix.cpp b/tests/test_eigen_matrix.cpp index b98eee492e..16aca3db8a 100644 --- a/tests/test_eigen_matrix.cpp +++ b/tests/test_eigen_matrix.cpp @@ -7,7 +7,7 @@ BSD-style license that can be found in the LICENSE file. */ -#include +#include #include #include "constructor_stats.h" diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 446d5ce10c..b16368876b 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -5,7 +5,7 @@ BSD-style license that can be found in the LICENSE file. */ -#include +#include #include "pybind11_tests.h" From 79ba4f342aaa253378bf7237eea1635dde553708 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Oct 2022 01:08:55 +0000 Subject: [PATCH 051/141] style: pre-commit fixes --- include/pybind11/eigen.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index f9e6dc7d89..899a6c4a86 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -13,4 +13,4 @@ #if EIGEN_VERSION_AT_LEAST(3, 3, 0) # include "eigen_tensor.h" -#endif \ No newline at end of file +#endif From 1a4b6a294c793e66acef9cc9176e176f49b0493d Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 5 Oct 2022 10:58:42 -0400 Subject: [PATCH 052/141] Fix cmake --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index ee0975bc1f..9f95702650 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,6 +120,8 @@ set(PYBIND11_HEADERS include/pybind11/complex.h include/pybind11/options.h include/pybind11/eigen.h + include/pybind11/eigen_matrix.h + include/pybind11/eigen_tensor.h include/pybind11/embed.h include/pybind11/eval.h include/pybind11/gil.h From e92231a34fd9c548c6a53b0e202971ddb27c89f6 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 5 Oct 2022 11:15:52 -0400 Subject: [PATCH 053/141] Improve casting --- include/pybind11/eigen_tensor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/eigen_tensor.h b/include/pybind11/eigen_tensor.h index f23a05efd4..aeee1c1358 100644 --- a/include/pybind11/eigen_tensor.h +++ b/include/pybind11/eigen_tensor.h @@ -47,7 +47,7 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) inline bool is_tensor_aligned(const void *data) { - return (std::size_t(data) % EIGEN_DEFAULT_ALIGN_BYTES) == 0; + return (reinterpret_cast(data) % EIGEN_DEFAULT_ALIGN_BYTES) == 0; } template @@ -146,7 +146,7 @@ struct get_tensor_descriptor { = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("], flags.writeable, ") - + const_name<(int) Type::Layout == (int) Eigen::RowMajor>("flags.c_contiguous]", + + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>("flags.c_contiguous]", "flags.f_contiguous]"); }; From 97358ea162e5d99bcb5a6995e72bf1bd370342ee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Oct 2022 15:19:11 +0000 Subject: [PATCH 054/141] style: pre-commit fixes --- include/pybind11/eigen_tensor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/eigen_tensor.h b/include/pybind11/eigen_tensor.h index aeee1c1358..c17e80e444 100644 --- a/include/pybind11/eigen_tensor.h +++ b/include/pybind11/eigen_tensor.h @@ -146,8 +146,8 @@ struct get_tensor_descriptor { = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("], flags.writeable, ") - + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>("flags.c_contiguous]", - "flags.f_contiguous]"); + + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>( + "flags.c_contiguous]", "flags.f_contiguous]"); }; template From 396f047ba48a6e2859bd79c5f924633d7890c666 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 5 Oct 2022 18:04:19 +0000 Subject: [PATCH 055/141] Add a ton more tests + refactor --- .gitignore | 1 + CMakeLists.txt | 4 +- include/pybind11/eigen.h | 4 +- .../{eigen_matrix.h => eigen/matrix.h} | 4 +- .../{eigen_tensor.h => eigen/tensor.h} | 45 ++++++++++--------- tests/extra_python_package/test_files.py | 9 ++-- tests/test_eigen_matrix.cpp | 2 +- tests/test_eigen_tensor.cpp | 24 +++++++++- tests/test_eigen_tensor.py | 43 +++++++++++++++--- 9 files changed, 99 insertions(+), 37 deletions(-) rename include/pybind11/{eigen_matrix.h => eigen/matrix.h} (99%) rename include/pybind11/{eigen_tensor.h => eigen/tensor.h} (92%) diff --git a/.gitignore b/.gitignore index 3cf4fbbda0..43d5094c96 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ pybind11Targets.cmake /pybind11/share/* /docs/_build/* .ipynb_checkpoints/ +tests/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f95702650..3284e21eb4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,8 +120,8 @@ set(PYBIND11_HEADERS include/pybind11/complex.h include/pybind11/options.h include/pybind11/eigen.h - include/pybind11/eigen_matrix.h - include/pybind11/eigen_tensor.h + include/pybind11/eigen/matrix.h + include/pybind11/eigen/tensor.h include/pybind11/embed.h include/pybind11/eval.h include/pybind11/gil.h diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 899a6c4a86..594659f5d6 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -9,8 +9,8 @@ #pragma once -#include "eigen_matrix.h" +#include "eigen/matrix.h" #if EIGEN_VERSION_AT_LEAST(3, 3, 0) -# include "eigen_tensor.h" +# include "eigen/tensor.h" #endif diff --git a/include/pybind11/eigen_matrix.h b/include/pybind11/eigen/matrix.h similarity index 99% rename from include/pybind11/eigen_matrix.h rename to include/pybind11/eigen/matrix.h index 348641fc43..7c1fd5fa34 100644 --- a/include/pybind11/eigen_matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -1,6 +1,6 @@ /* - pybind11/eigen.h: Transparent conversion for dense and sparse Eigen matrices + pybind11/eigen/matrix.h: Transparent conversion for dense and sparse Eigen matrices Copyright (c) 2016 Wenzel Jakob @@ -10,7 +10,7 @@ #pragma once -#include "numpy.h" +#include "../numpy.h" // Similar to comments & pragma block in eigen_tensor.h. PLEASE KEEP IN SYNC. /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. diff --git a/include/pybind11/eigen_tensor.h b/include/pybind11/eigen/tensor.h similarity index 92% rename from include/pybind11/eigen_tensor.h rename to include/pybind11/eigen/tensor.h index c17e80e444..ffbaaceab7 100644 --- a/include/pybind11/eigen_tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -1,6 +1,6 @@ /* - pybind11/eigen_tensor.h: Transparent conversion for Eigen tensors + pybind11/eigen/tensor.h: Transparent conversion for Eigen tensors All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. @@ -8,20 +8,8 @@ #pragma once -#include "numpy.h" +#include "../numpy.h" -// Similar to comments & pragma block in eigen_matrix.h. PLEASE KEEP IN SYNC. -/* HINT: To suppress warnings originating from the Eigen headers, use -isystem. - See also: - https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir - https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler -*/ - -// The C4127 suppression was introduced for Eigen 3.4.0. In theory we could -// make it version specific, or even remove it later, but considering that -// 1. C4127 is generally far more distracting than useful for modern template code, and -// 2. we definitely want to ignore any MSVC warnings originating from Eigen code, -// it is probably best to keep this around indefinitely. #if defined(_MSC_VER) # pragma warning(push) # pragma warning(disable : 4554) // Tensor.h warning @@ -65,6 +53,7 @@ struct eigen_tensor_helper {}; template struct eigen_tensor_helper> { using Type = Eigen::Tensor; + using ConstType = Eigen::Tensor; using ValidType = void; static Eigen::DSizes get_shape(const Type &f) { @@ -109,6 +98,7 @@ template , Options_, IndexType>> { using Type = Eigen::TensorFixedSize, Options_, IndexType>; + using ConstType = Eigen::TensorFixedSize, Options_, IndexType>; using ValidType = void; static constexpr Eigen::DSizes @@ -144,10 +134,10 @@ template struct get_tensor_descriptor { static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") + eigen_tensor_helper::dimensions_descriptor - + const_name("], flags.writeable, ") + + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("]") + + const_name::value>("", ", flags.writeable") + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>( - "flags.c_contiguous]", "flags.f_contiguous]"); + ", flags.c_contiguous]", ", flags.f_contiguous]"); }; template @@ -171,10 +161,9 @@ struct type_caster::ValidType> { } if (is_tensor_aligned(arr.data())) { - value = Eigen::TensorMap( - const_cast(arr.data()), shape); + value = Eigen::TensorMap(arr.data(), shape); } else { - value = Eigen::TensorMap(const_cast(arr.data()), shape); + value = Eigen::TensorMap(arr.data(), shape); } return true; @@ -297,6 +286,16 @@ struct type_caster::ValidType> { } }; +template ::value>> +const void* get_array_data_for_type(array& arr) { + return arr.data(); +} + +template ::value>> +void* get_array_data_for_type(array& arr) { + return arr.mutable_data(); +} + template struct type_caster, typename eigen_tensor_helper::ValidType> { @@ -331,8 +330,12 @@ struct type_caster, return false; } + if (!std::is_const::value && !arr.writeable()) { + return false; + } + value.reset( - new MapType(reinterpret_cast(arr.mutable_data()), shape)); + new MapType(static_cast(get_array_data_for_type(arr)), shape)); return true; } diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index e8bac488af..10e590a293 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -49,8 +49,6 @@ "include/pybind11/detail/class.h", "include/pybind11/detail/common.h", "include/pybind11/detail/descr.h", - "include/pybind11/detail/eigen_tensor.h", - "include/pybind11/detail/eigen_matrix.h", "include/pybind11/detail/init.h", "include/pybind11/detail/internals.h", "include/pybind11/detail/type_caster_base.h", @@ -61,6 +59,11 @@ "include/pybind11/stl/filesystem.h", } +eigen_headers = { + "include/pybind11/eigen/tensor.h", + "include/pybind11/eigen/matrix.h", +} + cmake_files = { "share/cmake/pybind11/FindPythonLibsNew.cmake", "share/cmake/pybind11/pybind11Common.cmake", @@ -84,7 +87,7 @@ "setup_helpers.py", } -headers = main_headers | detail_headers | stl_headers +headers = main_headers | detail_headers | stl_headers | eigen_headers src_files = headers | cmake_files | pkgconfig_files all_files = src_files | py_files diff --git a/tests/test_eigen_matrix.cpp b/tests/test_eigen_matrix.cpp index 16aca3db8a..b98eee492e 100644 --- a/tests/test_eigen_matrix.cpp +++ b/tests/test_eigen_matrix.cpp @@ -7,7 +7,7 @@ BSD-style license that can be found in the LICENSE file. */ -#include +#include #include #include "constructor_stats.h" diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index b16368876b..52e9951fcc 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -5,7 +5,7 @@ BSD-style license that can be found in the LICENSE file. */ -#include +#include #include "pybind11_tests.h" @@ -55,6 +55,8 @@ TEST_SUBMODULE(eigen_tensor, m) { m.def("move_tensor", []() { return get_tensor(); }); + m.def("move_const_tensor", []() -> const Eigen::Tensor { return get_const_tensor(); }); + m.def( "take_fixed_tensor", []() { @@ -69,10 +71,19 @@ TEST_SUBMODULE(eigen_tensor, m) { "take_tensor", []() { return new Eigen::Tensor(get_tensor()); }, py::return_value_policy::take_ownership); + + m.def("take_const_tensor", []() -> const Eigen::Tensor* { return new Eigen::Tensor(get_tensor());}, + py::return_value_policy::take_ownership); m.def( "reference_tensor", []() { return &get_tensor(); }, py::return_value_policy::reference); + m.def( + "reference_tensor_v2", []() -> Eigen::Tensor& { return get_tensor(); }, py::return_value_policy::reference); + + m.def( + "reference_tensor_internal", []() { return &get_tensor(); }, py::return_value_policy::reference_internal); + m.def( "reference_fixed_tensor", []() { return &get_tensor(); }, @@ -83,6 +94,11 @@ TEST_SUBMODULE(eigen_tensor, m) { []() { return &get_const_tensor(); }, py::return_value_policy::reference); + m.def( + "reference_const_tensor_v2", + []() -> const Eigen::Tensor& { return get_const_tensor(); }, + py::return_value_policy::reference); + m.def( "reference_view_of_tensor", []() { return Eigen::TensorMap>(get_tensor()); }, @@ -107,4 +123,10 @@ TEST_SUBMODULE(eigen_tensor, m) { "round_trip_aligned_view_tensor", [](Eigen::TensorMap, Eigen::Aligned> view) { return view; }, py::return_value_policy::reference); + + m.def( + "round_trip_const_view_tensor", + [](Eigen::TensorMap> view) { return Eigen::Tensor(view); }, + py::return_value_policy::move); + } diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 6ff04aa448..944b279e61 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -36,12 +36,44 @@ def test_convert_tensor_to_py(): assert_equal_tensor_ref(m.take_fixed_tensor()) assert_equal_tensor_ref(m.reference_tensor()) + assert_equal_tensor_ref(m.reference_tensor_v2()) assert_equal_tensor_ref(m.reference_fixed_tensor()) assert_equal_tensor_ref(m.reference_view_of_tensor()) assert_equal_tensor_ref(m.reference_view_of_fixed_tensor()) assert_equal_tensor_ref(m.reference_const_tensor(), writeable=False) - + assert_equal_tensor_ref(m.reference_const_tensor_v2(), writeable=False) + + +def test_bad_cpp_to_python_casts(): + with pytest.raises(Exception): + m.reference_tensor_internal() + + with pytest.raises(Exception): + m.move_const_tensor() + + with pytest.raises(Exception): + m.take_const_tensor() + + +def test_bad_python_to_cpp_casts(): + with pytest.raises(TypeError): + m.round_trip_tensor(np.zeros((2, 3))) + + with pytest.raises(TypeError): + m.round_trip_tensor(np.zeros(dtype=np.str_, shape=(2, 3, 1))) + + # Shape, dtype and the order need to be correct for a TensorMap cast + with pytest.raises(TypeError): + m.round_trip_view_tensor(np.zeros((3, 5, 2), dtype=np.float64, order="C")) + with pytest.raises(TypeError): + m.round_trip_view_tensor(np.zeros((3, 5, 2), dtype=np.float32, order="F")) + with pytest.raises(TypeError): + m.round_trip_view_tensor(np.zeros((3, 5), dtype=np.float64, order="F")) + with pytest.raises(TypeError): + temp = np.zeros((3, 5, 2), dtype=np.float64, order="F") + temp.setflags(write=False) + m.round_trip_view_tensor(temp) def test_references_actually_refer(): a = m.reference_tensor() @@ -60,11 +92,12 @@ def test_references_actually_refer(): def test_round_trip(): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) - - -def test_aligned_view(): assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) - + + copy = np.array(tensor_ref, dtype=np.float64, order="F") + assert_equal_tensor_ref(m.round_trip_view_tensor(copy)) + copy.setflags(write=False) + assert_equal_tensor_ref(m.round_trip_const_view_tensor(copy)) def test_round_trip_references_actually_refer(): # Need to create a copy that matches the type on the C side From 992795b4936eb925ed46f3098791bca1bfc68926 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 5 Oct 2022 18:12:16 +0000 Subject: [PATCH 056/141] Improve names --- include/pybind11/eigen/tensor.h | 9 +++++---- tests/test_eigen_tensor.py | 17 +++++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index ffbaaceab7..8460108afd 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -130,12 +130,12 @@ struct eigen_tensor_helper< } }; -template +template struct get_tensor_descriptor { static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("]") - + const_name::value>("", ", flags.writeable") + + const_name::value>("", ", flags.writeable") + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>( ", flags.c_contiguous]", ", flags.f_contiguous]"); }; @@ -143,7 +143,8 @@ struct get_tensor_descriptor { template struct type_caster::ValidType> { using Helper = eigen_tensor_helper; - PYBIND11_TYPE_CASTER(Type, get_tensor_descriptor::value); + static constexpr auto temp_name = get_tensor_descriptor::value; + PYBIND11_TYPE_CASTER(Type, temp_name); bool load(handle src, bool /*convert*/) { array_t()> arr( @@ -420,7 +421,7 @@ struct type_caster, std::unique_ptr value; public: - static constexpr auto name = get_tensor_descriptor::value; + static constexpr auto name = get_tensor_descriptor::value; explicit operator MapType *() { return value.get(); } explicit operator MapType &() { return *value; } explicit operator MapType &&() && { return std::move(*value); } diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 944b279e61..15dde0d855 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -113,16 +113,21 @@ def test_round_trip_references_actually_refer(): def test_doc_string(doc): assert ( doc(m.copy_tensor) - == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]" + == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]" ) assert ( doc(m.copy_fixed_tensor) - == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2], flags.writeable, flags.f_contiguous]" + == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2], flags.f_contiguous]" ) - - # Ideally this type signature wouldn't have flags.writeable, but - # I don't see a way to avoid it due to names being set at compile time ... assert ( doc(m.reference_const_tensor) - == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]" + == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]" + ) + assert ( + doc(m.round_trip_view_tensor) + == "round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]) -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]" + ) + assert ( + doc(m.round_trip_const_view_tensor) + == "round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]) -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]" ) From c364893de43b3d9e354053edf3e639abacd6791e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Oct 2022 18:18:15 +0000 Subject: [PATCH 057/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 27 +++++++++++++++++---------- tests/test_eigen_tensor.cpp | 26 ++++++++++++++++++-------- tests/test_eigen_tensor.py | 8 +++++--- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 8460108afd..c1598a62bc 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -98,7 +98,8 @@ template , Options_, IndexType>> { using Type = Eigen::TensorFixedSize, Options_, IndexType>; - using ConstType = Eigen::TensorFixedSize, Options_, IndexType>; + using ConstType + = Eigen::TensorFixedSize, Options_, IndexType>; using ValidType = void; static constexpr Eigen::DSizes @@ -134,10 +135,14 @@ template struct get_tensor_descriptor { static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("]") - + const_name::value>("", ", flags.writeable") - + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>( - ", flags.c_contiguous]", ", flags.f_contiguous]"); + + const_name("[") + eigen_tensor_helper::dimensions_descriptor + + const_name("]") + const_name + < AlwaysRead + || std::is_const::value + > ("", ", flags.writeable") + + const_name(Type::Layout) + == static_cast(Eigen::RowMajor)>(", flags.c_contiguous]", + ", flags.f_contiguous]"); }; template @@ -162,7 +167,8 @@ struct type_caster::ValidType> { } if (is_tensor_aligned(arr.data())) { - value = Eigen::TensorMap(arr.data(), shape); + value + = Eigen::TensorMap(arr.data(), shape); } else { value = Eigen::TensorMap(arr.data(), shape); } @@ -288,12 +294,12 @@ struct type_caster::ValidType> { }; template ::value>> -const void* get_array_data_for_type(array& arr) { +const void *get_array_data_for_type(array &arr) { return arr.data(); } template ::value>> -void* get_array_data_for_type(array& arr) { +void *get_array_data_for_type(array &arr) { return arr.mutable_data(); } @@ -335,8 +341,9 @@ struct type_caster, return false; } - value.reset( - new MapType(static_cast(get_array_data_for_type(arr)), shape)); + value.reset(new MapType(static_cast( + get_array_data_for_type(arr)), + shape)); return true; } diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 52e9951fcc..0adaa9423a 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -55,7 +55,8 @@ TEST_SUBMODULE(eigen_tensor, m) { m.def("move_tensor", []() { return get_tensor(); }); - m.def("move_const_tensor", []() -> const Eigen::Tensor { return get_const_tensor(); }); + m.def("move_const_tensor", + []() -> const Eigen::Tensor { return get_const_tensor(); }); m.def( "take_fixed_tensor", @@ -71,18 +72,26 @@ TEST_SUBMODULE(eigen_tensor, m) { "take_tensor", []() { return new Eigen::Tensor(get_tensor()); }, py::return_value_policy::take_ownership); - - m.def("take_const_tensor", []() -> const Eigen::Tensor* { return new Eigen::Tensor(get_tensor());}, + + m.def( + "take_const_tensor", + []() -> const Eigen::Tensor * { + return new Eigen::Tensor(get_tensor()); + }, py::return_value_policy::take_ownership); m.def( "reference_tensor", []() { return &get_tensor(); }, py::return_value_policy::reference); m.def( - "reference_tensor_v2", []() -> Eigen::Tensor& { return get_tensor(); }, py::return_value_policy::reference); + "reference_tensor_v2", + []() -> Eigen::Tensor & { return get_tensor(); }, + py::return_value_policy::reference); m.def( - "reference_tensor_internal", []() { return &get_tensor(); }, py::return_value_policy::reference_internal); + "reference_tensor_internal", + []() { return &get_tensor(); }, + py::return_value_policy::reference_internal); m.def( "reference_fixed_tensor", @@ -96,7 +105,7 @@ TEST_SUBMODULE(eigen_tensor, m) { m.def( "reference_const_tensor_v2", - []() -> const Eigen::Tensor& { return get_const_tensor(); }, + []() -> const Eigen::Tensor & { return get_const_tensor(); }, py::return_value_policy::reference); m.def( @@ -126,7 +135,8 @@ TEST_SUBMODULE(eigen_tensor, m) { m.def( "round_trip_const_view_tensor", - [](Eigen::TensorMap> view) { return Eigen::Tensor(view); }, + [](Eigen::TensorMap> view) { + return Eigen::Tensor(view); + }, py::return_value_policy::move); - } diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 15dde0d855..d168c84be4 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -43,12 +43,12 @@ def test_convert_tensor_to_py(): assert_equal_tensor_ref(m.reference_view_of_fixed_tensor()) assert_equal_tensor_ref(m.reference_const_tensor(), writeable=False) assert_equal_tensor_ref(m.reference_const_tensor_v2(), writeable=False) - + def test_bad_cpp_to_python_casts(): with pytest.raises(Exception): m.reference_tensor_internal() - + with pytest.raises(Exception): m.move_const_tensor() @@ -75,6 +75,7 @@ def test_bad_python_to_cpp_casts(): temp.setflags(write=False) m.round_trip_view_tensor(temp) + def test_references_actually_refer(): a = m.reference_tensor() temp = a[indices] @@ -93,12 +94,13 @@ def test_references_actually_refer(): def test_round_trip(): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) - + copy = np.array(tensor_ref, dtype=np.float64, order="F") assert_equal_tensor_ref(m.round_trip_view_tensor(copy)) copy.setflags(write=False) assert_equal_tensor_ref(m.round_trip_const_view_tensor(copy)) + def test_round_trip_references_actually_refer(): # Need to create a copy that matches the type on the C side copy = np.array(tensor_ref, dtype=np.float64, order="F") From 72ace1b813e61732a97194b9be2497f69a177389 Mon Sep 17 00:00:00 2001 From: Lalaland Date: Wed, 5 Oct 2022 11:28:34 -0700 Subject: [PATCH 058/141] Update include/pybind11/eigen/tensor.h Co-authored-by: Aaron Gokaslan --- include/pybind11/eigen/tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index c1598a62bc..7c00b1ba03 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -325,7 +325,7 @@ struct type_caster, } // Use temporary to avoid MSVC warning ... - bool is_aligned = (Options & Eigen::Aligned) != 0; + constexpr bool is_aligned = (Options & Eigen::Aligned) != 0; if (is_aligned && !is_tensor_aligned(arr.data())) { return false; } From 40a3976299646d25413ab5090b984b27139c3134 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 5 Oct 2022 18:24:09 +0000 Subject: [PATCH 059/141] Fix tests --- tests/extra_python_package/test_files.py | 1 + tests/test_eigen_tensor.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 10e590a293..e4e0c616b9 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -98,6 +98,7 @@ "pybind11/include/pybind11", "pybind11/include/pybind11/detail", "pybind11/include/pybind11/stl", + "pybind11/include/pybind11/eigen", "pybind11/share", "pybind11/share/cmake", "pybind11/share/cmake/pybind11", diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index d168c84be4..1ab9cf30da 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -127,9 +127,11 @@ def test_doc_string(doc): ) assert ( doc(m.round_trip_view_tensor) - == "round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]) -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]" + == ("round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous])" + + " -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]") ) assert ( doc(m.round_trip_const_view_tensor) - == "round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]) -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]" + == ("round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous])" + + " -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]") ) From 908da625160f4a02cc392dbb8251dae677f9c2de Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Oct 2022 18:31:50 +0000 Subject: [PATCH 060/141] style: pre-commit fixes --- tests/test_eigen_tensor.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 1ab9cf30da..79d5b92ced 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -125,13 +125,11 @@ def test_doc_string(doc): doc(m.reference_const_tensor) == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]" ) - assert ( - doc(m.round_trip_view_tensor) - == ("round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous])" + - " -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]") + assert doc(m.round_trip_view_tensor) == ( + "round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous])" + + " -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]" ) - assert ( - doc(m.round_trip_const_view_tensor) - == ("round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous])" + - " -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]") + assert doc(m.round_trip_const_view_tensor) == ( + "round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous])" + + " -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]" ) From e8c06c856cb06a06cfea26a529e0b8642f899bb5 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 5 Oct 2022 18:41:17 +0000 Subject: [PATCH 061/141] Update --- include/pybind11/eigen/tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 7c00b1ba03..c1598a62bc 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -325,7 +325,7 @@ struct type_caster, } // Use temporary to avoid MSVC warning ... - constexpr bool is_aligned = (Options & Eigen::Aligned) != 0; + bool is_aligned = (Options & Eigen::Aligned) != 0; if (is_aligned && !is_tensor_aligned(arr.data())) { return false; } From 2dd6c95e472bd208f69856706c4b8d45d98d3424 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 5 Oct 2022 20:07:07 +0000 Subject: [PATCH 062/141] Add a test to verify that strange numpy arrays work --- tests/test_eigen_tensor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 79d5b92ced..129d0614a7 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -100,6 +100,7 @@ def test_round_trip(): copy.setflags(write=False) assert_equal_tensor_ref(m.round_trip_const_view_tensor(copy)) + np.testing.assert_array_equal(tensor_ref[:, ::-1], m.round_trip_tensor(tensor_ref[:, ::-1, :])) def test_round_trip_references_actually_refer(): # Need to create a copy that matches the type on the C side From a3387c3039694eee147c4743a730cb0b3299d056 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 5 Oct 2022 20:18:54 +0000 Subject: [PATCH 063/141] Fix dumb compiler warning --- include/pybind11/eigen/tensor.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index c1598a62bc..38cb9836c9 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -337,7 +337,8 @@ struct type_caster, return false; } - if (!std::is_const::value && !arr.writeable()) { + bool needs_writeable = !std::is_const::value; + if (needs_writeable && !arr.writeable()) { return false; } From b41ff6e483cfbf0604fdf6f24f1cbb037211cd9d Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 5 Oct 2022 21:06:15 +0000 Subject: [PATCH 064/141] Better tests --- include/pybind11/eigen/tensor.h | 22 ++++---- tests/test_eigen_tensor.cpp | 98 +++++++++++++++++++++------------ tests/test_eigen_tensor.py | 73 ++++++++++++++---------- tools/setup_global.py.in | 4 +- tools/setup_main.py.in | 2 + 5 files changed, 122 insertions(+), 77 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 38cb9836c9..c8c9c17612 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -131,24 +131,26 @@ struct eigen_tensor_helper< } }; -template +template struct get_tensor_descriptor { + static constexpr auto details = + const_name::value> ("", ", flags.writeable") + + const_name(Type::Layout) + == static_cast(Eigen::RowMajor)>(", flags.c_contiguous", + ", flags.f_contiguous"); static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + eigen_tensor_helper::dimensions_descriptor - + const_name("]") + const_name - < AlwaysRead - || std::is_const::value - > ("", ", flags.writeable") - + const_name(Type::Layout) - == static_cast(Eigen::RowMajor)>(", flags.c_contiguous]", - ", flags.f_contiguous]"); + + const_name("]") + const_name( + details, + const_name("") + ) + const_name("]"); }; template struct type_caster::ValidType> { using Helper = eigen_tensor_helper; - static constexpr auto temp_name = get_tensor_descriptor::value; + static constexpr auto temp_name = get_tensor_descriptor::value; PYBIND11_TYPE_CASTER(Type, temp_name); bool load(handle src, bool /*convert*/) { @@ -429,7 +431,7 @@ struct type_caster, std::unique_ptr value; public: - static constexpr auto name = get_tensor_descriptor::value; + static constexpr auto name = get_tensor_descriptor::value; explicit operator MapType *() { return value.get(); } explicit operator MapType &() { return *value; } explicit operator MapType &&() && { return std::move(*value); } diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 0adaa9423a..f879b860d4 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -9,134 +9,160 @@ #include "pybind11_tests.h" + template void reset_tensor(M &x) { - for (int i = 0; i < x.size(); i++) { - x(i) = i; + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 5; j++) { + for (int k = 0; k < 2; k++) { + x(i, j, k) = i * (5 * 2) + j * 2 + k; + } + } } } -Eigen::Tensor &get_tensor() { - static Eigen::Tensor *x; +template +Eigen::Tensor &get_tensor() { + static Eigen::Tensor *x; if (!x) { - x = new Eigen::Tensor(3, 5, 2); + x = new Eigen::Tensor(3, 5, 2); reset_tensor(*x); } return *x; } -Eigen::TensorFixedSize> &get_fixed_tensor() { - static Eigen::TensorFixedSize> *x; +template +Eigen::TensorFixedSize, Options> &get_fixed_tensor() { + static Eigen::TensorFixedSize, Options> *x; if (!x) { - Eigen::aligned_allocator>> allocator; - x = new (allocator.allocate(1)) Eigen::TensorFixedSize>(); + Eigen::aligned_allocator, Options>> allocator; + x = new (allocator.allocate(1)) Eigen::TensorFixedSize, Options>(); reset_tensor(*x); } return *x; } -const Eigen::Tensor &get_const_tensor() { return get_tensor(); } +template +const Eigen::Tensor &get_const_tensor() { return get_tensor(); } + +template +void init_tensor_module(pybind11::module& m) { + const char* needed_options = ""; + if (Options == Eigen::ColMajor) { + needed_options = "F"; + } else { + needed_options = "C"; + } + m.attr("needed_options") = needed_options; -TEST_SUBMODULE(eigen_tensor, m) { m.def( - "copy_fixed_tensor", []() { return &get_fixed_tensor(); }, py::return_value_policy::copy); + "copy_fixed_tensor", []() { return &get_fixed_tensor(); }, py::return_value_policy::copy); m.def( - "copy_tensor", []() { return &get_tensor(); }, py::return_value_policy::copy); + "copy_tensor", []() { return &get_tensor(); }, py::return_value_policy::copy); m.def( - "copy_const_tensor", []() { return &get_const_tensor(); }, py::return_value_policy::copy); + "copy_const_tensor", []() { return &get_const_tensor(); }, py::return_value_policy::copy); - m.def("move_fixed_tensor", []() { return get_fixed_tensor(); }); + m.def("move_fixed_tensor", []() { return get_fixed_tensor(); }); - m.def("move_tensor", []() { return get_tensor(); }); + m.def("move_tensor", []() { return get_tensor(); }); m.def("move_const_tensor", - []() -> const Eigen::Tensor { return get_const_tensor(); }); + []() -> const Eigen::Tensor { return get_const_tensor(); }); m.def( "take_fixed_tensor", []() { - Eigen::aligned_allocator>> + Eigen::aligned_allocator, Options>> allocator; return new (allocator.allocate(1)) - Eigen::TensorFixedSize>(get_fixed_tensor()); + Eigen::TensorFixedSize, Options>(get_fixed_tensor()); }, py::return_value_policy::take_ownership); m.def( "take_tensor", - []() { return new Eigen::Tensor(get_tensor()); }, + []() { return new Eigen::Tensor(get_tensor()); }, py::return_value_policy::take_ownership); m.def( "take_const_tensor", - []() -> const Eigen::Tensor * { - return new Eigen::Tensor(get_tensor()); + []() -> const Eigen::Tensor * { + return new Eigen::Tensor(get_tensor()); }, py::return_value_policy::take_ownership); m.def( - "reference_tensor", []() { return &get_tensor(); }, py::return_value_policy::reference); + "reference_tensor", []() { return &get_tensor(); }, py::return_value_policy::reference); m.def( "reference_tensor_v2", - []() -> Eigen::Tensor & { return get_tensor(); }, + []() -> Eigen::Tensor & { return get_tensor(); }, py::return_value_policy::reference); m.def( "reference_tensor_internal", - []() { return &get_tensor(); }, + []() { return &get_tensor(); }, py::return_value_policy::reference_internal); m.def( "reference_fixed_tensor", - []() { return &get_tensor(); }, + []() { return &get_tensor(); }, py::return_value_policy::reference); m.def( "reference_const_tensor", - []() { return &get_const_tensor(); }, + []() { return &get_const_tensor(); }, py::return_value_policy::reference); m.def( "reference_const_tensor_v2", - []() -> const Eigen::Tensor & { return get_const_tensor(); }, + []() -> const Eigen::Tensor & { return get_const_tensor(); }, py::return_value_policy::reference); m.def( "reference_view_of_tensor", - []() { return Eigen::TensorMap>(get_tensor()); }, + []() { return Eigen::TensorMap>(get_tensor()); }, py::return_value_policy::reference); m.def( "reference_view_of_fixed_tensor", []() { - return Eigen::TensorMap>>( - get_fixed_tensor()); + return Eigen::TensorMap, Options>>( + get_fixed_tensor()); }, py::return_value_policy::reference); - m.def("round_trip_tensor", [](const Eigen::Tensor &tensor) { return tensor; }); + m.def("round_trip_tensor", [](const Eigen::Tensor &tensor) { return tensor; }); m.def( "round_trip_view_tensor", - [](Eigen::TensorMap> view) { return view; }, + [](Eigen::TensorMap> view) { return view; }, py::return_value_policy::reference); m.def( "round_trip_aligned_view_tensor", - [](Eigen::TensorMap, Eigen::Aligned> view) { return view; }, + [](Eigen::TensorMap, Eigen::Aligned> view) { return view; }, py::return_value_policy::reference); m.def( "round_trip_const_view_tensor", - [](Eigen::TensorMap> view) { - return Eigen::Tensor(view); + [](Eigen::TensorMap> view) { + return Eigen::Tensor(view); }, py::return_value_policy::move); + +} + +TEST_SUBMODULE(eigen_tensor, m) { + auto f_style = m.def_submodule("f_style"); + auto c_style = m.def_submodule("c_style"); + + init_tensor_module(f_style); + init_tensor_module(c_style); } diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 129d0614a7..42024f2cac 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -1,15 +1,16 @@ import pytest np = pytest.importorskip("numpy") -m = pytest.importorskip("pybind11_tests.eigen_tensor") +eigen_tensor = pytest.importorskip("pybind11_tests.eigen_tensor") -tensor_ref = np.array( - [ - [[0.0, 15.0], [3.0, 18.0], [6.0, 21.0], [9.0, 24.0], [12.0, 27.0]], - [[1.0, 16.0], [4.0, 19.0], [7.0, 22.0], [10.0, 25.0], [13.0, 28.0]], - [[2.0, 17.0], [5.0, 20.0], [8.0, 23.0], [11.0, 26.0], [14.0, 29.0]], - ], -) +submodules = [eigen_tensor.c_style, eigen_tensor.f_style] + +tensor_ref = np.zeros((3, 5, 2)) + +for i in range(3): + for j in range(5): + for k in range(2): + tensor_ref[i, j, k] = i * (5 * 2) + j * 2 + k indices = (2, 3, 1) @@ -23,8 +24,8 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=0): np.testing.assert_array_equal(mat, copy) - -def test_convert_tensor_to_py(): +@pytest.mark.parametrize('m', submodules) +def test_convert_tensor_to_py(m): assert_equal_tensor_ref(m.copy_tensor()) assert_equal_tensor_ref(m.copy_fixed_tensor()) assert_equal_tensor_ref(m.copy_const_tensor()) @@ -45,7 +46,8 @@ def test_convert_tensor_to_py(): assert_equal_tensor_ref(m.reference_const_tensor_v2(), writeable=False) -def test_bad_cpp_to_python_casts(): +@pytest.mark.parametrize('m', submodules) +def test_bad_cpp_to_python_casts(m): with pytest.raises(Exception): m.reference_tensor_internal() @@ -56,27 +58,33 @@ def test_bad_cpp_to_python_casts(): m.take_const_tensor() -def test_bad_python_to_cpp_casts(): +@pytest.mark.parametrize('m', submodules) +def test_bad_python_to_cpp_casts(m): with pytest.raises(TypeError): m.round_trip_tensor(np.zeros((2, 3))) with pytest.raises(TypeError): m.round_trip_tensor(np.zeros(dtype=np.str_, shape=(2, 3, 1))) + if m.needed_options == "F": + bad_options = "C" + else: + bad_options = "F" # Shape, dtype and the order need to be correct for a TensorMap cast with pytest.raises(TypeError): - m.round_trip_view_tensor(np.zeros((3, 5, 2), dtype=np.float64, order="C")) + m.round_trip_view_tensor(np.zeros((3, 5, 2), dtype=np.float64, order=bad_options)) with pytest.raises(TypeError): - m.round_trip_view_tensor(np.zeros((3, 5, 2), dtype=np.float32, order="F")) + m.round_trip_view_tensor(np.zeros((3, 5, 2), dtype=np.float32, order=m.needed_options)) with pytest.raises(TypeError): - m.round_trip_view_tensor(np.zeros((3, 5), dtype=np.float64, order="F")) + m.round_trip_view_tensor(np.zeros((3, 5), dtype=np.float64, order=m.needed_options)) with pytest.raises(TypeError): - temp = np.zeros((3, 5, 2), dtype=np.float64, order="F") + temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) temp.setflags(write=False) m.round_trip_view_tensor(temp) -def test_references_actually_refer(): +@pytest.mark.parametrize('m', submodules) +def test_references_actually_refer(m): a = m.reference_tensor() temp = a[indices] a[indices] = 100 @@ -91,20 +99,22 @@ def test_references_actually_refer(): assert_equal_tensor_ref(m.copy_const_tensor()) -def test_round_trip(): +@pytest.mark.parametrize('m', submodules) +def test_round_trip(m): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) - copy = np.array(tensor_ref, dtype=np.float64, order="F") + copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) assert_equal_tensor_ref(m.round_trip_view_tensor(copy)) copy.setflags(write=False) assert_equal_tensor_ref(m.round_trip_const_view_tensor(copy)) - np.testing.assert_array_equal(tensor_ref[:, ::-1], m.round_trip_tensor(tensor_ref[:, ::-1, :])) + np.testing.assert_array_equal(tensor_ref[:, ::-1, :], m.round_trip_tensor(tensor_ref[:, ::-1, :])) -def test_round_trip_references_actually_refer(): +@pytest.mark.parametrize('m', submodules) +def test_round_trip_references_actually_refer(m): # Need to create a copy that matches the type on the C side - copy = np.array(tensor_ref, dtype=np.float64, order="F") + copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) a = m.round_trip_view_tensor(copy) temp = a[indices] a[indices] = 100 @@ -113,24 +123,27 @@ def test_round_trip_references_actually_refer(): assert_equal_tensor_ref(copy) -def test_doc_string(doc): +@pytest.mark.parametrize('m', submodules) +def test_doc_string(m, doc): assert ( doc(m.copy_tensor) - == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]" + == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" ) assert ( doc(m.copy_fixed_tensor) - == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2], flags.f_contiguous]" + == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2]]" ) assert ( doc(m.reference_const_tensor) - == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]" + == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" ) + + order_flag = f'flags.{m.needed_options.lower()}_contiguous' assert doc(m.round_trip_view_tensor) == ( - "round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous])" - + " -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, flags.f_contiguous]" + f"round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}])" + + f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]" ) assert doc(m.round_trip_const_view_tensor) == ( - "round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous])" - + " -> numpy.ndarray[numpy.float64[?, ?, ?], flags.f_contiguous]" + f"round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], {order_flag}])" + + " -> numpy.ndarray[numpy.float64[?, ?, ?]]" ) diff --git a/tools/setup_global.py.in b/tools/setup_global.py.in index d91468c10c..a99dfdeb2d 100644 --- a/tools/setup_global.py.in +++ b/tools/setup_global.py.in @@ -28,9 +28,10 @@ class InstallHeadersNested(install_headers): main_headers = glob.glob("pybind11/include/pybind11/*.h") detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h") stl_headers = glob.glob("pybind11/include/pybind11/stl/*.h") +eigen_headers = glob.glob("pybind11/include/pybind11/eigen/*.h") cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake") pkgconfig_files = glob.glob("pybind11/share/pkgconfig/*.pc") -headers = main_headers + detail_headers + stl_headers +headers = main_headers + detail_headers + stl_headers + eigen_headers cmdclass = {"install_headers": InstallHeadersNested} $extra_cmd @@ -56,6 +57,7 @@ setup( (base + "include/pybind11", main_headers), (base + "include/pybind11/detail", detail_headers), (base + "include/pybind11/stl", stl_headers), + (base + "include/pybind11/eigen", eigen_headers), ], cmdclass=cmdclass, ) diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index 65198bdb66..f1be8e701f 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -16,6 +16,7 @@ setup( "pybind11.include.pybind11", "pybind11.include.pybind11.detail", "pybind11.include.pybind11.stl", + "pybind11.include.pybind11.eigen", "pybind11.share.cmake.pybind11", "pybind11.share.pkgconfig", ], @@ -24,6 +25,7 @@ setup( "pybind11.include.pybind11": ["*.h"], "pybind11.include.pybind11.detail": ["*.h"], "pybind11.include.pybind11.stl": ["*.h"], + "pybind11.include.pybind11.eigen": ["*.h"], "pybind11.share.cmake.pybind11": ["*.cmake"], "pybind11.share.pkgconfig": ["*.pc"], }, From 4a7179d500c9295a3b03912a54e1d821ccb98a56 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 5 Oct 2022 21:37:51 +0000 Subject: [PATCH 065/141] Better tests --- tests/test_eigen_tensor.cpp | 67 +++++++++++++++++++++++++++++++++++-- tests/test_eigen_tensor.py | 14 ++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index f879b860d4..8233a29113 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -33,6 +33,17 @@ Eigen::Tensor &get_tensor() { return *x; } +template +Eigen::TensorMap> &get_tensor_map() { + static Eigen::TensorMap> *x; + + if (!x) { + x = new Eigen::TensorMap>(get_tensor()); + } + + return *x; +} + template Eigen::TensorFixedSize, Options> &get_fixed_tensor() { static Eigen::TensorFixedSize, Options> *x; @@ -49,15 +60,30 @@ Eigen::TensorFixedSize, Options> &get_fixed_tensor template const Eigen::Tensor &get_const_tensor() { return get_tensor(); } +template +struct CustomExample { + CustomExample() { + member = get_tensor(); + } + + Eigen::Tensor member; +}; + + template void init_tensor_module(pybind11::module& m) { const char* needed_options = ""; - if (Options == Eigen::ColMajor) { + bool is_major = Options == Eigen::ColMajor; + if (is_major) { needed_options = "F"; } else { needed_options = "C"; } m.attr("needed_options") = needed_options; + + py::class_>(m, "CustomExample") + .def(py::init<>()) + .def_readonly("member", &CustomExample::member, py::return_value_policy::reference_internal); m.def( "copy_fixed_tensor", []() { return &get_fixed_tensor(); }, py::return_value_policy::copy); @@ -127,7 +153,44 @@ void init_tensor_module(pybind11::module& m) { m.def( "reference_view_of_tensor", - []() { return Eigen::TensorMap>(get_tensor()); }, + []() { + return get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v2", + []() -> const Eigen::TensorMap> { + return get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v3", + []() { + return &get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v4", + []() -> const Eigen::TensorMap>* { + return &get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v5", + []() -> Eigen::TensorMap>& { + return get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v6", + []() -> const Eigen::TensorMap>& { + return get_tensor_map(); + }, py::return_value_policy::reference); m.def( diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 42024f2cac..2970cc1f89 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -1,3 +1,4 @@ +import sys import pytest np = pytest.importorskip("numpy") @@ -30,6 +31,14 @@ def test_convert_tensor_to_py(m): assert_equal_tensor_ref(m.copy_fixed_tensor()) assert_equal_tensor_ref(m.copy_const_tensor()) + foo = m.CustomExample() + counts = sys.getrefcount(foo) + mem = foo.member + assert_equal_tensor_ref(mem, writeable=False) + new_counts = sys.getrefcount(foo) + assert new_counts == counts + 1 + assert_equal_tensor_ref(mem, writeable=False) + assert_equal_tensor_ref(m.move_tensor()) assert_equal_tensor_ref(m.move_fixed_tensor()) @@ -41,6 +50,11 @@ def test_convert_tensor_to_py(m): assert_equal_tensor_ref(m.reference_fixed_tensor()) assert_equal_tensor_ref(m.reference_view_of_tensor()) + assert_equal_tensor_ref(m.reference_view_of_tensor_v2(), writeable=False) + assert_equal_tensor_ref(m.reference_view_of_tensor_v3()) + assert_equal_tensor_ref(m.reference_view_of_tensor_v4(), writeable=False) + assert_equal_tensor_ref(m.reference_view_of_tensor_v5()) + assert_equal_tensor_ref(m.reference_view_of_tensor_v6(), writeable=False) assert_equal_tensor_ref(m.reference_view_of_fixed_tensor()) assert_equal_tensor_ref(m.reference_const_tensor(), writeable=False) assert_equal_tensor_ref(m.reference_const_tensor_v2(), writeable=False) From a754455a95ffc375b3f997fb3d3bf5b6ad35ebbc Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Thu, 6 Oct 2022 21:22:59 +0000 Subject: [PATCH 066/141] Fix tests --- include/pybind11/eigen/tensor.h | 2 ++ tests/test_eigen_tensor.cpp | 12 +++++------- tests/test_eigen_tensor.py | 12 +++++++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index c8c9c17612..ae784256f6 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -247,6 +247,8 @@ struct type_caster::ValidType> { case return_value_policy::take_ownership: if (std::is_const::value) { + // This cast is ugly, and might be UB in some cases, but we don't have an alterantive here as we must free that memory + Helper::free(const_cast(src)); pybind11_fail("Cannot take ownership of a const reference"); } diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 8233a29113..2025fb352d 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -62,11 +62,9 @@ const Eigen::Tensor &get_const_tensor() { return get_tensor< template struct CustomExample { - CustomExample() { - member = get_tensor(); - } + CustomExample(): member(get_tensor()) {} - Eigen::Tensor member; + const Eigen::Tensor member; }; @@ -99,7 +97,7 @@ void init_tensor_module(pybind11::module& m) { m.def("move_tensor", []() { return get_tensor(); }); m.def("move_const_tensor", - []() -> const Eigen::Tensor { return get_const_tensor(); }); + []() -> const Eigen::Tensor { return get_const_tensor(); }); //NOLINT m.def( "take_fixed_tensor", @@ -160,14 +158,14 @@ void init_tensor_module(pybind11::module& m) { m.def( "reference_view_of_tensor_v2", - []() -> const Eigen::TensorMap> { + []() -> const Eigen::TensorMap> { // NOLINT return get_tensor_map(); }, py::return_value_policy::reference); m.def( "reference_view_of_tensor_v3", - []() { + []() -> Eigen::TensorMap>* { return &get_tensor_map(); }, py::return_value_policy::reference); diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 2970cc1f89..b5d834039a 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -26,11 +26,7 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=0): np.testing.assert_array_equal(mat, copy) @pytest.mark.parametrize('m', submodules) -def test_convert_tensor_to_py(m): - assert_equal_tensor_ref(m.copy_tensor()) - assert_equal_tensor_ref(m.copy_fixed_tensor()) - assert_equal_tensor_ref(m.copy_const_tensor()) - +def test_reference_internal(m): foo = m.CustomExample() counts = sys.getrefcount(foo) mem = foo.member @@ -39,6 +35,12 @@ def test_convert_tensor_to_py(m): assert new_counts == counts + 1 assert_equal_tensor_ref(mem, writeable=False) +@pytest.mark.parametrize('m', submodules) +def test_convert_tensor_to_py(m): + assert_equal_tensor_ref(m.copy_tensor()) + assert_equal_tensor_ref(m.copy_fixed_tensor()) + assert_equal_tensor_ref(m.copy_const_tensor()) + assert_equal_tensor_ref(m.move_tensor()) assert_equal_tensor_ref(m.move_fixed_tensor()) From 9ae8a2fc3747cc9fb28001e9b6fda457c020a82d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 21:29:01 +0000 Subject: [PATCH 067/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 21 ++++---- tests/test_eigen_tensor.cpp | 94 +++++++++++++++++++-------------- tests/test_eigen_tensor.py | 41 ++++++++------ 3 files changed, 89 insertions(+), 67 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index ae784256f6..59a498f5c2 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -133,18 +133,14 @@ struct eigen_tensor_helper< template struct get_tensor_descriptor { - static constexpr auto details = - const_name::value> ("", ", flags.writeable") - + const_name(Type::Layout) - == static_cast(Eigen::RowMajor)>(", flags.c_contiguous", - ", flags.f_contiguous"); + static constexpr auto details + = const_name::value>("", ", flags.writeable") + + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>( + ", flags.c_contiguous", ", flags.f_contiguous"); static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") + eigen_tensor_helper::dimensions_descriptor - + const_name("]") + const_name( - details, - const_name("") - ) + const_name("]"); + + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("]") + + const_name(details, const_name("")) + const_name("]"); }; template @@ -247,8 +243,9 @@ struct type_caster::ValidType> { case return_value_policy::take_ownership: if (std::is_const::value) { - // This cast is ugly, and might be UB in some cases, but we don't have an alterantive here as we must free that memory - Helper::free(const_cast(src)); + // This cast is ugly, and might be UB in some cases, but we don't have an + // alterantive here as we must free that memory + Helper::free(const_cast(src)); pybind11_fail("Cannot take ownership of a const reference"); } diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 2025fb352d..d19982f2d0 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -9,7 +9,6 @@ #include "pybind11_tests.h" - template void reset_tensor(M &x) { for (int i = 0; i < 3; i++) { @@ -21,7 +20,7 @@ void reset_tensor(M &x) { } } -template +template Eigen::Tensor &get_tensor() { static Eigen::Tensor *x; @@ -33,7 +32,7 @@ Eigen::Tensor &get_tensor() { return *x; } -template +template Eigen::TensorMap> &get_tensor_map() { static Eigen::TensorMap> *x; @@ -44,33 +43,36 @@ Eigen::TensorMap> &get_tensor_map() { return *x; } -template +template Eigen::TensorFixedSize, Options> &get_fixed_tensor() { static Eigen::TensorFixedSize, Options> *x; if (!x) { - Eigen::aligned_allocator, Options>> allocator; - x = new (allocator.allocate(1)) Eigen::TensorFixedSize, Options>(); + Eigen::aligned_allocator, Options>> + allocator; + x = new (allocator.allocate(1)) + Eigen::TensorFixedSize, Options>(); reset_tensor(*x); } return *x; } -template -const Eigen::Tensor &get_const_tensor() { return get_tensor(); } +template +const Eigen::Tensor &get_const_tensor() { + return get_tensor(); +} -template +template struct CustomExample { - CustomExample(): member(get_tensor()) {} + CustomExample() : member(get_tensor()) {} const Eigen::Tensor member; }; - -template -void init_tensor_module(pybind11::module& m) { - const char* needed_options = ""; +template +void init_tensor_module(pybind11::module &m) { + const char *needed_options = ""; bool is_major = Options == Eigen::ColMajor; if (is_major) { needed_options = "F"; @@ -78,34 +80,43 @@ void init_tensor_module(pybind11::module& m) { needed_options = "C"; } m.attr("needed_options") = needed_options; - + py::class_>(m, "CustomExample") .def(py::init<>()) - .def_readonly("member", &CustomExample::member, py::return_value_policy::reference_internal); + .def_readonly("member", + &CustomExample::member, + py::return_value_policy::reference_internal); m.def( - "copy_fixed_tensor", []() { return &get_fixed_tensor(); }, py::return_value_policy::copy); + "copy_fixed_tensor", + []() { return &get_fixed_tensor(); }, + py::return_value_policy::copy); m.def( "copy_tensor", []() { return &get_tensor(); }, py::return_value_policy::copy); m.def( - "copy_const_tensor", []() { return &get_const_tensor(); }, py::return_value_policy::copy); + "copy_const_tensor", + []() { return &get_const_tensor(); }, + py::return_value_policy::copy); m.def("move_fixed_tensor", []() { return get_fixed_tensor(); }); m.def("move_tensor", []() { return get_tensor(); }); - m.def("move_const_tensor", - []() -> const Eigen::Tensor { return get_const_tensor(); }); //NOLINT + m.def("move_const_tensor", []() -> const Eigen::Tensor { + return get_const_tensor(); + }); // NOLINT m.def( "take_fixed_tensor", []() { - Eigen::aligned_allocator, Options>> + Eigen::aligned_allocator< + Eigen::TensorFixedSize, Options>> allocator; return new (allocator.allocate(1)) - Eigen::TensorFixedSize, Options>(get_fixed_tensor()); + Eigen::TensorFixedSize, Options>( + get_fixed_tensor()); }, py::return_value_policy::take_ownership); @@ -122,7 +133,9 @@ void init_tensor_module(pybind11::module& m) { py::return_value_policy::take_ownership); m.def( - "reference_tensor", []() { return &get_tensor(); }, py::return_value_policy::reference); + "reference_tensor", + []() { return &get_tensor(); }, + py::return_value_policy::reference); m.def( "reference_tensor_v2", @@ -151,55 +164,55 @@ void init_tensor_module(pybind11::module& m) { m.def( "reference_view_of_tensor", - []() { - return get_tensor_map(); - }, + []() { return get_tensor_map(); }, py::return_value_policy::reference); m.def( "reference_view_of_tensor_v2", - []() -> const Eigen::TensorMap> { // NOLINT - return get_tensor_map(); + []() -> const Eigen::TensorMap> { // NOLINT + return get_tensor_map(); }, py::return_value_policy::reference); m.def( "reference_view_of_tensor_v3", - []() -> Eigen::TensorMap>* { - return &get_tensor_map(); + []() -> Eigen::TensorMap> * { + return &get_tensor_map(); }, py::return_value_policy::reference); m.def( "reference_view_of_tensor_v4", - []() -> const Eigen::TensorMap>* { - return &get_tensor_map(); + []() -> const Eigen::TensorMap> * { + return &get_tensor_map(); }, py::return_value_policy::reference); m.def( "reference_view_of_tensor_v5", - []() -> Eigen::TensorMap>& { - return get_tensor_map(); + []() -> Eigen::TensorMap> & { + return get_tensor_map(); }, py::return_value_policy::reference); m.def( "reference_view_of_tensor_v6", - []() -> const Eigen::TensorMap>& { - return get_tensor_map(); + []() -> const Eigen::TensorMap> & { + return get_tensor_map(); }, py::return_value_policy::reference); m.def( "reference_view_of_fixed_tensor", []() { - return Eigen::TensorMap, Options>>( + return Eigen::TensorMap< + Eigen::TensorFixedSize, Options>>( get_fixed_tensor()); }, py::return_value_policy::reference); - m.def("round_trip_tensor", [](const Eigen::Tensor &tensor) { return tensor; }); + m.def("round_trip_tensor", + [](const Eigen::Tensor &tensor) { return tensor; }); m.def( "round_trip_view_tensor", @@ -208,7 +221,9 @@ void init_tensor_module(pybind11::module& m) { m.def( "round_trip_aligned_view_tensor", - [](Eigen::TensorMap, Eigen::Aligned> view) { return view; }, + [](Eigen::TensorMap, Eigen::Aligned> view) { + return view; + }, py::return_value_policy::reference); m.def( @@ -217,7 +232,6 @@ void init_tensor_module(pybind11::module& m) { return Eigen::Tensor(view); }, py::return_value_policy::move); - } TEST_SUBMODULE(eigen_tensor, m) { diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index b5d834039a..a82bb11119 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -1,4 +1,5 @@ import sys + import pytest np = pytest.importorskip("numpy") @@ -25,7 +26,8 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=0): np.testing.assert_array_equal(mat, copy) -@pytest.mark.parametrize('m', submodules) + +@pytest.mark.parametrize("m", submodules) def test_reference_internal(m): foo = m.CustomExample() counts = sys.getrefcount(foo) @@ -35,7 +37,8 @@ def test_reference_internal(m): assert new_counts == counts + 1 assert_equal_tensor_ref(mem, writeable=False) -@pytest.mark.parametrize('m', submodules) + +@pytest.mark.parametrize("m", submodules) def test_convert_tensor_to_py(m): assert_equal_tensor_ref(m.copy_tensor()) assert_equal_tensor_ref(m.copy_fixed_tensor()) @@ -62,7 +65,7 @@ def test_convert_tensor_to_py(m): assert_equal_tensor_ref(m.reference_const_tensor_v2(), writeable=False) -@pytest.mark.parametrize('m', submodules) +@pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): with pytest.raises(Exception): m.reference_tensor_internal() @@ -74,7 +77,7 @@ def test_bad_cpp_to_python_casts(m): m.take_const_tensor() -@pytest.mark.parametrize('m', submodules) +@pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): with pytest.raises(TypeError): m.round_trip_tensor(np.zeros((2, 3))) @@ -88,18 +91,24 @@ def test_bad_python_to_cpp_casts(m): bad_options = "F" # Shape, dtype and the order need to be correct for a TensorMap cast with pytest.raises(TypeError): - m.round_trip_view_tensor(np.zeros((3, 5, 2), dtype=np.float64, order=bad_options)) + m.round_trip_view_tensor( + np.zeros((3, 5, 2), dtype=np.float64, order=bad_options) + ) with pytest.raises(TypeError): - m.round_trip_view_tensor(np.zeros((3, 5, 2), dtype=np.float32, order=m.needed_options)) + m.round_trip_view_tensor( + np.zeros((3, 5, 2), dtype=np.float32, order=m.needed_options) + ) with pytest.raises(TypeError): - m.round_trip_view_tensor(np.zeros((3, 5), dtype=np.float64, order=m.needed_options)) + m.round_trip_view_tensor( + np.zeros((3, 5), dtype=np.float64, order=m.needed_options) + ) with pytest.raises(TypeError): temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) temp.setflags(write=False) m.round_trip_view_tensor(temp) -@pytest.mark.parametrize('m', submodules) +@pytest.mark.parametrize("m", submodules) def test_references_actually_refer(m): a = m.reference_tensor() temp = a[indices] @@ -115,7 +124,7 @@ def test_references_actually_refer(m): assert_equal_tensor_ref(m.copy_const_tensor()) -@pytest.mark.parametrize('m', submodules) +@pytest.mark.parametrize("m", submodules) def test_round_trip(m): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) @@ -125,9 +134,12 @@ def test_round_trip(m): copy.setflags(write=False) assert_equal_tensor_ref(m.round_trip_const_view_tensor(copy)) - np.testing.assert_array_equal(tensor_ref[:, ::-1, :], m.round_trip_tensor(tensor_ref[:, ::-1, :])) + np.testing.assert_array_equal( + tensor_ref[:, ::-1, :], m.round_trip_tensor(tensor_ref[:, ::-1, :]) + ) + -@pytest.mark.parametrize('m', submodules) +@pytest.mark.parametrize("m", submodules) def test_round_trip_references_actually_refer(m): # Need to create a copy that matches the type on the C side copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) @@ -139,11 +151,10 @@ def test_round_trip_references_actually_refer(m): assert_equal_tensor_ref(copy) -@pytest.mark.parametrize('m', submodules) +@pytest.mark.parametrize("m", submodules) def test_doc_string(m, doc): assert ( - doc(m.copy_tensor) - == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" + doc(m.copy_tensor) == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" ) assert ( doc(m.copy_fixed_tensor) @@ -154,7 +165,7 @@ def test_doc_string(m, doc): == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" ) - order_flag = f'flags.{m.needed_options.lower()}_contiguous' + order_flag = f"flags.{m.needed_options.lower()}_contiguous" assert doc(m.round_trip_view_tensor) == ( f"round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}])" + f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]" From ac263f0d8cb75419628af3ae4395a982a706231b Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Thu, 6 Oct 2022 21:42:58 +0000 Subject: [PATCH 068/141] More test fixes --- tests/test_eigen_tensor.cpp | 9 +++++++-- tests/test_eigen_tensor.py | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index d19982f2d0..e042fbfb4f 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -104,12 +104,15 @@ void init_tensor_module(pybind11::module &m) { m.def("move_tensor", []() { return get_tensor(); }); + // NOLINTBEGIN(readability-const-return-type) m.def("move_const_tensor", []() -> const Eigen::Tensor { return get_const_tensor(); - }); // NOLINT + }); + // NOLINTEND(readability-const-return-type) m.def( "take_fixed_tensor", + []() { Eigen::aligned_allocator< Eigen::TensorFixedSize, Options>> @@ -167,12 +170,14 @@ void init_tensor_module(pybind11::module &m) { []() { return get_tensor_map(); }, py::return_value_policy::reference); + // NOLINTBEGIN(readability-const-return-type) m.def( "reference_view_of_tensor_v2", - []() -> const Eigen::TensorMap> { // NOLINT + []() -> const Eigen::TensorMap> { return get_tensor_map(); }, py::return_value_policy::reference); + // NOLINTEND(readability-const-return-type) m.def( "reference_view_of_tensor_v3", diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index a82bb11119..f30c08b32a 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -29,6 +29,8 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=0): @pytest.mark.parametrize("m", submodules) def test_reference_internal(m): + if not hasattr(sys, "getrefcount"): + pytest.skip("No reference counting") foo = m.CustomExample() counts = sys.getrefcount(foo) mem = foo.member From 5265a815d53a2d7361e1e2c83810bedf83fb1fdb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 21:49:37 +0000 Subject: [PATCH 069/141] style: pre-commit fixes --- tests/test_eigen_tensor.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index e042fbfb4f..f196f01ccb 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -105,9 +105,8 @@ void init_tensor_module(pybind11::module &m) { m.def("move_tensor", []() { return get_tensor(); }); // NOLINTBEGIN(readability-const-return-type) - m.def("move_const_tensor", []() -> const Eigen::Tensor { - return get_const_tensor(); - }); + m.def("move_const_tensor", + []() -> const Eigen::Tensor { return get_const_tensor(); }); // NOLINTEND(readability-const-return-type) m.def( From 94775191b8a546912e0d7d89b8c78ff2d02ccdfb Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Thu, 6 Oct 2022 22:11:38 +0000 Subject: [PATCH 070/141] A ton more test coverage --- include/pybind11/eigen/tensor.h | 6 ++++-- tests/test_eigen_tensor.cpp | 12 +++++++++++- tests/test_eigen_tensor.py | 2 ++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 59a498f5c2..490fe5442f 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -431,8 +431,10 @@ struct type_caster, public: static constexpr auto name = get_tensor_descriptor::value; - explicit operator MapType *() { return value.get(); } - explicit operator MapType &() { return *value; } + explicit operator MapType *() { + return value.get(); } + explicit operator MapType &() { + return *value; } explicit operator MapType &&() && { return std::move(*value); } template diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index f196f01ccb..7344f0b230 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -166,7 +166,7 @@ void init_tensor_module(pybind11::module &m) { m.def( "reference_view_of_tensor", - []() { return get_tensor_map(); }, + []() -> Eigen::TensorMap> { return get_tensor_map(); }, py::return_value_policy::reference); // NOLINTBEGIN(readability-const-return-type) @@ -223,6 +223,16 @@ void init_tensor_module(pybind11::module &m) { [](Eigen::TensorMap> view) { return view; }, py::return_value_policy::reference); + m.def( + "round_trip_view_tensor_ref", + [](Eigen::TensorMap>& view) { return view; }, + py::return_value_policy::reference); + + m.def( + "round_trip_view_tensor_ptr", + [](Eigen::TensorMap>* view) { return view; }, + py::return_value_policy::reference); + m.def( "round_trip_aligned_view_tensor", [](Eigen::TensorMap, Eigen::Aligned> view) { diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index f30c08b32a..f9e895623d 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -133,6 +133,8 @@ def test_round_trip(m): copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) assert_equal_tensor_ref(m.round_trip_view_tensor(copy)) + assert_equal_tensor_ref(m.round_trip_view_tensor_ref(copy)) + assert_equal_tensor_ref(m.round_trip_view_tensor_ptr(copy)) copy.setflags(write=False) assert_equal_tensor_ref(m.round_trip_const_view_tensor(copy)) From 891dab85bed09650a572842b44cfc72c83918271 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 7 Oct 2022 02:06:32 +0000 Subject: [PATCH 071/141] Fix tests --- tests/test_eigen_tensor.cpp | 18 ++++++++++-------- tests/test_eigen_tensor.py | 7 +++++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 7344f0b230..c9970f1591 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -65,9 +65,10 @@ const Eigen::Tensor &get_const_tensor() { template struct CustomExample { - CustomExample() : member(get_tensor()) {} + CustomExample() : member(get_tensor()), view_member(member) {} - const Eigen::Tensor member; + Eigen::Tensor member; + Eigen::TensorMap> view_member; }; template @@ -85,6 +86,9 @@ void init_tensor_module(pybind11::module &m) { .def(py::init<>()) .def_readonly("member", &CustomExample::member, + py::return_value_policy::reference_internal) + .def_readonly("member_view", + &CustomExample::view_member, py::return_value_policy::reference_internal); m.def( @@ -104,10 +108,9 @@ void init_tensor_module(pybind11::module &m) { m.def("move_tensor", []() { return get_tensor(); }); - // NOLINTBEGIN(readability-const-return-type) m.def("move_const_tensor", + // NOLINTNEXTLINE(readability-const-return-type) []() -> const Eigen::Tensor { return get_const_tensor(); }); - // NOLINTEND(readability-const-return-type) m.def( "take_fixed_tensor", @@ -169,14 +172,13 @@ void init_tensor_module(pybind11::module &m) { []() -> Eigen::TensorMap> { return get_tensor_map(); }, py::return_value_policy::reference); - // NOLINTBEGIN(readability-const-return-type) m.def( "reference_view_of_tensor_v2", + // NOLINTNEXTLINE(readability-const-return-type) []() -> const Eigen::TensorMap> { - return get_tensor_map(); - }, + return get_tensor_map(); //NOLINT(readability-const-return-type) + }, // NOLINT(readability-const-return-type) py::return_value_policy::reference); - // NOLINTEND(readability-const-return-type) m.def( "reference_view_of_tensor_v3", diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index f9e895623d..2d70163ec1 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -28,16 +28,19 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=0): @pytest.mark.parametrize("m", submodules) -def test_reference_internal(m): +@pytest.mark.parametrize("member_name", ['member', 'member_view']) +def test_reference_internal(m, member_name): if not hasattr(sys, "getrefcount"): pytest.skip("No reference counting") foo = m.CustomExample() counts = sys.getrefcount(foo) - mem = foo.member + mem = getattr(foo, member_name) assert_equal_tensor_ref(mem, writeable=False) new_counts = sys.getrefcount(foo) assert new_counts == counts + 1 assert_equal_tensor_ref(mem, writeable=False) + del mem + assert sys.getrefcount(foo) == counts @pytest.mark.parametrize("m", submodules) From 18b06dcb0729188579d7100bd2d57c665e6cb970 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 22:17:30 +0000 Subject: [PATCH 072/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 6 ++---- tests/test_eigen_tensor.cpp | 8 +++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 490fe5442f..59a498f5c2 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -431,10 +431,8 @@ struct type_caster, public: static constexpr auto name = get_tensor_descriptor::value; - explicit operator MapType *() { - return value.get(); } - explicit operator MapType &() { - return *value; } + explicit operator MapType *() { return value.get(); } + explicit operator MapType &() { return *value; } explicit operator MapType &&() && { return std::move(*value); } template diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index c9970f1591..25712d98a6 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -169,7 +169,9 @@ void init_tensor_module(pybind11::module &m) { m.def( "reference_view_of_tensor", - []() -> Eigen::TensorMap> { return get_tensor_map(); }, + []() -> Eigen::TensorMap> { + return get_tensor_map(); + }, py::return_value_policy::reference); m.def( @@ -227,12 +229,12 @@ void init_tensor_module(pybind11::module &m) { m.def( "round_trip_view_tensor_ref", - [](Eigen::TensorMap>& view) { return view; }, + [](Eigen::TensorMap> &view) { return view; }, py::return_value_policy::reference); m.def( "round_trip_view_tensor_ptr", - [](Eigen::TensorMap>* view) { return view; }, + [](Eigen::TensorMap> *view) { return view; }, py::return_value_policy::reference); m.def( From 37230b86889a3cc83899ac673f7dfd90bab2c9ff Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 02:12:38 +0000 Subject: [PATCH 073/141] style: pre-commit fixes --- tests/test_eigen_tensor.cpp | 11 +++++------ tests/test_eigen_tensor.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 25712d98a6..8650635cf5 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -84,9 +84,8 @@ void init_tensor_module(pybind11::module &m) { py::class_>(m, "CustomExample") .def(py::init<>()) - .def_readonly("member", - &CustomExample::member, - py::return_value_policy::reference_internal) + .def_readonly( + "member", &CustomExample::member, py::return_value_policy::reference_internal) .def_readonly("member_view", &CustomExample::view_member, py::return_value_policy::reference_internal); @@ -109,7 +108,7 @@ void init_tensor_module(pybind11::module &m) { m.def("move_tensor", []() { return get_tensor(); }); m.def("move_const_tensor", - // NOLINTNEXTLINE(readability-const-return-type) + // NOLINTNEXTLINE(readability-const-return-type) []() -> const Eigen::Tensor { return get_const_tensor(); }); m.def( @@ -178,8 +177,8 @@ void init_tensor_module(pybind11::module &m) { "reference_view_of_tensor_v2", // NOLINTNEXTLINE(readability-const-return-type) []() -> const Eigen::TensorMap> { - return get_tensor_map(); //NOLINT(readability-const-return-type) - }, // NOLINT(readability-const-return-type) + return get_tensor_map(); // NOLINT(readability-const-return-type) + }, // NOLINT(readability-const-return-type) py::return_value_policy::reference); m.def( diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 2d70163ec1..ffe797fecd 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -28,7 +28,7 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=0): @pytest.mark.parametrize("m", submodules) -@pytest.mark.parametrize("member_name", ['member', 'member_view']) +@pytest.mark.parametrize("member_name", ["member", "member_view"]) def test_reference_internal(m, member_name): if not hasattr(sys, "getrefcount"): pytest.skip("No reference counting") From 185d3d5f951057a8670f7610d03065b11e860714 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 7 Oct 2022 02:18:20 +0000 Subject: [PATCH 074/141] Add back constexpr --- include/pybind11/eigen/tensor.h | 9 +++++---- tests/test_eigen_tensor.cpp | 3 +-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 59a498f5c2..16431fb596 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -326,8 +326,9 @@ struct type_caster, } // Use temporary to avoid MSVC warning ... - bool is_aligned = (Options & Eigen::Aligned) != 0; - if (is_aligned && !is_tensor_aligned(arr.data())) { + constexpr bool is_aligned = (Options & Eigen::Aligned) != 0; + + if (PYBIND11_SILENCE_MSVC_C4127(is_aligned && !is_tensor_aligned(arr.data()))) { return false; } @@ -338,8 +339,8 @@ struct type_caster, return false; } - bool needs_writeable = !std::is_const::value; - if (needs_writeable && !arr.writeable()) { + constexpr bool needs_writeable = !std::is_const::value; + if (PYBIND11_SILENCE_MSVC_C4127(needs_writeable && !arr.writeable())) { return false; } diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 8650635cf5..aad006aa1c 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -74,8 +74,7 @@ struct CustomExample { template void init_tensor_module(pybind11::module &m) { const char *needed_options = ""; - bool is_major = Options == Eigen::ColMajor; - if (is_major) { + if (PYBIND11_SILENCE_MSVC_C4127(Options == Eigen::ColMajor)) { needed_options = "F"; } else { needed_options = "C"; From c3b30a15b70926508dfa2b8a1a512b018c1d1989 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 7 Oct 2022 03:24:40 +0000 Subject: [PATCH 075/141] Another test --- tests/test_eigen_tensor.cpp | 3 +++ tests/test_eigen_tensor.py | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index aad006aa1c..e1dc36d82c 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -220,6 +220,9 @@ void init_tensor_module(pybind11::module &m) { m.def("round_trip_tensor", [](const Eigen::Tensor &tensor) { return tensor; }); + m.def("round_trip_fixed_tensor", + [](const Eigen::TensorFixedSize, Options> &tensor) { return tensor; }); + m.def( "round_trip_view_tensor", [](Eigen::TensorMap> view) { return view; }, diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index ffe797fecd..cf3f673dca 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -132,6 +132,7 @@ def test_references_actually_refer(m): @pytest.mark.parametrize("m", submodules) def test_round_trip(m): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) + assert_equal_tensor_ref(m.round_trip_fixed_tensor(tensor_ref)) assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) From ca043265ba5299f64d27061542662ae6852c1955 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 03:30:30 +0000 Subject: [PATCH 076/141] style: pre-commit fixes --- tests/test_eigen_tensor.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index e1dc36d82c..72034b95ad 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -221,7 +221,9 @@ void init_tensor_module(pybind11::module &m) { [](const Eigen::Tensor &tensor) { return tensor; }); m.def("round_trip_fixed_tensor", - [](const Eigen::TensorFixedSize, Options> &tensor) { return tensor; }); + [](const Eigen::TensorFixedSize, Options> &tensor) { + return tensor; + }); m.def( "round_trip_view_tensor", From b105dd20ddced371cffa92b16a2030df734a5dec Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Fri, 7 Oct 2022 09:24:11 -0400 Subject: [PATCH 077/141] Improve tests --- tests/test_eigen_tensor.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index cf3f673dca..4b74d15375 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -72,19 +72,22 @@ def test_convert_tensor_to_py(m): @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): - with pytest.raises(Exception): + with pytest.raises(RuntimeError) as exc_info: m.reference_tensor_internal() + assert str(exc_info.value) == 'Cannot use reference internal when there is no parent' - with pytest.raises(Exception): + with pytest.raises(RuntimeError) as exc_info: m.move_const_tensor() + assert str(exc_info.value) == 'Cannot move from a constant reference' - with pytest.raises(Exception): + with pytest.raises(RuntimeError) as exc_info: m.take_const_tensor() + assert str(exc_info.value) == 'Cannot take ownership of a const reference' @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): - with pytest.raises(TypeError): + with pytest.raises(TypeError) as exc_info: m.round_trip_tensor(np.zeros((2, 3))) with pytest.raises(TypeError): From 8636d0f235308ab4c1136ff6f0913a1530a20570 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Fri, 7 Oct 2022 09:25:36 -0400 Subject: [PATCH 078/141] Whoops --- tests/test_eigen_tensor.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 4b74d15375..4efd8fc53e 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -74,20 +74,22 @@ def test_convert_tensor_to_py(m): def test_bad_cpp_to_python_casts(m): with pytest.raises(RuntimeError) as exc_info: m.reference_tensor_internal() - assert str(exc_info.value) == 'Cannot use reference internal when there is no parent' + assert ( + str(exc_info.value) == "Cannot use reference internal when there is no parent" + ) with pytest.raises(RuntimeError) as exc_info: m.move_const_tensor() - assert str(exc_info.value) == 'Cannot move from a constant reference' + assert str(exc_info.value) == "Cannot move from a constant reference" with pytest.raises(RuntimeError) as exc_info: m.take_const_tensor() - assert str(exc_info.value) == 'Cannot take ownership of a const reference' + assert str(exc_info.value) == "Cannot take ownership of a const reference" @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): - with pytest.raises(TypeError) as exc_info: + with pytest.raises(TypeError): m.round_trip_tensor(np.zeros((2, 3))) with pytest.raises(TypeError): From bf36e9f3a14ee54ccf264c5debfc80f146babde6 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Fri, 7 Oct 2022 09:32:02 -0400 Subject: [PATCH 079/141] Less magic numbers --- tests/test_eigen_tensor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 4efd8fc53e..3e0d82e93c 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -9,9 +9,9 @@ tensor_ref = np.zeros((3, 5, 2)) -for i in range(3): - for j in range(5): - for k in range(2): +for i in range(tensor_ref.shape[0]): + for j in range(tensor_ref.shape[1]): + for k in range(tensor_ref.shape[2]): tensor_ref[i, j, k] = i * (5 * 2) + j * 2 + k indices = (2, 3, 1) From 9dd00c27883593f2e1a15fdc8a70c2b9bdaf08bf Mon Sep 17 00:00:00 2001 From: Lalaland Date: Fri, 7 Oct 2022 09:50:41 -0700 Subject: [PATCH 080/141] Update tests/test_eigen_tensor.py Co-authored-by: Sergiu Deitsch --- tests/test_eigen_tensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 3e0d82e93c..d7bd454a90 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -82,7 +82,7 @@ def test_bad_cpp_to_python_casts(m): m.move_const_tensor() assert str(exc_info.value) == "Cannot move from a constant reference" - with pytest.raises(RuntimeError) as exc_info: + with pytest.raises(RuntimeError, match='Cannot take ownership of a const reference'): m.take_const_tensor() assert str(exc_info.value) == "Cannot take ownership of a const reference" From aeeda4043b0eeb3ec80218985026085b95d4b1c3 Mon Sep 17 00:00:00 2001 From: Lalaland Date: Fri, 7 Oct 2022 09:51:11 -0700 Subject: [PATCH 081/141] Update tests/test_eigen_tensor.py Co-authored-by: Sergiu Deitsch --- tests/test_eigen_tensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index d7bd454a90..c7e279629c 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -7,7 +7,7 @@ submodules = [eigen_tensor.c_style, eigen_tensor.f_style] -tensor_ref = np.zeros((3, 5, 2)) +tensor_ref = np.empty((3, 5, 2), dtype=np.int_) for i in range(tensor_ref.shape[0]): for j in range(tensor_ref.shape[1]): From a2e915bea0cad13694636e48fcd3ba09d313b65e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 17:09:52 +0000 Subject: [PATCH 082/141] style: pre-commit fixes --- tests/test_eigen_tensor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index c7e279629c..bf1134dde0 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -82,7 +82,9 @@ def test_bad_cpp_to_python_casts(m): m.move_const_tensor() assert str(exc_info.value) == "Cannot move from a constant reference" - with pytest.raises(RuntimeError, match='Cannot take ownership of a const reference'): + with pytest.raises( + RuntimeError, match="Cannot take ownership of a const reference" + ): m.take_const_tensor() assert str(exc_info.value) == "Cannot take ownership of a const reference" From d46603d883874973a5df25def08c4a7dd9e3bb2f Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 7 Oct 2022 18:03:59 +0000 Subject: [PATCH 083/141] Fix tests --- tests/test_eigen_tensor.cpp | 7 +++++++ tests/test_eigen_tensor.py | 13 ++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 72034b95ad..e5ab17ac0a 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -135,6 +135,13 @@ void init_tensor_module(pybind11::module &m) { }, py::return_value_policy::take_ownership); + m.def( + "take_view_tensor", + []() -> const Eigen::TensorMap> * { + return new Eigen::TensorMap>(get_tensor()); + }, + py::return_value_policy::take_ownership); + m.def( "reference_tensor", []() { return &get_tensor(); }, diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index bf1134dde0..19797ecc1c 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -72,22 +72,21 @@ def test_convert_tensor_to_py(m): @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): - with pytest.raises(RuntimeError) as exc_info: + with pytest.raises(RuntimeError, match="Cannot use reference internal when there is no parent"): m.reference_tensor_internal() - assert ( - str(exc_info.value) == "Cannot use reference internal when there is no parent" - ) - with pytest.raises(RuntimeError) as exc_info: + with pytest.raises(RuntimeError, match="Cannot move from a constant reference"): m.move_const_tensor() - assert str(exc_info.value) == "Cannot move from a constant reference" with pytest.raises( RuntimeError, match="Cannot take ownership of a const reference" ): m.take_const_tensor() - assert str(exc_info.value) == "Cannot take ownership of a const reference" + with pytest.raises( + RuntimeError, match="Invalid return_value_policy for Eigen Map type, must be either reference or reference_internal" + ): + m.take_view_tensor() @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): From 55addec5426cd69ca680437fa823739f230968a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 18:10:30 +0000 Subject: [PATCH 084/141] style: pre-commit fixes --- tests/test_eigen_tensor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 19797ecc1c..439aa84213 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -72,7 +72,9 @@ def test_convert_tensor_to_py(m): @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): - with pytest.raises(RuntimeError, match="Cannot use reference internal when there is no parent"): + with pytest.raises( + RuntimeError, match="Cannot use reference internal when there is no parent" + ): m.reference_tensor_internal() with pytest.raises(RuntimeError, match="Cannot move from a constant reference"): @@ -84,10 +86,12 @@ def test_bad_cpp_to_python_casts(m): m.take_const_tensor() with pytest.raises( - RuntimeError, match="Invalid return_value_policy for Eigen Map type, must be either reference or reference_internal" + RuntimeError, + match="Invalid return_value_policy for Eigen Map type, must be either reference or reference_internal", ): m.take_view_tensor() + @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): with pytest.raises(TypeError): From 17eed83fdbeadd96f36359ca7c1bdf45b5c10728 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 7 Oct 2022 18:32:15 +0000 Subject: [PATCH 085/141] Fix memory leak --- include/pybind11/eigen/tensor.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 16431fb596..1a0770071b 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -410,6 +410,9 @@ struct type_caster, parent_object = reinterpret_borrow(parent); break; + case return_value_policy::take_ownership: + delete src; + //fallthrough default: // move, take_ownership don't make any sense for a ref/map: pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either " From 7a09efb263f483394ad40d732f3bd4e0dacc83fa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 18:39:34 +0000 Subject: [PATCH 086/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 1a0770071b..d13c3beba2 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -412,7 +412,7 @@ struct type_caster, case return_value_policy::take_ownership: delete src; - //fallthrough + // fallthrough default: // move, take_ownership don't make any sense for a ref/map: pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either " From 57a45e13f3658f54991966f583b30f682d511b8d Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 7 Oct 2022 20:05:37 +0000 Subject: [PATCH 087/141] Fix order --- tests/CMakeLists.txt | 10 +++++----- tests/extra_python_package/test_files.py | 14 +++++++------- tools/setup_global.py.in | 4 ++-- tools/setup_main.py.in | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 403f9a0aba..273508409d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -128,8 +128,8 @@ set(PYBIND11_TEST_FILES test_custom_type_casters test_custom_type_setup test_docstring_options - test_eigen_tensor test_eigen_matrix + test_eigen_tensor test_enum test_eval test_exceptions @@ -234,9 +234,9 @@ list(GET PYBIND11_EIGEN_VERSION_AND_HASH 1 PYBIND11_EIGEN_VERSION_HASH) # Check if Eigen is available; if not, remove from PYBIND11_TEST_FILES (but # keep it in PYBIND11_PYTEST_FILES, so that we get the "eigen is not installed" # skip message). -list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) +list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) if(PYBIND11_TEST_FILES_EIGEN_I EQUAL -1) - list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) endif() if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) # Try loading via newer Eigen's Eigen3Config first (bypassing tools/FindEigen3.cmake). @@ -293,12 +293,12 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) endif() message(STATUS "Building tests with Eigen v${EIGEN3_VERSION}") else() - list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) + list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() - list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index e4e0c616b9..1b72dab3fd 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -55,13 +55,13 @@ "include/pybind11/detail/typeid.h", } -stl_headers = { - "include/pybind11/stl/filesystem.h", -} - eigen_headers = { - "include/pybind11/eigen/tensor.h", "include/pybind11/eigen/matrix.h", + "include/pybind11/eigen/tensor.h", +} + +stl_headers = { + "include/pybind11/stl/filesystem.h", } cmake_files = { @@ -87,7 +87,7 @@ "setup_helpers.py", } -headers = main_headers | detail_headers | stl_headers | eigen_headers +headers = main_headers | detail_headers | eigen_headers | stl_headers src_files = headers | cmake_files | pkgconfig_files all_files = src_files | py_files @@ -97,8 +97,8 @@ "pybind11/include", "pybind11/include/pybind11", "pybind11/include/pybind11/detail", - "pybind11/include/pybind11/stl", "pybind11/include/pybind11/eigen", + "pybind11/include/pybind11/stl", "pybind11/share", "pybind11/share/cmake", "pybind11/share/cmake/pybind11", diff --git a/tools/setup_global.py.in b/tools/setup_global.py.in index a99dfdeb2d..885ac5c725 100644 --- a/tools/setup_global.py.in +++ b/tools/setup_global.py.in @@ -27,8 +27,8 @@ class InstallHeadersNested(install_headers): main_headers = glob.glob("pybind11/include/pybind11/*.h") detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h") -stl_headers = glob.glob("pybind11/include/pybind11/stl/*.h") eigen_headers = glob.glob("pybind11/include/pybind11/eigen/*.h") +stl_headers = glob.glob("pybind11/include/pybind11/stl/*.h") cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake") pkgconfig_files = glob.glob("pybind11/share/pkgconfig/*.pc") headers = main_headers + detail_headers + stl_headers + eigen_headers @@ -56,8 +56,8 @@ setup( (base + "share/pkgconfig", pkgconfig_files), (base + "include/pybind11", main_headers), (base + "include/pybind11/detail", detail_headers), - (base + "include/pybind11/stl", stl_headers), (base + "include/pybind11/eigen", eigen_headers), + (base + "include/pybind11/stl", stl_headers), ], cmdclass=cmdclass, ) diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index f1be8e701f..6358cc7b9b 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -15,8 +15,8 @@ setup( "pybind11", "pybind11.include.pybind11", "pybind11.include.pybind11.detail", - "pybind11.include.pybind11.stl", "pybind11.include.pybind11.eigen", + "pybind11.include.pybind11.stl", "pybind11.share.cmake.pybind11", "pybind11.share.pkgconfig", ], @@ -24,8 +24,8 @@ setup( "pybind11": ["py.typed"], "pybind11.include.pybind11": ["*.h"], "pybind11.include.pybind11.detail": ["*.h"], - "pybind11.include.pybind11.stl": ["*.h"], "pybind11.include.pybind11.eigen": ["*.h"], + "pybind11.include.pybind11.stl": ["*.h"], "pybind11.share.cmake.pybind11": ["*.cmake"], "pybind11.share.pkgconfig": ["*.pc"], }, From 9bdb36680b6c8f9a44189309c0aad83e59d93bf5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 20:11:45 +0000 Subject: [PATCH 088/141] style: pre-commit fixes --- tests/extra_python_package/test_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 1b72dab3fd..9a9bb1556a 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -87,7 +87,7 @@ "setup_helpers.py", } -headers = main_headers | detail_headers | eigen_headers | stl_headers +headers = main_headers | detail_headers | eigen_headers | stl_headers src_files = headers | cmake_files | pkgconfig_files all_files = src_files | py_files From 51bdba8bbf4ed39a34e6cdf55cfc6adcd4679312 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 7 Oct 2022 20:28:18 +0000 Subject: [PATCH 089/141] Add test to make sure unsafe casts fail --- tests/test_eigen_tensor.cpp | 3 +++ tests/test_eigen_tensor.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index e5ab17ac0a..67927b9282 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -227,6 +227,9 @@ void init_tensor_module(pybind11::module &m) { m.def("round_trip_tensor", [](const Eigen::Tensor &tensor) { return tensor; }); + m.def("round_trip_tensor2", + [](const Eigen::Tensor &tensor) { return tensor; }); + m.def("round_trip_fixed_tensor", [](const Eigen::TensorFixedSize, Options> &tensor) { return tensor; diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 439aa84213..6fdf4926b0 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -142,6 +142,12 @@ def test_references_actually_refer(m): @pytest.mark.parametrize("m", submodules) def test_round_trip(m): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) + + with pytest.raises(TypeError): + assert_equal_tensor_ref(m.round_trip_tensor2(tensor_ref)) + + assert_equal_tensor_ref(m.round_trip_tensor2(np.array(tensor_ref, dtype=np.int32))) + assert_equal_tensor_ref(m.round_trip_fixed_tensor(tensor_ref)) assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) From 9ef5aa7da258ad1f18552ab38ddda065389996c6 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 7 Oct 2022 20:31:01 +0000 Subject: [PATCH 090/141] Minor bug fix to work on 32 bit machines --- tests/test_eigen_tensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 6fdf4926b0..bf27d8e42c 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -7,7 +7,7 @@ submodules = [eigen_tensor.c_style, eigen_tensor.f_style] -tensor_ref = np.empty((3, 5, 2), dtype=np.int_) +tensor_ref = np.empty((3, 5, 2), dtype=np.int64) for i in range(tensor_ref.shape[0]): for j in range(tensor_ref.shape[1]): From ed0904071eace4df9065887855fe46305d1e392e Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 7 Oct 2022 21:33:58 +0000 Subject: [PATCH 091/141] Implement convert flag --- include/pybind11/eigen/tensor.h | 14 +++++++++++++- tests/test_eigen_tensor.cpp | 3 +++ tests/test_eigen_tensor.py | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index d13c3beba2..9bad929d36 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -149,7 +149,19 @@ struct type_caster::ValidType> { static constexpr auto temp_name = get_tensor_descriptor::value; PYBIND11_TYPE_CASTER(Type, temp_name); - bool load(handle src, bool /*convert*/) { + bool load(handle src, bool convert) { + if (!convert) { + array temp = array::ensure(src); + if (!temp) { + return false; + } + + if (!convert && !temp.dtype().is(dtype::of())) { + return false; + } + } + + array_t()> arr( reinterpret_borrow(src)); diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 67927b9282..2973c32b18 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -227,6 +227,9 @@ void init_tensor_module(pybind11::module &m) { m.def("round_trip_tensor", [](const Eigen::Tensor &tensor) { return tensor; }); + m.def("round_trip_tensor_noconvert", + [](const Eigen::Tensor &tensor) { return tensor; }, py::arg("tensor").noconvert()); + m.def("round_trip_tensor2", [](const Eigen::Tensor &tensor) { return tensor; }); diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index bf27d8e42c..eaa469efd0 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -100,6 +100,11 @@ def test_bad_python_to_cpp_casts(m): with pytest.raises(TypeError): m.round_trip_tensor(np.zeros(dtype=np.str_, shape=(2, 3, 1))) + with pytest.raises(TypeError): + m.round_trip_tensor_noconvert(tensor_ref) + + assert_equal_tensor_ref(m.round_trip_tensor_noconvert(tensor_ref.astype(np.float64))) + if m.needed_options == "F": bad_options = "C" else: @@ -109,14 +114,23 @@ def test_bad_python_to_cpp_casts(m): m.round_trip_view_tensor( np.zeros((3, 5, 2), dtype=np.float64, order=bad_options) ) + with pytest.raises(TypeError): m.round_trip_view_tensor( np.zeros((3, 5, 2), dtype=np.float32, order=m.needed_options) ) + with pytest.raises(TypeError): m.round_trip_view_tensor( np.zeros((3, 5), dtype=np.float64, order=m.needed_options) ) + + with pytest.raises(TypeError): + temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) + m.round_trip_view_tensor( + temp[:, ::-1, :], + ) + with pytest.raises(TypeError): temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) temp.setflags(write=False) From 8547ee0e01979801cfbeadb9c8041afd8f52ea53 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 21:39:52 +0000 Subject: [PATCH 092/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 3 +-- tests/test_eigen_tensor.cpp | 6 ++++-- tests/test_eigen_tensor.py | 4 +++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 9bad929d36..1afa7ca55d 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -155,13 +155,12 @@ struct type_caster::ValidType> { if (!temp) { return false; } - + if (!convert && !temp.dtype().is(dtype::of())) { return false; } } - array_t()> arr( reinterpret_borrow(src)); diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 2973c32b18..8a51028389 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -227,8 +227,10 @@ void init_tensor_module(pybind11::module &m) { m.def("round_trip_tensor", [](const Eigen::Tensor &tensor) { return tensor; }); - m.def("round_trip_tensor_noconvert", - [](const Eigen::Tensor &tensor) { return tensor; }, py::arg("tensor").noconvert()); + m.def( + "round_trip_tensor_noconvert", + [](const Eigen::Tensor &tensor) { return tensor; }, + py::arg("tensor").noconvert()); m.def("round_trip_tensor2", [](const Eigen::Tensor &tensor) { return tensor; }); diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index eaa469efd0..eac26e55ac 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -103,7 +103,9 @@ def test_bad_python_to_cpp_casts(m): with pytest.raises(TypeError): m.round_trip_tensor_noconvert(tensor_ref) - assert_equal_tensor_ref(m.round_trip_tensor_noconvert(tensor_ref.astype(np.float64))) + assert_equal_tensor_ref( + m.round_trip_tensor_noconvert(tensor_ref.astype(np.float64)) + ) if m.needed_options == "F": bad_options = "C" From a03ad096c1d141fae80221a59f97d1d37cc08f8b Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 7 Oct 2022 22:45:23 +0000 Subject: [PATCH 093/141] Switch to correct TensorMap const use --- include/pybind11/eigen/tensor.h | 45 ++++++++++++++++++--------------- tests/test_eigen_tensor.cpp | 3 ++- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 1afa7ca55d..16a194faaa 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -53,7 +53,6 @@ struct eigen_tensor_helper {}; template struct eigen_tensor_helper> { using Type = Eigen::Tensor; - using ConstType = Eigen::Tensor; using ValidType = void; static Eigen::DSizes get_shape(const Type &f) { @@ -98,8 +97,6 @@ template , Options_, IndexType>> { using Type = Eigen::TensorFixedSize, Options_, IndexType>; - using ConstType - = Eigen::TensorFixedSize, Options_, IndexType>; using ValidType = void; static constexpr Eigen::DSizes @@ -131,15 +128,15 @@ struct eigen_tensor_helper< } }; -template +template struct get_tensor_descriptor { static constexpr auto details - = const_name::value>("", ", flags.writeable") + = const_name(", flags.writeable", "") + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>( ", flags.c_contiguous", ", flags.f_contiguous"); static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") + eigen_tensor_helper::dimensions_descriptor + const_name("]") + + const_name("[") + eigen_tensor_helper::type>::dimensions_descriptor + const_name("]") + const_name(details, const_name("")) + const_name("]"); }; @@ -177,9 +174,9 @@ struct type_caster::ValidType> { if (is_tensor_aligned(arr.data())) { value - = Eigen::TensorMap(arr.data(), shape); + = Eigen::TensorMap(arr.data(), shape); } else { - value = Eigen::TensorMap(arr.data(), shape); + value = Eigen::TensorMap(arr.data(), shape); } return true; @@ -305,21 +302,26 @@ struct type_caster::ValidType> { } }; -template ::value>> -const void *get_array_data_for_type(array &arr) { - return arr.data(); + +template< class T > struct is_const_pointer {}; +template< class T > struct is_const_pointer : std::false_type {}; +template< class T > struct is_const_pointer: std::true_type {}; + +template ::value, bool> = true> +StoragePointerType get_array_data_for_type(array &arr) { + return reinterpret_cast(arr.data()); } -template ::value>> -void *get_array_data_for_type(array &arr) { - return arr.mutable_data(); +template ::value, bool> = true> +StoragePointerType get_array_data_for_type(array &arr) { + return reinterpret_cast(arr.mutable_data()); } template -struct type_caster, - typename eigen_tensor_helper::ValidType> { +struct type_caster, + typename eigen_tensor_helper::type>::ValidType> { using MapType = Eigen::TensorMap; - using Helper = eigen_tensor_helper; + using Helper = eigen_tensor_helper::type>; bool load(handle src, bool /*convert*/) { // Note that we have a lot more checks here as we want to make sure to avoid copies @@ -350,13 +352,12 @@ struct type_caster, return false; } - constexpr bool needs_writeable = !std::is_const::value; if (PYBIND11_SILENCE_MSVC_C4127(needs_writeable && !arr.writeable())) { return false; } - value.reset(new MapType(static_cast( - get_array_data_for_type(arr)), + value.reset(new MapType( + get_array_data_for_type(arr), shape)); return true; @@ -439,13 +440,15 @@ struct type_caster, return result.release(); } + + static constexpr bool needs_writeable = !is_const_pointer::value; protected: // TODO: Move to std::optional once std::optional has more support std::unique_ptr value; public: - static constexpr auto name = get_tensor_descriptor::value; + static constexpr auto name = get_tensor_descriptor::value; explicit operator MapType *() { return value.get(); } explicit operator MapType &() { return *value; } explicit operator MapType &&() && { return std::move(*value); } diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 8a51028389..3746882565 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -71,6 +71,7 @@ struct CustomExample { Eigen::TensorMap> view_member; }; + template void init_tensor_module(pybind11::module &m) { const char *needed_options = ""; @@ -264,7 +265,7 @@ void init_tensor_module(pybind11::module &m) { m.def( "round_trip_const_view_tensor", - [](Eigen::TensorMap> view) { + [](Eigen::TensorMap> view) { return Eigen::Tensor(view); }, py::return_value_policy::move); From db7d3906a4909546a0c5868949bcb708b94a9d85 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 22:51:34 +0000 Subject: [PATCH 094/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 37 +++++++++++++++++++-------------- tests/test_eigen_tensor.cpp | 1 - 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 16a194faaa..a16d71d8ef 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -136,8 +136,9 @@ struct get_tensor_descriptor { ", flags.c_contiguous", ", flags.f_contiguous"); static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") + eigen_tensor_helper::type>::dimensions_descriptor + const_name("]") - + const_name(details, const_name("")) + const_name("]"); + + const_name("[") + + eigen_tensor_helper::type>::dimensions_descriptor + + const_name("]") + const_name(details, const_name("")) + const_name("]"); }; template @@ -173,8 +174,7 @@ struct type_caster::ValidType> { } if (is_tensor_aligned(arr.data())) { - value - = Eigen::TensorMap(arr.data(), shape); + value = Eigen::TensorMap(arr.data(), shape); } else { value = Eigen::TensorMap(arr.data(), shape); } @@ -302,24 +302,29 @@ struct type_caster::ValidType> { } }; +template +struct is_const_pointer {}; +template +struct is_const_pointer : std::false_type {}; +template +struct is_const_pointer : std::true_type {}; -template< class T > struct is_const_pointer {}; -template< class T > struct is_const_pointer : std::false_type {}; -template< class T > struct is_const_pointer: std::true_type {}; - -template ::value, bool> = true> +template ::value, bool> = true> StoragePointerType get_array_data_for_type(array &arr) { return reinterpret_cast(arr.data()); } -template ::value, bool> = true> +template ::value, bool> = true> StoragePointerType get_array_data_for_type(array &arr) { return reinterpret_cast(arr.mutable_data()); } template -struct type_caster, - typename eigen_tensor_helper::type>::ValidType> { +struct type_caster< + Eigen::TensorMap, + typename eigen_tensor_helper::type>::ValidType> { using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper::type>; @@ -356,8 +361,7 @@ struct type_caster, return false; } - value.reset(new MapType( - get_array_data_for_type(arr), + value.reset(new MapType(get_array_data_for_type(arr), shape)); return true; @@ -440,8 +444,9 @@ struct type_caster, return result.release(); } - - static constexpr bool needs_writeable = !is_const_pointer::value; + + static constexpr bool needs_writeable + = !is_const_pointer::value; protected: // TODO: Move to std::optional once std::optional has more support diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 3746882565..0732b2c061 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -71,7 +71,6 @@ struct CustomExample { Eigen::TensorMap> view_member; }; - template void init_tensor_module(pybind11::module &m) { const char *needed_options = ""; From 105d973b10716360e0a59fbdfabcf4465567f277 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 00:13:28 +0000 Subject: [PATCH 095/141] Support older versions of eigen --- include/pybind11/eigen/tensor.h | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index a16d71d8ef..36912311c4 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -321,6 +321,19 @@ StoragePointerType get_array_data_for_type(array &arr) { return reinterpret_cast(arr.mutable_data()); } +template +struct get_storage_pointer_type; + +template +struct get_storage_pointer_type> { + using SPT = typename MapType::StoragePointerType; +}; + +template +struct get_storage_pointer_type> { + using SPT = typename MapType::PointerArgType; +}; + template struct type_caster< Eigen::TensorMap, @@ -361,8 +374,10 @@ struct type_caster< return false; } - value.reset(new MapType(get_array_data_for_type(arr), - shape)); + value.reset(new MapType( + get_array_data_for_type::SPT>(arr), + shape + )); return true; } @@ -444,9 +459,8 @@ struct type_caster< return result.release(); } - - static constexpr bool needs_writeable - = !is_const_pointer::value; + + static constexpr bool needs_writeable = !is_const_pointer::SPT>::value; protected: // TODO: Move to std::optional once std::optional has more support From c96a56d4a5225ce79e201701af8f09dfff9b1d12 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 00:45:10 +0000 Subject: [PATCH 096/141] Weird c++ compilers --- include/pybind11/eigen/tensor.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 36912311c4..d4ab81d07f 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -309,6 +309,7 @@ struct is_const_pointer : std::false_type {}; template struct is_const_pointer : std::true_type {}; + template ::value, bool> = true> StoragePointerType get_array_data_for_type(array &arr) { @@ -325,12 +326,12 @@ template struct get_storage_pointer_type; template -struct get_storage_pointer_type> { +struct get_storage_pointer_type> { using SPT = typename MapType::StoragePointerType; }; template -struct get_storage_pointer_type> { +struct get_storage_pointer_type> { using SPT = typename MapType::PointerArgType; }; From e236da277770929ebd2cf8c7e24bda7de4417b09 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 00:54:46 +0000 Subject: [PATCH 097/141] Fix Eigen bug --- include/pybind11/eigen/tensor.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index d4ab81d07f..984eaf717a 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -173,10 +173,18 @@ struct type_caster::ValidType> { return false; } + + #if EIGEN_VERSION_AT_LEAST(3, 4, 0) + auto data_pointer = arr.data(); + #else + // Handle Eigen bug + auto data_pointer = const_cast(arr.data()); + #endif + if (is_tensor_aligned(arr.data())) { - value = Eigen::TensorMap(arr.data(), shape); + value = Eigen::TensorMap(data_pointer, shape); } else { - value = Eigen::TensorMap(arr.data(), shape); + value = Eigen::TensorMap(data_pointer, shape); } return true; From 5343d8679f7ff4ad5ef4b9c1a29804f5ac60b512 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 01:08:46 +0000 Subject: [PATCH 098/141] Fix another eigen bug --- include/pybind11/eigen/tensor.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 984eaf717a..ceceb7bd4c 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -469,7 +469,12 @@ struct type_caster< return result.release(); } + #if EIGEN_VERSION_AT_LEAST(3, 4, 0) static constexpr bool needs_writeable = !is_const_pointer::SPT>::value; + #else + // Handle Eigen bug + static constexpr bool needs_writeable = !std::is_const::value; + #endif protected: // TODO: Move to std::optional once std::optional has more support From 7beab7b1eaf2e96ed9445625cc75af8f10242e8b Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 01:22:40 +0000 Subject: [PATCH 099/141] Yet another eigen bug --- include/pybind11/eigen/tensor.h | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index ceceb7bd4c..ad67b5ce2e 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -318,14 +318,19 @@ template struct is_const_pointer : std::true_type {}; -template ::value, bool> = true> +template = true> StoragePointerType get_array_data_for_type(array &arr) { + #if EIGEN_VERSION_AT_LEAST(3, 4, 0) return reinterpret_cast(arr.data()); + #else + // Handle Eigen bug + return reinterpret_cast(const_cast(arr.data())); + #endif } -template ::value, bool> = true> +template = true> StoragePointerType get_array_data_for_type(array &arr) { return reinterpret_cast(arr.mutable_data()); } @@ -383,8 +388,10 @@ struct type_caster< return false; } + auto result = get_array_data_for_type::SPT, needs_writeable>(arr); + value.reset(new MapType( - get_array_data_for_type::SPT>(arr), + result, shape )); From 9f4dfff8f95474d265ad6ad73f2b19b679807680 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 02:11:38 +0000 Subject: [PATCH 100/141] Potential flakes? --- include/pybind11/eigen/tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index ad67b5ce2e..b7a2f114eb 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -27,7 +27,7 @@ # pragma GCC diagnostic pop #endif -static_assert(EIGEN_VERSION_AT_LEAST(3, 3, 0), +static_assert(EIGEN_VERSION_AT_LEAST(3, 3,0), "Eigen Tensor support in pybind11 requires Eigen >= 3.3.0"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) From 2c904a0105769a8262555a5bc95484d89f8b9198 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 8 Oct 2022 02:17:37 +0000 Subject: [PATCH 101/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 51 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index b7a2f114eb..d5b2311363 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -27,7 +27,7 @@ # pragma GCC diagnostic pop #endif -static_assert(EIGEN_VERSION_AT_LEAST(3, 3,0), +static_assert(EIGEN_VERSION_AT_LEAST(3, 3, 0), "Eigen Tensor support in pybind11 requires Eigen >= 3.3.0"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -173,13 +173,12 @@ struct type_caster::ValidType> { return false; } - - #if EIGEN_VERSION_AT_LEAST(3, 4, 0) +#if EIGEN_VERSION_AT_LEAST(3, 4, 0) auto data_pointer = arr.data(); - #else +#else // Handle Eigen bug - auto data_pointer = const_cast(arr.data()); - #endif + auto data_pointer = const_cast(arr.data()); +#endif if (is_tensor_aligned(arr.data())) { value = Eigen::TensorMap(data_pointer, shape); @@ -317,25 +316,26 @@ struct is_const_pointer : std::false_type {}; template struct is_const_pointer : std::true_type {}; - -template = true> StoragePointerType get_array_data_for_type(array &arr) { - #if EIGEN_VERSION_AT_LEAST(3, 4, 0) +#if EIGEN_VERSION_AT_LEAST(3, 4, 0) return reinterpret_cast(arr.data()); - #else - // Handle Eigen bug - return reinterpret_cast(const_cast(arr.data())); - #endif +#else + // Handle Eigen bug + return reinterpret_cast(const_cast(arr.data())); +#endif } -template = true> StoragePointerType get_array_data_for_type(array &arr) { return reinterpret_cast(arr.mutable_data()); } -template +template struct get_storage_pointer_type; template @@ -388,12 +388,10 @@ struct type_caster< return false; } - auto result = get_array_data_for_type::SPT, needs_writeable>(arr); + auto result = get_array_data_for_type::SPT, + needs_writeable>(arr); - value.reset(new MapType( - result, - shape - )); + value.reset(new MapType(result, shape)); return true; } @@ -475,13 +473,14 @@ struct type_caster< return result.release(); } - - #if EIGEN_VERSION_AT_LEAST(3, 4, 0) - static constexpr bool needs_writeable = !is_const_pointer::SPT>::value; - #else - // Handle Eigen bug + +#if EIGEN_VERSION_AT_LEAST(3, 4, 0) + static constexpr bool needs_writeable + = !is_const_pointer::SPT>::value; +#else + // Handle Eigen bug static constexpr bool needs_writeable = !std::is_const::value; - #endif +#endif protected: // TODO: Move to std::optional once std::optional has more support From b7664b0a2887affc900cae87a8796fa653809bdb Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 18:23:40 +0000 Subject: [PATCH 102/141] Rerun tests with dummy exception to find out what is going on --- include/pybind11/eigen/tensor.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index d5b2311363..470cc420dd 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -397,10 +397,19 @@ struct type_caster< } static handle cast(MapType &&src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::take_ownership) { + pybind11_fail("wat"); + } return cast_impl(&src, policy, parent); } static handle cast(const MapType &&src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::take_ownership) { + + + + pybind11_fail("wat2"); + } return cast_impl(&src, policy, parent); } From d73069493f658d0e69aa2a0e28fe088a5c8e1016 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 18:36:30 +0000 Subject: [PATCH 103/141] Another dummy test run --- tests/test_eigen_tensor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index eac26e55ac..c2cab6c8c5 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -30,6 +30,7 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=0): @pytest.mark.parametrize("m", submodules) @pytest.mark.parametrize("member_name", ["member", "member_view"]) def test_reference_internal(m, member_name): + pytest.skip("Debug 7 second") if not hasattr(sys, "getrefcount"): pytest.skip("No reference counting") foo = m.CustomExample() @@ -72,6 +73,7 @@ def test_convert_tensor_to_py(m): @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): + pytest.skip("Debug 7") with pytest.raises( RuntimeError, match="Cannot use reference internal when there is no parent" ): From d5b73c7ab106b048a9dfb8ec6bf0551287895a3a Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 18:45:37 +0000 Subject: [PATCH 104/141] Ablate more --- tests/test_eigen_tensor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index c2cab6c8c5..57c3d537ac 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -46,6 +46,7 @@ def test_reference_internal(m, member_name): @pytest.mark.parametrize("m", submodules) def test_convert_tensor_to_py(m): + pytest.skip("Debug 7 second") assert_equal_tensor_ref(m.copy_tensor()) assert_equal_tensor_ref(m.copy_fixed_tensor()) assert_equal_tensor_ref(m.copy_const_tensor()) @@ -96,6 +97,7 @@ def test_bad_cpp_to_python_casts(m): @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): + pytest.skip("Debug 7 second") with pytest.raises(TypeError): m.round_trip_tensor(np.zeros((2, 3))) @@ -183,6 +185,7 @@ def test_round_trip(m): @pytest.mark.parametrize("m", submodules) def test_round_trip_references_actually_refer(m): + pytest.skip("Debug 7 second") # Need to create a copy that matches the type on the C side copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) a = m.round_trip_view_tensor(copy) From e85c92b31708425f01962c44f48b3185f0a96661 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 18:59:36 +0000 Subject: [PATCH 105/141] Found the broken test? --- include/pybind11/eigen/tensor.h | 1 + tests/test_eigen_tensor.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 470cc420dd..9e02df937e 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -241,6 +241,7 @@ struct type_caster::ValidType> { template static handle cast_impl(C *src, return_value_policy policy, handle parent) { + printf("Working with %d\n", (int) policy); object parent_object; bool writeable = false; switch (policy) { diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 57c3d537ac..0e5f9706f0 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -6,6 +6,7 @@ eigen_tensor = pytest.importorskip("pybind11_tests.eigen_tensor") submodules = [eigen_tensor.c_style, eigen_tensor.f_style] +submodules = [eigen_tensor.c_style] tensor_ref = np.empty((3, 5, 2), dtype=np.int64) @@ -145,6 +146,7 @@ def test_bad_python_to_cpp_casts(m): @pytest.mark.parametrize("m", submodules) def test_references_actually_refer(m): + pytest.skip("Debug 7 second") a = m.reference_tensor() temp = a[indices] a[indices] = 100 @@ -198,6 +200,7 @@ def test_round_trip_references_actually_refer(m): @pytest.mark.parametrize("m", submodules) def test_doc_string(m, doc): + pytest.skip("Debug 7 second") assert ( doc(m.copy_tensor) == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" ) From e4901333d6c7cb052a553f9ff7e9ed779512327e Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 19:11:10 +0000 Subject: [PATCH 106/141] One step closer --- include/pybind11/eigen/tensor.h | 1 + tests/test_eigen_tensor.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 9e02df937e..ec67961fe4 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -148,6 +148,7 @@ struct type_caster::ValidType> { PYBIND11_TYPE_CASTER(Type, temp_name); bool load(handle src, bool convert) { + printf("Loading it %d\n", convert); if (!convert) { array temp = array::ensure(src); if (!temp) { diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 0e5f9706f0..181d2e741d 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -163,7 +163,7 @@ def test_references_actually_refer(m): @pytest.mark.parametrize("m", submodules) def test_round_trip(m): - assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) + # assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) with pytest.raises(TypeError): assert_equal_tensor_ref(m.round_trip_tensor2(tensor_ref)) From 3fc8eeaf1efd5772146e829f48f0234ad56d7d51 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 19:20:29 +0000 Subject: [PATCH 107/141] one step further --- tests/test_eigen_tensor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 181d2e741d..33f3593eb7 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -165,8 +165,8 @@ def test_references_actually_refer(m): def test_round_trip(m): # assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) - with pytest.raises(TypeError): - assert_equal_tensor_ref(m.round_trip_tensor2(tensor_ref)) + # with pytest.raises(TypeError): + # assert_equal_tensor_ref(m.round_trip_tensor2(tensor_ref)) assert_equal_tensor_ref(m.round_trip_tensor2(np.array(tensor_ref, dtype=np.int32))) From 0c766c93ed9f2ca99fca46c0ef5272aa030f9c06 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 19:46:21 +0000 Subject: [PATCH 108/141] Double check --- tests/test_eigen_tensor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 33f3593eb7..06c0421f9d 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -170,6 +170,8 @@ def test_round_trip(m): assert_equal_tensor_ref(m.round_trip_tensor2(np.array(tensor_ref, dtype=np.int32))) + return + assert_equal_tensor_ref(m.round_trip_fixed_tensor(tensor_ref)) assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) From 08278ee2c1d6ec00f5cfac02ff185a7e6ca79413 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 20:20:37 +0000 Subject: [PATCH 109/141] one thing at a time --- tests/test_eigen_tensor.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 0732b2c061..b08c32bb1b 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -6,6 +6,7 @@ */ #include +#include #include "pybind11_tests.h" @@ -233,7 +234,9 @@ void init_tensor_module(pybind11::module &m) { py::arg("tensor").noconvert()); m.def("round_trip_tensor2", - [](const Eigen::Tensor &tensor) { return tensor; }); + [](const Eigen::Tensor &tensor) { + std::cerr<, Options> &tensor) { From 2291c5b5568ac2576aabe69f456665cf123eb595 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 20:45:56 +0000 Subject: [PATCH 110/141] Give up and disable the test --- include/pybind11/eigen/tensor.h | 16 ++++------------ tests/CMakeLists.txt | 9 +++++++++ tests/test_eigen_tensor.cpp | 8 +++----- tests/test_eigen_tensor.py | 17 +++-------------- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index ec67961fe4..342fda8e69 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -10,6 +10,10 @@ #include "../numpy.h" +#if defined(__GNUC__) +static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); +#endif + #if defined(_MSC_VER) # pragma warning(push) # pragma warning(disable : 4554) // Tensor.h warning @@ -69,17 +73,7 @@ struct eigen_tensor_helper struct helper> { -#if defined(__GNUC__) && __GNUC__ <= 4 - - // Hack to work around gcc 4.8 bugs. - static constexpr descr value - = concat(const_name(((void) Is, "?"))...); - -#else - static constexpr auto value = concat(const_name(((void) Is, "?"))...); - -#endif }; static constexpr auto dimensions_descriptor @@ -148,7 +142,6 @@ struct type_caster::ValidType> { PYBIND11_TYPE_CASTER(Type, temp_name); bool load(handle src, bool convert) { - printf("Loading it %d\n", convert); if (!convert) { array temp = array::ensure(src); if (!temp) { @@ -242,7 +235,6 @@ struct type_caster::ValidType> { template static handle cast_impl(C *src, return_value_policy policy, handle parent) { - printf("Working with %d\n", (int) policy); object parent_object; bool writeable = false; switch (policy) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 273508409d..6438e2a05b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -307,6 +307,15 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) endif() endif() +# Some code doesn't support gcc 4 +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() +endif() + + # Optional dependency for some tests (boost::variant is only supported with version >= 1.56) find_package(Boost 1.56) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index b08c32bb1b..579e118c95 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -6,15 +6,14 @@ */ #include -#include #include "pybind11_tests.h" template void reset_tensor(M &x) { - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 5; j++) { - for (int k = 0; k < 2; k++) { + for (int i = 0; i < x.dimension(0); i++) { + for (int j = 0; j < x.dimension(1); j++) { + for (int k = 0; k < x.dimension(2); k++) { x(i, j, k) = i * (5 * 2) + j * 2 + k; } } @@ -235,7 +234,6 @@ void init_tensor_module(pybind11::module &m) { m.def("round_trip_tensor2", [](const Eigen::Tensor &tensor) { - std::cerr< numpy.ndarray[numpy.float64[?, ?, ?]]" ) From 0e240049e22271e27197ec38cd0170b89d7ac580 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 20:53:47 +0000 Subject: [PATCH 111/141] Clang lies about being gcc --- include/pybind11/eigen/tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 342fda8e69..02c70f7646 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -10,7 +10,7 @@ #include "../numpy.h" -#if defined(__GNUC__) +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); #endif From 0f06d2f6b90e804e4cdc730600f73939809186e9 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sat, 8 Oct 2022 20:59:48 +0000 Subject: [PATCH 112/141] Oops, fix matrix test --- tests/test_eigen_matrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_eigen_matrix.cpp b/tests/test_eigen_matrix.cpp index b98eee492e..71e41d198a 100644 --- a/tests/test_eigen_matrix.cpp +++ b/tests/test_eigen_matrix.cpp @@ -7,7 +7,7 @@ BSD-style license that can be found in the LICENSE file. */ -#include +#include #include #include "constructor_stats.h" From a731b2163de52633cb626d0b2ff11d546f005c6d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 8 Oct 2022 21:05:45 +0000 Subject: [PATCH 113/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 14 ++++++-------- tests/CMakeLists.txt | 11 +++++------ tests/test_eigen_tensor.cpp | 3 +-- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 02c70f7646..2c12b71a09 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -391,19 +391,17 @@ struct type_caster< } static handle cast(MapType &&src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::take_ownership) { - pybind11_fail("wat"); - } + if (policy == return_value_policy::take_ownership) { + pybind11_fail("wat"); + } return cast_impl(&src, policy, parent); } static handle cast(const MapType &&src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::take_ownership) { - + if (policy == return_value_policy::take_ownership) { - - pybind11_fail("wat2"); - } + pybind11_fail("wat2"); + } return cast_impl(&src, policy, parent); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6438e2a05b..75a3a90072 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -308,14 +308,13 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) endif() # Some code doesn't support gcc 4 -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) - list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) - if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) - list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) - endif() +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() endif() - # Optional dependency for some tests (boost::variant is only supported with version >= 1.56) find_package(Boost 1.56) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 579e118c95..af02eb8c63 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -233,8 +233,7 @@ void init_tensor_module(pybind11::module &m) { py::arg("tensor").noconvert()); m.def("round_trip_tensor2", - [](const Eigen::Tensor &tensor) { - return tensor; }); + [](const Eigen::Tensor &tensor) { return tensor; }); m.def("round_trip_fixed_tensor", [](const Eigen::TensorFixedSize, Options> &tensor) { From 9744995f84f1534cfaf4a3f1901f1ee6fae1eb18 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Sun, 9 Oct 2022 01:55:30 +0000 Subject: [PATCH 114/141] Add tests to verify scalar conversions --- include/pybind11/eigen/tensor.h | 13 ++++++------- tests/test_eigen_tensor.cpp | 22 ++++++++++++++++++++++ tests/test_eigen_tensor.py | 15 +++++++++++++++ tests/test_numpy_array.cpp | 2 ++ tests/test_numpy_array.py | 5 +++++ 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 2c12b71a09..5bf98ecfb9 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -143,6 +143,9 @@ struct type_caster::ValidType> { bool load(handle src, bool convert) { if (!convert) { + if (!isinstance(src)) { + return false; + } array temp = array::ensure(src); if (!temp) { return false; @@ -351,6 +354,9 @@ struct type_caster< bool load(handle src, bool /*convert*/) { // Note that we have a lot more checks here as we want to make sure to avoid copies + if (!isinstance(src)) { + return false; + } auto arr = reinterpret_borrow(src); if ((arr.flags() & compute_array_flag_from_tensor()) == 0) { return false; @@ -391,17 +397,10 @@ struct type_caster< } static handle cast(MapType &&src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::take_ownership) { - pybind11_fail("wat"); - } return cast_impl(&src, policy, parent); } static handle cast(const MapType &&src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::take_ownership) { - - pybind11_fail("wat2"); - } return cast_impl(&src, policy, parent); } diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index af02eb8c63..4b01eb711d 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -268,6 +268,28 @@ void init_tensor_module(pybind11::module &m) { return Eigen::Tensor(view); }, py::return_value_policy::move); + + m.def( + "round_trip_rank_0", + [](const Eigen::Tensor& tensor) { + return tensor; + }, + py::return_value_policy::move); + + m.def( + "round_trip_rank_0_noconvert", + [](const Eigen::Tensor& tensor) { + return tensor; + }, py::arg("tensor").noconvert(), + py::return_value_policy::move); + + m.def( + "round_trip_rank_0_view", + [](Eigen::TensorMap>& tensor) { + return tensor; + }, + py::return_value_policy::reference); + } TEST_SUBMODULE(eigen_tensor, m) { diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 221769566c..1cde6a7cfc 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -177,6 +177,21 @@ def test_round_trip(m): tensor_ref[:, ::-1, :], m.round_trip_tensor(tensor_ref[:, ::-1, :]) ) + assert m.round_trip_rank_0(np.float64(3.5)) == 3.5 + assert m.round_trip_rank_0(3.5) == 3.5 + + with pytest.raises(TypeError): + m.round_trip_rank_0_noconvert(np.float64(3.5)) + + with pytest.raises(TypeError): + m.round_trip_rank_0_noconvert(3.5) + + with pytest.raises(TypeError): + m.round_trip_rank_0_view(np.float64(3.5)) + + with pytest.raises(TypeError): + m.round_trip_rank_0_view(3.5) + @pytest.mark.parametrize("m", submodules) def test_round_trip_references_actually_refer(m): diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index 69ddbe1ef2..d461a02c9e 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -521,4 +521,6 @@ TEST_SUBMODULE(numpy_array, sm) { sm.def("test_fmt_desc_double", [](const py::array_t &) {}); sm.def("test_fmt_desc_const_float", [](const py::array_t &) {}); sm.def("test_fmt_desc_const_double", [](const py::array_t &) {}); + + sm.def("round_trip_float", [](double d) {return d;}); } diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 504963b166..6f6ed36053 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -585,3 +585,8 @@ def test_dtype_refcount_leak(): m.ndim(a) after = getrefcount(dtype) assert after == before + +def test_round_trip_float(): + arr = np.zeros((), np.float64) + arr[()] = 37.2 + assert m.round_trip_float(arr) == 37.2 From 4dbd3e571ab6ab1d126dc3773e3b6c5d3c5a26ea Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 9 Oct 2022 02:41:28 +0000 Subject: [PATCH 115/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 10 +++++----- tests/test_eigen_tensor.cpp | 14 ++++---------- tests/test_numpy_array.cpp | 2 +- tests/test_numpy_array.py | 1 + 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 5bf98ecfb9..b47d74fdfc 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -143,9 +143,9 @@ struct type_caster::ValidType> { bool load(handle src, bool convert) { if (!convert) { - if (!isinstance(src)) { + if (!isinstance(src)) { return false; - } + } array temp = array::ensure(src); if (!temp) { return false; @@ -354,9 +354,9 @@ struct type_caster< bool load(handle src, bool /*convert*/) { // Note that we have a lot more checks here as we want to make sure to avoid copies - if (!isinstance(src)) { - return false; - } + if (!isinstance(src)) { + return false; + } auto arr = reinterpret_borrow(src); if ((arr.flags() & compute_array_flag_from_tensor()) == 0) { return false; diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 4b01eb711d..6c12a0e747 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -271,25 +271,19 @@ void init_tensor_module(pybind11::module &m) { m.def( "round_trip_rank_0", - [](const Eigen::Tensor& tensor) { - return tensor; - }, + [](const Eigen::Tensor &tensor) { return tensor; }, py::return_value_policy::move); m.def( "round_trip_rank_0_noconvert", - [](const Eigen::Tensor& tensor) { - return tensor; - }, py::arg("tensor").noconvert(), + [](const Eigen::Tensor &tensor) { return tensor; }, + py::arg("tensor").noconvert(), py::return_value_policy::move); m.def( "round_trip_rank_0_view", - [](Eigen::TensorMap>& tensor) { - return tensor; - }, + [](Eigen::TensorMap> &tensor) { return tensor; }, py::return_value_policy::reference); - } TEST_SUBMODULE(eigen_tensor, m) { diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index d461a02c9e..b118e2c6cc 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -522,5 +522,5 @@ TEST_SUBMODULE(numpy_array, sm) { sm.def("test_fmt_desc_const_float", [](const py::array_t &) {}); sm.def("test_fmt_desc_const_double", [](const py::array_t &) {}); - sm.def("round_trip_float", [](double d) {return d;}); + sm.def("round_trip_float", [](double d) { return d; }); } diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 6f6ed36053..cdec9ad60b 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -586,6 +586,7 @@ def test_dtype_refcount_leak(): after = getrefcount(dtype) assert after == before + def test_round_trip_float(): arr = np.zeros((), np.float64) arr[()] = 37.2 From a5998c5b0364280c92f3768aabc2d94e85d3895c Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 10 Oct 2022 23:53:04 +0000 Subject: [PATCH 116/141] Fix nits --- include/pybind11/eigen.h | 2 ++ include/pybind11/eigen/matrix.h | 1 - tests/test_eigen_tensor.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 594659f5d6..3866957d4a 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -9,6 +9,8 @@ #pragma once +#include + #include "eigen/matrix.h" #if EIGEN_VERSION_AT_LEAST(3, 3, 0) diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index 7c1fd5fa34..5f5ad3867b 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -1,4 +1,3 @@ - /* pybind11/eigen/matrix.h: Transparent conversion for dense and sparse Eigen matrices diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 1cde6a7cfc..8e7f0c648c 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -17,11 +17,11 @@ indices = (2, 3, 1) -def assert_equal_tensor_ref(mat, writeable=True, modified=0): +def assert_equal_tensor_ref(mat, writeable=True, modified=None): assert mat.flags.writeable == writeable copy = np.array(tensor_ref) - if modified != 0: + if modified is not None: copy[indices] = modified np.testing.assert_array_equal(mat, copy) From 5946c86ff40dde892ba014c74b17d6e4bc2834ce Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 18:12:38 +0000 Subject: [PATCH 117/141] Support no_array --- include/pybind11/eigen/tensor.h | 22 +- tests/CMakeLists.txt | 1 + tests/test_eigen_tensor.cpp | 288 +------------------------ tests/test_eigen_tensor.include.cpp | 302 +++++++++++++++++++++++++++ tests/test_eigen_tensor.py | 7 +- tests/test_eigen_tensor_no_array.cpp | 12 ++ 6 files changed, 341 insertions(+), 291 deletions(-) create mode 100644 tests/test_eigen_tensor.include.cpp create mode 100644 tests/test_eigen_tensor_no_array.cpp diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index b47d74fdfc..644952e361 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -135,6 +135,16 @@ struct get_tensor_descriptor { + const_name("]") + const_name(details, const_name("")) + const_name("]"); }; +template +std::vector convert_dsizes_to_vector(const Eigen::DSizes& arr) { + std::vector result(size); + for (size_t i = 0; i < size; i++) { + result[i] = arr[i]; + } + + return result; +} + template struct type_caster::ValidType> { using Helper = eigen_tensor_helper; @@ -164,7 +174,9 @@ struct type_caster::ValidType> { } Eigen::DSizes shape; - std::copy(arr.shape(), arr.shape() + Type::NumIndices, shape.begin()); + for (size_t i = 0; i < Type::NumIndices; i++) { + shape[i] = arr.shape()[i]; + } if (!Helper::is_correct_shape(shape)) { return false; @@ -296,7 +308,7 @@ struct type_caster::ValidType> { } auto result = array_t()>( - Helper::get_shape(*src), src->data(), parent_object); + convert_dsizes_to_vector(Helper::get_shape(*src)), src->data(), parent_object); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; @@ -378,7 +390,9 @@ struct type_caster< } Eigen::DSizes shape; - std::copy(arr.shape(), arr.shape() + Type::NumIndices, shape.begin()); + for (size_t i = 0; i < Type::NumIndices; i++) { + shape[i] = arr.shape()[i]; + } if (!Helper::is_correct_shape(shape)) { return false; @@ -465,7 +479,7 @@ struct type_caster< } auto result = array_t()>( - Helper::get_shape(*src), src->data(), parent_object); + convert_dsizes_to_vector(Helper::get_shape(*src)), src->data(), parent_object); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 75a3a90072..8d5cd4e0b9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -130,6 +130,7 @@ set(PYBIND11_TEST_FILES test_docstring_options test_eigen_matrix test_eigen_tensor + test_eigen_tensor_no_array.cpp test_enum test_eval test_exceptions diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 6c12a0e747..de7a87f66b 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -5,291 +5,7 @@ BSD-style license that can be found in the LICENSE file. */ -#include -#include "pybind11_tests.h" +constexpr const char* TEST_MODULE_NAME = "eigen_tensor"; -template -void reset_tensor(M &x) { - for (int i = 0; i < x.dimension(0); i++) { - for (int j = 0; j < x.dimension(1); j++) { - for (int k = 0; k < x.dimension(2); k++) { - x(i, j, k) = i * (5 * 2) + j * 2 + k; - } - } - } -} - -template -Eigen::Tensor &get_tensor() { - static Eigen::Tensor *x; - - if (!x) { - x = new Eigen::Tensor(3, 5, 2); - reset_tensor(*x); - } - - return *x; -} - -template -Eigen::TensorMap> &get_tensor_map() { - static Eigen::TensorMap> *x; - - if (!x) { - x = new Eigen::TensorMap>(get_tensor()); - } - - return *x; -} - -template -Eigen::TensorFixedSize, Options> &get_fixed_tensor() { - static Eigen::TensorFixedSize, Options> *x; - - if (!x) { - Eigen::aligned_allocator, Options>> - allocator; - x = new (allocator.allocate(1)) - Eigen::TensorFixedSize, Options>(); - reset_tensor(*x); - } - - return *x; -} - -template -const Eigen::Tensor &get_const_tensor() { - return get_tensor(); -} - -template -struct CustomExample { - CustomExample() : member(get_tensor()), view_member(member) {} - - Eigen::Tensor member; - Eigen::TensorMap> view_member; -}; - -template -void init_tensor_module(pybind11::module &m) { - const char *needed_options = ""; - if (PYBIND11_SILENCE_MSVC_C4127(Options == Eigen::ColMajor)) { - needed_options = "F"; - } else { - needed_options = "C"; - } - m.attr("needed_options") = needed_options; - - py::class_>(m, "CustomExample") - .def(py::init<>()) - .def_readonly( - "member", &CustomExample::member, py::return_value_policy::reference_internal) - .def_readonly("member_view", - &CustomExample::view_member, - py::return_value_policy::reference_internal); - - m.def( - "copy_fixed_tensor", - []() { return &get_fixed_tensor(); }, - py::return_value_policy::copy); - - m.def( - "copy_tensor", []() { return &get_tensor(); }, py::return_value_policy::copy); - - m.def( - "copy_const_tensor", - []() { return &get_const_tensor(); }, - py::return_value_policy::copy); - - m.def("move_fixed_tensor", []() { return get_fixed_tensor(); }); - - m.def("move_tensor", []() { return get_tensor(); }); - - m.def("move_const_tensor", - // NOLINTNEXTLINE(readability-const-return-type) - []() -> const Eigen::Tensor { return get_const_tensor(); }); - - m.def( - "take_fixed_tensor", - - []() { - Eigen::aligned_allocator< - Eigen::TensorFixedSize, Options>> - allocator; - return new (allocator.allocate(1)) - Eigen::TensorFixedSize, Options>( - get_fixed_tensor()); - }, - py::return_value_policy::take_ownership); - - m.def( - "take_tensor", - []() { return new Eigen::Tensor(get_tensor()); }, - py::return_value_policy::take_ownership); - - m.def( - "take_const_tensor", - []() -> const Eigen::Tensor * { - return new Eigen::Tensor(get_tensor()); - }, - py::return_value_policy::take_ownership); - - m.def( - "take_view_tensor", - []() -> const Eigen::TensorMap> * { - return new Eigen::TensorMap>(get_tensor()); - }, - py::return_value_policy::take_ownership); - - m.def( - "reference_tensor", - []() { return &get_tensor(); }, - py::return_value_policy::reference); - - m.def( - "reference_tensor_v2", - []() -> Eigen::Tensor & { return get_tensor(); }, - py::return_value_policy::reference); - - m.def( - "reference_tensor_internal", - []() { return &get_tensor(); }, - py::return_value_policy::reference_internal); - - m.def( - "reference_fixed_tensor", - []() { return &get_tensor(); }, - py::return_value_policy::reference); - - m.def( - "reference_const_tensor", - []() { return &get_const_tensor(); }, - py::return_value_policy::reference); - - m.def( - "reference_const_tensor_v2", - []() -> const Eigen::Tensor & { return get_const_tensor(); }, - py::return_value_policy::reference); - - m.def( - "reference_view_of_tensor", - []() -> Eigen::TensorMap> { - return get_tensor_map(); - }, - py::return_value_policy::reference); - - m.def( - "reference_view_of_tensor_v2", - // NOLINTNEXTLINE(readability-const-return-type) - []() -> const Eigen::TensorMap> { - return get_tensor_map(); // NOLINT(readability-const-return-type) - }, // NOLINT(readability-const-return-type) - py::return_value_policy::reference); - - m.def( - "reference_view_of_tensor_v3", - []() -> Eigen::TensorMap> * { - return &get_tensor_map(); - }, - py::return_value_policy::reference); - - m.def( - "reference_view_of_tensor_v4", - []() -> const Eigen::TensorMap> * { - return &get_tensor_map(); - }, - py::return_value_policy::reference); - - m.def( - "reference_view_of_tensor_v5", - []() -> Eigen::TensorMap> & { - return get_tensor_map(); - }, - py::return_value_policy::reference); - - m.def( - "reference_view_of_tensor_v6", - []() -> const Eigen::TensorMap> & { - return get_tensor_map(); - }, - py::return_value_policy::reference); - - m.def( - "reference_view_of_fixed_tensor", - []() { - return Eigen::TensorMap< - Eigen::TensorFixedSize, Options>>( - get_fixed_tensor()); - }, - py::return_value_policy::reference); - - m.def("round_trip_tensor", - [](const Eigen::Tensor &tensor) { return tensor; }); - - m.def( - "round_trip_tensor_noconvert", - [](const Eigen::Tensor &tensor) { return tensor; }, - py::arg("tensor").noconvert()); - - m.def("round_trip_tensor2", - [](const Eigen::Tensor &tensor) { return tensor; }); - - m.def("round_trip_fixed_tensor", - [](const Eigen::TensorFixedSize, Options> &tensor) { - return tensor; - }); - - m.def( - "round_trip_view_tensor", - [](Eigen::TensorMap> view) { return view; }, - py::return_value_policy::reference); - - m.def( - "round_trip_view_tensor_ref", - [](Eigen::TensorMap> &view) { return view; }, - py::return_value_policy::reference); - - m.def( - "round_trip_view_tensor_ptr", - [](Eigen::TensorMap> *view) { return view; }, - py::return_value_policy::reference); - - m.def( - "round_trip_aligned_view_tensor", - [](Eigen::TensorMap, Eigen::Aligned> view) { - return view; - }, - py::return_value_policy::reference); - - m.def( - "round_trip_const_view_tensor", - [](Eigen::TensorMap> view) { - return Eigen::Tensor(view); - }, - py::return_value_policy::move); - - m.def( - "round_trip_rank_0", - [](const Eigen::Tensor &tensor) { return tensor; }, - py::return_value_policy::move); - - m.def( - "round_trip_rank_0_noconvert", - [](const Eigen::Tensor &tensor) { return tensor; }, - py::arg("tensor").noconvert(), - py::return_value_policy::move); - - m.def( - "round_trip_rank_0_view", - [](Eigen::TensorMap> &tensor) { return tensor; }, - py::return_value_policy::reference); -} - -TEST_SUBMODULE(eigen_tensor, m) { - auto f_style = m.def_submodule("f_style"); - auto c_style = m.def_submodule("c_style"); - - init_tensor_module(f_style); - init_tensor_module(c_style); -} +#include "test_eigen_tensor.include.cpp" diff --git a/tests/test_eigen_tensor.include.cpp b/tests/test_eigen_tensor.include.cpp new file mode 100644 index 0000000000..6fa032ec3a --- /dev/null +++ b/tests/test_eigen_tensor.include.cpp @@ -0,0 +1,302 @@ +/* + tests/eigen_tensor.cpp -- automatic conversion of Eigen Tensor + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include + +#include "pybind11_tests.h" + +namespace { + +template +void reset_tensor(M &x) { + for (int i = 0; i < x.dimension(0); i++) { + for (int j = 0; j < x.dimension(1); j++) { + for (int k = 0; k < x.dimension(2); k++) { + x(i, j, k) = i * (5 * 2) + j * 2 + k; + } + } + } +} + +template +Eigen::Tensor &get_tensor() { + static Eigen::Tensor *x; + + if (!x) { + x = new Eigen::Tensor(3, 5, 2); + reset_tensor(*x); + } + + return *x; +} + +template +Eigen::TensorMap> &get_tensor_map() { + static Eigen::TensorMap> *x; + + if (!x) { + x = new Eigen::TensorMap>(get_tensor()); + } + + return *x; +} + +template +Eigen::TensorFixedSize, Options> &get_fixed_tensor() { + static Eigen::TensorFixedSize, Options> *x; + + if (!x) { + Eigen::aligned_allocator, Options>> + allocator; + x = new (allocator.allocate(1)) + Eigen::TensorFixedSize, Options>(); + reset_tensor(*x); + } + + return *x; +} + +template +const Eigen::Tensor &get_const_tensor() { + return get_tensor(); +} + +template +struct CustomExample { + CustomExample() : member(get_tensor()), view_member(member) {} + + Eigen::Tensor member; + Eigen::TensorMap> view_member; +}; + +template +void init_tensor_module(pybind11::module &m) { + const char *needed_options = ""; + if (PYBIND11_SILENCE_MSVC_C4127(Options == Eigen::ColMajor)) { + needed_options = "F"; + } else { + needed_options = "C"; + } + m.attr("needed_options") = needed_options; + + py::class_>(m, "CustomExample") + .def(py::init<>()) + .def_readonly( + "member", &CustomExample::member, py::return_value_policy::reference_internal) + .def_readonly("member_view", + &CustomExample::view_member, + py::return_value_policy::reference_internal); + + m.def( + "copy_fixed_tensor", + []() { return &get_fixed_tensor(); }, + py::return_value_policy::copy); + + m.def( + "copy_tensor", []() { return &get_tensor(); }, py::return_value_policy::copy); + + m.def( + "copy_const_tensor", + []() { return &get_const_tensor(); }, + py::return_value_policy::copy); + + m.def("move_fixed_tensor", []() { return get_fixed_tensor(); }); + + m.def("move_tensor", []() { return get_tensor(); }); + + m.def("move_const_tensor", + // NOLINTNEXTLINE(readability-const-return-type) + []() -> const Eigen::Tensor { return get_const_tensor(); }); + + m.def( + "take_fixed_tensor", + + []() { + Eigen::aligned_allocator< + Eigen::TensorFixedSize, Options>> + allocator; + return new (allocator.allocate(1)) + Eigen::TensorFixedSize, Options>( + get_fixed_tensor()); + }, + py::return_value_policy::take_ownership); + + m.def( + "take_tensor", + []() { return new Eigen::Tensor(get_tensor()); }, + py::return_value_policy::take_ownership); + + m.def( + "take_const_tensor", + []() -> const Eigen::Tensor * { + return new Eigen::Tensor(get_tensor()); + }, + py::return_value_policy::take_ownership); + + m.def( + "take_view_tensor", + []() -> const Eigen::TensorMap> * { + return new Eigen::TensorMap>(get_tensor()); + }, + py::return_value_policy::take_ownership); + + m.def( + "reference_tensor", + []() { return &get_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_tensor_v2", + []() -> Eigen::Tensor & { return get_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_tensor_internal", + []() { return &get_tensor(); }, + py::return_value_policy::reference_internal); + + m.def( + "reference_fixed_tensor", + []() { return &get_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_const_tensor", + []() { return &get_const_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_const_tensor_v2", + []() -> const Eigen::Tensor & { return get_const_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor", + []() -> Eigen::TensorMap> { + return get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v2", + // NOLINTNEXTLINE(readability-const-return-type) + []() -> const Eigen::TensorMap> { + return get_tensor_map(); // NOLINT(readability-const-return-type) + }, // NOLINT(readability-const-return-type) + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v3", + []() -> Eigen::TensorMap> * { + return &get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v4", + []() -> const Eigen::TensorMap> * { + return &get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v5", + []() -> Eigen::TensorMap> & { + return get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v6", + []() -> const Eigen::TensorMap> & { + return get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_fixed_tensor", + []() { + return Eigen::TensorMap< + Eigen::TensorFixedSize, Options>>( + get_fixed_tensor()); + }, + py::return_value_policy::reference); + + m.def("round_trip_tensor", + [](const Eigen::Tensor &tensor) { return tensor; }); + + m.def( + "round_trip_tensor_noconvert", + [](const Eigen::Tensor &tensor) { return tensor; }, + py::arg("tensor").noconvert()); + + m.def("round_trip_tensor2", + [](const Eigen::Tensor &tensor) { return tensor; }); + + m.def("round_trip_fixed_tensor", + [](const Eigen::TensorFixedSize, Options> &tensor) { + return tensor; + }); + + m.def( + "round_trip_view_tensor", + [](Eigen::TensorMap> view) { return view; }, + py::return_value_policy::reference); + + m.def( + "round_trip_view_tensor_ref", + [](Eigen::TensorMap> &view) { return view; }, + py::return_value_policy::reference); + + m.def( + "round_trip_view_tensor_ptr", + [](Eigen::TensorMap> *view) { return view; }, + py::return_value_policy::reference); + + m.def( + "round_trip_aligned_view_tensor", + [](Eigen::TensorMap, Eigen::Aligned> view) { + return view; + }, + py::return_value_policy::reference); + + m.def( + "round_trip_const_view_tensor", + [](Eigen::TensorMap> view) { + return Eigen::Tensor(view); + }, + py::return_value_policy::move); + + m.def( + "round_trip_rank_0", + [](const Eigen::Tensor &tensor) { return tensor; }, + py::return_value_policy::move); + + m.def( + "round_trip_rank_0_noconvert", + [](const Eigen::Tensor &tensor) { return tensor; }, + py::arg("tensor").noconvert(), + py::return_value_policy::move); + + m.def( + "round_trip_rank_0_view", + [](Eigen::TensorMap> &tensor) { return tensor; }, + py::return_value_policy::reference); +} + + +void test_module(py::module_ &); +test_initializer name(TEST_MODULE_NAME, test_module); +void test_module(py::module_ &m) { + auto f_style = m.def_submodule("f_style"); + auto c_style = m.def_submodule("c_style"); + + init_tensor_module(f_style); + init_tensor_module(c_style); +} + +} diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 8e7f0c648c..653bd2f9be 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -4,8 +4,13 @@ np = pytest.importorskip("numpy") eigen_tensor = pytest.importorskip("pybind11_tests.eigen_tensor") - submodules = [eigen_tensor.c_style, eigen_tensor.f_style] +try: + from pybind11_tests import eigen_tensor_no_array + submodules += [eigen_tensor_no_array.c_style, eigen_tensor_no_array.f_style] +except ImportError: + print("Could not load no_array tests", sys.stderr) + pass tensor_ref = np.empty((3, 5, 2), dtype=np.int64) diff --git a/tests/test_eigen_tensor_no_array.cpp b/tests/test_eigen_tensor_no_array.cpp new file mode 100644 index 0000000000..418ea4964a --- /dev/null +++ b/tests/test_eigen_tensor_no_array.cpp @@ -0,0 +1,12 @@ +/* + tests/eigen_tensor.cpp -- automatic conversion of Eigen Tensor + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + + +constexpr const char* TEST_MODULE_NAME = "eigen_tensor_no_array"; +#define EIGEN_AVOID_STL_ARRAY + +#include "test_eigen_tensor.include.cpp" From fdbf0040c8b7ec599ceefccf6482ff0644eb35d8 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 18:27:34 +0000 Subject: [PATCH 118/141] Fix tests --- tests/test_eigen_tensor.cpp | 4 +++- tests/test_eigen_tensor.include.cpp | 2 +- tests/test_eigen_tensor_no_array.cpp | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index de7a87f66b..9e9675f225 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -8,4 +8,6 @@ constexpr const char* TEST_MODULE_NAME = "eigen_tensor"; -#include "test_eigen_tensor.include.cpp" +#define TEST_NAMESPACE eigen_tensor + +#include "test_eigen_tensor.include.cpp" \ No newline at end of file diff --git a/tests/test_eigen_tensor.include.cpp b/tests/test_eigen_tensor.include.cpp index 6fa032ec3a..68c13931a5 100644 --- a/tests/test_eigen_tensor.include.cpp +++ b/tests/test_eigen_tensor.include.cpp @@ -9,7 +9,7 @@ #include "pybind11_tests.h" -namespace { +namespace TEST_NAMESPACE { template void reset_tensor(M &x) { diff --git a/tests/test_eigen_tensor_no_array.cpp b/tests/test_eigen_tensor_no_array.cpp index 418ea4964a..a6fff7875a 100644 --- a/tests/test_eigen_tensor_no_array.cpp +++ b/tests/test_eigen_tensor_no_array.cpp @@ -9,4 +9,6 @@ constexpr const char* TEST_MODULE_NAME = "eigen_tensor_no_array"; #define EIGEN_AVOID_STL_ARRAY +#define TEST_NAMESPACE eigen_tensor_no_array + #include "test_eigen_tensor.include.cpp" From 49aaf0c21c71798a22ddd6f7c6ef7e2ac2406497 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Oct 2022 18:33:41 +0000 Subject: [PATCH 119/141] style: pre-commit fixes --- include/pybind11/eigen.h | 4 ++-- include/pybind11/eigen/tensor.h | 4 ++-- tests/test_eigen_tensor.cpp | 5 ++--- tests/test_eigen_tensor.include.cpp | 7 +++---- tests/test_eigen_tensor.py | 1 + tests/test_eigen_tensor_no_array.cpp | 5 ++--- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 3866957d4a..51308054bd 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -9,10 +9,10 @@ #pragma once -#include - #include "eigen/matrix.h" +#include + #if EIGEN_VERSION_AT_LEAST(3, 3, 0) # include "eigen/tensor.h" #endif diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 644952e361..8c4da3e794 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -135,8 +135,8 @@ struct get_tensor_descriptor { + const_name("]") + const_name(details, const_name("")) + const_name("]"); }; -template -std::vector convert_dsizes_to_vector(const Eigen::DSizes& arr) { +template +std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { std::vector result(size); for (size_t i = 0; i < size; i++) { result[i] = arr[i]; diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 9e9675f225..00f553e1a3 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -5,9 +5,8 @@ BSD-style license that can be found in the LICENSE file. */ - -constexpr const char* TEST_MODULE_NAME = "eigen_tensor"; +constexpr const char *TEST_MODULE_NAME = "eigen_tensor"; #define TEST_NAMESPACE eigen_tensor -#include "test_eigen_tensor.include.cpp" \ No newline at end of file +#include "test_eigen_tensor.include.cpp" diff --git a/tests/test_eigen_tensor.include.cpp b/tests/test_eigen_tensor.include.cpp index 68c13931a5..417e0de71b 100644 --- a/tests/test_eigen_tensor.include.cpp +++ b/tests/test_eigen_tensor.include.cpp @@ -288,9 +288,8 @@ void init_tensor_module(pybind11::module &m) { py::return_value_policy::reference); } - -void test_module(py::module_ &); -test_initializer name(TEST_MODULE_NAME, test_module); +void test_module(py::module_ &); +test_initializer name(TEST_MODULE_NAME, test_module); void test_module(py::module_ &m) { auto f_style = m.def_submodule("f_style"); auto c_style = m.def_submodule("c_style"); @@ -299,4 +298,4 @@ void test_module(py::module_ &m) { init_tensor_module(c_style); } -} +} // namespace TEST_NAMESPACE diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 653bd2f9be..f7c81cb20e 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -7,6 +7,7 @@ submodules = [eigen_tensor.c_style, eigen_tensor.f_style] try: from pybind11_tests import eigen_tensor_no_array + submodules += [eigen_tensor_no_array.c_style, eigen_tensor_no_array.f_style] except ImportError: print("Could not load no_array tests", sys.stderr) diff --git a/tests/test_eigen_tensor_no_array.cpp b/tests/test_eigen_tensor_no_array.cpp index a6fff7875a..294ac81e7a 100644 --- a/tests/test_eigen_tensor_no_array.cpp +++ b/tests/test_eigen_tensor_no_array.cpp @@ -5,9 +5,8 @@ BSD-style license that can be found in the LICENSE file. */ - -constexpr const char* TEST_MODULE_NAME = "eigen_tensor_no_array"; -#define EIGEN_AVOID_STL_ARRAY +constexpr const char *TEST_MODULE_NAME = "eigen_tensor_no_array"; +#define EIGEN_AVOID_STL_ARRAY #define TEST_NAMESPACE eigen_tensor_no_array From d17a9f55197c138151395c33c921954fb2d13e1e Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 18:39:08 +0000 Subject: [PATCH 120/141] Silence compiler warning --- include/pybind11/eigen/tensor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 8c4da3e794..be83dc914b 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -138,7 +138,7 @@ struct get_tensor_descriptor { template std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { std::vector result(size); - for (size_t i = 0; i < size; i++) { + for (int i = 0; i < size; i++) { result[i] = arr[i]; } @@ -175,7 +175,7 @@ struct type_caster::ValidType> { Eigen::DSizes shape; for (size_t i = 0; i < Type::NumIndices; i++) { - shape[i] = arr.shape()[i]; + shape[i] = arr.shape()[i ]; } if (!Helper::is_correct_shape(shape)) { From 7ab836423b87fb42da33ba596e5b91f89ea3da29 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 18:45:50 +0000 Subject: [PATCH 121/141] Improve build system for ancient compilers --- include/pybind11/eigen.h | 12 +++++++++++- tests/CMakeLists.txt | 8 ++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 51308054bd..79e540c305 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -13,6 +13,16 @@ #include -#if EIGEN_VERSION_AT_LEAST(3, 3, 0) +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) +# if __GNUC__ < 5 +# define PYBIND11_CANT_INCLUDE_TENSOR +# endif +#endif + +#if !EIGEN_VERSION_AT_LEAST(3, 3, 0) +#define PYBIND11_CANT_INCLUDE_TENSOR +#endif + +#ifndef PYBIND11_CANT_INCLUDE_TENSOR # include "eigen/tensor.h" #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8d5cd4e0b9..75da0a537c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -303,6 +303,10 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() + list(FIND PYBIND11_TEST_FILES test_eigen_tensor_no_array.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() message( STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN=ON on CMake 3.11+ to download") endif() @@ -314,6 +318,10 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_L if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() + list(FIND PYBIND11_TEST_FILES test_eigen_tensor_no_array.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() endif() # Optional dependency for some tests (boost::variant is only supported with version >= 1.56) From fff0c96edd541381bfcf07e6b25df17aaadbd65c Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 18:56:23 +0000 Subject: [PATCH 122/141] Make clang happy --- include/pybind11/eigen/tensor.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index be83dc914b..d41d84af34 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -137,12 +137,16 @@ struct get_tensor_descriptor { template std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { - std::vector result(size); - for (int i = 0; i < size; i++) { - result[i] = arr[i]; - } + if (size == 0) { + return {}; + } else { + std::vector result(size); + for (size_t i = 0; i < size; i++) { + result[i] = arr[i]; + } - return result; + return result; + } } template From 17dc4abf32ca11d5f2d273907dfa350dde2e1339 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 19:08:55 +0000 Subject: [PATCH 123/141] Make gcc happy --- include/pybind11/eigen/tensor.h | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index d41d84af34..ef2fde664d 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -137,16 +137,22 @@ struct get_tensor_descriptor { template std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { - if (size == 0) { - return {}; - } else { - std::vector result(size); - for (size_t i = 0; i < size; i++) { - result[i] = arr[i]; - } + std::vector result(size); + +#if defined(__GNUC__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wtype-limits" +#endif - return result; + for (size_t i = 0; i < size; i++) { + result[i] = arr[i]; } + +#if defined(__GNUC__) +# pragma GCC diagnostic pop +#endif + + return result; } template From 645634d5b0f3b7afcdb409df21fd5217f0219027 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 20:22:55 +0000 Subject: [PATCH 124/141] Implement Skylion's suggestions --- include/pybind11/eigen/tensor.h | 32 +++++++++---------- tests/test_eigen_tensor.cpp | 2 +- ...nsor.include.cpp => test_eigen_tensor.ipp} | 0 tests/test_eigen_tensor_no_array.cpp | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) rename tests/{test_eigen_tensor.include.cpp => test_eigen_tensor.ipp} (100%) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index ef2fde664d..b4daeb80f4 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -155,6 +155,17 @@ std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { return result; } +template +Eigen::DSizes get_shape_for_array(const array& arr) { + Eigen::DSizes result; + const T* shape = arr.shape(); + for (size_t i = 0; i < size; i++) { + result[i] = shape[i]; + } + + return result; +} + template struct type_caster::ValidType> { using Helper = eigen_tensor_helper; @@ -182,11 +193,7 @@ struct type_caster::ValidType> { if (arr.ndim() != Type::NumIndices) { return false; } - - Eigen::DSizes shape; - for (size_t i = 0; i < Type::NumIndices; i++) { - shape[i] = arr.shape()[i ]; - } + auto shape = get_shape_for_array(arr); if (!Helper::is_correct_shape(shape)) { return false; @@ -328,12 +335,8 @@ struct type_caster::ValidType> { } }; -template -struct is_const_pointer {}; -template -struct is_const_pointer : std::false_type {}; -template -struct is_const_pointer : std::true_type {}; +template +using is_pointer_to_const = std::is_const::type>; template shape; - for (size_t i = 0; i < Type::NumIndices; i++) { - shape[i] = arr.shape()[i]; - } + auto shape = get_shape_for_array(arr); if (!Helper::is_correct_shape(shape)) { return false; @@ -500,7 +500,7 @@ struct type_caster< #if EIGEN_VERSION_AT_LEAST(3, 4, 0) static constexpr bool needs_writeable - = !is_const_pointer::SPT>::value; + = !is_pointer_to_const::SPT>::value; #else // Handle Eigen bug static constexpr bool needs_writeable = !std::is_const::value; diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 00f553e1a3..556c16bf8d 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -9,4 +9,4 @@ constexpr const char *TEST_MODULE_NAME = "eigen_tensor"; #define TEST_NAMESPACE eigen_tensor -#include "test_eigen_tensor.include.cpp" +#include "test_eigen_tensor.ipp" diff --git a/tests/test_eigen_tensor.include.cpp b/tests/test_eigen_tensor.ipp similarity index 100% rename from tests/test_eigen_tensor.include.cpp rename to tests/test_eigen_tensor.ipp diff --git a/tests/test_eigen_tensor_no_array.cpp b/tests/test_eigen_tensor_no_array.cpp index 294ac81e7a..a4078c8af4 100644 --- a/tests/test_eigen_tensor_no_array.cpp +++ b/tests/test_eigen_tensor_no_array.cpp @@ -10,4 +10,4 @@ constexpr const char *TEST_MODULE_NAME = "eigen_tensor_no_array"; #define TEST_NAMESPACE eigen_tensor_no_array -#include "test_eigen_tensor.include.cpp" +#include "test_eigen_tensor.ipp" From cd40d6893a2bb8233be9be02298ce4079fd8c587 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 20:31:52 +0000 Subject: [PATCH 125/141] Fix warning --- include/pybind11/eigen/tensor.h | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index b4daeb80f4..598d008075 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -135,22 +135,20 @@ struct get_tensor_descriptor { + const_name("]") + const_name(details, const_name("")) + const_name("]"); }; -template -std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { - std::vector result(size); - #if defined(__GNUC__) # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wtype-limits" #endif +template +std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { + std::vector result(size); + + for (size_t i = 0; i < size; i++) { result[i] = arr[i]; } -#if defined(__GNUC__) -# pragma GCC diagnostic pop -#endif return result; } @@ -166,6 +164,10 @@ Eigen::DSizes get_shape_for_array(const array& arr) { return result; } +#if defined(__GNUC__) +# pragma GCC diagnostic pop +#endif + template struct type_caster::ValidType> { using Helper = eigen_tensor_helper; From e9639fcd78fdb6dd92ace525d3ed311820ef6bb3 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 20:50:12 +0000 Subject: [PATCH 126/141] Inline const pointer check --- include/pybind11/eigen/tensor.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 598d008075..8527ee55da 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -337,8 +337,6 @@ struct type_caster::ValidType> { } }; -template -using is_pointer_to_const = std::is_const::type>; template ::SPT>::value; + = !std::is_const::SPT>::type>::value; #else // Handle Eigen bug static constexpr bool needs_writeable = !std::is_const::value; From e69a79e7ae582b85ca976d9f145ab54ffc9b6687 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 20:53:16 +0000 Subject: [PATCH 127/141] Implement suggestions --- include/pybind11/eigen/tensor.h | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 8527ee55da..6e1e120cb0 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -298,13 +298,6 @@ struct type_caster::ValidType> { break; case return_value_policy::copy: -#if defined(__clang_major__) && __clang_major__ <= 3 - // Hack to work around clang bugs - { parent_object = {}; } -#else - parent_object = {}; -#endif - writeable = true; break; @@ -415,7 +408,7 @@ struct type_caster< auto result = get_array_data_for_type::SPT, needs_writeable>(arr); - value.reset(new MapType(result, shape)); + value.reset(new MapType(std::move(result), std::move(shape))); return true; } From dc86d350a38d8b5c2b75a05c2d6f76533b48616b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Oct 2022 20:59:15 +0000 Subject: [PATCH 128/141] style: pre-commit fixes --- include/pybind11/eigen.h | 8 ++++---- include/pybind11/eigen/tensor.h | 22 ++++++++-------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 79e540c305..ed76608158 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -14,13 +14,13 @@ #include #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) -# if __GNUC__ < 5 -# define PYBIND11_CANT_INCLUDE_TENSOR -# endif +# if __GNUC__ < 5 +# define PYBIND11_CANT_INCLUDE_TENSOR +# endif #endif #if !EIGEN_VERSION_AT_LEAST(3, 3, 0) -#define PYBIND11_CANT_INCLUDE_TENSOR +# define PYBIND11_CANT_INCLUDE_TENSOR #endif #ifndef PYBIND11_CANT_INCLUDE_TENSOR diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 6e1e120cb0..089b636e70 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -144,22 +144,20 @@ template std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { std::vector result(size); - for (size_t i = 0; i < size; i++) { result[i] = arr[i]; } - return result; } template -Eigen::DSizes get_shape_for_array(const array& arr) { - Eigen::DSizes result; - const T* shape = arr.shape(); - for (size_t i = 0; i < size; i++) { - result[i] = shape[i]; - } +Eigen::DSizes get_shape_for_array(const array &arr) { + Eigen::DSizes result; + const T *shape = arr.shape(); + for (size_t i = 0; i < size; i++) { + result[i] = shape[i]; + } return result; } @@ -330,7 +328,6 @@ struct type_caster::ValidType> { } }; - template = true> @@ -491,13 +488,10 @@ struct type_caster< return result.release(); } - - #if EIGEN_VERSION_AT_LEAST(3, 4, 0) - static constexpr bool needs_writeable - = !std::is_const::SPT>::type>::value; + static constexpr bool needs_writeable = !std::is_const::SPT>::type>::value; #else // Handle Eigen bug static constexpr bool needs_writeable = !std::is_const::value; From 5ea37fd5fa9d149126237083f85e2ca0b201e12f Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 21:43:02 +0000 Subject: [PATCH 129/141] Improve tests --- include/pybind11/eigen.h | 4 +++- include/pybind11/eigen/tensor.h | 6 +++++- tests/CMakeLists.txt | 6 +++--- tests/test_eigen_tensor.cpp | 2 +- ...t_eigen_tensor.ipp => test_eigen_tensor.inl} | 7 +++---- tests/test_eigen_tensor.py | 17 ++++++++++++----- ...pp => test_eigen_tensor_avoid_stl_array.cpp} | 6 +++--- 7 files changed, 30 insertions(+), 18 deletions(-) rename tests/{test_eigen_tensor.ipp => test_eigen_tensor.inl} (95%) rename tests/{test_eigen_tensor_no_array.cpp => test_eigen_tensor_avoid_stl_array.cpp} (59%) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index ed76608158..f038c4ceae 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -11,7 +11,6 @@ #include "eigen/matrix.h" -#include #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) # if __GNUC__ < 5 @@ -19,6 +18,9 @@ # endif #endif +// IWYU correctness. Relies on the warning suppressions in eigen/matrix.h for simplicity. +#include + #if !EIGEN_VERSION_AT_LEAST(3, 3, 0) # define PYBIND11_CANT_INCLUDE_TENSOR #endif diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 089b636e70..ca0d03a543 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -135,6 +135,11 @@ struct get_tensor_descriptor { + const_name("]") + const_name(details, const_name("")) + const_name("]"); }; +// When EIGEN_AVOID_STL_ARRAY is defined, Eigen::DSizes does not have the begin() member function. +// Falling back to a simple loop works around this issue. +// +// We need to disble the type-limits warning for the inner loop when size = 0. + #if defined(__GNUC__) # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wtype-limits" @@ -385,7 +390,6 @@ struct type_caster< return false; } - // Use temporary to avoid MSVC warning ... constexpr bool is_aligned = (Options & Eigen::Aligned) != 0; if (PYBIND11_SILENCE_MSVC_C4127(is_aligned && !is_tensor_aligned(arr.data()))) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 75da0a537c..2ef349a7eb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -130,7 +130,7 @@ set(PYBIND11_TEST_FILES test_docstring_options test_eigen_matrix test_eigen_tensor - test_eigen_tensor_no_array.cpp + test_eigen_tensor_avoid_stl_array.cpp test_enum test_eval test_exceptions @@ -303,7 +303,7 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() - list(FIND PYBIND11_TEST_FILES test_eigen_tensor_no_array.cpp PYBIND11_TEST_FILES_EIGEN_I) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor_avoid_stl_array.cpp PYBIND11_TEST_FILES_EIGEN_I) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() @@ -318,7 +318,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_L if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() - list(FIND PYBIND11_TEST_FILES test_eigen_tensor_no_array.cpp PYBIND11_TEST_FILES_EIGEN_I) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor_avoid_stl_array.cpp PYBIND11_TEST_FILES_EIGEN_I) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 556c16bf8d..9f6d10a027 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -9,4 +9,4 @@ constexpr const char *TEST_MODULE_NAME = "eigen_tensor"; #define TEST_NAMESPACE eigen_tensor -#include "test_eigen_tensor.ipp" +#include "test_eigen_tensor.inl" diff --git a/tests/test_eigen_tensor.ipp b/tests/test_eigen_tensor.inl similarity index 95% rename from tests/test_eigen_tensor.ipp rename to tests/test_eigen_tensor.inl index 417e0de71b..3026a303cd 100644 --- a/tests/test_eigen_tensor.ipp +++ b/tests/test_eigen_tensor.inl @@ -104,13 +104,12 @@ void init_tensor_module(pybind11::module &m) { []() { return &get_const_tensor(); }, py::return_value_policy::copy); - m.def("move_fixed_tensor", []() { return get_fixed_tensor(); }); + m.def("move_fixed_tensor_copy", []() -> Eigen::TensorFixedSize, Options> { return get_fixed_tensor(); }, py::return_value_policy::move); - m.def("move_tensor", []() { return get_tensor(); }); + m.def("move_tensor_copy", []() -> Eigen::Tensor { return get_tensor();}, py::return_value_policy::move); m.def("move_const_tensor", - // NOLINTNEXTLINE(readability-const-return-type) - []() -> const Eigen::Tensor { return get_const_tensor(); }); + []() -> const Eigen::Tensor& { return get_const_tensor(); }, py::return_value_policy::move); m.def( "take_fixed_tensor", diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index f7c81cb20e..ac0f19b580 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -6,11 +6,11 @@ eigen_tensor = pytest.importorskip("pybind11_tests.eigen_tensor") submodules = [eigen_tensor.c_style, eigen_tensor.f_style] try: - from pybind11_tests import eigen_tensor_no_array + from pybind11_tests import eigen_tensor_avoid_stl_array as avoid - submodules += [eigen_tensor_no_array.c_style, eigen_tensor_no_array.f_style] + submodules += [avoid.c_style, avoid.f_style] except ImportError: - print("Could not load no_array tests", sys.stderr) + print("Could not load avoid stl array tests", sys.stderr) pass tensor_ref = np.empty((3, 5, 2), dtype=np.int64) @@ -36,6 +36,7 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=None): @pytest.mark.parametrize("m", submodules) @pytest.mark.parametrize("member_name", ["member", "member_view"]) def test_reference_internal(m, member_name): + if not hasattr(sys, "getrefcount"): pytest.skip("No reference counting") foo = m.CustomExample() @@ -51,12 +52,13 @@ def test_reference_internal(m, member_name): @pytest.mark.parametrize("m", submodules) def test_convert_tensor_to_py(m): + assert_equal_tensor_ref(m.copy_tensor()) assert_equal_tensor_ref(m.copy_fixed_tensor()) assert_equal_tensor_ref(m.copy_const_tensor()) - assert_equal_tensor_ref(m.move_tensor()) - assert_equal_tensor_ref(m.move_fixed_tensor()) + assert_equal_tensor_ref(m.move_tensor_copy()) + assert_equal_tensor_ref(m.move_fixed_tensor_copy()) assert_equal_tensor_ref(m.take_tensor()) assert_equal_tensor_ref(m.take_fixed_tensor()) @@ -78,6 +80,7 @@ def test_convert_tensor_to_py(m): @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): + with pytest.raises( RuntimeError, match="Cannot use reference internal when there is no parent" ): @@ -100,6 +103,7 @@ def test_bad_cpp_to_python_casts(m): @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): + with pytest.raises(TypeError): m.round_trip_tensor(np.zeros((2, 3))) @@ -147,6 +151,7 @@ def test_bad_python_to_cpp_casts(m): @pytest.mark.parametrize("m", submodules) def test_references_actually_refer(m): + a = m.reference_tensor() temp = a[indices] a[indices] = 100 @@ -163,6 +168,7 @@ def test_references_actually_refer(m): @pytest.mark.parametrize("m", submodules) def test_round_trip(m): + assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) with pytest.raises(TypeError): @@ -201,6 +207,7 @@ def test_round_trip(m): @pytest.mark.parametrize("m", submodules) def test_round_trip_references_actually_refer(m): + # Need to create a copy that matches the type on the C side copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) a = m.round_trip_view_tensor(copy) diff --git a/tests/test_eigen_tensor_no_array.cpp b/tests/test_eigen_tensor_avoid_stl_array.cpp similarity index 59% rename from tests/test_eigen_tensor_no_array.cpp rename to tests/test_eigen_tensor_avoid_stl_array.cpp index a4078c8af4..8b829dff5d 100644 --- a/tests/test_eigen_tensor_no_array.cpp +++ b/tests/test_eigen_tensor_avoid_stl_array.cpp @@ -5,9 +5,9 @@ BSD-style license that can be found in the LICENSE file. */ -constexpr const char *TEST_MODULE_NAME = "eigen_tensor_no_array"; +constexpr const char *TEST_MODULE_NAME = "eigen_tensor_avoid_stl_array"; #define EIGEN_AVOID_STL_ARRAY -#define TEST_NAMESPACE eigen_tensor_no_array +#define TEST_NAMESPACE eigen_tensor_avoid_stl_array -#include "test_eigen_tensor.ipp" +#include "test_eigen_tensor.inl" From a53f0c710e2e521b627778369d3261ad829949ff Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 11 Oct 2022 21:49:35 +0000 Subject: [PATCH 130/141] Typo --- include/pybind11/eigen/tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index ca0d03a543..ebca6bfa8f 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -138,7 +138,7 @@ struct get_tensor_descriptor { // When EIGEN_AVOID_STL_ARRAY is defined, Eigen::DSizes does not have the begin() member function. // Falling back to a simple loop works around this issue. // -// We need to disble the type-limits warning for the inner loop when size = 0. +// We need to disable the type-limits warning for the inner loop when size = 0. #if defined(__GNUC__) # pragma GCC diagnostic push From b7e00581e3a2e5b04909cf4db380af3cf0579c34 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Oct 2022 21:56:42 +0000 Subject: [PATCH 131/141] style: pre-commit fixes --- include/pybind11/eigen.h | 1 - include/pybind11/eigen/tensor.h | 6 +++--- tests/CMakeLists.txt | 3 ++- tests/test_eigen_tensor.inl | 18 ++++++++++++++---- tests/test_eigen_tensor.py | 14 +++++++------- 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index f038c4ceae..2a191c5401 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -11,7 +11,6 @@ #include "eigen/matrix.h" - #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) # if __GNUC__ < 5 # define PYBIND11_CANT_INCLUDE_TENSOR diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index ebca6bfa8f..fe37bb1667 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -135,9 +135,9 @@ struct get_tensor_descriptor { + const_name("]") + const_name(details, const_name("")) + const_name("]"); }; -// When EIGEN_AVOID_STL_ARRAY is defined, Eigen::DSizes does not have the begin() member function. -// Falling back to a simple loop works around this issue. -// +// When EIGEN_AVOID_STL_ARRAY is defined, Eigen::DSizes does not have the begin() member +// function. Falling back to a simple loop works around this issue. +// // We need to disable the type-limits warning for the inner loop when size = 0. #if defined(__GNUC__) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2ef349a7eb..491f215cef 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -303,7 +303,8 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() - list(FIND PYBIND11_TEST_FILES test_eigen_tensor_avoid_stl_array.cpp PYBIND11_TEST_FILES_EIGEN_I) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor_avoid_stl_array.cpp + PYBIND11_TEST_FILES_EIGEN_I) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() diff --git a/tests/test_eigen_tensor.inl b/tests/test_eigen_tensor.inl index 3026a303cd..47ef66aaea 100644 --- a/tests/test_eigen_tensor.inl +++ b/tests/test_eigen_tensor.inl @@ -104,12 +104,22 @@ void init_tensor_module(pybind11::module &m) { []() { return &get_const_tensor(); }, py::return_value_policy::copy); - m.def("move_fixed_tensor_copy", []() -> Eigen::TensorFixedSize, Options> { return get_fixed_tensor(); }, py::return_value_policy::move); + m.def( + "move_fixed_tensor_copy", + []() -> Eigen::TensorFixedSize, Options> { + return get_fixed_tensor(); + }, + py::return_value_policy::move); - m.def("move_tensor_copy", []() -> Eigen::Tensor { return get_tensor();}, py::return_value_policy::move); + m.def( + "move_tensor_copy", + []() -> Eigen::Tensor { return get_tensor(); }, + py::return_value_policy::move); - m.def("move_const_tensor", - []() -> const Eigen::Tensor& { return get_const_tensor(); }, py::return_value_policy::move); + m.def( + "move_const_tensor", + []() -> const Eigen::Tensor & { return get_const_tensor(); }, + py::return_value_policy::move); m.def( "take_fixed_tensor", diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index ac0f19b580..1f050ec1e5 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -36,7 +36,7 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=None): @pytest.mark.parametrize("m", submodules) @pytest.mark.parametrize("member_name", ["member", "member_view"]) def test_reference_internal(m, member_name): - + if not hasattr(sys, "getrefcount"): pytest.skip("No reference counting") foo = m.CustomExample() @@ -52,7 +52,7 @@ def test_reference_internal(m, member_name): @pytest.mark.parametrize("m", submodules) def test_convert_tensor_to_py(m): - + assert_equal_tensor_ref(m.copy_tensor()) assert_equal_tensor_ref(m.copy_fixed_tensor()) assert_equal_tensor_ref(m.copy_const_tensor()) @@ -80,7 +80,7 @@ def test_convert_tensor_to_py(m): @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): - + with pytest.raises( RuntimeError, match="Cannot use reference internal when there is no parent" ): @@ -103,7 +103,7 @@ def test_bad_cpp_to_python_casts(m): @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): - + with pytest.raises(TypeError): m.round_trip_tensor(np.zeros((2, 3))) @@ -151,7 +151,7 @@ def test_bad_python_to_cpp_casts(m): @pytest.mark.parametrize("m", submodules) def test_references_actually_refer(m): - + a = m.reference_tensor() temp = a[indices] a[indices] = 100 @@ -168,7 +168,7 @@ def test_references_actually_refer(m): @pytest.mark.parametrize("m", submodules) def test_round_trip(m): - + assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) with pytest.raises(TypeError): @@ -207,7 +207,7 @@ def test_round_trip(m): @pytest.mark.parametrize("m", submodules) def test_round_trip_references_actually_refer(m): - + # Need to create a copy that matches the type on the C side copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) a = m.round_trip_view_tensor(copy) From 3a95d46aabeb27bc315f316f9c7d23afc9c90331 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 12 Oct 2022 00:10:30 +0000 Subject: [PATCH 132/141] Support Google's build environment --- tests/test_eigen_tensor.cpp | 8 ++++++-- tests/test_eigen_tensor.inl | 6 +++--- tests/test_eigen_tensor_avoid_stl_array.cpp | 7 +++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 9f6d10a027..64d27c9ce4 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -5,8 +5,12 @@ BSD-style license that can be found in the LICENSE file. */ -constexpr const char *TEST_MODULE_NAME = "eigen_tensor"; +constexpr const char *test_eigen_tensor_module_name = "eigen_tensor"; -#define TEST_NAMESPACE eigen_tensor +#define PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE eigen_tensor + +#ifdef EIGEN_AVOID_STL_ARRAY +#undef EIGEN_AVOID_STL_ARRAY +#endif #include "test_eigen_tensor.inl" diff --git a/tests/test_eigen_tensor.inl b/tests/test_eigen_tensor.inl index 47ef66aaea..1721f4a803 100644 --- a/tests/test_eigen_tensor.inl +++ b/tests/test_eigen_tensor.inl @@ -9,7 +9,7 @@ #include "pybind11_tests.h" -namespace TEST_NAMESPACE { +namespace PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE { template void reset_tensor(M &x) { @@ -298,7 +298,7 @@ void init_tensor_module(pybind11::module &m) { } void test_module(py::module_ &); -test_initializer name(TEST_MODULE_NAME, test_module); +test_initializer name(test_eigen_tensor_module_name, test_module); void test_module(py::module_ &m) { auto f_style = m.def_submodule("f_style"); auto c_style = m.def_submodule("c_style"); @@ -307,4 +307,4 @@ void test_module(py::module_ &m) { init_tensor_module(c_style); } -} // namespace TEST_NAMESPACE +} // namespace PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE diff --git a/tests/test_eigen_tensor_avoid_stl_array.cpp b/tests/test_eigen_tensor_avoid_stl_array.cpp index 8b829dff5d..6b0098308d 100644 --- a/tests/test_eigen_tensor_avoid_stl_array.cpp +++ b/tests/test_eigen_tensor_avoid_stl_array.cpp @@ -5,9 +5,12 @@ BSD-style license that can be found in the LICENSE file. */ -constexpr const char *TEST_MODULE_NAME = "eigen_tensor_avoid_stl_array"; +constexpr const char *test_eigen_tensor_module_name = "eigen_tensor_avoid_stl_array"; + +#ifndef EIGEN_AVOID_STL_ARRAY #define EIGEN_AVOID_STL_ARRAY +#endif -#define TEST_NAMESPACE eigen_tensor_avoid_stl_array +#define PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE eigen_tensor_avoid_stl_array #include "test_eigen_tensor.inl" From 7516e9921b5f985f508552d0e8f0ea8e14a03d8d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Oct 2022 00:16:33 +0000 Subject: [PATCH 133/141] style: pre-commit fixes --- tests/test_eigen_tensor.cpp | 2 +- tests/test_eigen_tensor_avoid_stl_array.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp index 64d27c9ce4..40b4940053 100644 --- a/tests/test_eigen_tensor.cpp +++ b/tests/test_eigen_tensor.cpp @@ -10,7 +10,7 @@ constexpr const char *test_eigen_tensor_module_name = "eigen_tensor"; #define PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE eigen_tensor #ifdef EIGEN_AVOID_STL_ARRAY -#undef EIGEN_AVOID_STL_ARRAY +# undef EIGEN_AVOID_STL_ARRAY #endif #include "test_eigen_tensor.inl" diff --git a/tests/test_eigen_tensor_avoid_stl_array.cpp b/tests/test_eigen_tensor_avoid_stl_array.cpp index 6b0098308d..58bedf62dd 100644 --- a/tests/test_eigen_tensor_avoid_stl_array.cpp +++ b/tests/test_eigen_tensor_avoid_stl_array.cpp @@ -8,7 +8,7 @@ constexpr const char *test_eigen_tensor_module_name = "eigen_tensor_avoid_stl_array"; #ifndef EIGEN_AVOID_STL_ARRAY -#define EIGEN_AVOID_STL_ARRAY +# define EIGEN_AVOID_STL_ARRAY #endif #define PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE eigen_tensor_avoid_stl_array From 4870a3845784edb61d56f57ef903e8c4119cab78 Mon Sep 17 00:00:00 2001 From: Lalaland Date: Wed, 12 Oct 2022 15:16:37 -0700 Subject: [PATCH 134/141] Update include/pybind11/eigen/tensor.h Co-authored-by: Aaron Gokaslan --- include/pybind11/eigen/tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index fe37bb1667..255968a559 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -483,7 +483,7 @@ struct type_caster< } auto result = array_t()>( - convert_dsizes_to_vector(Helper::get_shape(*src)), src->data(), parent_object); + convert_dsizes_to_vector(Helper::get_shape(*src)), src->data(), std::move(parent_object)); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; From 51d4a46464c26526ea7160207af8ee472c82d1c8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Oct 2022 22:19:48 +0000 Subject: [PATCH 135/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 255968a559..20a94f38f4 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -483,7 +483,9 @@ struct type_caster< } auto result = array_t()>( - convert_dsizes_to_vector(Helper::get_shape(*src)), src->data(), std::move(parent_object)); + convert_dsizes_to_vector(Helper::get_shape(*src)), + src->data(), + std::move(parent_object)); if (!writeable) { array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; From 96e08023746798b81a161e03ad704a7be18e5529 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 12 Oct 2022 22:27:21 +0000 Subject: [PATCH 136/141] Test cleanup per Skylion --- include/pybind11/eigen/tensor.h | 4 +-- tests/test_eigen_tensor.py | 61 ++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 20a94f38f4..abac120805 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -392,7 +392,7 @@ struct type_caster< constexpr bool is_aligned = (Options & Eigen::Aligned) != 0; - if (PYBIND11_SILENCE_MSVC_C4127(is_aligned && !is_tensor_aligned(arr.data()))) { + if (PYBIND11_SILENCE_MSVC_C4127(is_aligned) && !is_tensor_aligned(arr.data())) { return false; } @@ -402,7 +402,7 @@ struct type_caster< return false; } - if (PYBIND11_SILENCE_MSVC_C4127(needs_writeable && !arr.writeable())) { + if (PYBIND11_SILENCE_MSVC_C4127(needs_writeable) && !arr.writeable()) { return false; } diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 1f050ec1e5..7eb959dbca 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -10,7 +10,6 @@ submodules += [avoid.c_style, avoid.f_style] except ImportError: - print("Could not load avoid stl array tests", sys.stderr) pass tensor_ref = np.empty((3, 5, 2), dtype=np.int64) @@ -22,6 +21,10 @@ indices = (2, 3, 1) +def test_import_avoid_stl_array(): + pytest.importorskip("pybind11_tests.eigen_tensor_avoid_stl_array") + assert len(submodules) == 4 + def assert_equal_tensor_ref(mat, writeable=True, modified=None): assert mat.flags.writeable == writeable @@ -49,34 +52,38 @@ def test_reference_internal(m, member_name): del mem assert sys.getrefcount(foo) == counts +assert_equal_funcs = [ + "copy_tensor", + "copy_fixed_tensor", + "copy_const_tensor", + "move_tensor_copy", + "move_fixed_tensor_copy", + "take_tensor", + "take_fixed_tensor", + "reference_tensor", + "reference_tensor_v2", + "reference_fixed_tensor", + "reference_view_of_tensor", + "reference_view_of_tensor_v3", + "reference_view_of_tensor_v5", + "reference_view_of_fixed_tensor", +] + +assert_equal_const_funcs = [ + "reference_view_of_tensor_v2", + "reference_view_of_tensor_v4", + "reference_view_of_tensor_v6", + "reference_const_tensor", + "reference_const_tensor_v2", +] + +functions_with_write_flag = [(a, True) for a in assert_equal_funcs] + [(a, False) for a in assert_equal_const_funcs] @pytest.mark.parametrize("m", submodules) -def test_convert_tensor_to_py(m): - - assert_equal_tensor_ref(m.copy_tensor()) - assert_equal_tensor_ref(m.copy_fixed_tensor()) - assert_equal_tensor_ref(m.copy_const_tensor()) - - assert_equal_tensor_ref(m.move_tensor_copy()) - assert_equal_tensor_ref(m.move_fixed_tensor_copy()) - - assert_equal_tensor_ref(m.take_tensor()) - assert_equal_tensor_ref(m.take_fixed_tensor()) - - assert_equal_tensor_ref(m.reference_tensor()) - assert_equal_tensor_ref(m.reference_tensor_v2()) - assert_equal_tensor_ref(m.reference_fixed_tensor()) - - assert_equal_tensor_ref(m.reference_view_of_tensor()) - assert_equal_tensor_ref(m.reference_view_of_tensor_v2(), writeable=False) - assert_equal_tensor_ref(m.reference_view_of_tensor_v3()) - assert_equal_tensor_ref(m.reference_view_of_tensor_v4(), writeable=False) - assert_equal_tensor_ref(m.reference_view_of_tensor_v5()) - assert_equal_tensor_ref(m.reference_view_of_tensor_v6(), writeable=False) - assert_equal_tensor_ref(m.reference_view_of_fixed_tensor()) - assert_equal_tensor_ref(m.reference_const_tensor(), writeable=False) - assert_equal_tensor_ref(m.reference_const_tensor_v2(), writeable=False) - +@pytest.mark.parametrize("func_name_and_flag", functions_with_write_flag) +def test_convert_tensor_to_py(m, func_name_and_flag): + func_name, write_flag = func_name_and_flag + assert_equal_tensor_ref(getattr(m, func_name)(), writeable=write_flag) @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): From 74d50897bbec795a2b3d01f1b141da1e01d30795 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 12 Oct 2022 22:31:45 +0000 Subject: [PATCH 137/141] Switch to remvove_cv_t --- include/pybind11/eigen/tensor.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index abac120805..86c62f778e 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -131,7 +131,7 @@ struct get_tensor_descriptor { static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") - + eigen_tensor_helper::type>::dimensions_descriptor + + eigen_tensor_helper>::dimensions_descriptor + const_name("]") + const_name(details, const_name("")) + const_name("]"); }; @@ -368,9 +368,9 @@ struct get_storage_pointer_type struct type_caster< Eigen::TensorMap, - typename eigen_tensor_helper::type>::ValidType> { + typename eigen_tensor_helper>::ValidType> { using MapType = Eigen::TensorMap; - using Helper = eigen_tensor_helper::type>; + using Helper = eigen_tensor_helper>; bool load(handle src, bool /*convert*/) { // Note that we have a lot more checks here as we want to make sure to avoid copies From 39b44780791f7ac740bc325c25435b7ccedc92fe Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 12 Oct 2022 22:35:47 +0000 Subject: [PATCH 138/141] Cleaner test --- tests/test_eigen_tensor.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 7eb959dbca..3571b091ce 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -77,13 +77,12 @@ def test_reference_internal(m, member_name): "reference_const_tensor_v2", ] -functions_with_write_flag = [(a, True) for a in assert_equal_funcs] + [(a, False) for a in assert_equal_const_funcs] @pytest.mark.parametrize("m", submodules) -@pytest.mark.parametrize("func_name_and_flag", functions_with_write_flag) -def test_convert_tensor_to_py(m, func_name_and_flag): - func_name, write_flag = func_name_and_flag - assert_equal_tensor_ref(getattr(m, func_name)(), writeable=write_flag) +@pytest.mark.parametrize("func_name", assert_equal_funcs + assert_equal_const_funcs) +def test_convert_tensor_to_py(m, func_name): + writeable = func_name in assert_equal_funcs + assert_equal_tensor_ref(getattr(m, func_name)(), writeable=writeable) @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): From bf6ae7a642079a75e8abb880916c7425b2cc70e7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Oct 2022 22:41:49 +0000 Subject: [PATCH 139/141] style: pre-commit fixes --- include/pybind11/eigen/tensor.h | 8 +++----- tests/test_eigen_tensor.py | 3 +++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 86c62f778e..3cc5214e98 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -130,8 +130,7 @@ struct get_tensor_descriptor { ", flags.c_contiguous", ", flags.f_contiguous"); static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") - + eigen_tensor_helper>::dimensions_descriptor + + const_name("[") + eigen_tensor_helper>::dimensions_descriptor + const_name("]") + const_name(details, const_name("")) + const_name("]"); }; @@ -366,9 +365,8 @@ struct get_storage_pointer_type -struct type_caster< - Eigen::TensorMap, - typename eigen_tensor_helper>::ValidType> { +struct type_caster, + typename eigen_tensor_helper>::ValidType> { using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper>; diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 3571b091ce..44d93c2392 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -21,6 +21,7 @@ indices = (2, 3, 1) + def test_import_avoid_stl_array(): pytest.importorskip("pybind11_tests.eigen_tensor_avoid_stl_array") assert len(submodules) == 4 @@ -52,6 +53,7 @@ def test_reference_internal(m, member_name): del mem assert sys.getrefcount(foo) == counts + assert_equal_funcs = [ "copy_tensor", "copy_fixed_tensor", @@ -84,6 +86,7 @@ def test_convert_tensor_to_py(m, func_name): writeable = func_name in assert_equal_funcs assert_equal_tensor_ref(getattr(m, func_name)(), writeable=writeable) + @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): From 8b43c4d033527fb003699be285978e191f7a4150 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Tue, 18 Oct 2022 19:09:39 +0000 Subject: [PATCH 140/141] Remove tensor from eigen.h, update tests --- include/pybind11/eigen.h | 19 +----------------- include/pybind11/eigen/tensor.h | 1 - tests/test_eigen_tensor.inl | 25 ++++++++++++++++++++++- tests/test_eigen_tensor.py | 35 +++++++++++++++++++++------------ 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 2a191c5401..80e1221f9b 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -9,21 +9,4 @@ #pragma once -#include "eigen/matrix.h" - -#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) -# if __GNUC__ < 5 -# define PYBIND11_CANT_INCLUDE_TENSOR -# endif -#endif - -// IWYU correctness. Relies on the warning suppressions in eigen/matrix.h for simplicity. -#include - -#if !EIGEN_VERSION_AT_LEAST(3, 3, 0) -# define PYBIND11_CANT_INCLUDE_TENSOR -#endif - -#ifndef PYBIND11_CANT_INCLUDE_TENSOR -# include "eigen/tensor.h" -#endif +#include "eigen/matrix.h" \ No newline at end of file diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 3cc5214e98..a823c0f397 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -1,4 +1,3 @@ - /* pybind11/eigen/tensor.h: Transparent conversion for Eigen tensors diff --git a/tests/test_eigen_tensor.inl b/tests/test_eigen_tensor.inl index 1721f4a803..09b35fa134 100644 --- a/tests/test_eigen_tensor.inl +++ b/tests/test_eigen_tensor.inl @@ -5,7 +5,7 @@ BSD-style license that can be found in the LICENSE file. */ -#include +#include #include "pybind11_tests.h" @@ -22,6 +22,20 @@ void reset_tensor(M &x) { } } +template +bool check_tensor(M &x) { + for (int i = 0; i < x.dimension(0); i++) { + for (int j = 0; j < x.dimension(1); j++) { + for (int k = 0; k < x.dimension(2); k++) { + if (x(i, j, k) != (i * (5 * 2) + j * 2 + k)) { + return false; + } + } + } + } + return true; +} + template Eigen::Tensor &get_tensor() { static Eigen::Tensor *x; @@ -83,6 +97,15 @@ void init_tensor_module(pybind11::module &m) { } m.attr("needed_options") = needed_options; + m.def("setup", []() { + reset_tensor(get_tensor()); + reset_tensor(get_fixed_tensor()); + }); + + m.def("is_ok", []() { + return check_tensor(get_tensor()) && check_tensor(get_fixed_tensor()); + }); + py::class_>(m, "CustomExample") .def(py::init<>()) .def_readonly( diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 44d93c2392..350a862102 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -21,7 +21,16 @@ indices = (2, 3, 1) +@pytest.fixture(autouse=True) +def cleanup(): + for module in submodules: + module.setup() + yield + + for module in submodules: + assert module.is_ok() + def test_import_avoid_stl_array(): pytest.importorskip("pybind11_tests.eigen_tensor_avoid_stl_array") assert len(submodules) == 4 @@ -113,13 +122,13 @@ def test_bad_cpp_to_python_casts(m): @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_tensor\(\): incompatible function arguments'): m.round_trip_tensor(np.zeros((2, 3))) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^Cannot cast array data from dtype'): m.round_trip_tensor(np.zeros(dtype=np.str_, shape=(2, 3, 1))) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_tensor_noconvert\(\): incompatible function arguments'): m.round_trip_tensor_noconvert(tensor_ref) assert_equal_tensor_ref( @@ -131,28 +140,28 @@ def test_bad_python_to_cpp_casts(m): else: bad_options = "F" # Shape, dtype and the order need to be correct for a TensorMap cast - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_view_tensor\(\): incompatible function arguments'): m.round_trip_view_tensor( np.zeros((3, 5, 2), dtype=np.float64, order=bad_options) ) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_view_tensor\(\): incompatible function arguments'): m.round_trip_view_tensor( np.zeros((3, 5, 2), dtype=np.float32, order=m.needed_options) ) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_view_tensor\(\): incompatible function arguments'): m.round_trip_view_tensor( np.zeros((3, 5), dtype=np.float64, order=m.needed_options) ) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_view_tensor\(\): incompatible function arguments'): temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) m.round_trip_view_tensor( temp[:, ::-1, :], ) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_view_tensor\(\): incompatible function arguments'): temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) temp.setflags(write=False) m.round_trip_view_tensor(temp) @@ -180,7 +189,7 @@ def test_round_trip(m): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match='^Cannot cast array data from'): assert_equal_tensor_ref(m.round_trip_tensor2(tensor_ref)) assert_equal_tensor_ref(m.round_trip_tensor2(np.array(tensor_ref, dtype=np.int32))) @@ -201,16 +210,16 @@ def test_round_trip(m): assert m.round_trip_rank_0(np.float64(3.5)) == 3.5 assert m.round_trip_rank_0(3.5) == 3.5 - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_rank_0_noconvert\(\): incompatible function arguments'): m.round_trip_rank_0_noconvert(np.float64(3.5)) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_rank_0_noconvert\(\): incompatible function arguments'): m.round_trip_rank_0_noconvert(3.5) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_rank_0_view\(\): incompatible function arguments'): m.round_trip_rank_0_view(np.float64(3.5)) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match=r'^round_trip_rank_0_view\(\): incompatible function arguments'): m.round_trip_rank_0_view(3.5) From 4880e0b7a709a3ef0dc3fa24d06d87532a3bb542 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 18 Oct 2022 19:18:40 +0000 Subject: [PATCH 141/141] style: pre-commit fixes --- include/pybind11/eigen.h | 2 +- tests/test_eigen_tensor.py | 55 ++++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 80e1221f9b..273b9c9308 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -9,4 +9,4 @@ #pragma once -#include "eigen/matrix.h" \ No newline at end of file +#include "eigen/matrix.h" diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index 350a862102..5ee5fa01b2 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -21,6 +21,7 @@ indices = (2, 3, 1) + @pytest.fixture(autouse=True) def cleanup(): for module in submodules: @@ -30,7 +31,8 @@ def cleanup(): for module in submodules: assert module.is_ok() - + + def test_import_avoid_stl_array(): pytest.importorskip("pybind11_tests.eigen_tensor_avoid_stl_array") assert len(submodules) == 4 @@ -122,13 +124,18 @@ def test_bad_cpp_to_python_casts(m): @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): - with pytest.raises(TypeError, match=r'^round_trip_tensor\(\): incompatible function arguments'): + with pytest.raises( + TypeError, match=r"^round_trip_tensor\(\): incompatible function arguments" + ): m.round_trip_tensor(np.zeros((2, 3))) - with pytest.raises(TypeError, match=r'^Cannot cast array data from dtype'): + with pytest.raises(TypeError, match=r"^Cannot cast array data from dtype"): m.round_trip_tensor(np.zeros(dtype=np.str_, shape=(2, 3, 1))) - with pytest.raises(TypeError, match=r'^round_trip_tensor_noconvert\(\): incompatible function arguments'): + with pytest.raises( + TypeError, + match=r"^round_trip_tensor_noconvert\(\): incompatible function arguments", + ): m.round_trip_tensor_noconvert(tensor_ref) assert_equal_tensor_ref( @@ -140,28 +147,38 @@ def test_bad_python_to_cpp_casts(m): else: bad_options = "F" # Shape, dtype and the order need to be correct for a TensorMap cast - with pytest.raises(TypeError, match=r'^round_trip_view_tensor\(\): incompatible function arguments'): + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): m.round_trip_view_tensor( np.zeros((3, 5, 2), dtype=np.float64, order=bad_options) ) - with pytest.raises(TypeError, match=r'^round_trip_view_tensor\(\): incompatible function arguments'): + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): m.round_trip_view_tensor( np.zeros((3, 5, 2), dtype=np.float32, order=m.needed_options) ) - with pytest.raises(TypeError, match=r'^round_trip_view_tensor\(\): incompatible function arguments'): + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): m.round_trip_view_tensor( np.zeros((3, 5), dtype=np.float64, order=m.needed_options) ) - with pytest.raises(TypeError, match=r'^round_trip_view_tensor\(\): incompatible function arguments'): + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) m.round_trip_view_tensor( temp[:, ::-1, :], ) - with pytest.raises(TypeError, match=r'^round_trip_view_tensor\(\): incompatible function arguments'): + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) temp.setflags(write=False) m.round_trip_view_tensor(temp) @@ -189,7 +206,7 @@ def test_round_trip(m): assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) - with pytest.raises(TypeError, match='^Cannot cast array data from'): + with pytest.raises(TypeError, match="^Cannot cast array data from"): assert_equal_tensor_ref(m.round_trip_tensor2(tensor_ref)) assert_equal_tensor_ref(m.round_trip_tensor2(np.array(tensor_ref, dtype=np.int32))) @@ -210,16 +227,26 @@ def test_round_trip(m): assert m.round_trip_rank_0(np.float64(3.5)) == 3.5 assert m.round_trip_rank_0(3.5) == 3.5 - with pytest.raises(TypeError, match=r'^round_trip_rank_0_noconvert\(\): incompatible function arguments'): + with pytest.raises( + TypeError, + match=r"^round_trip_rank_0_noconvert\(\): incompatible function arguments", + ): m.round_trip_rank_0_noconvert(np.float64(3.5)) - with pytest.raises(TypeError, match=r'^round_trip_rank_0_noconvert\(\): incompatible function arguments'): + with pytest.raises( + TypeError, + match=r"^round_trip_rank_0_noconvert\(\): incompatible function arguments", + ): m.round_trip_rank_0_noconvert(3.5) - with pytest.raises(TypeError, match=r'^round_trip_rank_0_view\(\): incompatible function arguments'): + with pytest.raises( + TypeError, match=r"^round_trip_rank_0_view\(\): incompatible function arguments" + ): m.round_trip_rank_0_view(np.float64(3.5)) - with pytest.raises(TypeError, match=r'^round_trip_rank_0_view\(\): incompatible function arguments'): + with pytest.raises( + TypeError, match=r"^round_trip_rank_0_view\(\): incompatible function arguments" + ): m.round_trip_rank_0_view(3.5)