Generic containers/iterators for shape/strides

This adds support for constructing `buffer_info` and `array`s using
arbitrary containers or iterators instead of requiring a vector.

On its own, this adds quite a few templated constructors which isn't all
that desirable, but it's needed by PR pybind#782 to preserve backwards
compatibility for previous pybind11 versions that accept a
`std::vector<size_t>` for strides.  Given a choice between duplicating
all the stride-taking constructors (accepting both a ssize_t and size_t
vector) and making the whole interface more general, I think this is a
better approach.

I also seem to recall some discussion a few months ago about wanting
something like this, but I can't find the issue/PR where it was
mentioned.

Author: jagerman

include/pybind11/buffer_info.h

@@ -25,26 +25,37 @@ struct buffer_info {
 
     buffer_info() { }
 
+    template <typename ShapeIt, typename StridesIt,
+             typename = detail::enable_if_t<detail::is_input_iterator<ShapeIt>::value && detail::is_input_iterator<StridesIt>::value>>
     buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t ndim,
-                const std::vector<size_t> &shape, const std::vector<size_t> &strides)
-        : ptr(ptr), itemsize(itemsize), size(1), format(format),
-          ndim(ndim), shape(shape), strides(strides) {
+            ShapeIt shape_first, ShapeIt shape_last, StridesIt strides_first, StridesIt strides_last)
+        : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim),
+          shape(shape_first, shape_last), strides(strides_first, strides_last) {
+        if (ndim != shape.size() || ndim != strides.size())
+            pybind11_fail("buffer_info: ndim doesn't match shape and/or strides length");
         for (size_t i = 0; i < ndim; ++i)
             size *= shape[i];
     }
 
+    template <typename Shape, typename Strides,
+             typename = decltype(std::begin(std::declval<const Shape &>())),
+             typename = decltype(std::begin(std::declval<const Shape &>()))>
+    buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t ndim,
+                const Shape &shape, const Strides &strides)
+    : buffer_info(ptr, itemsize, format, ndim, std::begin(shape), std::end(shape), std::begin(strides), std::end(strides)) { }
+
+    template <typename T1, typename T2>
+    buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t ndim,
+            const std::initializer_list<T1> &shape, const std::initializer_list<T2> &strides)
+    : buffer_info(ptr, itemsize, format, ndim, shape.begin(), shape.end(), strides.begin(), strides.end()) { }
+
     buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t size)
-    : buffer_info(ptr, itemsize, format, 1, std::vector<size_t> { size },
-                  std::vector<size_t> { itemsize }) { }
-
-    explicit buffer_info(Py_buffer *view, bool ownview = true)
-        : ptr(view->buf), itemsize((size_t) view->itemsize), size(1), format(view->format),
-          ndim((size_t) view->ndim), shape((size_t) view->ndim), strides((size_t) view->ndim), view(view), ownview(ownview) {
-        for (size_t i = 0; i < (size_t) view->ndim; ++i) {
-            shape[i] = (size_t) view->shape[i];
-            strides[i] = (size_t) view->strides[i];
-            size *= shape[i];
-        }
+    : buffer_info(ptr, itemsize, format, 1, { size }, { itemsize }) { }
+
+    explicit buffer_info(Py_buffer *view, bool ownview_in = true)
+    : buffer_info(view->buf, (size_t) view->itemsize, view->format, (size_t) view->ndim,
+            view->shape, view->shape + view->ndim, view->strides, view->strides + view->ndim) {
+        ownview = ownview_in;
     }
 
     buffer_info(const buffer_info &) = delete;

include/pybind11/common.h

@@ -490,6 +490,12 @@ struct is_instantiation<Class, Class<Us...>> : std::true_type { };
 /// Check if T is std::shared_ptr<U> where U can be anything
 template <typename T> using is_shared_ptr = is_instantiation<std::shared_ptr, T>;
 
+/// Check if T looks like an input iterator
+template <typename T, typename = void> struct is_input_iterator : std::false_type {};
+template <typename T>
+struct is_input_iterator<T, void_t<decltype(*std::declval<T &>()), decltype(++std::declval<T &>())>>
+    : std::true_type {};
+
 /// Ignore that a variable is unused in compiler warnings
 inline void ignore_unused(const int *) { }
 

include/pybind11/eigen.h

@@ -200,18 +200,13 @@ template <typename Type_> struct EigenProps {
 // otherwise it'll make a copy.  writeable lets you turn off the writeable flag for the array.
 template <typename props> handle eigen_array_cast(typename props::Type const &src, handle base = handle(), bool writeable = true) {
     constexpr size_t elem_size = sizeof(typename props::Scalar);
-    std::vector<size_t> shape, strides;
-    if (props::vector) {
-        shape.push_back(src.size());
-        strides.push_back(elem_size * src.innerStride());
-    }
-    else {
-        shape.push_back(src.rows());
-        shape.push_back(src.cols());
-        strides.push_back(elem_size * src.rowStride());
-        strides.push_back(elem_size * src.colStride());
-    }
-    array a(std::move(shape), std::move(strides), src.data(), base);
+    array a;
+    if (props::vector)
+        a = array({ src.size() }, { elem_size * src.innerStride() }, src.data(), base);
+    else
+        a = array({ src.rows(), src.cols() }, { elem_size * src.rowStride(), elem_size * src.colStride() },
+                  src.data(), base);
+
     if (!writeable)
         array_proxy(a.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_;
 

include/pybind11/numpy.h

@@ -455,12 +455,15 @@ class array : public buffer {
 
     array() : array(0, static_cast<const double *>(nullptr)) {}
 
-    array(const pybind11::dtype &dt, const std::vector<size_t> &shape,
-          const std::vector<size_t> &strides, const void *ptr = nullptr,
-          handle base = handle()) {
+    template <typename ShapeIt, typename StridesIt,
+             typename = detail::enable_if_t<detail::is_input_iterator<ShapeIt>::value && detail::is_input_iterator<StridesIt>::value>>
+    array(const pybind11::dtype &dt, ShapeIt shape_first, ShapeIt shape_last, StridesIt strides_first, StridesIt strides_last,
+            const void *ptr = nullptr, handle base = handle()) {
         auto& api = detail::npy_api::get();
+
+        std::vector<Py_intptr_t> shape(shape_first, shape_last), strides(strides_first, strides_last);
         auto ndim = shape.size();
-        if (shape.size() != strides.size())
+        if (ndim != strides.size())
             pybind11_fail("NumPy: shape ndim doesn't match strides ndim");
         auto descr = dt;
 
@@ -475,9 +478,7 @@ class array : public buffer {
         }
 
         auto tmp = reinterpret_steal<object>(api.PyArray_NewFromDescr_(
-            api.PyArray_Type_, descr.release().ptr(), (int) ndim,
-            reinterpret_cast<Py_intptr_t *>(const_cast<size_t*>(shape.data())),
-            reinterpret_cast<Py_intptr_t *>(const_cast<size_t*>(strides.data())),
+            api.PyArray_Type_, descr.release().ptr(), (int) ndim, shape.data(), strides.data(),
             const_cast<void *>(ptr), flags, nullptr));
         if (!tmp)
             pybind11_fail("NumPy: unable to create array!");
@@ -491,27 +492,59 @@ class array : public buffer {
         m_ptr = tmp.release().ptr();
     }
 
-    array(const pybind11::dtype &dt, const std::vector<size_t> &shape,
+    template <typename Shape, typename Strides,
+             typename = decltype(std::begin(std::declval<const Shape &>())),
+             typename = decltype(std::begin(std::declval<const Strides &>()))>
+    array(const pybind11::dtype &dt, const Shape &shape, const Strides &strides,
           const void *ptr = nullptr, handle base = handle())
-        : array(dt, shape, default_strides(shape, dt.itemsize()), ptr, base) { }
+        : array(dt, std::begin(shape), std::end(shape), std::begin(strides), std::end(strides), ptr, base) { }
 
-    array(const pybind11::dtype &dt, size_t count, const void *ptr = nullptr,
-          handle base = handle())
-        : array(dt, std::vector<size_t>{ count }, ptr, base) { }
+    template <typename TSh, typename TSt>
+    array(const pybind11::dtype &dt, const std::initializer_list<TSh> &shape, const std::initializer_list<TSt> &strides,
+          const void *ptr = nullptr, handle base = handle())
+        : array(dt, std::begin(shape), std::end(shape), std::begin(strides), std::end(strides), ptr, base) { }
 
-    template<typename T> array(const std::vector<size_t>& shape,
-                               const std::vector<size_t>& strides,
-                               const T* ptr, handle base = handle())
-    : array(pybind11::dtype::of<T>(), shape, strides, (const void *) ptr, base) { }
+    template <typename Shape, typename = decltype(std::begin(std::declval<const Shape &>()))>
+    array(const pybind11::dtype &dt, const Shape &shape, const void *ptr = nullptr, handle base = handle())
+        : array(dt, shape, default_strides({std::begin(shape), std::end(shape)}, dt.itemsize()), ptr, base) { }
 
-    template <typename T>
-    array(const std::vector<size_t> &shape, const T *ptr,
+    template <typename TSh>
+    array(const pybind11::dtype &dt, const std::initializer_list<TSh> &shape, const void *ptr = nullptr, handle base = handle())
+        : array(dt, shape, default_strides({std::begin(shape), std::end(shape)}, dt.itemsize()), ptr, base) { }
+
+    array(const pybind11::dtype &dt, size_t count, const void *ptr = nullptr,
           handle base = handle())
-        : array(shape, default_strides(shape, sizeof(T)), ptr, base) { }
+        : array(dt, { count }, ptr, base) { }
+
+    template <typename T, typename ShapeIt, typename StridesIt,
+             typename = detail::enable_if_t<detail::is_input_iterator<ShapeIt>::value && detail::is_input_iterator<StridesIt>::value>>
+    array(ShapeIt shape_first, ShapeIt shape_last, StridesIt strides_first, StridesIt strides_last,
+            const T *ptr = nullptr, handle base = handle())
+        : array(pybind11::dtype::of<T>(), std::move(shape_first), std::move(shape_last),
+                std::move(strides_first), std::move(strides_last), ptr, base) { }
+
+    template <typename T, typename Shape, typename Strides,
+             typename = decltype(std::begin(std::declval<const Shape &>())),
+             typename = decltype(std::begin(std::declval<const Strides &>()))>
+    array(const Shape &shape, const Strides &strides, const T *ptr, handle base = handle())
+        : array(pybind11::dtype::of<T>(), shape, strides, ptr, base) { }
+
+    template <typename T, typename TSh, typename TSt>
+    array(const std::initializer_list<TSh> &shape, const std::initializer_list<TSt> &strides,
+            const T *ptr, handle base = handle())
+        : array(pybind11::dtype::of<T>(), shape, strides, ptr, base) { }
+
+    template <typename T, typename Shape, typename = decltype(std::begin(std::declval<const Shape &>()))>
+    array(const Shape &shape, const T *ptr, handle base = handle())
+        : array(shape, default_strides({std::begin(shape), std::end(shape)}, sizeof(T)), ptr, base) { }
+
+    template <typename T, typename TSh>
+    array(const std::initializer_list<TSh> &shape, const T *ptr, handle base = handle())
+        : array(shape, default_strides({std::begin(shape), std::end(shape)}, sizeof(T)), ptr, base) { }
 
     template <typename T>
     array(size_t count, const T *ptr, handle base = handle())
-        : array(std::vector<size_t>{ count }, ptr, base) { }
+        : array({ count }, ptr, base) { }
 
     explicit array(const buffer_info &info)
     : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { }
@@ -729,13 +762,29 @@ template <typename T, int ExtraFlags = array::forcecast> class array_t : public
 
     explicit array_t(const buffer_info& info) : array(info) { }
 
-    array_t(const std::vector<size_t> &shape,
-            const std::vector<size_t> &strides, const T *ptr = nullptr,
-            handle base = handle())
+    template <typename ShapeIt, typename StridesIt,
+             typename = detail::enable_if_t<detail::is_input_iterator<ShapeIt>::value && detail::is_input_iterator<StridesIt>::value>>
+    array_t(ShapeIt shape_first, ShapeIt shape_last, StridesIt strides_first, StridesIt strides_last,
+            const T *ptr = nullptr, handle base = handle())
+        : array(std::move(shape_first), std::move(shape_last), std::move(strides_first), std::move(strides_last), ptr, base) { }
+
+    template <typename Shape, typename Strides,
+             typename = decltype(std::begin(std::declval<const Shape &>())),
+             typename = decltype(std::begin(std::declval<const Strides &>()))>
+    array_t(const Shape &shape, const Strides &strides, const T *ptr = nullptr, handle base = handle())
+        : array(shape, strides, ptr, base) { }
+
+    template <typename TSh, typename TSt>
+    array_t(const std::initializer_list<TSh> &shape, const std::initializer_list<TSt> &strides,
+            const T *ptr = nullptr, handle base = handle())
         : array(shape, strides, ptr, base) { }
 
-    explicit array_t(const std::vector<size_t> &shape, const T *ptr = nullptr,
-            handle base = handle())
+    template <typename Shape, typename = decltype(std::begin(std::declval<const Shape &>()))>
+    explicit array_t(const Shape &shape, const T *ptr = nullptr, handle base = handle())
+        : array(shape, ptr, base) { }
+
+    template <typename TSh>
+    explicit array_t(const std::initializer_list<TSh> &shape, const T *ptr = nullptr, handle base = handle())
         : array(shape, ptr, base) { }
 
     explicit array_t(size_t count, const T *ptr = nullptr, handle base = handle())

tests/test_numpy_array.cpp

@@ -13,7 +13,6 @@
 #include <pybind11/stl.h>
 
 #include <cstdint>
-#include <vector>
 
 using arr = py::array;
 using arr_t = py::array_t<uint16_t, 0>;
@@ -119,8 +118,8 @@ test_initializer numpy_array([](py::module &m) {
     sm.def("wrap", [](py::array a) {
         return py::array(
             a.dtype(),
-            std::vector<size_t>(a.shape(), a.shape() + a.ndim()),
-            std::vector<size_t>(a.strides(), a.strides() + a.ndim()),
+            a.shape(), a.shape() + a.ndim(),
+            a.strides(), a.strides() + a.ndim(),
             a.data(),
             a
         );