pickling support (fixes #144)

Author: wjakob

CMakeLists.txt

@@ -120,6 +120,7 @@ set(PYBIND11_EXAMPLES
   example/example12.cpp
   example/example13.cpp
   example/example14.cpp
+  example/example15.cpp
   example/issues.cpp
 )
 

docs/advanced.rst

@@ -841,7 +841,7 @@ objects (e.g. a NumPy matrix).
     The file :file:`example/example7.cpp` contains a complete example that
     demonstrates using the buffer protocol with pybind11 in more detail.
 
-.. [#f1] https://docs.python.org/3/c-api/buffer.html
+.. [#f1] http://docs.python.org/3/c-api/buffer.html
 
 NumPy support
 =============
@@ -1184,3 +1184,70 @@ set of admissible operations.
 
     The file :file:`example/example14.cpp` contains a complete example that
     demonstrates how to create opaque types using pybind11 in more detail.
+
+Pickling support
+================
+
+Python's ``pickle`` module provides a powerful facility to serialize and
+de-serialize a Python object graph into a binary data stream. To pickle and
+unpickle C++ classes using pybind11, two additional functions most be provided.
+Suppose the class in question has the following signature:
+
+.. code-block:: cpp
+
+    class Pickleable {
+    public:
+        Pickleable(const std::string &value) : m_value(value) { }
+        const std::string &value() const { return m_value; }
+
+        void setExtra(int extra) { m_extra = extra; }
+        int extra() const { return m_extra; }
+    private:
+        std::string m_value;
+        int m_extra = 0;
+    };
+
+The binding code including the requisite ``__setstate__`` and ``__getstate__`` methods [#f2]_
+looks as follows:
+
+.. code-block:: cpp
+
+    py::class_<Pickleable>(m, "Pickleable")
+        .def(py::init<std::string>())
+        .def("value", &Pickleable::value)
+        .def("extra", &Pickleable::extra)
+        .def("setExtra", &Pickleable::setExtra)
+        .def("__getstate__", [](const Pickleable &p) {
+            /* Return a tuple that fully encodes the state of the object */
+            return py::make_tuple(p.value(), p.extra());
+        })
+        .def("__setstate__", [](Pickleable &p, py::tuple t) {
+            if (t.size() != 2)
+                throw std::runtime_error("Invalid state!");
+
+            /* Invoke the constructor (need to use in-place version) */
+            new (&p) Pickleable(t[0].cast<std::string>());
+
+            /* Assign any additional state */
+            p.setExtra(t[1].cast<int>());
+        });
+
+An instance can now be pickled as follows:
+
+.. code-block:: python
+
+    import cPickle # important
+    p = Pickleable("test_value")
+    p.setExtra(15)
+    data = cPickle.dumps(p, -1)
+
+Note that only the cPickle module is supported for now. It is important to
+request usage of the highest protocol version using the ``-1`` argument to
+``dumps``.
+
+.. seealso::
+
+    The file :file:`example/example15.cpp` contains a complete example that
+    demonstrates how to pickle and unpickle types using pybind11 in more detail.
+
+.. [#f2] http://docs.python.org/3/library/pickle.html#pickling-class-instances

docs/changelog.rst

@@ -5,8 +5,10 @@ Changelog
 
 1.5 (not yet released)
 ----------------------
+* Pickling support
+* Added a variadic ``make_tuple()`` function
 * Address a rare issue that could confuse the current virtual function dispatcher
-* Documentation improvements: import issues, symbol visibility, limitations
+* Documentation improvements: import issues, symbol visibility, pickling, limitations
 
 1.4 (April 7, 2016)
 --------------------------

example/example.cpp

@@ -23,6 +23,7 @@ void init_ex11(py::module &);
 void init_ex12(py::module &);
 void init_ex13(py::module &);
 void init_ex14(py::module &);
+void init_ex15(py::module &);
 void init_issues(py::module &);
 
 PYBIND11_PLUGIN(example) {
@@ -42,6 +43,7 @@ PYBIND11_PLUGIN(example) {
     init_ex12(m);
     init_ex13(m);
     init_ex14(m);
+    init_ex15(m);
     init_issues(m);
 
     return m.ptr();

example/example15.cpp

@@ -0,0 +1,51 @@
+/*
+    example/example15.cpp -- pickle support
+
+    Copyright (c) 2015 Wenzel Jakob <[email protected]>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#include "example.h"
+
+class Pickleable {
+public:
+    Pickleable(const std::string &value) : m_value(value) { }
+    const std::string &value() const { return m_value; }
+
+    void setExtra1(int extra1) { m_extra1 = extra1; }
+    void setExtra2(int extra2) { m_extra2 = extra2; }
+    int extra1() const { return m_extra1; }
+    int extra2() const { return m_extra2; }
+private:
+    std::string m_value;
+    int m_extra1 = 0;
+    int m_extra2 = 0;
+};
+
+void init_ex15(py::module &m) {
+    py::class_<Pickleable>(m, "Pickleable")
+        .def(py::init<std::string>())
+        .def("value", &Pickleable::value)
+        .def("extra1", &Pickleable::extra1)
+        .def("extra2", &Pickleable::extra2)
+        .def("setExtra1", &Pickleable::setExtra1)
+        .def("setExtra2", &Pickleable::setExtra2)
+        // For details on the methods below, refer to
+        // http://docs.python.org/3/library/pickle.html#pickling-class-instances
+        .def("__getstate__", [](const Pickleable &p) {
+            /* Return a tuple that fully encodes the state of the object */
+            return py::make_tuple(p.value(), p.extra1(), p.extra2());
+        })
+        .def("__setstate__", [](Pickleable &p, py::tuple t) {
+            if (t.size() != 3)
+                throw std::runtime_error("Invalid state!");
+            /* Invoke the constructor (need to use in-place version) */
+            new (&p) Pickleable(t[0].cast<std::string>());
+
+            /* Assign any additional state */
+            p.setExtra1(t[1].cast<int>());
+            p.setExtra2(t[2].cast<int>());
+        });
+}

example/example15.py

@@ -0,0 +1,18 @@
+from __future__ import print_function
+import sys
+
+sys.path.append('.')
+
+from example import Pickleable
+
+import cPickle as pickle  # Only cPickle is supported for now
+
+p = Pickleable("test_value")
+p.setExtra1(15)
+p.setExtra2(48)
+
+data = pickle.dumps(p, -1)  # -1 is important (use highest protocol version)
+print("%s %i %i" % (p.value(), p.extra1(), p.extra2()))
+
+p2 = pickle.loads(data)
+print("%s %i %i" % (p2.value(), p2.extra1(), p2.extra2()))

example/example15.ref

@@ -0,0 +1,2 @@
+test_value 15 48
+test_value 15 48

include/pybind11/pytypes.h

@@ -142,6 +142,8 @@ class accessor {
         return result;
     }
 
+    template <typename T> inline T cast() const { return operator object().cast<T>(); }
+
     operator bool() const {
         if (attr) {
             return (bool) PyObject_HasAttr(obj.ptr(), key.ptr());
@@ -161,18 +163,23 @@ class accessor {
 struct list_accessor {
 public:
     list_accessor(handle list, size_t index) : list(list), index(index) { }
+
     void operator=(list_accessor o) { return operator=(object(o)); }
+
     void operator=(const handle &o) {
         // PyList_SetItem steals a reference to 'o'
         if (PyList_SetItem(list.ptr(), (ssize_t) index, o.inc_ref().ptr()) < 0)
             pybind11_fail("Unable to assign value in Python list!");
     }
+
     operator object() const {
         PyObject *result = PyList_GetItem(list.ptr(), (ssize_t) index);
         if (!result)
             pybind11_fail("Unable to retrieve value from Python list!");
         return object(result, true);
     }
+
+    template <typename T> inline T cast() const { return operator object().cast<T>(); }
 private:
     handle list;
     size_t index;
@@ -181,18 +188,23 @@ struct list_accessor {
 struct tuple_accessor {
 public:
     tuple_accessor(handle tuple, size_t index) : tuple(tuple), index(index) { }
+
     void operator=(tuple_accessor o) { return operator=(object(o)); }
+
     void operator=(const handle &o) {
         // PyTuple_SetItem steals a referenceto 'o'
         if (PyTuple_SetItem(tuple.ptr(), (ssize_t) index, o.inc_ref().ptr()) < 0)
             pybind11_fail("Unable to assign value in Python tuple!");
     }
+
     operator object() const {
         PyObject *result = PyTuple_GetItem(tuple.ptr(), (ssize_t) index);
         if (!result)
             pybind11_fail("Unable to retrieve value from Python tuple!");
         return object(result, true);
     }
+
+    template <typename T> inline T cast() const { return operator object().cast<T>(); }
 private:
     handle tuple;
     size_t index;