Split pybind11 into headers and cpp files to speedup compilation

As discussed at #2322, the split
is opt-in, and has been observed to speed up the compilation of the gem5
project by around 40% from 25 minutes to 15 minutes.

If the PYBIND11_DECLARATIONS_ONLY is not defined, then the .cpp files are
included in the .h files, and everything works header-only as was the case
before this commit.

If PYBIND11_DECLARATIONS_ONLY=1, then only declarations are made visible
from, the header files, and the user must the user must compile the
pybind11 .cpp files, also using PYBIND11_DECLARATIONS_ONLY=1, into objects
and add link those into the final link. This commit also updates CMakeLists
to automate building those object files.

This commit has been tested as follows.

First, the original build and pytest (without PYBIND11_DECLARATIONS_ONLY)
work as before:

mkdir build
cmake ..
make -j `nproc`
make pytest

TODO: if this commit gets traction, I will try to add a test for the
PYBIND11_DECLARATIONS_ONLY version too, otherwise it will likely break.
I'd just re-run the entire pybind with PYBIND11_DECLARATIONS_ONLY as well
every time.

The commit has also been tested with the following minimal manual example:

class_test.cpp

```
struct ClassTest {
    ClassTest(const std::string &name) : name(name) {}
    void setName(const std::string &name_) { name = name_; }
    const std::string &getName() const { return name; }
    std::string name;
};

struct ClassTestDerived : ClassTest {
    ClassTestDerived(const std::string &name, const std::string &name2) :
        ClassTest(name), name2(name2) {}
    std::string getName2() { return name + name2 + "2"; }
    std::string name2;
};

namespace py = pybind11;

PYBIND11_MODULE(class_test, m) {
    m.doc() = "pybind11 example plugin";
    py::class_<ClassTest>(m, "ClassTest")
        .def(py::init<const std::string &>())
        .def("setName", &ClassTest::setName)
        .def("getName", &ClassTest::getName)
        .def_readwrite("name", &ClassTest::name);
    py::class_<ClassTestDerived, ClassTest>(m, "ClassTestDerived")
        .def(py::init<const std::string &, const std::string &>())
        .def("getName2", &ClassTestDerived::getName2)
        .def_readwrite("name", &ClassTestDerived::name);
}
```

class_test_main.py

```
import class_test

my_class_test = class_test.ClassTest('abc');
assert(my_class_test.getName() == 'abc')
my_class_test.setName('012')
assert(my_class_test.name == '012')

my_class_test_derived = class_test.ClassTestDerived('abc', 'def');
assert(my_class_test_derived.getName2() == 'abcdef2')
```

without PYBIND11_DECLARATIONS_ONLY (`python3-config --cflags`) is opened up
and hacked to point to the custom pybind11 source:

```
g++ \
  -save-temps \
  -I/usr/include/python3.8 \
  -I/home/ciro/git/pybind11/include \
  -Wno-unused-result \
  -Wsign-compare \
  -g \
  -fdebug-prefix-map=/build/python3.8-fKk4GY/python3.8-3.8.2=. \
  -specs=/usr/share/dpkg/no-pie-compile.specs \
  -fstack-protector \
  -Wformat \
  -Werror=format-security  \
  -DNDEBUG \
  -g \
  -fwrapv \
  -O3 \
  -Wall \
  -shared \
  -std=c++11 \
  -fPIC class_test.cpp \
  -o class_test`python3-config --extension-suffix` \
  `python3-config --libs` \
;
./class_test_main.py
```

with PYBIND11_DECLARATIONS_ONLY:

```
g++ \
  -DPYBIND11_DECLARATIONS_ONLY=1 \
  -save-temps \
  -I/usr/include/python3.8 \
  -I/home/ciro/git/pybind11/include \
  -Wno-unused-result \
  -Wsign-compare \
  -g \
  -fdebug-prefix-map=/build/python3.8-fKk4GY/python3.8-3.8.2=. \
  -specs=/usr/share/dpkg/no-pie-compile.specs \
  -fstack-protector \
  -Wformat \
  -Werror=format-security  \
  -DNDEBUG \
  -g \
  -fwrapv \
  -O3 \
  -Wall \
  -shared \
  -std=c++11 \
  -fPIC class_test.cpp \
  /home/ciro/git/pybind11/build/libpybind11.a \
  -o class_test`python3-config --extension-suffix` \
  `python3-config --libs` \
;
./class_test_main.py
```

Author: cirosantilli2

CMakeLists.txt

@@ -161,6 +161,24 @@ add_library(pybind11::headers ALIAS pybind11_headers) # easier to use/remember
 
 include("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11Common.cmake")
 
+option(PYBIND11_PRECOMPILE "Precompile parts of pybind11" ON)
+
+if(PYBIND11_PRECOMPILE)
+  file(GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
+  if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy")
+    # Pypy does not support embedding.
+    list(REMOVE_ITEM SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/embed.cpp)
+  endif()
+  # Produce libpybind11.a, which contains position independent object files.
+  # that may be used to create a shared library.
+  add_library(pybind11_lib STATIC ${SOURCES} ${PYBIND11_HEADERS})
+  set_property(TARGET pybind11_lib PROPERTY POSITION_INDEPENDENT_CODE ON)
+  target_compile_definitions(pybind11_lib PUBLIC -DPYBIND11_DECLARATIONS_ONLY=1)
+  add_library(pybind11::lib ALIAS pybind11_lib)
+  target_link_libraries(pybind11_lib PRIVATE pybind11::pybind11)
+  target_link_libraries(pybind11_lib PUBLIC pybind11::headers)
+endif()
+
 if(NOT PYBIND11_MASTER_PROJECT AND NOT pybind11_FIND_QUIETLY)
   message(STATUS "Using pybind11: (version \"${pybind11_VERSION}\" ${pybind11_VERSION_TYPE})")
 endif()

docs/Doxyfile

@@ -11,6 +11,7 @@ XML_PROGRAMLISTING     = YES
 MACRO_EXPANSION        = YES
 EXPAND_ONLY_PREDEF     = YES
 EXPAND_AS_DEFINED      = PYBIND11_RUNTIME_EXCEPTION
+EXCLUDE_PATTERNS       = *-inl.h
 
 ALIASES                = "rst=\verbatim embed:rst"
 ALIASES               += "endrst=\endverbatim"
@@ -19,4 +20,5 @@ QUIET                  = YES
 WARNINGS               = YES
 WARN_IF_UNDOCUMENTED   = NO
 PREDEFINED             = DOXYGEN_SHOULD_SKIP_THIS \
-                         PY_MAJOR_VERSION=3
+                         PY_MAJOR_VERSION=3 \
+                         PYBIND11_INLINE=

include/pybind11/attr-inl.h

@@ -0,0 +1,48 @@
+#include "attr.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+PYBIND11_INLINE void type_record::add_base(const std::type_info &base, void *(*caster)(void *)) {
+    auto base_info = detail::get_type_info(base, false);
+    if (!base_info) {
+        std::string tname(base.name());
+        detail::clean_type_id(tname);
+        pybind11_fail("generic_type: type \"" + std::string(name) +
+                        "\" referenced unknown base type \"" + tname + "\"");
+    }
+
+    if (default_holder != base_info->default_holder) {
+        std::string tname(base.name());
+        detail::clean_type_id(tname);
+        pybind11_fail("generic_type: type \"" + std::string(name) + "\" " +
+                (default_holder ? "does not have" : "has") +
+                " a non-default holder type while its base \"" + tname + "\" " +
+                (base_info->default_holder ? "does not" : "does"));
+    }
+
+    bases.append((PyObject *) base_info->type);
+
+    if (base_info->type->tp_dictoffset != 0)
+        dynamic_attr = true;
+
+    if (caster)
+        base_info->implicit_casts.emplace_back(type, caster);
+}
+
+PYBIND11_INLINE function_call::function_call(const function_record &f, handle p) :
+        func(f), parent(p) {
+    args.reserve(f.nargs);
+    args_convert.reserve(f.nargs);
+}
+
+PYBIND11_INLINE void process_kwonly_arg(const arg &a, function_record *r) {
+    if (!a.name || strlen(a.name) == 0)
+        pybind11_fail("arg(): cannot specify an unnamed argument after an kwonly() annotation");
+    ++r->nargs_kwonly;
+}
+
+PYBIND11_NAMESPACE_END(detail)
+
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/attr.h

@@ -119,7 +119,7 @@ enum op_id : int;
 enum op_type : int;
 struct undefined_t;
 template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t> struct op_;
-inline void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret);
+void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret);
 
 /// Internal data structure which holds metadata about a keyword argument
 struct argument_record {
@@ -267,40 +267,9 @@ struct type_record {
     /// Is the class inheritable from python classes?
     bool is_final : 1;
 
-    PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *)) {
-        auto base_info = detail::get_type_info(base, false);
-        if (!base_info) {
-            std::string tname(base.name());
-            detail::clean_type_id(tname);
-            pybind11_fail("generic_type: type \"" + std::string(name) +
-                          "\" referenced unknown base type \"" + tname + "\"");
-        }
-
-        if (default_holder != base_info->default_holder) {
-            std::string tname(base.name());
-            detail::clean_type_id(tname);
-            pybind11_fail("generic_type: type \"" + std::string(name) + "\" " +
-                    (default_holder ? "does not have" : "has") +
-                    " a non-default holder type while its base \"" + tname + "\" " +
-                    (base_info->default_holder ? "does not" : "does"));
-        }
-
-        bases.append((PyObject *) base_info->type);
-
-        if (base_info->type->tp_dictoffset != 0)
-            dynamic_attr = true;
-
-        if (caster)
-            base_info->implicit_casts.emplace_back(type, caster);
-    }
+    PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *));
 };
 
-inline function_call::function_call(const function_record &f, handle p) :
-        func(f), parent(p) {
-    args.reserve(f.nargs);
-    args_convert.reserve(f.nargs);
-}
-
 /// Tag for a new-style `__init__` defined in `detail/init.h`
 struct is_new_style_constructor { };
 
@@ -366,11 +335,7 @@ template <> struct process_attribute<is_new_style_constructor> : process_attribu
     static void init(const is_new_style_constructor &, function_record *r) { r->is_new_style_constructor = true; }
 };
 
-inline void process_kwonly_arg(const arg &a, function_record *r) {
-    if (!a.name || strlen(a.name) == 0)
-        pybind11_fail("arg(): cannot specify an unnamed argument after an kwonly() annotation");
-    ++r->nargs_kwonly;
-}
+void process_kwonly_arg(const arg &a, function_record *r);
 
 /// Process a keyword argument attribute (*without* a default value)
 template <> struct process_attribute<arg> : process_attribute_default<arg> {
@@ -526,3 +491,7 @@ constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) {
 
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)
+
+#if !defined(PYBIND11_DECLARATIONS_ONLY)
+#include "attr-inl.h"
+#endif

include/pybind11/buffer_info-inl.h

@@ -0,0 +1,44 @@
+#include "buffer_info.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+
+PYBIND11_INLINE buffer_info::buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim,
+            detail::any_container<ssize_t> shape_in, detail::any_container<ssize_t> strides_in, bool readonly)
+: ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim),
+    shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) {
+    if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size())
+        pybind11_fail("buffer_info: ndim doesn't match shape and/or strides length");
+    for (size_t i = 0; i < (size_t) ndim; ++i)
+        size *= shape[i];
+}
+
+PYBIND11_INLINE buffer_info::buffer_info(Py_buffer *view, bool ownview)
+: buffer_info(view->buf, view->itemsize, view->format, view->ndim,
+        {view->shape, view->shape + view->ndim}, {view->strides, view->strides + view->ndim}, view->readonly) {
+    this->m_view = view;
+    this->ownview = ownview;
+}
+
+PYBIND11_INLINE buffer_info::buffer_info(buffer_info &&other) {
+    (*this) = std::move(other);
+}
+
+PYBIND11_INLINE buffer_info& buffer_info::operator=(buffer_info &&rhs) {
+    ptr = rhs.ptr;
+    itemsize = rhs.itemsize;
+    size = rhs.size;
+    format = std::move(rhs.format);
+    ndim = rhs.ndim;
+    shape = std::move(rhs.shape);
+    strides = std::move(rhs.strides);
+    std::swap(m_view, rhs.m_view);
+    std::swap(ownview, rhs.ownview);
+    readonly = rhs.readonly;
+    return *this;
+}
+
+PYBIND11_INLINE buffer_info::~buffer_info() {
+    if (m_view && ownview) { PyBuffer_Release(m_view); delete m_view; }
+}
+
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/buffer_info.h

@@ -27,14 +27,7 @@ struct buffer_info {
     buffer_info() { }
 
     buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim,
-                detail::any_container<ssize_t> shape_in, detail::any_container<ssize_t> strides_in, bool readonly=false)
-    : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim),
-      shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) {
-        if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size())
-            pybind11_fail("buffer_info: ndim doesn't match shape and/or strides length");
-        for (size_t i = 0; i < (size_t) ndim; ++i)
-            size *= shape[i];
-    }
+                detail::any_container<ssize_t> shape_in, detail::any_container<ssize_t> strides_in, bool readonly=false);
 
     template <typename T>
     buffer_info(T *ptr, detail::any_container<ssize_t> shape_in, detail::any_container<ssize_t> strides_in, bool readonly=false)
@@ -51,37 +44,16 @@ struct buffer_info {
     buffer_info(const T *ptr, ssize_t size, bool readonly=true)
     : buffer_info(const_cast<T*>(ptr), sizeof(T), format_descriptor<T>::format(), size, readonly) { }
 
-    explicit buffer_info(Py_buffer *view, bool ownview = true)
-    : buffer_info(view->buf, view->itemsize, view->format, view->ndim,
-            {view->shape, view->shape + view->ndim}, {view->strides, view->strides + view->ndim}, view->readonly) {
-        this->m_view = view;
-        this->ownview = ownview;
-    }
+    explicit buffer_info(Py_buffer *view, bool ownview = true);
 
     buffer_info(const buffer_info &) = delete;
     buffer_info& operator=(const buffer_info &) = delete;
 
-    buffer_info(buffer_info &&other) {
-        (*this) = std::move(other);
-    }
+    buffer_info(buffer_info &&other);
 
-    buffer_info& operator=(buffer_info &&rhs) {
-        ptr = rhs.ptr;
-        itemsize = rhs.itemsize;
-        size = rhs.size;
-        format = std::move(rhs.format);
-        ndim = rhs.ndim;
-        shape = std::move(rhs.shape);
-        strides = std::move(rhs.strides);
-        std::swap(m_view, rhs.m_view);
-        std::swap(ownview, rhs.ownview);
-        readonly = rhs.readonly;
-        return *this;
-    }
+    buffer_info& operator=(buffer_info &&rhs);
 
-    ~buffer_info() {
-        if (m_view && ownview) { PyBuffer_Release(m_view); delete m_view; }
-    }
+    ~buffer_info();
 
     Py_buffer *view() const { return m_view; }
     Py_buffer *&view() { return m_view; }
@@ -114,3 +86,7 @@ template <typename T> struct compare_buffer_info<T, detail::enable_if_t<std::is_
 
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)
+
+#if !defined(PYBIND11_DECLARATIONS_ONLY)
+#include "buffer_info-inl.h"
+#endif