Fix stubgen regressions

Fix pybind11 stubgen tests so that they report failures

Author: chadrik

misc/test-stubgenc.sh

@@ -24,7 +24,8 @@ function stubgenc_test() {
     # Compare generated stubs to expected ones
     if ! git diff --exit-code "$STUBGEN_OUTPUT_FOLDER";
     then
-        EXIT=$?
+        echo "exit status $?"
+        EXIT=1
     fi
 }
 

mypy/stubdoc.py

@@ -374,7 +374,8 @@ def infer_ret_type_sig_from_docstring(docstr: str, name: str) -> str | None:
 
 def infer_ret_type_sig_from_anon_docstring(docstr: str) -> str | None:
     """Convert signature in form of "(self: TestClass, arg0) -> int" to their return type."""
-    return infer_ret_type_sig_from_docstring("stub" + docstr.strip(), "stub")
+    lines = ["stub" + line.strip() for line in docstr.splitlines() if line.strip().startswith("(")]
+    return infer_ret_type_sig_from_docstring("".join(lines), "stub")
 
 
 def parse_signature(sig: str) -> tuple[str, list[str], list[str]] | None:

mypy/stubgen.py

@@ -1629,6 +1629,7 @@ def generate_stubs(options: Options) -> None:
                 doc_dir=options.doc_dir,
                 include_private=options.include_private,
                 export_less=options.export_less,
+                include_docstrings=options.include_docstrings,
             )
     num_modules = len(all_modules)
     if not options.quiet and num_modules > 0:

mypy/stubgenc.py

@@ -126,10 +126,12 @@ def get_property_type(self, default_type: str | None, ctx: FunctionContext) -> s
         """Infer property type from docstring or docstring signature."""
         if ctx.docstring is not None:
             inferred = infer_ret_type_sig_from_anon_docstring(ctx.docstring)
-            if not inferred:
-                inferred = infer_ret_type_sig_from_docstring(ctx.docstring, ctx.name)
-            if not inferred:
-                inferred = infer_prop_type_from_docstring(ctx.docstring)
+            if inferred:
+                return inferred
+            inferred = infer_ret_type_sig_from_docstring(ctx.docstring, ctx.name)
+            if inferred:
+                return inferred
+            inferred = infer_prop_type_from_docstring(ctx.docstring)
             return inferred
         else:
             return None
@@ -590,9 +592,31 @@ def generate_function_stub(
                 if inferred[0].args and inferred[0].args[0].name == "cls":
                     decorators.append("@classmethod")
 
-        output.extend(self.format_func_def(inferred, decorators=decorators, docstring=docstring))
+        output.extend(
+            self.format_func_def(
+                inferred, decorators=decorators, docstring=self._indent_docstring(docstring)
+            )
+        )
         self._fix_iter(ctx, inferred, output)
 
+    def _indent_docstring(self, docstring: str) -> str:
+        """Fix indentation of docstring extracted from pybind11 or other binding generators."""
+        lines = docstring.splitlines(keepends=True)
+        indent = self._indent + "    "
+        if len(lines) > 1:
+            if not all(line.startswith(indent) or not line.strip() for line in lines):
+                # if the docstring is not indented, then indent all but the first line
+                for i, line in enumerate(lines[1:]):
+                    if line.strip():
+                        lines[i + 1] = indent + line
+        # if there's a trailing newline, add a final line to visually indent the quoted docstring
+        if lines[-1].endswith("\n"):
+            if len(lines) > 1:
+                lines.append(indent)
+            else:
+                lines[-1] = lines[-1][:-1]
+        return "".join(lines)
+
     def _fix_iter(
         self, ctx: FunctionContext, inferred: list[FunctionSig], output: list[str]
     ) -> None:

test-data/pybind11_mypy_demo/src/main.cpp

@@ -134,14 +134,17 @@ void bind_basics(py::module& basics) {
     .def(py::init<double, double>(), py::arg("x"), py::arg("y"))
     .def("distance_to", py::overload_cast<double, double>(&Point::distance_to, py::const_), py::arg("x"), py::arg("y"))
     .def("distance_to", py::overload_cast<const Point&>(&Point::distance_to, py::const_), py::arg("other"))
-    .def_readwrite("x", &Point::x)
+    // Note that the trailing newline is required because the generated docstring
+    // is concatenated to the signature, e.g.:
+    // 'some docstring\n(self: pybind11_mypy_demo.basics.Point) -> float\n'
+    .def_readwrite("x", &Point::x, "some docstring\n")
     .def_property("y",
         [](Point& self){ return self.y; },
         [](Point& self, double value){ self.y = value; }
     )
     .def_property_readonly("length", &Point::length)
     .def_property_readonly_static("x_axis", [](py::object cls){return Point::x_axis;})
-    .def_property_readonly_static("y_axis", [](py::object cls){return Point::y_axis;})
+    .def_property_readonly_static("y_axis", [](py::object cls){return Point::y_axis;}, "another docstring")
     .def_readwrite_static("length_unit", &Point::length_unit)
     .def_property_static("angle_unit",
         [](py::object& /*cls*/){ return Point::angle_unit; },

test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/__init__.pyi

@@ -0,0 +1 @@
+from . import basics as basics

test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/basics.pyi

@@ -1,7 +1,7 @@
-from typing import ClassVar
+from typing import ClassVar, overload
 
-from typing import overload
 PI: float
+__version__: str
 
 class Point:
     class AngleUnit:
@@ -13,8 +13,6 @@ class Point:
             """__init__(self: pybind11_mypy_demo.basics.Point.AngleUnit, value: int) -> None"""
         def __eq__(self, other: object) -> bool:
             """__eq__(self: object, other: object) -> bool"""
-        def __getstate__(self) -> int:
-            """__getstate__(self: object) -> int"""
         def __hash__(self) -> int:
             """__hash__(self: object) -> int"""
         def __index__(self) -> int:
@@ -23,8 +21,6 @@ class Point:
             """__int__(self: pybind11_mypy_demo.basics.Point.AngleUnit) -> int"""
         def __ne__(self, other: object) -> bool:
             """__ne__(self: object, other: object) -> bool"""
-        def __setstate__(self, state: int) -> None:
-            """__setstate__(self: pybind11_mypy_demo.basics.Point.AngleUnit, state: int) -> None"""
         @property
         def name(self) -> str: ...
         @property
@@ -40,8 +36,6 @@ class Point:
             """__init__(self: pybind11_mypy_demo.basics.Point.LengthUnit, value: int) -> None"""
         def __eq__(self, other: object) -> bool:
             """__eq__(self: object, other: object) -> bool"""
-        def __getstate__(self) -> int:
-            """__getstate__(self: object) -> int"""
         def __hash__(self) -> int:
             """__hash__(self: object) -> int"""
         def __index__(self) -> int:
@@ -50,8 +44,6 @@ class Point:
             """__int__(self: pybind11_mypy_demo.basics.Point.LengthUnit) -> int"""
         def __ne__(self, other: object) -> bool:
             """__ne__(self: object, other: object) -> bool"""
-        def __setstate__(self, state: int) -> None:
-            """__setstate__(self: pybind11_mypy_demo.basics.Point.LengthUnit, state: int) -> None"""
         @property
         def name(self) -> str: ...
         @property
@@ -70,43 +62,49 @@ class Point:
 
         1. __init__(self: pybind11_mypy_demo.basics.Point) -> None
 
-        2. __init__(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> None"""
+        2. __init__(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> None
+        """
     @overload
     def __init__(self, x: float, y: float) -> None:
         """__init__(*args, **kwargs)
         Overloaded function.
 
         1. __init__(self: pybind11_mypy_demo.basics.Point) -> None
 
-        2. __init__(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> None"""
+        2. __init__(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> None
+        """
     @overload
     def distance_to(self, x: float, y: float) -> float:
         """distance_to(*args, **kwargs)
         Overloaded function.
 
         1. distance_to(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> float
 
-        2. distance_to(self: pybind11_mypy_demo.basics.Point, other: pybind11_mypy_demo.basics.Point) -> float"""
+        2. distance_to(self: pybind11_mypy_demo.basics.Point, other: pybind11_mypy_demo.basics.Point) -> float
+        """
     @overload
     def distance_to(self, other: Point) -> float:
         """distance_to(*args, **kwargs)
         Overloaded function.
 
         1. distance_to(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> float
 
-        2. distance_to(self: pybind11_mypy_demo.basics.Point, other: pybind11_mypy_demo.basics.Point) -> float"""
+        2. distance_to(self: pybind11_mypy_demo.basics.Point, other: pybind11_mypy_demo.basics.Point) -> float
+        """
     @property
     def length(self) -> float: ...
 
 def answer() -> int:
     '''answer() -> int
 
-    answer docstring, with end quote"'''
+    answer docstring, with end quote"
+    '''
 def midpoint(left: float, right: float) -> float:
     """midpoint(left: float, right: float) -> float"""
 def sum(arg0: int, arg1: int) -> int:
     '''sum(arg0: int, arg1: int) -> int
 
-    multiline docstring test, edge case quotes """\'\'\''''
+    multiline docstring test, edge case quotes """\'\'\'
+    '''
 def weighted_midpoint(left: float, right: float, alpha: float = ...) -> float:
     """weighted_midpoint(left: float, right: float, alpha: float = 0.5) -> float"""