A way to register additional test targets and support .py only tests. (#3590)

* A way to register additional test targets.

* Support specifying tests with extension.

* Ensure TEST_OVERRIDE is backwards compatible.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Ensure regex is non greedy.

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: iwanders

.github/CONTRIBUTING.md

@@ -159,8 +159,9 @@ tests with these targets:
 * `test_cmake_build`: Install / subdirectory tests
 
 If you want to build just a subset of tests, use
-`-DPYBIND11_TEST_OVERRIDE="test_callbacks.cpp;test_pickling.cpp"`. If this is
-empty, all tests will be built.
+`-DPYBIND11_TEST_OVERRIDE="test_callbacks;test_pickling"`. If this is
+empty, all tests will be built. Tests are specified without an extension if they need both a .py and
+.cpp file.
 
 You may also pass flags to the `pytest` target by editing `tests/pytest.ini` or
 by using the `PYTEST_ADDOPTS` environment variable

tests/CMakeLists.txt

@@ -19,18 +19,25 @@ endif()
 # Only needed for CMake < 3.5 support
 include(CMakeParseArguments)
 
-# Filter out items; print an optional message if any items filtered
+# Filter out items; print an optional message if any items filtered. This ignores extensions.
 #
 # Usage:
 #   pybind11_filter_tests(LISTNAME file1.cpp file2.cpp ... MESSAGE "")
 #
 macro(pybind11_filter_tests LISTNAME)
   cmake_parse_arguments(ARG "" "MESSAGE" "" ${ARGN})
   set(PYBIND11_FILTER_TESTS_FOUND OFF)
+  # Make a list of the test without any extensions, for easier filtering.
+  set(_TMP_ACTUAL_LIST "${${LISTNAME}};") # enforce ';' at the end to allow matching last item.
+  string(REGEX REPLACE "\\.[^.;]*;" ";" LIST_WITHOUT_EXTENSIONS "${_TMP_ACTUAL_LIST}")
   foreach(filename IN LISTS ARG_UNPARSED_ARGUMENTS)
-    list(FIND ${LISTNAME} ${filename} _FILE_FOUND)
+    string(REGEX REPLACE "\\.[^.]*$" "" filename_no_ext ${filename})
+    # Search in the list without extensions.
+    list(FIND LIST_WITHOUT_EXTENSIONS ${filename_no_ext} _FILE_FOUND)
     if(_FILE_FOUND GREATER -1)
-      list(REMOVE_AT ${LISTNAME} ${_FILE_FOUND})
+      list(REMOVE_AT ${LISTNAME} ${_FILE_FOUND}) # And remove from the list with extensions.
+      list(REMOVE_AT LIST_WITHOUT_EXTENSIONS ${_FILE_FOUND}
+      )# And our search list, to ensure it is in sync.
       set(PYBIND11_FILTER_TESTS_FOUND ON)
     endif()
   endforeach()
@@ -47,6 +54,18 @@ macro(possibly_uninitialized)
   endforeach()
 endmacro()
 
+# Function to add additional targets if any of the provided tests are found.
+# Needles; Specifies the test names to look for.
+# Additions; Specifies the additional test targets to add when any of the needles are found.
+macro(tests_extra_targets needles additions)
+  # Add the index for this relation to the index extra targets map.
+  list(LENGTH PYBIND11_TEST_EXTRA_TARGETS PYBIND11_TEST_EXTRA_TARGETS_LEN)
+  list(APPEND PYBIND11_TEST_EXTRA_TARGETS ${PYBIND11_TEST_EXTRA_TARGETS_LEN})
+  # Add the test names to look for, and the associated test target additions.
+  set(PYBIND11_TEST_EXTRA_TARGETS_NEEDLES_${PYBIND11_TEST_EXTRA_TARGETS_LEN} ${needles})
+  set(PYBIND11_TEST_EXTRA_TARGETS_ADDITION_${PYBIND11_TEST_EXTRA_TARGETS_LEN} ${additions})
+endmacro()
+
 # New Python support
 if(DEFINED Python_EXECUTABLE)
   set(PYTHON_EXECUTABLE "${Python_EXECUTABLE}")
@@ -92,55 +111,67 @@ if(PYBIND11_CUDA_TESTS)
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
 endif()
 
-# Full set of test files (you can override these; see below)
+# Full set of test files (you can override these; see below, overrides ignore extension)
+# Any test that has no extension is both .py and .cpp, so 'foo' will add 'foo.cpp' and 'foo.py'.
+# Any test that has an extension is exclusively that and handled as such.
 set(PYBIND11_TEST_FILES
-    test_async.cpp
-    test_buffers.cpp
-    test_builtin_casters.cpp
-    test_call_policies.cpp
-    test_callbacks.cpp
-    test_chrono.cpp
-    test_class.cpp
-    test_const_name.cpp
-    test_constants_and_functions.cpp
-    test_copy_move.cpp
-    test_custom_type_casters.cpp
-    test_custom_type_setup.cpp
-    test_docstring_options.cpp
-    test_eigen.cpp
-    test_enum.cpp
-    test_eval.cpp
-    test_exceptions.cpp
-    test_factory_constructors.cpp
-    test_gil_scoped.cpp
-    test_iostream.cpp
-    test_kwargs_and_defaults.cpp
-    test_local_bindings.cpp
-    test_methods_and_attributes.cpp
-    test_modules.cpp
-    test_multiple_inheritance.cpp
-    test_numpy_array.cpp
-    test_numpy_dtypes.cpp
-    test_numpy_vectorize.cpp
-    test_opaque_types.cpp
-    test_operator_overloading.cpp
-    test_pickling.cpp
-    test_pytypes.cpp
-    test_sequences_and_iterators.cpp
-    test_smart_ptr.cpp
-    test_stl.cpp
-    test_stl_binders.cpp
-    test_tagbased_polymorphic.cpp
-    test_thread.cpp
-    test_union.cpp
-    test_virtual_functions.cpp)
+    test_async
+    test_buffers
+    test_builtin_casters
+    test_call_policies
+    test_callbacks
+    test_chrono
+    test_class
+    test_const_name
+    test_constants_and_functions
+    test_copy_move
+    test_custom_type_casters
+    test_custom_type_setup
+    test_docstring_options
+    test_eigen
+    test_enum
+    test_eval
+    test_exceptions
+    test_factory_constructors
+    test_gil_scoped
+    test_iostream
+    test_kwargs_and_defaults
+    test_local_bindings
+    test_methods_and_attributes
+    test_modules
+    test_multiple_inheritance
+    test_numpy_array
+    test_numpy_dtypes
+    test_numpy_vectorize
+    test_opaque_types
+    test_operator_overloading
+    test_pickling
+    test_pytypes
+    test_sequences_and_iterators
+    test_smart_ptr
+    test_stl
+    test_stl_binders
+    test_tagbased_polymorphic
+    test_thread
+    test_union
+    test_virtual_functions)
 
 # Invoking cmake with something like:
 #     cmake -DPYBIND11_TEST_OVERRIDE="test_callbacks.cpp;test_pickling.cpp" ..
 # lets you override the tests that get compiled and run.  You can restore to all tests with:
 #     cmake -DPYBIND11_TEST_OVERRIDE= ..
 if(PYBIND11_TEST_OVERRIDE)
-  set(PYBIND11_TEST_FILES ${PYBIND11_TEST_OVERRIDE})
+  # Instead of doing a direct override here, we iterate over the overrides without extension and
+  # match them against entries from the PYBIND11_TEST_FILES, anything that not matches goes into the filter list.
+  string(REGEX REPLACE "\\.[^.;]*;" ";" TEST_OVERRIDE_NO_EXT "${PYBIND11_TEST_OVERRIDE};")
+  string(REGEX REPLACE "\\.[^.;]*;" ";" TEST_FILES_NO_EXT "${PYBIND11_TEST_FILES};")
+  # This allows the override to be done with extensions, preserving backwards compatibility.
+  foreach(test_name ${TEST_FILES_NO_EXT})
+    if(NOT ${test_name} IN_LIST TEST_OVERRIDE_NO_EXT
+    )# If not in the whitelist, add to be filtered out.
+      list(APPEND PYBIND11_TEST_FILTER ${test_name})
+    endif()
+  endforeach()
 endif()
 
 # You can also filter tests:
@@ -162,15 +193,34 @@ if(PYBIND11_CUDA_TESTS)
     "Skipping test_constants_and_functions due to incompatible exception specifications")
 endif()
 
-string(REPLACE ".cpp" ".py" PYBIND11_PYTEST_FILES "${PYBIND11_TEST_FILES}")
+# Now that the test filtering is complete, we need to split the list into the test for PYTEST
+# and the list for the cpp targets.
+set(PYBIND11_CPPTEST_FILES "")
+set(PYBIND11_PYTEST_FILES "")
+
+foreach(test_name ${PYBIND11_TEST_FILES})
+  if(test_name MATCHES "\\.py$") # Ends in .py, purely python test.
+    list(APPEND PYBIND11_PYTEST_FILES ${test_name})
+  elseif(test_name MATCHES "\\.cpp$") # Ends in .cpp, purely cpp test.
+    list(APPEND PYBIND11_CPPTEST_FILES ${test_name})
+  elseif(NOT test_name MATCHES "\\.") # No extension specified, assume both, add extension.
+    list(APPEND PYBIND11_PYTEST_FILES ${test_name}.py)
+    list(APPEND PYBIND11_CPPTEST_FILES ${test_name}.cpp)
+  else()
+    message(WARNING "Unhanded test extension in test: ${test_name}")
+  endif()
+endforeach()
+set(PYBIND11_TEST_FILES ${PYBIND11_CPPTEST_FILES})
+list(SORT PYBIND11_PYTEST_FILES)
 
 # Contains the set of test files that require pybind11_cross_module_tests to be
 # built; if none of these are built (i.e. because TEST_OVERRIDE is used and
 # doesn't include them) the second module doesn't get built.
-set(PYBIND11_CROSS_MODULE_TESTS test_exceptions.py test_local_bindings.py test_stl.py
-                                test_stl_binders.py)
+tests_extra_targets("test_exceptions.py;test_local_bindings.py;test_stl.py;test_stl_binders.py"
+                    "pybind11_cross_module_tests")
 
-set(PYBIND11_CROSS_MODULE_GIL_TESTS test_gil_scoped.py)
+# And add additional targets for other tests.
+tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils")
 
 set(PYBIND11_EIGEN_REPO
     "https://gitlab.com/libeigen/eigen.git"
@@ -353,21 +403,17 @@ endfunction()
 
 set(test_targets pybind11_tests)
 
-# Build pybind11_cross_module_tests if any test_whatever.py are being built that require it
-foreach(t ${PYBIND11_CROSS_MODULE_TESTS})
-  list(FIND PYBIND11_PYTEST_FILES ${t} i)
-  if(i GREATER -1)
-    list(APPEND test_targets pybind11_cross_module_tests)
-    break()
-  endif()
-endforeach()
-
-foreach(t ${PYBIND11_CROSS_MODULE_GIL_TESTS})
-  list(FIND PYBIND11_PYTEST_FILES ${t} i)
-  if(i GREATER -1)
-    list(APPEND test_targets cross_module_gil_utils)
-    break()
-  endif()
+# Check if any tests need extra targets by iterating through the mappings registered.
+foreach(i ${PYBIND11_TEST_EXTRA_TARGETS})
+  foreach(needle ${PYBIND11_TEST_EXTRA_TARGETS_NEEDLES_${i}})
+    if(needle IN_LIST PYBIND11_PYTEST_FILES)
+      # Add all the additional targets to the test list. List join in newer cmake.
+      foreach(extra_target ${PYBIND11_TEST_EXTRA_TARGETS_ADDITION_${i}})
+        list(APPEND test_targets ${extra_target})
+      endforeach()
+      break() # Breaks out of the needle search, continues with the next mapping.
+    endif()
+  endforeach()
 endforeach()
 
 # Support CUDA testing by forcing the target file to compile with NVCC