From a0cc4cf3669b73ee5532540655ba47fe026fe954 Mon Sep 17 00:00:00 2001 From: Ameya Vikram Singh Date: Mon, 10 Feb 2025 10:01:24 +0530 Subject: [PATCH 1/4] Initial support for `DISCOVERY_EXTRA_ARGS` for pytest command line support during test discovery. --- cmake/FindPytest.cmake | 3 ++- cmake/PytestAddTests.cmake | 2 +- doc/api_reference.rst | 21 +++++++++++++++ test/06-extra-args/CMakeLists.txt | 45 ++++++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/cmake/FindPytest.cmake b/cmake/FindPytest.cmake index 480b2f5..d9bb693 100644 --- a/cmake/FindPytest.cmake +++ b/cmake/FindPytest.cmake @@ -58,7 +58,7 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) cmake_parse_arguments( PARSE_ARGV 1 "" "STRIP_PARAM_BRACKETS;INCLUDE_FILE_PATH;BUNDLE_TESTS" "WORKING_DIRECTORY;TRIM_FROM_NAME;TRIM_FROM_FULL_NAME" - "LIBRARY_PATH_PREPEND;PYTHON_PATH_PREPEND;ENVIRONMENT;PROPERTIES;DEPENDS;EXTRA_ARGS" + "LIBRARY_PATH_PREPEND;PYTHON_PATH_PREPEND;ENVIRONMENT;PROPERTIES;DEPENDS;EXTRA_ARGS;DISCOVERY_EXTRA_ARGS" ) # Set platform-specific library path environment variable. @@ -133,6 +133,7 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) -D "TEST_PROPERTIES=${_PROPERTIES}" -D "CTEST_FILE=${_tests_file}" -D "EXTRA_ARGS=${_EXTRA_ARGS}" + -D "DISCOVERY_EXTRA_ARGS=${_DISCOVERY_EXTRA_ARGS}" -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/PytestAddTests.cmake") # Create a custom target to run the tests. diff --git a/cmake/PytestAddTests.cmake b/cmake/PytestAddTests.cmake index c13a7e1..0935df2 100644 --- a/cmake/PytestAddTests.cmake +++ b/cmake/PytestAddTests.cmake @@ -69,7 +69,7 @@ if(CMAKE_SCRIPT_MODE_FILE) execute_process( COMMAND "${PYTEST_EXECUTABLE}" --collect-only -q - --rootdir=${WORKING_DIRECTORY} . + --rootdir=${WORKING_DIRECTORY} ${DISCOVERY_EXTRA_ARGS} . OUTPUT_VARIABLE _output_lines ERROR_VARIABLE _output_lines OUTPUT_STRIP_TRAILING_WHITESPACE diff --git a/doc/api_reference.rst b/doc/api_reference.rst index d35bbb4..15c9d23 100644 --- a/doc/api_reference.rst +++ b/doc/api_reference.rst @@ -24,6 +24,7 @@ API Reference [STRIP_PARAM_BRACKETS] [BUNDLE_TESTS] [EXTRA_ARGS arg1 arg2...] + [DISCOVERY_EXTRA_ARGS arg1 arg2...] ) The options are: @@ -200,6 +201,26 @@ API Reference `Pytest Command-Line Flags `_ + * ``DISCOVERY_EXTRA_ARGS`` + + List of extra arguments to pass on the :term:`Pytest` command line for + the test discovery:: + + pytest_discover_tests( + ... + DISCOVERY_EXTRA_ARGS "--capture=no" + ) + + .. seealso:: + + `Pytest Command-Line Flags + `_ + + .. note:: + Due to possibility of changing the :term:`Pytest` test discovery + output format with certain command line options, ensure no such + combination are utilized or handled explicitly. + .. note:: This function works similarly to the `gtest_discover_tests diff --git a/test/06-extra-args/CMakeLists.txt b/test/06-extra-args/CMakeLists.txt index e1bac5e..7702ef2 100644 --- a/test/06-extra-args/CMakeLists.txt +++ b/test/06-extra-args/CMakeLists.txt @@ -7,7 +7,50 @@ find_package(Pytest REQUIRED) enable_testing() pytest_discover_tests( - TestExtraArgs + TestExtraArgs.extraArgs EXTRA_ARGS "--capture=no" "--cmdopt=demo" DEPENDS test_extra_args.py ) + +set(EXPECTED + "TestExtraArgs.extraArgs.test_requires_no_capture" + "TestExtraArgs.extraArgs.test_requires_args" +) + +add_test( + NAME TestExtraArgs.Validate + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestExtraArgs.extraArgs" + -D "EXPECTED=${EXPECTED}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/../01-modify-name/compare_discovered_tests.cmake +) + +# Negative Test which suppresses test discovery +pytest_discover_tests( + TestDiscoveryArgsSupressDiscovery + DISCOVERY_EXTRA_ARGS "-v" + DEPENDS test_extra_args.py +) + +# Test which re-enables test discovery due to usage of "-v" command line flag +# during test discovery +pytest_discover_tests( + TestDiscoveryArgsDiscovery + EXTRA_ARGS "--capture=no" "--cmdopt=discover" + DISCOVERY_EXTRA_ARGS "-v" "-q" + INCLUDE_FILE_PATH + DEPENDS test_extra_args.py +) + +set(EXPECTED + "TestDiscoveryArgsDiscovery.test_extra_args.test_requires_no_capture" + "TestDiscoveryArgsDiscovery.test_extra_args.test_requires_args" +) + +add_test( + NAME TestDiscoveryArgsDiscovery.Validate + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestDiscoveryArgsDiscovery.test_extra_args" + -D "EXPECTED=${EXPECTED}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/../01-modify-name/compare_discovered_tests.cmake +) From c79a92cf55d5a18052c5c3cdb4223964bef41c8d Mon Sep 17 00:00:00 2001 From: Ameya Vikram Singh Date: Mon, 10 Feb 2025 10:15:08 +0530 Subject: [PATCH 2/4] Add test to validate test discovery suppression Test is expected to fail --- test/06-extra-args/CMakeLists.txt | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/06-extra-args/CMakeLists.txt b/test/06-extra-args/CMakeLists.txt index 7702ef2..736b4b3 100644 --- a/test/06-extra-args/CMakeLists.txt +++ b/test/06-extra-args/CMakeLists.txt @@ -27,11 +27,27 @@ add_test( # Negative Test which suppresses test discovery pytest_discover_tests( - TestDiscoveryArgsSupressDiscovery + TestDiscoveryArgsSupressDiscovery.discoverExtraArgs DISCOVERY_EXTRA_ARGS "-v" DEPENDS test_extra_args.py ) +set(EXPECTED + "" +) + +add_test( + NAME TestDiscoveryArgsSupressDiscovery.Validate + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestDiscoveryArgsSupressDiscovery.discoverExtraArgs" + -D "EXPECTED=${EXPECTED}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/../01-modify-name/compare_discovered_tests.cmake +) + +set_tests_properties( + TestDiscoveryArgsSupressDiscovery.Validate PROPERTIES WILL_FAIL TRUE +) + # Test which re-enables test discovery due to usage of "-v" command line flag # during test discovery pytest_discover_tests( From dbaa5138b1a64e9f33db3634b9dff9727809c036 Mon Sep 17 00:00:00 2001 From: Ameya Vikram Singh Date: Mon, 10 Feb 2025 10:23:25 +0530 Subject: [PATCH 3/4] Update DISCOVER_EXTRA_ARGS --- test/06-extra-args/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/06-extra-args/CMakeLists.txt b/test/06-extra-args/CMakeLists.txt index 736b4b3..169608b 100644 --- a/test/06-extra-args/CMakeLists.txt +++ b/test/06-extra-args/CMakeLists.txt @@ -53,7 +53,7 @@ set_tests_properties( pytest_discover_tests( TestDiscoveryArgsDiscovery EXTRA_ARGS "--capture=no" "--cmdopt=discover" - DISCOVERY_EXTRA_ARGS "-v" "-q" + DISCOVERY_EXTRA_ARGS "--capture=no" "-v" "-q" INCLUDE_FILE_PATH DEPENDS test_extra_args.py ) From cdb1de1828cdb7e0d4fd5f3df43024128a1dd741 Mon Sep 17 00:00:00 2001 From: Jeremy Retailleau Date: Sun, 16 Feb 2025 15:48:38 -0800 Subject: [PATCH 4/4] Update logic, test and documentation - Ensure that discovery arguments are passed when using BUNDLE_TESTS option; - Update unit tests; - Update documentation. --- cmake/PytestAddTests.cmake | 4 ++ doc/api_reference.rst | 10 +-- doc/release/release_notes.rst | 9 +++ test/06-extra-args/CMakeLists.txt | 65 ++++--------------- test/06-extra-args/conftest.py | 6 -- .../custom_args/test_custom_args.py | 7 ++ .../{test_extra_args.py => test_args.py} | 9 --- 7 files changed, 37 insertions(+), 73 deletions(-) create mode 100644 test/06-extra-args/custom_args/test_custom_args.py rename test/06-extra-args/{test_extra_args.py => test_args.py} (50%) diff --git a/cmake/PytestAddTests.cmake b/cmake/PytestAddTests.cmake index 0935df2..095b3d0 100644 --- a/cmake/PytestAddTests.cmake +++ b/cmake/PytestAddTests.cmake @@ -27,6 +27,10 @@ if(CMAKE_SCRIPT_MODE_FILE) endforeach() # Handle EXTRA_ARGS for individual tests + if(BUNDLE_TESTS) + list(PREPEND EXTRA_ARGS ${DISCOVERY_EXTRA_ARGS}) + endif() + set(EXTRA_ARGS_WRAPPED) foreach(arg IN LISTS EXTRA_ARGS) list(APPEND EXTRA_ARGS_WRAPPED "[==[${arg}]==]") diff --git a/doc/api_reference.rst b/doc/api_reference.rst index 15c9d23..712e8ea 100644 --- a/doc/api_reference.rst +++ b/doc/api_reference.rst @@ -208,7 +208,7 @@ API Reference pytest_discover_tests( ... - DISCOVERY_EXTRA_ARGS "--capture=no" + DISCOVERY_EXTRA_ARGS "-k=expression" ) .. seealso:: @@ -216,10 +216,10 @@ API Reference `Pytest Command-Line Flags `_ - .. note:: - Due to possibility of changing the :term:`Pytest` test discovery - output format with certain command line options, ensure no such - combination are utilized or handled explicitly. + .. warning:: + + This option may affect test discovery. Some arguments can change + output format and break the test collection logic. Use with caution! .. note:: diff --git a/doc/release/release_notes.rst b/doc/release/release_notes.rst index 568f025..f453aed 100644 --- a/doc/release/release_notes.rst +++ b/doc/release/release_notes.rst @@ -4,6 +4,15 @@ Release Notes ************* +.. release:: Upcoming + + .. change:: changed + + Added ``DISCOVERY_EXTRA_ARGS`` option to the + :func:`pytest_discover_tests` function, allowing custom arguments to be + passed to the :term:`Pytest` command during test collection. + Thanks :github_user:`AmeyaVS`! + .. release:: 0.12.0 :date: 2025-02-09 diff --git a/test/06-extra-args/CMakeLists.txt b/test/06-extra-args/CMakeLists.txt index 169608b..7c9b662 100644 --- a/test/06-extra-args/CMakeLists.txt +++ b/test/06-extra-args/CMakeLists.txt @@ -7,66 +7,25 @@ find_package(Pytest REQUIRED) enable_testing() pytest_discover_tests( - TestExtraArgs.extraArgs + TestExtraArgs EXTRA_ARGS "--capture=no" "--cmdopt=demo" - DEPENDS test_extra_args.py ) -set(EXPECTED - "TestExtraArgs.extraArgs.test_requires_no_capture" - "TestExtraArgs.extraArgs.test_requires_args" -) - -add_test( - NAME TestExtraArgs.Validate - COMMAND ${CMAKE_COMMAND} - -D "TEST_PREFIX=TestExtraArgs.extraArgs" - -D "EXPECTED=${EXPECTED}" - -P ${CMAKE_CURRENT_SOURCE_DIR}/../01-modify-name/compare_discovered_tests.cmake -) - -# Negative Test which suppresses test discovery pytest_discover_tests( - TestDiscoveryArgsSupressDiscovery.discoverExtraArgs - DISCOVERY_EXTRA_ARGS "-v" - DEPENDS test_extra_args.py + TestDiscoveryExtraArgs.IgnoreCustomArgs + EXTRA_ARGS "--capture=no" + DISCOVERY_EXTRA_ARGS "--ignore=custom_args" ) -set(EXPECTED - "" -) - -add_test( - NAME TestDiscoveryArgsSupressDiscovery.Validate - COMMAND ${CMAKE_COMMAND} - -D "TEST_PREFIX=TestDiscoveryArgsSupressDiscovery.discoverExtraArgs" - -D "EXPECTED=${EXPECTED}" - -P ${CMAKE_CURRENT_SOURCE_DIR}/../01-modify-name/compare_discovered_tests.cmake -) - -set_tests_properties( - TestDiscoveryArgsSupressDiscovery.Validate PROPERTIES WILL_FAIL TRUE -) - -# Test which re-enables test discovery due to usage of "-v" command line flag -# during test discovery pytest_discover_tests( - TestDiscoveryArgsDiscovery - EXTRA_ARGS "--capture=no" "--cmdopt=discover" - DISCOVERY_EXTRA_ARGS "--capture=no" "-v" "-q" - INCLUDE_FILE_PATH - DEPENDS test_extra_args.py + TestDiscoveryExtraArgs.OnlyCustomArgs + EXTRA_ARGS "--cmdopt=demo" + DISCOVERY_EXTRA_ARGS "-k=custom" ) -set(EXPECTED - "TestDiscoveryArgsDiscovery.test_extra_args.test_requires_no_capture" - "TestDiscoveryArgsDiscovery.test_extra_args.test_requires_args" -) - -add_test( - NAME TestDiscoveryArgsDiscovery.Validate - COMMAND ${CMAKE_COMMAND} - -D "TEST_PREFIX=TestDiscoveryArgsDiscovery.test_extra_args" - -D "EXPECTED=${EXPECTED}" - -P ${CMAKE_CURRENT_SOURCE_DIR}/../01-modify-name/compare_discovered_tests.cmake +pytest_discover_tests( + TestDiscoveryExtraArgs.OnlyCustomArgsBundled + EXTRA_ARGS "--cmdopt=demo" + DISCOVERY_EXTRA_ARGS "-k=custom" + BUNDLE_TESTS ) diff --git a/test/06-extra-args/conftest.py b/test/06-extra-args/conftest.py index 2d52afa..4be0ed0 100644 --- a/test/06-extra-args/conftest.py +++ b/test/06-extra-args/conftest.py @@ -1,10 +1,4 @@ -import pytest - def pytest_addoption(parser): parser.addoption( "--cmdopt", action="store", help="custom options" ) - -@pytest.fixture -def cmdopt(request): - return request.config.getoption("--cmdopt") diff --git a/test/06-extra-args/custom_args/test_custom_args.py b/test/06-extra-args/custom_args/test_custom_args.py new file mode 100644 index 0000000..c52502b --- /dev/null +++ b/test/06-extra-args/custom_args/test_custom_args.py @@ -0,0 +1,7 @@ +import pytest + +def test_requires_cmdopt(request): + """Fails unless '--cmdopt=' is passed to pytest.""" + if request.config.getoption("--cmdopt") is None: + pytest.fail("Test requires '--cmdopt' to properly execute the test.") + assert True diff --git a/test/06-extra-args/test_extra_args.py b/test/06-extra-args/test_args.py similarity index 50% rename from test/06-extra-args/test_extra_args.py rename to test/06-extra-args/test_args.py index fe62a0c..52e1ec9 100644 --- a/test/06-extra-args/test_extra_args.py +++ b/test/06-extra-args/test_args.py @@ -5,12 +5,3 @@ def test_requires_no_capture(request): if "--capture=no" not in request.config.invocation_params.args: pytest.fail("Test requires '--capture=no' to properly display output.") assert True - - -def test_requires_args(cmdopt): - """Fails unless '--cmdopt=' is passed to pytest.""" - if cmdopt != None: - print(f"Option Value: cmdopt: {cmdopt}") - else: - pytest.fail("Test requires '--cmdopt=' to properly execute the test.") - pass