From 8f79fd755449663881c899f91fc9af81787f886d Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 14 Oct 2025 17:44:22 +0100 Subject: [PATCH 01/11] Odd developer documentation improvements. --- docs/details/developer_notes.rst | 15 ++++++++++++++- docs/userdocs/getting_started/installation.rst | 5 +++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/details/developer_notes.rst b/docs/details/developer_notes.rst index d0b052af..d791db7b 100644 --- a/docs/details/developer_notes.rst +++ b/docs/details/developer_notes.rst @@ -29,6 +29,19 @@ with a ``towncrier`` command-line command: * N.B. for this to work well, every change should be identified with a matching github issue. If there are multiple associated PRs, they should all be linked to the issue. +.. _developer_install: + +Developer Installation +---------------------- +For an editable installation, make a Python environment containing at least **numpy, +netCDF4, dask and pip**. It is also highly recommended to get +`towncrier `_ and +`pre-commit `_. +(and enable pre-commit with ``$ pre-commit install``). + +Then, cd to your checkout, and ``$ pip install -e .``. +This should result in an editable development installation. + Documentation build ------------------- @@ -83,7 +96,7 @@ Release actions #. create a new env with test dependencies - * ``$ conda create -n ncdtmp python=3.11 iris xarray filelock requests pytest pip`` + * ``$ conda create -n ncdtmp python=3.13 iris xarray filelock requests pytest pip`` * ( N.B. 'filelock' and 'requests' are *test dependencies* of iris ) #. install the new package with diff --git a/docs/userdocs/getting_started/installation.rst b/docs/userdocs/getting_started/installation.rst index 2c805c10..fb64ccfe 100644 --- a/docs/userdocs/getting_started/installation.rst +++ b/docs/userdocs/getting_started/installation.rst @@ -29,3 +29,8 @@ Check install > + +Developer Installation +---------------------- +To work on changes to the ncdata code, you will need an "editable installation". +See : :ref:`developer_install`. From 5f374973b3f99e9d228da83965a6fba5e515d477 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 15 Oct 2025 14:34:23 +0100 Subject: [PATCH 02/11] Typo in dev docs. --- docs/details/developer_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/details/developer_notes.rst b/docs/details/developer_notes.rst index d791db7b..965adf14 100644 --- a/docs/details/developer_notes.rst +++ b/docs/details/developer_notes.rst @@ -19,7 +19,7 @@ with a ``towncrier`` command-line command: * "feat": user features * "doc": documentation changes * "bug": bug fixes - * "def": general developer-relevant changes + * "dev": general developer-relevant changes * "misc": miscellaneous (For reference, these categories are configured in ``pyproject.toml``). From f5a04604e0b15d3f71c4c6bf990090d74304f5de Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 15 Oct 2025 16:18:58 +0100 Subject: [PATCH 03/11] Various additional change fragments. --- docs/Makefile | 3 +-- docs/changelog_fragments/173.dev.rst | 1 + docs/changelog_fragments/174.doc.rst | 1 + docs/changelog_fragments/175.dev.rst | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 docs/changelog_fragments/173.dev.rst create mode 100644 docs/changelog_fragments/174.doc.rst create mode 100644 docs/changelog_fragments/175.dev.rst diff --git a/docs/Makefile b/docs/Makefile index a5c0035c..14c187c1 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -19,8 +19,7 @@ allapi: sphinx-apidoc -Mfe -o ./details/api ../lib/ncdata towncrier: - if [ -e changelog_fragments/*.rst ]; then towncrier build --yes; fi - + if [[ "$$(ls changelog_fragments)" != "" ]]; then towncrier build --yes; fi # Tweaked "make html", which restores the changelog state after docs build. html-keeplog: html diff --git a/docs/changelog_fragments/173.dev.rst b/docs/changelog_fragments/173.dev.rst new file mode 100644 index 00000000..b795db7b --- /dev/null +++ b/docs/changelog_fragments/173.dev.rst @@ -0,0 +1 @@ +Fix xarray 2025.09.1 problem. diff --git a/docs/changelog_fragments/174.doc.rst b/docs/changelog_fragments/174.doc.rst new file mode 100644 index 00000000..8cfdde19 --- /dev/null +++ b/docs/changelog_fragments/174.doc.rst @@ -0,0 +1 @@ +Document how to create a developer installation. diff --git a/docs/changelog_fragments/175.dev.rst b/docs/changelog_fragments/175.dev.rst new file mode 100644 index 00000000..2fbd7332 --- /dev/null +++ b/docs/changelog_fragments/175.dev.rst @@ -0,0 +1 @@ +Pinned python for now, since 3.14 causes problems with Iris (notably). From 274a56637a4b865add37a678ce9d375341313a4d Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 16 Oct 2025 16:29:46 +0100 Subject: [PATCH 04/11] First attempt at homebrew doctest runner. --- .github/workflows/ci-tests.yml | 6 +- tools/check_doctest.py | 18 ++++ tools/run_doctests.py | 156 +++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 tools/check_doctest.py create mode 100755 tools/run_doctests.py diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 5e8baa09..46a34f89 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -83,11 +83,9 @@ jobs: - name: "Run doctests: Docs" if: matrix.session == 'doctests-docs' run: | - cd docs - pytest --doctest-glob="*.rst" --doctest-continue-on-failure + tools/run_doctests.py $(find ./docs -iname '*.rst') -va - name: "Run doctests: API" if: matrix.session == 'doctests-api' run: | - cd lib - pytest --doctest-modules --doctest-continue-on-failure + tools/run_doctests.py ncdata -va diff --git a/tools/check_doctest.py b/tools/check_doctest.py new file mode 100644 index 00000000..29030b82 --- /dev/null +++ b/tools/check_doctest.py @@ -0,0 +1,18 @@ +from doctest import ELLIPSIS as ELLIPSIS_FLAG +from run_doctests import run_doctest_paths, _parser, parserargs_as_kwargs + +# tstargs = ['ncdata', '-da', '--options', 'verbose=1'] + +tstargs = ['/home/users/patrick.peglar/git/ncdata/docs/userdocs/user_guide/howtos.rst', '--options', 'verbose=1'] + + +args = _parser.parse_args(tstargs) +kwargs = parserargs_as_kwargs(args) +# if not "options" in kwargs: +# kwargs["options"] = "ELLIPSIS=1" +run_doctest_paths(**kwargs) + +# +# Currently good: +# $ tools/run_doctests.py docs/userdocs/getting_started/introduction.rst -o "optionflags=8" +# $ tools/run_doctests.py docs/userdocs/getting_started/*.rst -vo "optionflags=8" diff --git a/tools/run_doctests.py b/tools/run_doctests.py new file mode 100755 index 00000000..645c39b5 --- /dev/null +++ b/tools/run_doctests.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +import argparse +import doctest +import importlib +from pathlib import Path +import pkgutil +import sys + +def list_modules_recursive(module_importname: str, include_private: bool = False): + module_names = [module_importname] + # Identify module from its import path (no import -> fail back to caller) + module = importlib.import_module(module_importname) + # Get the filepath of the module base directory + module_filepath = str(Path(module.__file__).parent) + for _, name, ispkg in pkgutil.iter_modules([module_filepath]): + if not name.startswith("_") or include_private: + submodule_name = module_importname + "." + name + module_names.append(submodule_name) + if ispkg: + module_names.extend(list_modules_recursive( + submodule_name, include_private=include_private + )) + # I don't know why there are duplicates, but there can be. + result = [] + for name in module_names: + if name not in result: + # For some reason, some things get listed twice. + result.append(name) + + return result + + +def process_options(opt_str:str) -> dict[str, str]: + # First collapse spaces around equals signs." + while " =" in opt_str: + opt_str = opt_str.replace(" =", "=") + while "= " in opt_str: + opt_str = opt_str.replace("= ", "=") + # Collapse (remaining) duplicate spaces + while " " in opt_str: + opt_str = opt_str.replace(" ", " ") + # Split on spaces, and split each one on "=" expecting a simple name=val form + opts_dict = {} + if opt_str: # N.B. to avoid unexpected behaviour from "".split() + for setting_str in opt_str.split(" "): + try: + name, val = setting_str.split("=") + + # Translate certain things (but do not exec!!) + bool_vals = {"true": True, "false": False} + if val.isdigit(): + val = int(val) + elif val.lower() in bool_vals: + val = bool_vals[val.lower()] + + except ValueError: + msg = f"Invalid option setting {setting_str!r}, expected 'name=value' only." + raise ValueError(msg) + + opts_dict[name] = val + + return opts_dict + + +def run_doctest_paths( + paths: list[str], + opts_str:str, + verbose:bool = False, + dry_run:bool=False, + do_all:bool=False +): + if verbose: + print( + "RUNNING run_doctest(" + f"paths={paths!r}" + f", opts_str={opts_str!r}" + f", verbose={verbose!r}" + f", dry_run={dry_run!r}" + f", do_all={do_all!r}" + ")" + ) + if dry_run: + verbose = True + opts_kwargs = process_options(opts_str) + finished = False + try: + module_paths = [] + for path in paths: + module_paths += list_modules_recursive(path, include_private=do_all) + for path in module_paths: + if verbose: + print(f"\ndoctest.testmod: {path!r}") + if not dry_run: + module = importlib.import_module(path) + doctest.testmod(module, **opts_kwargs) + finished = True + except (ImportError, ModuleNotFoundError, TypeError): + # TODO: this list is "awkward" ! + pass + + if not finished: + # Module search failed : treat paths as (documentation) filepaths instead + # Fix options : TODO this is not very clever, think of something better?? + if not "module_relative" in opts_kwargs: + opts_kwargs["module_relative"] = False + if not "optionflags" in opts_kwargs: + default_flags = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE + opts_kwargs["optionflags"] = default_flags + for path in paths: + if verbose: + print(f"\ndoctest.testfile: {path!r}") + if not dry_run: + doctest.testfile(path, **opts_kwargs) + + +_parser = argparse.ArgumentParser( + prog="run_doctests", + description="Runs doctests in docs files, or docstrings in packages." +) +_parser.add_argument( + "-o", "--options", nargs="?", + help="doctest options settings (as a string).", + type=str, default="" +) +_parser.add_argument( + "-v", "--verbose", action="store_true", + help="Show actions." +) +_parser.add_argument( + "-d", "--dryrun", action="store_true", + help="Only print the names of modules/files which *would* be tested." +) +_parser.add_argument( + "-a", "--all", action="store_true", + help="If set, include private files/modules " +) +_parser.add_argument( + "paths", nargs="*", + help="docs filepaths, or module paths (not both).", + type=str, default=[] +) + +def parserargs_as_kwargs(args): + return dict( + paths=args.paths, + opts_str=args.options, + verbose=args.verbose, + dry_run=args.dryrun, + do_all=args.all + ) + + +if __name__ == '__main__': + args = _parser.parse_args(sys.argv[1:]) + kwargs = parserargs_as_kwargs(args) + run_doctest_paths(**kwargs) From fd9a1cae362b5e7eeaa7af3d81100e89207c1610 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 20 Oct 2025 10:00:32 +0100 Subject: [PATCH 05/11] Random errors. --- .../userdocs/getting_started/introduction.rst | 2 +- lib/ncdata/utils/_dim_indexing.py | 3 +- tools/check_doctest.py | 6 +- tools/run_doctests.py | 62 ++++++++++++------- 4 files changed, 47 insertions(+), 26 deletions(-) diff --git a/docs/userdocs/getting_started/introduction.rst b/docs/userdocs/getting_started/introduction.rst index 0a4a51d8..6f9306b9 100644 --- a/docs/userdocs/getting_started/introduction.rst +++ b/docs/userdocs/getting_started/introduction.rst @@ -40,7 +40,7 @@ and :attr:`~ncdata.NcData.attributes`: >>> data >>> print(data) - >>> dim = NcDimension("x", 3) diff --git a/lib/ncdata/utils/_dim_indexing.py b/lib/ncdata/utils/_dim_indexing.py index 42b94aec..d3721093 100644 --- a/lib/ncdata/utils/_dim_indexing.py +++ b/lib/ncdata/utils/_dim_indexing.py @@ -3,6 +3,7 @@ import dask.array as da import numpy as np + from ncdata import NcData @@ -34,7 +35,7 @@ def index_by_dimensions( >>> data = NcData(dimensions=[NcDimension(nn, 10) for nn in ("time", "levels")]) >>> data1 = index_by_dimensions(data, time=slice(0, 10)) # equivalent to [:10] - >>> data2 = index_by_dimensions(data, levels=[1,2,5]) + >>> data2 = index_by_dimension(data, levels=[1,2,5]) >>> data3 = index_by_dimensions(data, time=3, levels=slice(2, 10, 3)) Notes diff --git a/tools/check_doctest.py b/tools/check_doctest.py index 29030b82..98e8f74b 100644 --- a/tools/check_doctest.py +++ b/tools/check_doctest.py @@ -3,7 +3,11 @@ # tstargs = ['ncdata', '-da', '--options', 'verbose=1'] -tstargs = ['/home/users/patrick.peglar/git/ncdata/docs/userdocs/user_guide/howtos.rst', '--options', 'verbose=1'] +tstargs = [ + "/home/users/patrick.peglar/git/ncdata/docs/userdocs/user_guide/howtos.rst", + "--options", + "verbose=1", +] args = _parser.parse_args(tstargs) diff --git a/tools/run_doctests.py b/tools/run_doctests.py index 645c39b5..ee645375 100755 --- a/tools/run_doctests.py +++ b/tools/run_doctests.py @@ -6,7 +6,10 @@ import pkgutil import sys -def list_modules_recursive(module_importname: str, include_private: bool = False): + +def list_modules_recursive( + module_importname: str, include_private: bool = False +): module_names = [module_importname] # Identify module from its import path (no import -> fail back to caller) module = importlib.import_module(module_importname) @@ -17,9 +20,11 @@ def list_modules_recursive(module_importname: str, include_private: bool = False submodule_name = module_importname + "." + name module_names.append(submodule_name) if ispkg: - module_names.extend(list_modules_recursive( - submodule_name, include_private=include_private - )) + module_names.extend( + list_modules_recursive( + submodule_name, include_private=include_private + ) + ) # I don't know why there are duplicates, but there can be. result = [] for name in module_names: @@ -30,7 +35,7 @@ def list_modules_recursive(module_importname: str, include_private: bool = False return result -def process_options(opt_str:str) -> dict[str, str]: +def process_options(opt_str: str) -> dict[str, str]: # First collapse spaces around equals signs." while " =" in opt_str: opt_str = opt_str.replace(" =", "=") @@ -64,10 +69,10 @@ def process_options(opt_str:str) -> dict[str, str]: def run_doctest_paths( paths: list[str], - opts_str:str, - verbose:bool = False, - dry_run:bool=False, - do_all:bool=False + opts_str: str, + verbose: bool = False, + dry_run: bool = False, + do_all: bool = False, ): if verbose: print( @@ -86,7 +91,9 @@ def run_doctest_paths( try: module_paths = [] for path in paths: - module_paths += list_modules_recursive(path, include_private=do_all) + module_paths += list_modules_recursive( + path, include_private=do_all + ) for path in module_paths: if verbose: print(f"\ndoctest.testmod: {path!r}") @@ -115,42 +122,51 @@ def run_doctest_paths( _parser = argparse.ArgumentParser( prog="run_doctests", - description="Runs doctests in docs files, or docstrings in packages." + description="Runs doctests in docs files, or docstrings in packages.", ) _parser.add_argument( - "-o", "--options", nargs="?", + "-o", + "--options", + nargs="?", help="doctest options settings (as a string).", - type=str, default="" + type=str, + default="", ) _parser.add_argument( - "-v", "--verbose", action="store_true", - help="Show actions." + "-v", "--verbose", action="store_true", help="Show actions." ) _parser.add_argument( - "-d", "--dryrun", action="store_true", - help="Only print the names of modules/files which *would* be tested." + "-d", + "--dryrun", + action="store_true", + help="Only print the names of modules/files which *would* be tested.", ) _parser.add_argument( - "-a", "--all", action="store_true", - help="If set, include private files/modules " + "-a", + "--all", + action="store_true", + help="If set, include private files/modules ", ) _parser.add_argument( - "paths", nargs="*", + "paths", + nargs="*", help="docs filepaths, or module paths (not both).", - type=str, default=[] + type=str, + default=[], ) + def parserargs_as_kwargs(args): return dict( paths=args.paths, opts_str=args.options, verbose=args.verbose, dry_run=args.dryrun, - do_all=args.all + do_all=args.all, ) -if __name__ == '__main__': +if __name__ == "__main__": args = _parser.parse_args(sys.argv[1:]) kwargs = parserargs_as_kwargs(args) run_doctest_paths(**kwargs) From 44a161fa774bfcdfc6d16d2ccf75c3e5ee47f42d Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 20 Oct 2025 12:26:52 +0100 Subject: [PATCH 06/11] Various improvements + fixes. --- .github/workflows/ci-tests.yml | 4 +- tools/check_doctest.py | 11 +- tools/run_doctests.py | 181 +++++++++++++++++++++------------ 3 files changed, 124 insertions(+), 72 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 46a34f89..99b4a4f3 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -83,9 +83,9 @@ jobs: - name: "Run doctests: Docs" if: matrix.session == 'doctests-docs' run: | - tools/run_doctests.py $(find ./docs -iname '*.rst') -va + tools/run_doctests.py -v $(find ./docs -iname '*.rst') - name: "Run doctests: API" if: matrix.session == 'doctests-api' run: | - tools/run_doctests.py ncdata -va + tools/run_doctests.py -v -mr ncdata diff --git a/tools/check_doctest.py b/tools/check_doctest.py index 98e8f74b..1a1a847f 100644 --- a/tools/check_doctest.py +++ b/tools/check_doctest.py @@ -3,12 +3,13 @@ # tstargs = ['ncdata', '-da', '--options', 'verbose=1'] -tstargs = [ - "/home/users/patrick.peglar/git/ncdata/docs/userdocs/user_guide/howtos.rst", - "--options", - "verbose=1", -] +# tstargs = [ +# "/home/users/patrick.peglar/git/ncdata/docs/userdocs/user_guide/howtos.rst", +# "--options", +# "verbose=1", +# ] +tstargs = ["-mvr", "ncdata.iris"] args = _parser.parse_args(tstargs) kwargs = parserargs_as_kwargs(args) diff --git a/tools/run_doctests.py b/tools/run_doctests.py index ee645375..ef0278a2 100755 --- a/tools/run_doctests.py +++ b/tools/run_doctests.py @@ -8,23 +8,25 @@ def list_modules_recursive( - module_importname: str, include_private: bool = False + module_importname: str, include_private: bool = True ): module_names = [module_importname] # Identify module from its import path (no import -> fail back to caller) module = importlib.import_module(module_importname) # Get the filepath of the module base directory - module_filepath = str(Path(module.__file__).parent) - for _, name, ispkg in pkgutil.iter_modules([module_filepath]): - if not name.startswith("_") or include_private: - submodule_name = module_importname + "." + name - module_names.append(submodule_name) - if ispkg: - module_names.extend( - list_modules_recursive( - submodule_name, include_private=include_private + module_filepath = Path(module.__file__) + if module_filepath.name == "__init__.py": + search_filepath = str(module_filepath.parent) + for _, name, ispkg in pkgutil.iter_modules([search_filepath]): + if not name.startswith("_") or include_private: + submodule_name = module_importname + "." + name + module_names.append(submodule_name) + if ispkg: + module_names.extend( + list_modules_recursive( + submodule_name, include_private=include_private + ) ) - ) # I don't know why there are duplicates, but there can be. result = [] for name in module_names: @@ -36,22 +38,17 @@ def list_modules_recursive( def process_options(opt_str: str) -> dict[str, str]: - # First collapse spaces around equals signs." - while " =" in opt_str: - opt_str = opt_str.replace(" =", "=") - while "= " in opt_str: - opt_str = opt_str.replace("= ", "=") - # Collapse (remaining) duplicate spaces - while " " in opt_str: - opt_str = opt_str.replace(" ", " ") - # Split on spaces, and split each one on "=" expecting a simple name=val form + """Convert the "-o/--options" arg into a **kwargs for the doctest function call.""" + # Remove all spaces (think they are never needed). + opt_str = opt_str.replace(" ", "") + # Split on commas, and split each one on "=" expecting a simple name=val form opts_dict = {} - if opt_str: # N.B. to avoid unexpected behaviour from "".split() - for setting_str in opt_str.split(" "): + if opt_str: # N.B. to avoid unexpected behaviour: "".split() --> [""] + for setting_str in opt_str.split(","): try: name, val = setting_str.split("=") - # Translate certain things (but do not exec!!) + # Detect + translate numberic and boolean values. bool_vals = {"true": True, "false": False} if val.isdigit(): val = int(val) @@ -69,71 +66,121 @@ def process_options(opt_str: str) -> dict[str, str]: def run_doctest_paths( paths: list[str], - opts_str: str, + paths_are_modules:bool = False, + recurse_modules: bool = False, + include_private_modules: bool = False, + option_kwargs: dict = {}, verbose: bool = False, dry_run: bool = False, - do_all: bool = False, + stop_on_failure: bool = False, ): + n_total_fails, n_total_tests, n_paths_tested = 0, 0, 0 if verbose: print( "RUNNING run_doctest(" f"paths={paths!r}" - f", opts_str={opts_str!r}" + f", paths_are_modules={paths_are_modules!r}" + f", option_kwargs={option_kwargs!r}" f", verbose={verbose!r}" f", dry_run={dry_run!r}" - f", do_all={do_all!r}" + f", stop_on_failure={stop_on_failure!r}" + f", include_private={include_private_modules!r}" ")" ) if dry_run: verbose = True - opts_kwargs = process_options(opts_str) - finished = False - try: - module_paths = [] - for path in paths: - module_paths += list_modules_recursive( - path, include_private=do_all - ) - for path in module_paths: - if verbose: - print(f"\ndoctest.testmod: {path!r}") - if not dry_run: - module = importlib.import_module(path) - doctest.testmod(module, **opts_kwargs) - finished = True - except (ImportError, ModuleNotFoundError, TypeError): - # TODO: this list is "awkward" ! - pass - - if not finished: - # Module search failed : treat paths as (documentation) filepaths instead + + if paths_are_modules: + doctest_function = doctest.testmod + if recurse_modules: + module_paths = [] + for path in paths: + module_paths += list_modules_recursive( + path, include_private=include_private_modules + ) + paths = module_paths + + else: # paths are filepaths + doctest_function = doctest.testfile # Fix options : TODO this is not very clever, think of something better?? - if not "module_relative" in opts_kwargs: - opts_kwargs["module_relative"] = False - if not "optionflags" in opts_kwargs: + if not "module_relative" in option_kwargs: + option_kwargs["module_relative"] = False + if not "optionflags" in option_kwargs: default_flags = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE - opts_kwargs["optionflags"] = default_flags - for path in paths: - if verbose: - print(f"\ndoctest.testfile: {path!r}") - if not dry_run: - doctest.testfile(path, **opts_kwargs) + option_kwargs["optionflags"] = default_flags + + for path in paths: + if verbose: + print(f"\n-----\ndoctest.{doctest_function.__name__}: {path!r}") + if not dry_run: + if paths_are_modules: + arg = importlib.import_module(path) + else: + arg = path + n_fails, n_tests = doctest_function(arg, **option_kwargs) + n_total_fails += n_fails + n_total_tests += n_tests + n_paths_tested += 1 + if n_total_fails > 0 and stop_on_failure: + break + + if verbose or n_total_fails > 0: + # Print a final report + msgs = ["", "=====", "run_doctest: FINAL REPORT"] + if dry_run: + msgs += ["(DRY RUN: no actual tests)"] + elif stop_on_failure and n_total_fails > 0: + msgs += ["(FAIL FAST: stopped at first target with errors)"] + + msgs += [ + f" paths tested = {n_paths_tested}", + f" tests completed = {n_total_tests}", + f" errors = {n_total_fails}", + "" + ] + if n_total_fails > 0: + msgs += ["FAILED."] + else: + msgs += ["OK."] + + print('\n'.join(msgs)) + + return n_total_fails _parser = argparse.ArgumentParser( prog="run_doctests", description="Runs doctests in docs files, or docstrings in packages.", ) +_parser.add_argument( + "-m", "--module", action="store_true", + help="Paths are module paths (xx.yy.zz), instead of filepaths." +) +_parser.add_argument( + "-r", + "--recursive", + action="store_true", + help="If set, include submodules (only applies with -m).", +) +_parser.add_argument( + "-p", + "--publiconly", + action="store_true", + help="If set, exclude private modules (only applies with -m and -r)", +) _parser.add_argument( "-o", "--options", nargs="?", - help="doctest options settings (as a string).", + help=( + "doctest function kwargs (string)" + ", e.g. \"report=False, raise_on_error=True, optionflags=8\"." + ), type=str, default="", ) _parser.add_argument( - "-v", "--verbose", action="store_true", help="Show actions." + "-v", "--verbose", action="store_true", help="Show details of each action." ) _parser.add_argument( "-d", @@ -142,10 +189,10 @@ def run_doctest_paths( help="Only print the names of modules/files which *would* be tested.", ) _parser.add_argument( - "-a", - "--all", + "-f", + "--stop-on-fail", action="store_true", - help="If set, include private files/modules ", + help="If set, stop at the first path with an error (else continue to test all).", ) _parser.add_argument( "paths", @@ -159,14 +206,18 @@ def run_doctest_paths( def parserargs_as_kwargs(args): return dict( paths=args.paths, - opts_str=args.options, + paths_are_modules=args.module, + recurse_modules=args.recursive, + include_private_modules=not args.publiconly, + option_kwargs=process_options(args.options), verbose=args.verbose, dry_run=args.dryrun, - do_all=args.all, + stop_on_failure=args.stop_on_fail, ) if __name__ == "__main__": args = _parser.parse_args(sys.argv[1:]) kwargs = parserargs_as_kwargs(args) - run_doctest_paths(**kwargs) + n_errs = run_doctest_paths(**kwargs) + exit(1 if n_errs > 0 else 0) From d1b7213d17c9d96f2326149079a1fa6a8a3f8d03 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 21 Oct 2025 10:32:00 +0100 Subject: [PATCH 07/11] Added excludes; in-doctest exception handling; mask warnings (for now?). --- tools/check_doctest.py | 6 +++- tools/run_doctests.py | 79 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 69 insertions(+), 16 deletions(-) diff --git a/tools/check_doctest.py b/tools/check_doctest.py index 1a1a847f..dade8e88 100644 --- a/tools/check_doctest.py +++ b/tools/check_doctest.py @@ -9,7 +9,11 @@ # "verbose=1", # ] -tstargs = ["-mvr", "ncdata.iris"] +# tstargs = ["-mvr", "iris.coords", "-o", "verbose=True"] + +# tstargs = ["-mvr", "iris.tests.unit.fileformats.netcdf", "-e", "attribute_handlers"] + +tstargs = ["-mvr", "iris._combine", "-o", "raise_on_error=True"] args = _parser.parse_args(tstargs) kwargs = parserargs_as_kwargs(args) diff --git a/tools/run_doctests.py b/tools/run_doctests.py index ef0278a2..e048fcec 100755 --- a/tools/run_doctests.py +++ b/tools/run_doctests.py @@ -2,24 +2,40 @@ import argparse import doctest import importlib +import traceback from pathlib import Path import pkgutil import sys +import warnings def list_modules_recursive( - module_importname: str, include_private: bool = True + module_importname: str, include_private: bool = True, + exclude_matches: list[str] = [] ): module_names = [module_importname] # Identify module from its import path (no import -> fail back to caller) - module = importlib.import_module(module_importname) - # Get the filepath of the module base directory - module_filepath = Path(module.__file__) - if module_filepath.name == "__init__.py": - search_filepath = str(module_filepath.parent) - for _, name, ispkg in pkgutil.iter_modules([search_filepath]): - if not name.startswith("_") or include_private: + try: + error = None + module = importlib.import_module(module_importname) + except Exception as exc: + print(f"\n\nIMPORT FAILED: {module_importname}\n") + error = exc + + if error is None: + # Add sub-modules to the list + # Get the filepath of the module base directory + module_filepath = Path(module.__file__) + if module_filepath.name == "__init__.py": + search_filepath = str(module_filepath.parent) + for _, name, ispkg in pkgutil.iter_modules([search_filepath]): + if name.startswith("_") and not include_private: + continue + submodule_name = module_importname + "." + name + if any(match in submodule_name for match in exclude_matches): + continue + module_names.append(submodule_name) if ispkg: module_names.extend( @@ -27,6 +43,7 @@ def list_modules_recursive( submodule_name, include_private=include_private ) ) + # I don't know why there are duplicates, but there can be. result = [] for name in module_names: @@ -69,6 +86,7 @@ def run_doctest_paths( paths_are_modules:bool = False, recurse_modules: bool = False, include_private_modules: bool = False, + exclude_matches: list[str] = [], option_kwargs: dict = {}, verbose: bool = False, dry_run: bool = False, @@ -80,23 +98,28 @@ def run_doctest_paths( "RUNNING run_doctest(" f"paths={paths!r}" f", paths_are_modules={paths_are_modules!r}" + f", recurse_modules={recurse_modules!r}" + f", include_private_modules={include_private_modules!r}" + f", exclude_matches={exclude_matches!r}" f", option_kwargs={option_kwargs!r}" f", verbose={verbose!r}" f", dry_run={dry_run!r}" f", stop_on_failure={stop_on_failure!r}" - f", include_private={include_private_modules!r}" ")" ) if dry_run: verbose = True + warnings.simplefilter("ignore") + if paths_are_modules: doctest_function = doctest.testmod if recurse_modules: module_paths = [] for path in paths: module_paths += list_modules_recursive( - path, include_private=include_private_modules + path, include_private=include_private_modules, + exclude_matches=exclude_matches ) paths = module_paths @@ -113,14 +136,33 @@ def run_doctest_paths( if verbose: print(f"\n-----\ndoctest.{doctest_function.__name__}: {path!r}") if not dry_run: + op_fail = None if paths_are_modules: - arg = importlib.import_module(path) + try: + arg = importlib.import_module(path) + except Exception as exc: + op_fail = exc else: arg = path - n_fails, n_tests = doctest_function(arg, **option_kwargs) - n_total_fails += n_fails - n_total_tests += n_tests - n_paths_tested += 1 + + if op_fail is None: + try: + n_fails, n_tests = doctest_function(arg, **option_kwargs) + n_total_fails += n_fails + n_total_tests += n_tests + n_paths_tested += 1 + except Exception as exc: + op_fail = exc + + if op_fail is not None: + n_total_fails += 1 + print(f"\n\nERROR occurred at {path!r}: {op_fail}\n") + if isinstance(op_fail, doctest.UnexpectedException): + # This is what happens with "-o raise_on_error=True", which is the + # Python call equivalent of "-o FAIL_FAST" in the doctest CLI. + print(f"Doctest caught exception: {op_fail}") + traceback.print_exception(*op_fail.exc_info) + if n_total_fails > 0 and stop_on_failure: break @@ -168,6 +210,12 @@ def run_doctest_paths( action="store_true", help="If set, exclude private modules (only applies with -m and -r)", ) +_parser.add_argument( + "-e", + "--exclude", + action="append", + help="Match fragments of paths to exclude.", +) _parser.add_argument( "-o", "--options", @@ -209,6 +257,7 @@ def parserargs_as_kwargs(args): paths_are_modules=args.module, recurse_modules=args.recursive, include_private_modules=not args.publiconly, + exclude_matches=args.exclude or [], option_kwargs=process_options(args.options), verbose=args.verbose, dry_run=args.dryrun, From 506dc26065ede9eae081eee68566b98d1c5d9885 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 21 Oct 2025 11:05:49 +0100 Subject: [PATCH 08/11] Flag paths with errors; stop doctests getting verbose from sys.argv. --- tools/run_doctests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/run_doctests.py b/tools/run_doctests.py index e048fcec..d98fb219 100755 --- a/tools/run_doctests.py +++ b/tools/run_doctests.py @@ -128,6 +128,8 @@ def run_doctest_paths( # Fix options : TODO this is not very clever, think of something better?? if not "module_relative" in option_kwargs: option_kwargs["module_relative"] = False + if not "verbose" in option_kwargs: + option_kwargs["verbose"] = False if not "optionflags" in option_kwargs: default_flags = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE option_kwargs["optionflags"] = default_flags @@ -151,6 +153,8 @@ def run_doctest_paths( n_total_fails += n_fails n_total_tests += n_tests n_paths_tested += 1 + if n_fails: + print(f"\nERRORS from doctests in path: {arg}\n") except Exception as exc: op_fail = exc From 6bef4254ce1c87dd15b24d6c81c0e56e9749bc47 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 21 Oct 2025 11:29:02 +0100 Subject: [PATCH 09/11] Apply options processing to both module+file modes. --- tools/run_doctests.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tools/run_doctests.py b/tools/run_doctests.py index d98fb219..313485ed 100755 --- a/tools/run_doctests.py +++ b/tools/run_doctests.py @@ -54,7 +54,7 @@ def list_modules_recursive( return result -def process_options(opt_str: str) -> dict[str, str]: +def process_options(opt_str: str, paths_are_modules: bool = True) -> dict[str, str]: """Convert the "-o/--options" arg into a **kwargs for the doctest function call.""" # Remove all spaces (think they are never needed). opt_str = opt_str.replace(" ", "") @@ -78,6 +78,17 @@ def process_options(opt_str: str) -> dict[str, str]: opts_dict[name] = val + # Post-process to "fix" options, especially to correct defaults + # TODO this is not very clever, think of something better?? + if not paths_are_modules: + if not "module_relative" in opts_dict: + opts_dict["module_relative"] = False + if not "verbose" in opts_dict: + opts_dict["verbose"] = False + if not "optionflags" in opts_dict: + default_flags = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE + opts_dict["optionflags"] = default_flags + return opts_dict @@ -125,14 +136,6 @@ def run_doctest_paths( else: # paths are filepaths doctest_function = doctest.testfile - # Fix options : TODO this is not very clever, think of something better?? - if not "module_relative" in option_kwargs: - option_kwargs["module_relative"] = False - if not "verbose" in option_kwargs: - option_kwargs["verbose"] = False - if not "optionflags" in option_kwargs: - default_flags = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE - option_kwargs["optionflags"] = default_flags for path in paths: if verbose: @@ -262,7 +265,7 @@ def parserargs_as_kwargs(args): recurse_modules=args.recursive, include_private_modules=not args.publiconly, exclude_matches=args.exclude or [], - option_kwargs=process_options(args.options), + option_kwargs=process_options(args.options, args.module), verbose=args.verbose, dry_run=args.dryrun, stop_on_failure=args.stop_on_fail, From 6e27328bae3082d97a3b7f29c9472e4638ba6fda Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 22 Oct 2025 13:37:04 +0100 Subject: [PATCH 10/11] Add filepath globbing; review docs. --- .github/workflows/ci-tests.yml | 2 +- tools/check_doctest.py | 2 + tools/run_doctests.py | 184 +++++++++++++++++++++++---------- 3 files changed, 132 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 99b4a4f3..046317d8 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -83,7 +83,7 @@ jobs: - name: "Run doctests: Docs" if: matrix.session == 'doctests-docs' run: | - tools/run_doctests.py -v $(find ./docs -iname '*.rst') + tools/run_doctests.py -v "./docs/**/*.rst" - name: "Run doctests: API" if: matrix.session == 'doctests-api' diff --git a/tools/check_doctest.py b/tools/check_doctest.py index dade8e88..6e845d26 100644 --- a/tools/check_doctest.py +++ b/tools/check_doctest.py @@ -15,6 +15,8 @@ tstargs = ["-mvr", "iris._combine", "-o", "raise_on_error=True"] +tstargs = ["-r", "../docs/userdocs/**/*.rst", "-e", "started.rst"] + args = _parser.parse_args(tstargs) kwargs = parserargs_as_kwargs(args) # if not "options" in kwargs: diff --git a/tools/run_doctests.py b/tools/run_doctests.py index 313485ed..a60a90b3 100755 --- a/tools/run_doctests.py +++ b/tools/run_doctests.py @@ -2,6 +2,7 @@ import argparse import doctest import importlib +import os import traceback from pathlib import Path import pkgutil @@ -13,6 +14,10 @@ def list_modules_recursive( module_importname: str, include_private: bool = True, exclude_matches: list[str] = [] ): + """Find all the submodules of a given module. + + Also filter with private and exclude controls. + """ module_names = [module_importname] # Identify module from its import path (no import -> fail back to caller) try: @@ -40,7 +45,8 @@ def list_modules_recursive( if ispkg: module_names.extend( list_modules_recursive( - submodule_name, include_private=include_private + submodule_name, include_private=include_private, + exclude_matches=exclude_matches, ) ) @@ -54,6 +60,38 @@ def list_modules_recursive( return result +def list_filepaths_recursive( + file_path: str, + exclude_matches: list[str] = [] +) -> list[Path]: + """Expand globs to a list of filepaths. + + Also filter with exclude controls. + """ + actual_paths: list[Path] = [] + segments = file_path.split("/") + i_wilds = [ + index for index, segment in enumerate(segments) + if any(char in segment for char in "*?[") + ] + if len(i_wilds) == 0: + actual_paths.append(Path(file_path)) + else: + i_first_wild = i_wilds[0] + base_path = Path("/".join(segments[:i_first_wild])) + file_spec = "/".join(segments[i_first_wild:]) + # This is the magic bit! expand with globs, '**' enabling recursive + actual_paths += list(base_path.glob(file_spec)) + + # Also apply exclude and private filters to results + result = [ + path for path in actual_paths + if not any(match in str(path) for match in exclude_matches) + and not path.name.startswith("_") + ] + return result + + def process_options(opt_str: str, paths_are_modules: bool = True) -> dict[str, str]: """Convert the "-o/--options" arg into a **kwargs for the doctest function call.""" # Remove all spaces (think they are never needed). @@ -65,7 +103,7 @@ def process_options(opt_str: str, paths_are_modules: bool = True) -> dict[str, s try: name, val = setting_str.split("=") - # Detect + translate numberic and boolean values. + # Detect + translate numeric and boolean values. bool_vals = {"true": True, "false": False} if val.isdigit(): val = int(val) @@ -98,12 +136,13 @@ def run_doctest_paths( recurse_modules: bool = False, include_private_modules: bool = False, exclude_matches: list[str] = [], - option_kwargs: dict = {}, + doctest_kwargs: dict = {}, verbose: bool = False, dry_run: bool = False, stop_on_failure: bool = False, ): n_total_fails, n_total_tests, n_paths_tested = 0, 0, 0 + if verbose: print( "RUNNING run_doctest(" @@ -112,15 +151,17 @@ def run_doctest_paths( f", recurse_modules={recurse_modules!r}" f", include_private_modules={include_private_modules!r}" f", exclude_matches={exclude_matches!r}" - f", option_kwargs={option_kwargs!r}" + f", doctest_kwargs={doctest_kwargs!r}" f", verbose={verbose!r}" f", dry_run={dry_run!r}" f", stop_on_failure={stop_on_failure!r}" ")" ) + if dry_run: verbose = True + # For now at least, simply discard ALL warnings. warnings.simplefilter("ignore") if paths_are_modules: @@ -133,45 +174,54 @@ def run_doctest_paths( exclude_matches=exclude_matches ) paths = module_paths - - else: # paths are filepaths + else: + # paths are filepaths doctest_function = doctest.testfile + filepaths = [] + for path in paths: + filepaths += list_filepaths_recursive( + path, + exclude_matches=exclude_matches + ) + paths = filepaths for path in paths: if verbose: print(f"\n-----\ndoctest.{doctest_function.__name__}: {path!r}") - if not dry_run: - op_fail = None - if paths_are_modules: - try: - arg = importlib.import_module(path) - except Exception as exc: - op_fail = exc - else: - arg = path - - if op_fail is None: - try: - n_fails, n_tests = doctest_function(arg, **option_kwargs) - n_total_fails += n_fails - n_total_tests += n_tests - n_paths_tested += 1 - if n_fails: - print(f"\nERRORS from doctests in path: {arg}\n") - except Exception as exc: - op_fail = exc - - if op_fail is not None: - n_total_fails += 1 - print(f"\n\nERROR occurred at {path!r}: {op_fail}\n") - if isinstance(op_fail, doctest.UnexpectedException): - # This is what happens with "-o raise_on_error=True", which is the - # Python call equivalent of "-o FAIL_FAST" in the doctest CLI. - print(f"Doctest caught exception: {op_fail}") - traceback.print_exception(*op_fail.exc_info) - - if n_total_fails > 0 and stop_on_failure: - break + if dry_run: + continue + + op_fail = None + if paths_are_modules: + try: + arg = importlib.import_module(path) + except Exception as exc: + op_fail = exc + else: + arg = path + + if op_fail is None: + try: + n_fails, n_tests = doctest_function(arg, **doctest_kwargs) + n_total_fails += n_fails + n_total_tests += n_tests + n_paths_tested += 1 + if n_fails: + print(f"\nERRORS in path: {arg}\n") + except Exception as exc: + op_fail = exc + + if op_fail is not None: + n_total_fails += 1 + print(f"\n\nERROR occurred at {path!r}: {op_fail}\n") + if isinstance(op_fail, doctest.UnexpectedException): + # E.G. this is what happens with "-o raise_on_error=True", which is + # the Python call equivalent of "-o FAIL_FAST" in the doctest CLI. + print(f"Doctest caught exception: {op_fail}") + traceback.print_exception(*op_fail.exc_info) + + if n_total_fails > 0 and stop_on_failure: + break if verbose or n_total_fails > 0: # Print a final report @@ -179,7 +229,7 @@ def run_doctest_paths( if dry_run: msgs += ["(DRY RUN: no actual tests)"] elif stop_on_failure and n_total_fails > 0: - msgs += ["(FAIL FAST: stopped at first target with errors)"] + msgs += ["(FAIL FAST: stopped at first path with errors)"] msgs += [ f" paths tested = {n_paths_tested}", @@ -197,57 +247,77 @@ def run_doctest_paths( return n_total_fails +_help_extra_lines = """\ +Notes: + * file paths support glob patterns '* ? [] **' (** to include subdirectories) + * N.B. use ** to include subdirectories + * N.B. usually requires quotes, to avoid shell expansion + * module paths do *not* support globs + * but --recurse includes all submodules + * \"--exclude\" patterns are a simple substring to match (not a glob/regexp) + +Examples: + $ run_doctests \"docs/**/*.rst\" # test all document sources + $ run_doctests \"docs/user*/**/*.rst\" -e detail # skip filepaths containing key string + $ run_doctests -mr mymod # test module + all submodules + $ run_doctests -mr mymod.util -e maths -e fun.err # skip module paths with substrings + $ run_doctests -mr mymod -o verbose=true # make doctest print each test +""" + + _parser = argparse.ArgumentParser( prog="run_doctests", - description="Runs doctests in docs files, or docstrings in packages.", + description="Run doctests in docs files, or docstrings in packages.", + epilog=_help_extra_lines, + formatter_class=argparse.RawDescriptionHelpFormatter, ) _parser.add_argument( "-m", "--module", action="store_true", - help="Paths are module paths (xx.yy.zz), instead of filepaths." + help="paths are module paths (xx.yy.zz), instead of filepaths." ) _parser.add_argument( "-r", - "--recursive", + "--recurse", action="store_true", - help="If set, include submodules (only applies with -m).", + help="include submodules (only applies with -m).", ) _parser.add_argument( "-p", "--publiconly", action="store_true", - help="If set, exclude private modules (only applies with -m and -r)", + help="exclude module names beginning '_' (only applies with -m and -r)", ) _parser.add_argument( "-e", "--exclude", action="append", - help="Match fragments of paths to exclude.", + help="exclude paths containing substring (may appear multiple times).", ) _parser.add_argument( "-o", "--options", nargs="?", help=( - "doctest function kwargs (string)" - ", e.g. \"report=False, raise_on_error=True, optionflags=8\"." + "kwargs (Python) for doctest call" + ", e.g. \"raise_on_error=True,optionflags=8\"." ), type=str, default="", ) _parser.add_argument( - "-v", "--verbose", action="store_true", help="Show details of each action." + "-v", "--verbose", action="store_true", help="show details of each operation." ) _parser.add_argument( "-d", "--dryrun", action="store_true", - help="Only print the names of modules/files which *would* be tested.", + help="only print names of modules/files which *would* be tested.", ) _parser.add_argument( "-f", "--stop-on-fail", action="store_true", - help="If set, stop at the first path with an error (else continue to test all).", + help="stop at the first path with an error (else continue to test all).", ) _parser.add_argument( "paths", @@ -262,10 +332,10 @@ def parserargs_as_kwargs(args): return dict( paths=args.paths, paths_are_modules=args.module, - recurse_modules=args.recursive, + recurse_modules=args.recurse, include_private_modules=not args.publiconly, exclude_matches=args.exclude or [], - option_kwargs=process_options(args.options, args.module), + doctest_kwargs=process_options(args.options, args.module), verbose=args.verbose, dry_run=args.dryrun, stop_on_failure=args.stop_on_fail, @@ -274,6 +344,10 @@ def parserargs_as_kwargs(args): if __name__ == "__main__": args = _parser.parse_args(sys.argv[1:]) - kwargs = parserargs_as_kwargs(args) - n_errs = run_doctest_paths(**kwargs) - exit(1 if n_errs > 0 else 0) + if not args.paths: + _parser.print_help() + else: + kwargs = parserargs_as_kwargs(args) + n_errs = run_doctest_paths(**kwargs) + if n_errs > 0: + exit(1) From 65edafa8e24650356d064469e45c82b5600e8ff9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Oct 2025 12:51:54 +0000 Subject: [PATCH 11/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- lib/ncdata/utils/_dim_indexing.py | 1 - tools/run_doctests.py | 48 +++++++++++++++++++------------ 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/lib/ncdata/utils/_dim_indexing.py b/lib/ncdata/utils/_dim_indexing.py index d3721093..ca4b16d3 100644 --- a/lib/ncdata/utils/_dim_indexing.py +++ b/lib/ncdata/utils/_dim_indexing.py @@ -3,7 +3,6 @@ import dask.array as da import numpy as np - from ncdata import NcData diff --git a/tools/run_doctests.py b/tools/run_doctests.py index a60a90b3..948a1ea5 100755 --- a/tools/run_doctests.py +++ b/tools/run_doctests.py @@ -11,8 +11,9 @@ def list_modules_recursive( - module_importname: str, include_private: bool = True, - exclude_matches: list[str] = [] + module_importname: str, + include_private: bool = True, + exclude_matches: list[str] = [], ): """Find all the submodules of a given module. @@ -45,7 +46,8 @@ def list_modules_recursive( if ispkg: module_names.extend( list_modules_recursive( - submodule_name, include_private=include_private, + submodule_name, + include_private=include_private, exclude_matches=exclude_matches, ) ) @@ -61,8 +63,7 @@ def list_modules_recursive( def list_filepaths_recursive( - file_path: str, - exclude_matches: list[str] = [] + file_path: str, exclude_matches: list[str] = [] ) -> list[Path]: """Expand globs to a list of filepaths. @@ -71,7 +72,8 @@ def list_filepaths_recursive( actual_paths: list[Path] = [] segments = file_path.split("/") i_wilds = [ - index for index, segment in enumerate(segments) + index + for index, segment in enumerate(segments) if any(char in segment for char in "*?[") ] if len(i_wilds) == 0: @@ -85,14 +87,17 @@ def list_filepaths_recursive( # Also apply exclude and private filters to results result = [ - path for path in actual_paths + path + for path in actual_paths if not any(match in str(path) for match in exclude_matches) and not path.name.startswith("_") ] return result -def process_options(opt_str: str, paths_are_modules: bool = True) -> dict[str, str]: +def process_options( + opt_str: str, paths_are_modules: bool = True +) -> dict[str, str]: """Convert the "-o/--options" arg into a **kwargs for the doctest function call.""" # Remove all spaces (think they are never needed). opt_str = opt_str.replace(" ", "") @@ -132,7 +137,7 @@ def process_options(opt_str: str, paths_are_modules: bool = True) -> dict[str, s def run_doctest_paths( paths: list[str], - paths_are_modules:bool = False, + paths_are_modules: bool = False, recurse_modules: bool = False, include_private_modules: bool = False, exclude_matches: list[str] = [], @@ -170,8 +175,9 @@ def run_doctest_paths( module_paths = [] for path in paths: module_paths += list_modules_recursive( - path, include_private=include_private_modules, - exclude_matches=exclude_matches + path, + include_private=include_private_modules, + exclude_matches=exclude_matches, ) paths = module_paths else: @@ -180,8 +186,7 @@ def run_doctest_paths( filepaths = [] for path in paths: filepaths += list_filepaths_recursive( - path, - exclude_matches=exclude_matches + path, exclude_matches=exclude_matches ) paths = filepaths @@ -235,14 +240,14 @@ def run_doctest_paths( f" paths tested = {n_paths_tested}", f" tests completed = {n_total_tests}", f" errors = {n_total_fails}", - "" + "", ] if n_total_fails > 0: msgs += ["FAILED."] else: msgs += ["OK."] - print('\n'.join(msgs)) + print("\n".join(msgs)) return n_total_fails @@ -272,8 +277,10 @@ def run_doctest_paths( formatter_class=argparse.RawDescriptionHelpFormatter, ) _parser.add_argument( - "-m", "--module", action="store_true", - help="paths are module paths (xx.yy.zz), instead of filepaths." + "-m", + "--module", + action="store_true", + help="paths are module paths (xx.yy.zz), instead of filepaths.", ) _parser.add_argument( "-r", @@ -299,13 +306,16 @@ def run_doctest_paths( nargs="?", help=( "kwargs (Python) for doctest call" - ", e.g. \"raise_on_error=True,optionflags=8\"." + ', e.g. "raise_on_error=True,optionflags=8".' ), type=str, default="", ) _parser.add_argument( - "-v", "--verbose", action="store_true", help="show details of each operation." + "-v", + "--verbose", + action="store_true", + help="show details of each operation.", ) _parser.add_argument( "-d",