From 9ded13b6649bb6d2cbfc1fbbb958e6c2c3a275c5 Mon Sep 17 00:00:00 2001 From: Philip A Cook Date: Tue, 2 Dec 2025 10:23:48 -0500 Subject: [PATCH 01/10] CI: Update CI workflow Fixes: 1. Cache data based on the files antspynet/utilities/get_antsxnet_data.py and antspynet/utilities/get_pretrained_network.py. These are what actually determine what gets downloaded. Caching on the hash of download_all_data.py will result in cache hits even if data changes. 2. Only update cache once per job. Don't do it in the build matrix. Avoids race conditions and duplicating cache jobs. 3. Specify branches in run conditions, avoids running twice on PRs 4. Build from pyproject.toml, not modified requirements.txt. This allows testing on recent python. Maybe need a separate test for requirements.txt install, if it's still needed --- .github/workflows/test.yml | 116 ++++++++++++++++++++++++++++++------- 1 file changed, 96 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1655b8..d86989a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,22 +2,101 @@ name: ANTsPyNet Unit Tests on: push: + branches: [master] pull_request: + branches: [master] + workflow_dispatch: env: - USE_SOURCE_BUILD: false # Set to 'false' to use PyPI version of ANTsPy + USE_ANTSPY_SOURCE_BUILD: false jobs: + + ########################################### + # Job 1 — Build or Restore ANTsXNet Data Cache + ########################################### + prepare-data: + name: Prepare ANTsXNet Data + runs-on: ubuntu-22.04 + outputs: + cache-key: ${{ steps.compute-key.outputs.cache-key }} + cache-hit: ${{ steps.restore-cache.outputs.cache-hit }} + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Compute cache key + id: compute-key + run: | + KEY="antsxnet-${{ hashFiles( + 'antspynet/utilities/get_antsxnet_data.py', + 'antspynet/utilities/get_pretrained_network.py' + ) }}" + echo "cache-key=$KEY" >> "$GITHUB_OUTPUT" + + - name: Try restoring ANTsXNet data cache + id: restore-cache + uses: actions/cache/restore@v4 + with: + key: ${{ steps.compute-key.outputs.cache-key }} + path: ${{ runner.temp }}/ANTsXNet + + - name: Cache hit — skip download + if: steps.restore-cache.outputs.cache-hit == 'true' + run: echo "Cache HIT — will use cached ANTsXNet data." + + - name: Set up Python 3.11 + if: steps.restore-cache.outputs.cache-hit != 'true' + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install ANTsPy for data download + if: steps.restore-cache.outputs.cache-hit != 'true' + run: | + if [ "$USE_ANTSPY_SOURCE_BUILD" = "true" ]; then + echo "Installing ANTsPy from source..." + git clone https://github.com/ANTsX/ANTsPy.git + pip install scikit-build-core pybind11 nanobind + pip install ./ANTsPy --no-build-isolation --no-deps + else + echo "Installing ANTsPy from PyPI..." + pip install antspyx + fi + + - name: Download ANTsXNet models (cache miss) + if: steps.restore-cache.outputs.cache-hit != 'true' + run: | + pip install -e . + python download_all_data.py --strict + # Move to cache save location + mkdir -p ${{ runner.temp }}/ANTsXNet + cp -r ~/.keras/ANTsXNet/* ${{ runner.temp }}/ANTsXNet/ + + - name: Save newly downloaded ANTsXNet data to cache + if: steps.restore-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + key: ${{ steps.compute-key.outputs.cache-key }} + path: ${{ runner.temp }}/ANTsXNet + + + ########################################### + # Job 2 — Test Matrix + ########################################### test: name: Test on Python ${{ matrix.python-version }} runs-on: ubuntu-22.04 + needs: prepare-data strategy: + fail-fast: false matrix: - python-version: [3.11] + python-version: ["3.11", "3.12", "3.13"] steps: - - name: Checkout ANTsPyNet + - name: Checkout repository uses: actions/checkout@v3 - name: Set up Python @@ -30,35 +109,32 @@ jobs: sudo apt-get update sudo apt-get install -y cmake build-essential git - - name: Install ANTsPy (source or PyPI) + - name: Install ANTsPy run: | - if [ "$USE_SOURCE_BUILD" = "true" ]; then - echo "🔧 Installing ANTsPy from GitHub source..." + if [ "$USE_ANTSPY_SOURCE_BUILD" = "true" ]; then + echo "Installing ANTsPy from source..." git clone https://github.com/ANTsX/ANTsPy.git - pip install scikit-build-core pybind11 nanobind # 🔑 install build backend dependencies + pip install scikit-build-core pybind11 nanobind pip install ./ANTsPy --no-build-isolation --no-deps else - echo "📦 Installing ANTsPy from PyPI..." + echo "Installing ANTsPy from PyPI..." pip install antspyx fi - - name: Cache ANTsXNet model/data downloads - uses: actions/cache@v4 + - name: Restore ANTsXNet model cache for tests + uses: actions/cache/restore@v4 with: - path: ~/.keras/ANTsXNet - key: antsxnet-${{ runner.os }}-${{ hashFiles('**/download_all_data.py') }} - restore-keys: | - antsxnet-${{ runner.os }}- + key: ${{ needs.prepare-data.outputs.cache-key }} + path: ${{ runner.temp }}/ANTsXNet - - name: Install ANTsPyNet and dependencies + - name: Move cached data into ~/.keras run: | - pip install -e . - sed -i '/antspyx==/d' requirements.txt - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + mkdir -p ~/.keras/ANTsXNet + cp -r ${{ runner.temp }}/ANTsXNet/* ~/.keras/ANTsXNet/ - - name: Pre-download ANTsXNet models and data + - name: Install ANTsPyNet run: | - python download_all_data.py --strict + pip install -e . - name: Run tests (individually via pytest) run: | From 5c1da63f044de32f602e984417c1a0d342ee07df Mon Sep 17 00:00:00 2001 From: Philip A Cook Date: Tue, 2 Dec 2025 10:57:53 -0500 Subject: [PATCH 02/10] CI: Need to conserve runner disk space --- .github/workflows/test.yml | 18 +++++------------- download_all_data.py | 8 ++++++-- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d86989a..b314ade 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ on: env: USE_ANTSPY_SOURCE_BUILD: false - + ANTSXNET_CACHE_DIR: ${{ runner.temp }}/ANTsXNet jobs: ########################################### @@ -40,7 +40,7 @@ jobs: uses: actions/cache/restore@v4 with: key: ${{ steps.compute-key.outputs.cache-key }} - path: ${{ runner.temp }}/ANTsXNet + path: ${{ env.ANTSXNET_CACHE_DIR }} - name: Cache hit — skip download if: steps.restore-cache.outputs.cache-hit == 'true' @@ -69,17 +69,14 @@ jobs: if: steps.restore-cache.outputs.cache-hit != 'true' run: | pip install -e . - python download_all_data.py --strict - # Move to cache save location - mkdir -p ${{ runner.temp }}/ANTsXNet - cp -r ~/.keras/ANTsXNet/* ${{ runner.temp }}/ANTsXNet/ + python download_all_data.py --strict --cache-dir ${{ env.ANTSXNET_CACHE_DIR }} - name: Save newly downloaded ANTsXNet data to cache if: steps.restore-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: key: ${{ steps.compute-key.outputs.cache-key }} - path: ${{ runner.temp }}/ANTsXNet + path: ${{ env.ANTSXNET_CACHE_DIR }} ########################################### @@ -125,12 +122,7 @@ jobs: uses: actions/cache/restore@v4 with: key: ${{ needs.prepare-data.outputs.cache-key }} - path: ${{ runner.temp }}/ANTsXNet - - - name: Move cached data into ~/.keras - run: | - mkdir -p ~/.keras/ANTsXNet - cp -r ${{ runner.temp }}/ANTsXNet/* ~/.keras/ANTsXNet/ + path: ~/.keras/ANTsXNet - name: Install ANTsPyNet run: | diff --git a/download_all_data.py b/download_all_data.py index 34ed319..1cae420 100644 --- a/download_all_data.py +++ b/download_all_data.py @@ -2,8 +2,11 @@ import argparse import sys -def download_all_data(strict=False): +def download_all_data(strict=False, cache_dir=None): print("Downloading data files from get_antsxnet_data...") + if cache_dir is not None: + antspynet.set_antsxnet_cache_directory(cache_dir) + print(f"Using custom cache directory: {cache_dir}") try: data_keys = antspynet.get_antsxnet_data("show") for key in data_keys: @@ -44,10 +47,11 @@ def download_all_data(strict=False): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--strict", action="store_true", help="Exit on first failed download.") + parser.add_argument("--cache-dir", type=str, help="Custom cache directory for downloads.", default=None) args = parser.parse_args() try: - download_all_data(strict=args.strict) + download_all_data(strict=args.strict, cache_dir=args.cache_dir) except Exception as e: print(f"\nAborted due to error: {e}") sys.exit(1) From 880f0cefbae34a55afae0c9df384777bcb254f30 Mon Sep 17 00:00:00 2001 From: Philip A Cook Date: Tue, 2 Dec 2025 11:00:35 -0500 Subject: [PATCH 03/10] CI: Move env into job --- .github/workflows/test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b314ade..aa0baac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ on: env: USE_ANTSPY_SOURCE_BUILD: false - ANTSXNET_CACHE_DIR: ${{ runner.temp }}/ANTsXNet + jobs: ########################################### @@ -18,6 +18,8 @@ jobs: prepare-data: name: Prepare ANTsXNet Data runs-on: ubuntu-22.04 + env: + ANTSXNET_CACHE_DIR: ${{ runner.temp }}/ANTsXNet outputs: cache-key: ${{ steps.compute-key.outputs.cache-key }} cache-hit: ${{ steps.restore-cache.outputs.cache-hit }} @@ -85,6 +87,8 @@ jobs: test: name: Test on Python ${{ matrix.python-version }} runs-on: ubuntu-22.04 + env: + ANTSXNET_CACHE_DIR: ${{ runner.temp }}/ANTsXNet needs: prepare-data strategy: From 486db4e4d001258335291fbd30d51354a448da02 Mon Sep 17 00:00:00 2001 From: Philip A Cook Date: Tue, 2 Dec 2025 11:05:57 -0500 Subject: [PATCH 04/10] CI: Clean up cache after check --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa0baac..561bee0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,9 @@ jobs: - name: Cache hit — skip download if: steps.restore-cache.outputs.cache-hit == 'true' - run: echo "Cache HIT — will use cached ANTsXNet data." + run: | + echo "Cache HIT — will use cached ANTsXNet data." + rm -rf ${{ env.ANTSXNET_CACHE_DIR }} - name: Set up Python 3.11 if: steps.restore-cache.outputs.cache-hit != 'true' @@ -87,8 +89,6 @@ jobs: test: name: Test on Python ${{ matrix.python-version }} runs-on: ubuntu-22.04 - env: - ANTSXNET_CACHE_DIR: ${{ runner.temp }}/ANTsXNet needs: prepare-data strategy: From 5049b4443d091902d43eebd61889fe9e28ebd318 Mon Sep 17 00:00:00 2001 From: Philip A Cook Date: Tue, 2 Dec 2025 11:07:46 -0500 Subject: [PATCH 05/10] CI: Go back to hard-coded cache location Tired of guessing where I can put an environment variable --- .github/workflows/test.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 561bee0..ad34a7c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,8 +18,7 @@ jobs: prepare-data: name: Prepare ANTsXNet Data runs-on: ubuntu-22.04 - env: - ANTSXNET_CACHE_DIR: ${{ runner.temp }}/ANTsXNet + outputs: cache-key: ${{ steps.compute-key.outputs.cache-key }} cache-hit: ${{ steps.restore-cache.outputs.cache-hit }} @@ -42,13 +41,13 @@ jobs: uses: actions/cache/restore@v4 with: key: ${{ steps.compute-key.outputs.cache-key }} - path: ${{ env.ANTSXNET_CACHE_DIR }} + path: ${{ runner.temp }}/ANTsXNet - name: Cache hit — skip download if: steps.restore-cache.outputs.cache-hit == 'true' run: | echo "Cache HIT — will use cached ANTsXNet data." - rm -rf ${{ env.ANTSXNET_CACHE_DIR }} + rm -rf ${{ runner.temp }}/ANTsXNet - name: Set up Python 3.11 if: steps.restore-cache.outputs.cache-hit != 'true' @@ -73,14 +72,14 @@ jobs: if: steps.restore-cache.outputs.cache-hit != 'true' run: | pip install -e . - python download_all_data.py --strict --cache-dir ${{ env.ANTSXNET_CACHE_DIR }} + python download_all_data.py --strict --cache-dir ${{ runner.temp }}/ANTsXNet - name: Save newly downloaded ANTsXNet data to cache if: steps.restore-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: key: ${{ steps.compute-key.outputs.cache-key }} - path: ${{ env.ANTSXNET_CACHE_DIR }} + path: ${{ runner.temp }}/ANTsXNet ########################################### From b8d5901d51dda8fee1a3c3e848d71388c0e6be07 Mon Sep 17 00:00:00 2001 From: Philip A Cook Date: Tue, 2 Dec 2025 13:43:36 -0500 Subject: [PATCH 06/10] CI: Switch from cache to artifacts Not great but something --- .github/workflows/test.yml | 50 ++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad34a7c..d80aceb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,15 @@ jobs: ########################################### # Job 1 — Build or Restore ANTsXNet Data Cache + # + # This job downloads all ANTsXNet data and pretrained models and uploads as an artifact + # The data is too big for Github's cache and runner disk space to handle + # + # But we still avoid independent downloads from figshare for each build matrix job + # ########################################### prepare-data: - name: Prepare ANTsXNet Data + name: Prepare ANTsXNet data and models runs-on: ubuntu-22.04 outputs: @@ -36,27 +42,12 @@ jobs: ) }}" echo "cache-key=$KEY" >> "$GITHUB_OUTPUT" - - name: Try restoring ANTsXNet data cache - id: restore-cache - uses: actions/cache/restore@v4 - with: - key: ${{ steps.compute-key.outputs.cache-key }} - path: ${{ runner.temp }}/ANTsXNet - - - name: Cache hit — skip download - if: steps.restore-cache.outputs.cache-hit == 'true' - run: | - echo "Cache HIT — will use cached ANTsXNet data." - rm -rf ${{ runner.temp }}/ANTsXNet - - name: Set up Python 3.11 - if: steps.restore-cache.outputs.cache-hit != 'true' uses: actions/setup-python@v5 with: python-version: "3.11" - name: Install ANTsPy for data download - if: steps.restore-cache.outputs.cache-hit != 'true' run: | if [ "$USE_ANTSPY_SOURCE_BUILD" = "true" ]; then echo "Installing ANTsPy from source..." @@ -68,18 +59,17 @@ jobs: pip install antspyx fi - - name: Download ANTsXNet models (cache miss) - if: steps.restore-cache.outputs.cache-hit != 'true' + - name: Download ANTsXNet data and models run: | pip install -e . python download_all_data.py --strict --cache-dir ${{ runner.temp }}/ANTsXNet - - name: Save newly downloaded ANTsXNet data to cache - if: steps.restore-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 + - name: Upload data to artifact + uses: actions/upload-artifact@v4 with: - key: ${{ steps.compute-key.outputs.cache-key }} - path: ${{ runner.temp }}/ANTsXNet + name: ANTSXNet-data + path: ${{ env.ANTSXNET_CACHE_DIR }} + retention-days: 1 ########################################### @@ -99,6 +89,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 + - name: Download ANTsXNet data and models artifact + uses: actions/download-artifact@v4 + with: + name: ANTSXNet-data + path: ~/.keras/ANTsXNet + - name: Set up Python uses: actions/setup-python@v5 with: @@ -121,12 +117,6 @@ jobs: pip install antspyx fi - - name: Restore ANTsXNet model cache for tests - uses: actions/cache/restore@v4 - with: - key: ${{ needs.prepare-data.outputs.cache-key }} - path: ~/.keras/ANTsXNet - - name: Install ANTsPyNet run: | pip install -e . @@ -134,6 +124,8 @@ jobs: - name: Run tests (individually via pytest) run: | pip install pytest pytest-xdist pytest-forked psutil + echo "DEBUG" + echo ${{ ls -la ~/.keras/ANTsXNet }} for f in tests/test_*.py; do echo "🔍 Running $f" python -c "import psutil; print('Memory before:', psutil.virtual_memory().used // (1024*1024), 'MB')" From 2850ea2fa65d313dee98a9cc0cbec04e454e3f16 Mon Sep 17 00:00:00 2001 From: Philip A Cook Date: Tue, 2 Dec 2025 13:46:00 -0500 Subject: [PATCH 07/10] CI: wrong syntax --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d80aceb..a0769a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,7 +125,7 @@ jobs: run: | pip install pytest pytest-xdist pytest-forked psutil echo "DEBUG" - echo ${{ ls -la ~/.keras/ANTsXNet }} + ls -la ~/.keras/ANTsXNet for f in tests/test_*.py; do echo "🔍 Running $f" python -c "import psutil; print('Memory before:', psutil.virtual_memory().used // (1024*1024), 'MB')" From 0523ba98c927e279312501e10f218175a4a1cf1d Mon Sep 17 00:00:00 2001 From: Philip A Cook Date: Tue, 2 Dec 2025 14:14:33 -0500 Subject: [PATCH 08/10] CI: artifact path --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a0769a3..ef868cd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: ANTSXNet-data - path: ${{ env.ANTSXNET_CACHE_DIR }} + path: ${{ runner.temp }}/ANTsXNet retention-days: 1 From 1e2eb0036df67933399900d827b698d8107dd7e6 Mon Sep 17 00:00:00 2001 From: Philip A Cook Date: Tue, 2 Dec 2025 14:53:16 -0500 Subject: [PATCH 09/10] CI: remove debug output --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef868cd..a94286c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -124,8 +124,6 @@ jobs: - name: Run tests (individually via pytest) run: | pip install pytest pytest-xdist pytest-forked psutil - echo "DEBUG" - ls -la ~/.keras/ANTsXNet for f in tests/test_*.py; do echo "🔍 Running $f" python -c "import psutil; print('Memory before:', psutil.virtual_memory().used // (1024*1024), 'MB')" From c0c3bef209c1ea8877d5748cc8c6d5632ebc8db5 Mon Sep 17 00:00:00 2001 From: Philip A Cook Date: Tue, 13 Jan 2026 16:21:31 -0500 Subject: [PATCH 10/10] CI: Test for cache hits --- .github/workflows/test.yml | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a94286c..97a5584 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,7 @@ jobs: outputs: cache-key: ${{ steps.compute-key.outputs.cache-key }} - cache-hit: ${{ steps.restore-cache.outputs.cache-hit }} + cache-hit: ${{ steps.check-cache.outputs.cache-hit }} steps: - name: Checkout repository @@ -49,27 +49,43 @@ jobs: - name: Install ANTsPy for data download run: | - if [ "$USE_ANTSPY_SOURCE_BUILD" = "true" ]; then - echo "Installing ANTsPy from source..." - git clone https://github.com/ANTsX/ANTsPy.git - pip install scikit-build-core pybind11 nanobind - pip install ./ANTsPy --no-build-isolation --no-deps + echo "Installing ANTsPy from PyPI for cache preparation" + pip install antspyx + + # Try to restore an existing artifact for this key + - name: Restore ANTsXNet data artifact (if present) + id: restore-cache + uses: actions/download-artifact@v4 + continue-on-error: true # don't fail if correct cached data is not found + with: + name: ANTSXNet-data-${{ steps.compute-key.outputs.cache-key }} + path: ${{ runner.temp }}/ANTsXNet + + # Convert the restore step outcome into a boolean output + - name: Check cache hit + id: check-cache + run: | + if [ "${{ steps.restore-cache.outcome }}" = "success" ]; then + echo "cache-hit=true" >> "$GITHUB_OUTPUT" else - echo "Installing ANTsPy from PyPI..." - pip install antspyx + echo "cache-hit=false" >> "$GITHUB_OUTPUT" fi + # Download from figshare if we didn't restore an artifact - name: Download ANTsXNet data and models + if: steps.restore-cache.outcome != 'success' run: | pip install -e . python download_all_data.py --strict --cache-dir ${{ runner.temp }}/ANTsXNet + # Upload if we did a download above - name: Upload data to artifact + if: steps.restore-cache.outcome != 'success' uses: actions/upload-artifact@v4 with: - name: ANTSXNet-data + name: ANTSXNet-data-${{ steps.compute-key.outputs.cache-key }} path: ${{ runner.temp }}/ANTsXNet - retention-days: 1 + retention-days: 14 ########################################### @@ -92,7 +108,7 @@ jobs: - name: Download ANTsXNet data and models artifact uses: actions/download-artifact@v4 with: - name: ANTSXNet-data + name: ANTSXNet-data-${{ needs.prepare-data.outputs.cache-key }} path: ~/.keras/ANTsXNet - name: Set up Python