run.yml

name: Run Benchpark and Simple Benchmark Suite
on:
  workflow_call:
    secrets:
      BENCHPARK_CODECOV_TOKEN:
        required: true
jobs:
  check_errors:
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout Benchpark
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
      - name: Add needed Python libs
        run: |
          pip install -r ./requirements.txt
      - name: Run negative tests via helper
        shell: bash
        run: |
          fail() { echo "TEST FAILED: $*" >&2; exit 1; }

          # one-time setup
          ./bin/benchpark system init --dest dane llnl-cluster

          # 1) Non-MPIOnlyExperiment Test
          set +e
          # babelstream is not an MPIOnlyExperiment
          stderr_output="$({ ./bin/benchpark experiment init dane babelstream; } 2>&1)"
          status=$?
          set -e
          [[ $status -ne 0 ]] || fail "Expected non-zero exit status, got $status"
          # Check for the specific BenchparkError message
          expected="cannot run with MPI only"
          grep -Fq "$expected" <<< "$stderr_output" && echo "failed as expected: $expected" || fail "Expected error message not found in stderr. Got: $stderr_output"

          # 2) Multiple scaling options
          set +e
          # Cannot use both +strong and +weak
          stderr_output="$({ ./bin/benchpark experiment init dane kripke+strong+weak; } 2>&1)"
          status=$?
          set -e
          [[ $status -ne 0 ]] || fail "Expected non-zero exit status, got $status"
          # Check for the specific BenchparkError message
          expected="spec cannot specify multiple scaling options"
          grep -Fq "$expected" <<<"$stderr_output" && echo "failed as expected: $expected" || fail "Expected error message not found in stderr. Got: $stderr_output"
  saxpy:
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout Benchpark
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
      - name: Add needed Python libs
        run: |
          pip install -r ./requirements.txt
      - name: Build Saxpy Workspace
        run: |
          ./bin/benchpark system init --dest=generic-x86 generic-x86
          ./bin/benchpark experiment init --dest=saxpy/openmp generic-x86 saxpy+openmp
          ./bin/benchpark setup generic-x86/saxpy/openmp workspace/
      - name: Setup Ramble & Spack
        run: |
          . workspace/setup.sh
          spack mirror add ci-buildcache oci://ghcr.io/llnl/benchpark-binary-cache
          spack config add "packages:all:target:[x86_64_v3]"
          env | grep SPACK >> "$GITHUB_ENV"
          env | grep RAMBLE >> "$GITHUB_ENV"
          echo "PATH=$PATH" >> "$GITHUB_ENV"
      - name: Setup Saxpy Workspace
        working-directory: ./workspace/generic-x86/saxpy/openmp/workspace/
        run: |
          ramble \
            --workspace-dir . \
            --disable-logger \
            -c config:spack_flags:install:'--no-check-signature' \
            workspace setup
      - name: Run Saxpy Experiments
        working-directory: ./workspace/generic-x86/saxpy/openmp/workspace/
        run: |
          ramble \
            --workspace-dir . \
            --disable-logger \
            on \
            --executor '{execute_experiment}' \
            --where '{n_nodes} == 1'
      - name: Analyze Saxpy Results
        working-directory: ./workspace/generic-x86/saxpy/openmp/workspace/
        run: |
          ramble \
            --workspace-dir . \
            --disable-logger \
            workspace analyze
      - name: Archive Experiment Workspace
        working-directory: ./workspace/generic-x86/saxpy/openmp/workspace/
        if: always()
        run: |
          ramble \
            --workspace-dir . \
            --disable-logger \
            workspace archive
      - name: Upload Workspace Archive as CI Artifact
        if: always()
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4
        with:
          name: workspace-archive
          path: ./workspace/generic-x86/saxpy/openmp/workspace/archive/**
      - name: Upload Binaries to CI Cache
        if: github.ref == 'refs/heads/develop'
        run: |
          spack mirror set \
            --push \
            --oci-username ${{ github.actor }} \
            --oci-password "${{ secrets.GITHUB_TOKEN }}" \
            ci-buildcache
          spack buildcache push \
            -j $(($(nproc) + 1)) \
            --base-image ubuntu:24.04 \
            --unsigned \
            --update-index ci-buildcache \
            $(spack find --format '/{hash}')
      - name: Test Execution Using Binary
        run: |
          OUTPUT=$(spack find --paths saxpy)
          SAXPY_PATH=$(echo "$OUTPUT" | grep 'saxpy@' | awk '{print $2}')/bin/saxpy
          echo "Extracted Path: $SAXPY_PATH"
          BENCHPARK_RUN_COVERAGE=binary ./bin/benchpark experiment init --dest=saxpy/binary generic-x86 saxpy+openmp package_manager="user-managed" append_path="$SAXPY_PATH"
          ./bin/benchpark setup generic-x86/saxpy/binary workspace-binary/
          ./workspace-binary/ramble/bin/ramble \
            --workspace-dir \
            /home/runner/work/benchpark/benchpark/workspace-binary/generic-x86/saxpy/binary/workspace \
            workspace setup
          ./workspace-binary/ramble/bin/ramble \
            --workspace-dir \
            /home/runner/work/benchpark/benchpark/workspace-binary/generic-x86/saxpy/binary/workspace \
            on \
            --executor '{execute_experiment}' \
            --where '{n_nodes} == 1'
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.BENCHPARK_CODECOV_TOKEN }}
          directory: ./coverage-data-binary
          flags: dryrunexperiments-binary
          verbose: true
          fail_ci_if_error: true
  dryrunexperiments:
    runs-on: ubuntu-24.04
    strategy:
      # Don't fail other running testtype tests
      fail-fast: false
      matrix:
        testtype:
          - cuda
          - rocm
          - openmp
          - mpi
          - strong
          - weak
          - throughput
          - modifiers
    steps:
      - name: Checkout Benchpark
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
      - name: Add needed Python libs
        run: |
          pip install -r ./requirements.txt
      - name: Run Dryruns
        run: |
          ./bin/benchpark bootstrap
          BENCHPARK_RUN_COVERAGE=${{ matrix.testtype }} ./bin/benchpark-python .github/utils/dryruns.py --test ${{ matrix.testtype }}
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.BENCHPARK_CODECOV_TOKEN }}
          directory: ./coverage-data-${{ matrix.testtype }}
          flags: dryrunexperiments-${{ matrix.testtype }}
          verbose: true
          fail_ci_if_error: true
  pytest:
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout Benchpark
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
      - name: Add needed Python libs
        run: |
          pip install -r ./requirements.txt
      - name: Run
        run: |-
          ./bin/benchpark unit-test
  test_bootstrap:
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout Benchpark
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
      - name: Add needed Python libs
        run: |
          pip install -r ./requirements.txt
      - name: Run
        run: |-
          output=$(./bin/benchpark configure --bootstrap-location . && ./bin/benchpark bootstrap 2>&1)

          # 1. Check the path is present
          echo "$output" | grep -q "/home/runner/work/benchpark/benchpark/.benchpark/ramble" \
            || { echo "Expected path not found"; exit 1; }

          # 2. Check "Cloning Ramble" appears exactly once
          count=$(echo "$output" | grep -c "Cloning Ramble")
          if [[ "$count" -ne 1 ]]; then
            echo "'Cloning Ramble' appeared $count times (expected 1)"
            exit 1
          fi
  validate_docs:
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
      - name: Setup Python
        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548
        with:
          python-version: '3.11'
          cache: pip
          cache-dependency-path: .github/workflows/requirements/docs.txt
      - name: Install deps
        run: |
          pip install -r .github/workflows/requirements/docs.txt
      - name: Test Docs Scripts
        run: |-
          cd docs/
          python generate-sys-defs-list.py
          python generate-benchmark-list.py