diff --git a/.github/workflows/automated-ci.yml b/.github/workflows/automated-ci.yml deleted file mode 100644 index 7e2ec7de0..000000000 --- a/.github/workflows/automated-ci.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: CI -on: - pull_request: - types: [opened, synchronize, reopened] - -jobs: - lint_format: - uses: ./.github/workflows/lint_format.yml - test_unit: - uses: ./.github/workflows/test_unit.yml - rule_tester: - if: github.event.action == 'opened' - uses: ./.github/workflows/rule-tester.yml - secrets: inherit - with: - environment: DEV - test_rule_editor_preview: - if: github.event.action == 'opened' - needs: rule_tester - uses: ./.github/workflows/test_rule_editor_preview.yml - secrets: inherit - with: - pr_ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/build-binary.yml b/.github/workflows/build-binary.yml index 5a82dabb8..8a257da1d 100644 --- a/.github/workflows/build-binary.yml +++ b/.github/workflows/build-binary.yml @@ -12,9 +12,9 @@ jobs: build: runs-on: ${{ inputs.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 id: pysetup with: python-version: "3.12" @@ -35,7 +35,7 @@ jobs: if: runner.os == 'Windows' run: pyinstaller --onedir --contents-directory "." core.py --icon=resources/assets/CORE_logo_sm.ico --dist ./dist/output/${{ inputs.name }} --collect-submodules pyreadstat --add-data="$env:pythonLocation\Lib\site-packages\xmlschema\schemas;xmlschema/schemas" --hidden-import numpy --hidden-import numpy.core._methods --hidden-import numpy.lib.format --add-data="resources/cache;resources/cache" --add-data="resources/templates;resources/templates" --add-data="resources/schema;resources/schema" --add-data="resources/datasets;resources/datasets" --add-data="resources/jsonata;resources/jsonata" - name: Archive Binary - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ inputs.name }} path: dist/output/${{ inputs.name }}/ diff --git a/.github/workflows/build_version.yml b/.github/workflows/build-version.yml similarity index 94% rename from .github/workflows/build_version.yml rename to .github/workflows/build-version.yml index 1066e9001..f23872f85 100644 --- a/.github/workflows/build_version.yml +++ b/.github/workflows/build-version.yml @@ -8,12 +8,12 @@ jobs: custom-executable-build: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: repository: cdisc-org/cdisc-rules-engine ref: main - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" - name: Install dependencies @@ -48,7 +48,7 @@ jobs: mv core cdisc-core-$(date +%Y%m%d) echo "Executable ready: $(ls cdisc-core-*)" - name: Upload Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: cdisc-core-ubuntu-22.04 path: dist/output/core-ubuntu-22.04/core/ @@ -62,12 +62,12 @@ jobs: # custom-executable-build-windows: # runs-on: windows-latest # steps: -# - uses: actions/checkout@v4 +# - uses: actions/checkout@v6 # with: # repository: cdisc-org/cdisc-rules-engine # ref: main # - name: Set up Python -# uses: actions/setup-python@v5 +# uses: actions/setup-python@v6 # with: # python-version: "3.12" # - name: Install dependencies @@ -91,7 +91,7 @@ jobs: # Rename-Item core.exe "cdisc-core-$date.exe" # Write-Output "Executable ready: $(Get-ChildItem cdisc-core-*)" # - name: Upload Artifact -# uses: actions/upload-artifact@v4 +# uses: actions/upload-artifact@v6 # with: # name: cdisc-core-windows # path: dist/output/core-windows/core/ @@ -106,12 +106,12 @@ jobs: # custom-executable-build-macos: # runs-on: macos-latest # steps: -# - uses: actions/checkout@v4 +# - uses: actions/checkout@v6 # with: # repository: cdisc-org/cdisc-rules-engine # ref: main # - name: Set up Python -# uses: actions/setup-python@v5 +# uses: actions/setup-python@v6 # with: # python-version: "3.12" # - name: Install dependencies @@ -146,7 +146,7 @@ jobs: # mv core cdisc-core-$(date +%Y%m%d) # echo "Executable ready: $(ls cdisc-core-*)" # - name: Upload Artifact -# uses: actions/upload-artifact@v4 +# uses: actions/upload-artifact@v6 # with: # name: cdisc-core-macos # path: dist/output/core-macos/core/ diff --git a/.github/workflows/rule-tester.yml b/.github/workflows/deploy-rule-tester.yml similarity index 75% rename from .github/workflows/rule-tester.yml rename to .github/workflows/deploy-rule-tester.yml index c5ec8c701..bb6b2c14c 100644 --- a/.github/workflows/rule-tester.yml +++ b/.github/workflows/deploy-rule-tester.yml @@ -1,23 +1,20 @@ name: Deploy Rule Tester Endpoint on: - workflow_call: - inputs: - environment: - required: true - type: string - workflow_dispatch: inputs: environment: required: true default: DEV type: environment - + pull_request: push: branches: - main +permissions: + contents: read + env: creds: '{"clientId":"${{ vars.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ vars.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ vars.AZURE_TENANT_ID }}"}' functionAppName: cdisc-library-conformance-rules-generator-dev @@ -26,9 +23,9 @@ env: jobs: deploy_rule_tester: runs-on: ubuntu-latest - environment: ${{ inputs.environment || 'PROD' }} + environment: ${{ github.event_name == 'pull_request' && 'DEV' || inputs.environment || 'PROD' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: azure/login@v2 with: creds: ${{ env.creds }} @@ -38,7 +35,6 @@ jobs: app-name: ${{ env.functionAppName }} mask-inputs: false app-settings-json: '{"WEBSITE_ENABLE_SYNC_UPDATE_SITE": "1"}' # wait for this step to complete before the webapps-deploy step - general-settings-json: '{"linuxFxVersion": "PYTHON|${{ env.PYTHON_VERSION }}"}' slot-name: ${{ vars.AZURE_WEBAPP_SLOT }} @@ -47,3 +43,9 @@ jobs: with: app-name: ${{ env.functionAppName }} slot-name: ${{ vars.AZURE_WEBAPP_SLOT }} + + test_rule_editor_preview: + needs: deploy_rule_tester + uses: ./.github/workflows/test-rule-editor-preview.yml + secrets: inherit + if: contains(fromJSON('["push", "pull_request"]'), github.event_name) diff --git a/.github/workflows/docker_release.yml b/.github/workflows/docker-release.yml similarity index 98% rename from .github/workflows/docker_release.yml rename to .github/workflows/docker-release.yml index aef7c620e..d927ee21e 100644 --- a/.github/workflows/docker_release.yml +++ b/.github/workflows/docker-release.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: ref: ${{ needs.check-if-latest.outputs.version }} diff --git a/.github/workflows/lint-format.yml b/.github/workflows/lint-format.yml new file mode 100644 index 000000000..1e7be35ec --- /dev/null +++ b/.github/workflows/lint-format.yml @@ -0,0 +1,75 @@ +name: Check Lint and Format +on: + workflow_dispatch: + pull_request: + push: + branches: + - main +permissions: + contents: read + +jobs: + get_changed_files: + runs-on: ubuntu-latest + outputs: + py: ${{ steps.changes.outputs.py_all_changed_files }} + pretty: ${{ steps.changes.outputs.pretty_all_changed_files }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Get changed files + id: changes + uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + with: + files_yaml: | + py: + - '**.py' + pretty: + - '**.json' + - '**.md' + - '**.yaml' + - '**.yml' + separator: " " + + check_python_files: + runs-on: ubuntu-latest + needs: get_changed_files + # only run if there are changed files + if: ${{needs.get_changed_files.outputs.py}} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install linters + run: | + pip install black flake8 -c requirements-dev.txt + - name: Run flake8 + run: | + flake8 ${{needs.get_changed_files.outputs.py}} --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 ${{needs.get_changed_files.outputs.py}} --ignore E203,W503 --count --statistics + - name: Run black + run: | + black --check ${{needs.get_changed_files.outputs.py}} + + check_prettier_files: + runs-on: ubuntu-latest + needs: get_changed_files + # only run if there are changed files + if: ${{needs.get_changed_files.outputs.pretty}} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: actions/setup-node@v6 + - name: Run prettier + run: | + npm i prettier + npx prettier --check ${{needs.get_changed_files.outputs.pretty}} diff --git a/.github/workflows/lint_format.yml b/.github/workflows/lint_format.yml deleted file mode 100644 index 644be249d..000000000 --- a/.github/workflows/lint_format.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Check Lint and Format -on: - # Can't be workflow_dispatch because it uses github.event.pull_request.base.sha - workflow_call: - -jobs: - get_changed_files: - runs-on: ubuntu-latest - outputs: - py: ${{ steps.changes.outputs.py }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: "0" - - name: Get changed files - id: changes - run: | - echo "py=$(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} | grep .py$ | xargs)" >> $GITHUB_OUTPUT - - lint: - runs-on: ubuntu-latest - needs: get_changed_files - # only run there are changed files - if: ${{needs.get_changed_files.outputs.py}} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: "0" - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install linters - run: | - pip install black flake8 -c requirements-dev.txt - - name: Run flake8 - run: | - flake8 ${{needs.get_changed_files.outputs.py}} --count --select=E9,F63,F7,F82 --show-source --statistics - flake8 ${{needs.get_changed_files.outputs.py}} --ignore E203,W503 --count --statistics - - name: Run black - run: | - black --check ${{needs.get_changed_files.outputs.py}} diff --git a/.github/workflows/prerelease-update-cache.yml b/.github/workflows/prerelease-update-cache.yml index 91822624b..ba225f5d1 100644 --- a/.github/workflows/prerelease-update-cache.yml +++ b/.github/workflows/prerelease-update-cache.yml @@ -9,7 +9,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: # https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent#generating-a-new-ssh-key # git bash: ssh-keygen -t ed25519 -C "github-actions@cdisc.org" @@ -18,7 +18,7 @@ jobs: ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/.github/workflows/prerelease-update-version.yml b/.github/workflows/prerelease-update-version.yml index 69b525592..f0a9a07fb 100644 --- a/.github/workflows/prerelease-update-version.yml +++ b/.github/workflows/prerelease-update-version.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: # https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent#generating-a-new-ssh-key # git bash: ssh-keygen -t ed25519 -C "github-actions@cdisc.org" @@ -27,7 +27,7 @@ jobs: ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 805ac4ddf..f3b8258e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,7 +31,7 @@ jobs: contents: write steps: - name: Download all artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: path: artifacts - name: Create release zip files @@ -61,7 +61,7 @@ jobs: with: name: core-ubuntu-22.04-tar - name: Upload Ubuntu 22.04 tar.gz - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: core-ubuntu-22.04-tarball path: core-ubuntu-22.04.tar.gz @@ -78,7 +78,7 @@ jobs: # Upload zips as artifact for the PyPI job - name: Upload Release Assets for PyPI - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: release-artifacts path: | @@ -94,9 +94,9 @@ jobs: id-token: write steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: "3.12" - name: Install dependencies @@ -118,7 +118,7 @@ jobs: build-and-push-docker: needs: [create-release-assets, deploy-PyPi] - uses: ./.github/workflows/docker_release.yml + uses: ./.github/workflows/docker-release.yml with: version: ${{ github.event.release.tag_name }} secrets: inherit diff --git a/.github/workflows/test_release.yml b/.github/workflows/test-release.yml similarity index 94% rename from .github/workflows/test_release.yml rename to .github/workflows/test-release.yml index 1a844d3ff..bd3a60802 100644 --- a/.github/workflows/test_release.yml +++ b/.github/workflows/test-release.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download all artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: path: artifacts - name: Create individual release zip files @@ -58,7 +58,7 @@ jobs: with: name: core-ubuntu-22.04-tar - name: Upload Ubuntu 22.04 tar.gz - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: core-ubuntu-22.04-tarball path: core-ubuntu-22.04.tar.gz @@ -71,10 +71,10 @@ jobs: id-token: write steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/.github/workflows/test_rule_editor_preview.yml b/.github/workflows/test-rule-editor-preview.yml similarity index 71% rename from .github/workflows/test_rule_editor_preview.yml rename to .github/workflows/test-rule-editor-preview.yml index 57492fe16..1523e932a 100644 --- a/.github/workflows/test_rule_editor_preview.yml +++ b/.github/workflows/test-rule-editor-preview.yml @@ -2,15 +2,10 @@ name: Run Tests on Preview Deployment on: workflow_call: - inputs: - pr_ref: - description: "Git reference to checkout (for PRs)" - required: false - type: string workflow_dispatch: inputs: pr_number: - description: "PR number to test (leave empty for current branch)" + description: "PR number to test (for forked repositories) (leave empty for current branch)" required: false type: string @@ -20,12 +15,12 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: - ref: ${{ inputs.pr_ref || (inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number)) || github.ref }} + ref: ${{ (inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number)) || github.ref }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.12 @@ -47,7 +42,7 @@ jobs: - name: Upload screenshot (always runs) if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: selenium-screenshot path: login_screenshot.png diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test-suite.yml similarity index 95% rename from .github/workflows/test_suite.yml rename to .github/workflows/test-suite.yml index 59732139e..407ce9743 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test-suite.yml @@ -1,19 +1,16 @@ name: CORE Test Suite Validation on: - workflow_call: - inputs: - pr_ref: - description: "Git reference to checkout (for PRs)" - required: false - type: string workflow_dispatch: inputs: pr_number: - description: "PR number to test (leave empty for current branch)" + description: "PR number to test (for forked repositories) (leave empty for current branch)" required: false type: string pull_request_target: - types: [opened, synchronize, reopened] + push: + branches: + - main + env: DATASET_SIZE_THRESHOLD: 1000000000 jobs: @@ -21,18 +18,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Get Engine Changes - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: - ref: ${{ inputs.pr_ref || (inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number)) || github.event.pull_request.head.sha || github.ref }} + ref: ${{ (inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number)) || github.event.pull_request.head.sha || github.ref }} - name: Get Test Suite Repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: repository: cdisc-org/CORE_Test_Suite path: CORE_Test_Suite token: ${{ secrets.ACCESS_TOKEN }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" - name: Install dependencies @@ -218,7 +215,7 @@ jobs: ####################### - name: Upload test results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: comparison-results path: | diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test-unit.yml similarity index 80% rename from .github/workflows/test_unit.yml rename to .github/workflows/test-unit.yml index d27fc62bf..8b3207bca 100644 --- a/.github/workflows/test_unit.yml +++ b/.github/workflows/test-unit.yml @@ -1,14 +1,17 @@ name: Unit Tests on: - workflow_call: workflow_dispatch: + pull_request: + push: + branches: + - main jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" - name: Install requirements diff --git a/.gitignore b/.gitignore index ecf4c8a1c..67ce5235a 100644 --- a/.gitignore +++ b/.gitignore @@ -150,5 +150,4 @@ resources/cache/custom_rules.pkl #ignore custom build artifacts /build-output -.github/workflows/build_version.yml Dockerfile.build