From 819e8f70f00b54d3142cd10bc865334e3e2f6f33 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 4 Nov 2022 21:14:40 -0700 Subject: [PATCH 001/159] Updates the actions to pull from the new branch, updates to add new self hosted runner. Updates to add gitguardian api --- .github/workflows/build.yaml | 6 ++++-- .github/workflows/deploy.yaml | 6 +++--- .github/workflows/gitguardian.yaml | 21 +++++++++++++++++++++ .github/workflows/lint.yaml | 4 ++-- .github/workflows/retag.yaml | 6 +++--- .github/workflows/test.yaml | 4 ++-- 6 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/gitguardian.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 25ac19b..1cc6dcf 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -40,11 +40,11 @@ env: jobs: build: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@update-to-new-actions-library with: tag: ${{ inputs.tag }} - name: Checkout code @@ -53,6 +53,8 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + # with: + # enpoint: mycontext - name: Github Container Login uses: docker/login-action@v2 with: diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 829e1fa..3d41afe 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -17,8 +17,8 @@ env: REGISTRY: ghcr.io jobs: - deployment: - runs-on: ubuntu-latest + deploy: + runs-on: self-hosted container: dtzar/helm-kubectl:3.9.4 environment: ${{ inputs.environment }} env: @@ -43,7 +43,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@update-to-new-actions-library with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/gitguardian.yaml b/.github/workflows/gitguardian.yaml new file mode 100644 index 0000000..092ad82 --- /dev/null +++ b/.github/workflows/gitguardian.yaml @@ -0,0 +1,21 @@ +name: GitGuardian scan + +on: [push, pull_request] + +jobs: + scan: + name: GitGuardian scan + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 # fetch all history so multiple commits can be scanned + - name: GitGuardian scan + uses: GitGuardian/ggshield-action@master + env: + GITHUB_PUSH_BEFORE_SHA: ${{ github.event.before }} + GITHUB_PUSH_BASE_SHA: ${{ github.event.base }} + GITHUB_PULL_BASE_SHA: ${{ github.event.pull_request.base.sha }} + GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} \ No newline at end of file diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d2f5576..5691f6d 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -22,11 +22,11 @@ env: jobs: lint: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@update-to-new-actions-library with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index f6b9e7e..66c8017 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -22,14 +22,14 @@ env: jobs: retag: - runs-on: ubuntu-latest + runs-on: self-hosted if: ${{ github.ref == 'refs/heads/main' }} steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@update-to-new-actions-library with: - tag: ${{ inputs.tag }} + tag: ${{ inputs.tag }} || ${{ env.TAG }} - name: Retag web to latest uses: koraykoska/retag-docker-image@0.2.4 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d432f05..c87b7a4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -21,7 +21,7 @@ env: jobs: test: - runs-on: ubuntu-latest + runs-on: self-hosted strategy: fail-fast: false matrix: @@ -41,7 +41,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@update-to-new-actions-library with: tag: ${{ inputs.tag }} - name: Checkout code From 4bb51229f05bdd00513219d646318d04f51ee219 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 4 Nov 2022 21:21:40 -0700 Subject: [PATCH 002/159] Updates the actions to pull from the new branch, updates to add new self hosted runner. Updates to add gitguardian api --- .github/workflows/build.yaml | 2 +- .github/workflows/deploy.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/retag.yaml | 2 +- .github/workflows/test.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1cc6dcf..0e6b6d5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -44,7 +44,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@update-to-new-actions-library + uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 3d41afe..2fedc81 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -43,7 +43,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@update-to-new-actions-library + uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 5691f6d..6f9c841 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -26,7 +26,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@update-to-new-actions-library + uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index 66c8017..6139dba 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -27,7 +27,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@update-to-new-actions-library + uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} || ${{ env.TAG }} - name: Retag web to latest diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c87b7a4..99bb600 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -41,7 +41,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@update-to-new-actions-library + uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - name: Checkout code From 58f6ab468accfc7948938b44b39257f2a4523ddf Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Fri, 4 Nov 2022 21:26:29 -0700 Subject: [PATCH 003/159] Update gitguardian.yaml update workflow call and workflow dispatch --- .github/workflows/gitguardian.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gitguardian.yaml b/.github/workflows/gitguardian.yaml index 092ad82..c569314 100644 --- a/.github/workflows/gitguardian.yaml +++ b/.github/workflows/gitguardian.yaml @@ -1,6 +1,8 @@ name: GitGuardian scan -on: [push, pull_request] +on: + workflow_dispatch: + workflow_call: jobs: scan: @@ -18,4 +20,4 @@ jobs: GITHUB_PUSH_BASE_SHA: ${{ github.event.base }} GITHUB_PULL_BASE_SHA: ${{ github.event.pull_request.base.sha }} GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} \ No newline at end of file + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} From 70c2b81e459eff7858ca0ff4a080d2d34a2e6cd7 Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Fri, 4 Nov 2022 21:34:45 -0700 Subject: [PATCH 004/159] add registry env variable env: REGISTRY: ghcr.io --- .github/workflows/gitguardian.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gitguardian.yaml b/.github/workflows/gitguardian.yaml index c569314..17417a8 100644 --- a/.github/workflows/gitguardian.yaml +++ b/.github/workflows/gitguardian.yaml @@ -1,9 +1,11 @@ name: GitGuardian scan - on: workflow_dispatch: workflow_call: +env: + REGISTRY: ghcr.io + jobs: scan: name: GitGuardian scan From 876bea78f12d63c93da21a009b3dd7f07e78b1e5 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 4 Nov 2022 22:12:22 -0700 Subject: [PATCH 005/159] update with endpoint defined --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0e6b6d5..ac48f9c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -53,8 +53,8 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - # with: - # enpoint: mycontext + with: + enpoint: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Github Container Login uses: docker/login-action@v2 with: From 1972ee964972787ff5b8d2e258bdb757058baa6e Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 4 Nov 2022 22:21:20 -0700 Subject: [PATCH 006/159] tweaking per documentation here: https://github.com/actions-runner-controller/actions-runner-controller/issues/893#issuecomment-1128602058 --- .github/workflows/build.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ac48f9c..70711fc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -52,9 +52,12 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx + run: >- + docker context create builders uses: docker/setup-buildx-action@v2 with: - enpoint: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + version: latest + endpoint: builders - name: Github Container Login uses: docker/login-action@v2 with: From 591e7f62c68531b90a8d610e15282b50e958527b Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 4 Nov 2022 22:24:59 -0700 Subject: [PATCH 007/159] tweaks --- .github/workflows/build.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 70711fc..bea9ff5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -46,17 +46,18 @@ jobs: name: Setup uses: scientist-softserv/actions/setup-env@self-hosted-runner with: - tag: ${{ inputs.tag }} + tag: ${{ inputs.tag }} || ${{ env.TAG }} - name: Checkout code uses: actions/checkout@v3 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx + - name: Create Docker Context run: >- docker context create builders + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: - version: latest + version: ${{ inputs.tag }} || ${{ env.TAG }} endpoint: builders - name: Github Container Login uses: docker/login-action@v2 From c11f93cb264f3b258000f1570c1f04c1037598db Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 4 Nov 2022 22:27:55 -0700 Subject: [PATCH 008/159] tweaks --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bea9ff5..64533ed 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -46,7 +46,7 @@ jobs: name: Setup uses: scientist-softserv/actions/setup-env@self-hosted-runner with: - tag: ${{ inputs.tag }} || ${{ env.TAG }} + tag: ${{ inputs.tag }} - name: Checkout code uses: actions/checkout@v3 - name: Set up QEMU @@ -57,7 +57,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: - version: ${{ inputs.tag }} || ${{ env.TAG }} + version: ${{ env.TAG }} endpoint: builders - name: Github Container Login uses: docker/login-action@v2 From 3949f9667ad94be22509426603d316ea86982992 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 4 Nov 2022 22:29:45 -0700 Subject: [PATCH 009/159] tweaks --- .github/workflows/build.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 64533ed..17e5b45 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -57,7 +57,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: - version: ${{ env.TAG }} endpoint: builders - name: Github Container Login uses: docker/login-action@v2 From f99ea4366551dea2f08e0e3d58c2050b89970981 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 00:29:25 -0700 Subject: [PATCH 010/159] updates for testing the lint action error --- .github/workflows/lint.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 6f9c841..28d3215 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -31,13 +31,17 @@ jobs: tag: ${{ inputs.tag }} - name: Checkout code uses: actions/checkout@v3 + with: + repository: ${{ github.repository }} + path: ${{ github.event.repository.name }} - name: Github Container Login uses: docker/login-action@v2 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull image to prevent build + - name: Pull image to prevent build # line 1 (line 45?): docker-compose: command not found + # Error: Process completed with exit code 127. run: >- touch .env.development; touch .env; From 3310169e09a01760b19b471dc139a0cecf491b5d Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 11:47:07 -0700 Subject: [PATCH 011/159] adding tmate debugging session to troubleshoot error with running docker-compose while using self-hosted runners --- .github/workflows/lint.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 28d3215..6e74e28 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -31,22 +31,23 @@ jobs: tag: ${{ inputs.tag }} - name: Checkout code uses: actions/checkout@v3 - with: - repository: ${{ github.repository }} - path: ${{ github.event.repository.name }} - name: Github Container Login uses: docker/login-action@v2 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull image to prevent build # line 1 (line 45?): docker-compose: command not found - # Error: Process completed with exit code 127. + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + with: + limit-access-to-actor: true + - name: Pull image to prevent build run: >- touch .env.development; touch .env; docker-compose pull web - - name: Pull worker image to prevent build + - name: Pull worker image to prevent build run: >- docker-compose pull worker if: ${{ inputs.worker }} From ac45d5b4b3901692d3d89c19f0281c3865b8370b Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Sat, 5 Nov 2022 12:03:11 -0700 Subject: [PATCH 012/159] fix copy pasta fix copy pasta --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 6e74e28..3bb7cce 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -47,7 +47,7 @@ jobs: touch .env.development; touch .env; docker-compose pull web - - name: Pull worker image to prevent build + - name: Pull worker image to prevent build run: >- docker-compose pull worker if: ${{ inputs.worker }} From 65c87b7ac01216457311e744b0f4d4421b9e8aa8 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 12:06:41 -0700 Subject: [PATCH 013/159] found typo --- setup-env/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index f24bc7f..a047f1d 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -17,7 +17,7 @@ runs: shell: bash - name: Downcase repo run: | - echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} + echo "REPO_LOWER=${REPO}" >> ${GITHUB_ENV} env: REPO: "${{ github.repository }}" shell: bash From 2d39229705d01499db9baeec0f8bf619681248d6 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 12:18:19 -0700 Subject: [PATCH 014/159] gathering more inforamtion to troubleshoot --- .github/workflows/lint.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 3bb7cce..d49c943 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -44,6 +44,8 @@ jobs: limit-access-to-actor: true - name: Pull image to prevent build run: >- + ls -la; + docker ps; touch .env.development; touch .env; docker-compose pull web From d3503546c8dd925309a7b369388c34611967d7ef Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 13:29:51 -0700 Subject: [PATCH 015/159] tweaks to see where we are and how we can checkout the package registry's image --- .github/workflows/lint.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d49c943..347a29b 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -44,15 +44,16 @@ jobs: limit-access-to-actor: true - name: Pull image to prevent build run: >- - ls -la; - docker ps; touch .env.development; touch .env; - docker-compose pull web + ls -la; + docker ps; + docker pull ghcr.io/scientist-softserv/arce:${{ env.TAG }}; + docker ps - name: Pull worker image to prevent build - run: >- - docker-compose pull worker if: ${{ inputs.worker }} + run: >- + docker pull ghcr.io/scientist-softserv/arce/worker:${{ env.TAG }}; - name: Run Rubocop run: docker-compose run web bundle exec rubocop --parallel --format progress - name: Publish Test Report From 37edfb4b5295f008e298c90da8fa6a68c44ce529 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 14:06:04 -0700 Subject: [PATCH 016/159] updates to get more information while troubleshooting --- .github/workflows/lint.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 347a29b..79c5872 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -46,16 +46,14 @@ jobs: run: >- touch .env.development; touch .env; - ls -la; - docker ps; - docker pull ghcr.io/scientist-softserv/arce:${{ env.TAG }}; - docker ps + echo "${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }}"; + docker pull ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} - name: Pull worker image to prevent build if: ${{ inputs.worker }} run: >- - docker pull ghcr.io/scientist-softserv/arce/worker:${{ env.TAG }}; + docker pull ${{ env.REGISTRY }}/${{ github.repository }}/worker:${{ env.TAG }}; - name: Run Rubocop - run: docker-compose run web bundle exec rubocop --parallel --format progress + run: docker run web bundle exec rubocop --parallel --format progress - name: Publish Test Report uses: mikepenz/action-junit-report@v3 if: always() # always run even if the previous step fails From d3fcbb6d57bffb40b67f249880061ddebbb5c89c Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 14:29:53 -0700 Subject: [PATCH 017/159] more testing and tweaking --- .github/workflows/lint.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 79c5872..eaea4ae 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -46,14 +46,13 @@ jobs: run: >- touch .env.development; touch .env; - echo "${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }}"; docker pull ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} - name: Pull worker image to prevent build - if: ${{ inputs.worker }} + if: ${{ inputs.worker == 'true' }} run: >- - docker pull ${{ env.REGISTRY }}/${{ github.repository }}/worker:${{ env.TAG }}; + docker pull ${{ env.REGISTRY }}/${{ github.repository }}/worker:${{ env.TAG }} - name: Run Rubocop - run: docker run web bundle exec rubocop --parallel --format progress + run: bundle exec rubocop --parallel --format progress - name: Publish Test Report uses: mikepenz/action-junit-report@v3 if: always() # always run even if the previous step fails From 6c067c14f59c191280a1cb42f4ee0a0b28f871c5 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 14:58:59 -0700 Subject: [PATCH 018/159] works locally, lets see if it works in CI now --- .github/workflows/lint.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index eaea4ae..36ae639 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -52,7 +52,8 @@ jobs: run: >- docker pull ${{ env.REGISTRY }}/${{ github.repository }}/worker:${{ env.TAG }} - name: Run Rubocop - run: bundle exec rubocop --parallel --format progress + run: >- + docker run ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bundle exec rubocop --parallel --format progress - name: Publish Test Report uses: mikepenz/action-junit-report@v3 if: always() # always run even if the previous step fails From 0a52684f0bb2fd721848d5e09959f44e3d450f59 Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Sat, 5 Nov 2022 15:29:08 -0700 Subject: [PATCH 019/159] remove tmate remove tmat --- .github/workflows/lint.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 36ae639..f66ef52 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -37,11 +37,6 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - with: - limit-access-to-actor: true - name: Pull image to prevent build run: >- touch .env.development; From ecad26ae6a9b113371ad0895c238407e3f2e51ea Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 17:29:45 -0700 Subject: [PATCH 020/159] testing fail_on_failure flag and updates the docker compose syntax to just docker. --- .github/workflows/test.yaml | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 99bb600..55b245a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,6 +8,10 @@ on: tag: required: false type: string + confdir: + required: false + type: string + workflow_call: inputs: worker: @@ -34,7 +38,7 @@ jobs: ci_node_index: [0, 1, 2] env: ALLOW_ANONYMOUS_LOGIN: "yes" - CONFDIR: "/app/samvera/hyrax-webapp/solr/config" + CONFDIR: ${{ inputs.confdir }} DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml @@ -43,7 +47,7 @@ jobs: name: Setup uses: scientist-softserv/actions/setup-env@self-hosted-runner with: - tag: ${{ inputs.tag }} + tag: ${{ inputs.tag || env.TAG }} - name: Checkout code uses: actions/checkout@v3 - name: Github Container Login @@ -61,23 +65,24 @@ jobs: run: >- touch .env.development; touch .env; - docker-compose pull web + docker pull ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} - name: Pull worker image to prevent build - run: >- - docker-compose pull worker - if: ${{ inputs.worker }} - - name: Start containers - run: docker-compose up -d web + if: ${{ inputs.worker == 'true' }} + run: docker pull ${{ env.REGISTRY }}/${{ github.repository }}/worker:${{ env.TAG }} - name: Setup solr run: >- - docker-compose exec -T web bash -c - "solrcloud-upload-configset.sh /app/samvera/hyrax-webapp/solr/config && + docker exec -T ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c + "solrcloud-upload-configset.sh ${{ env.CONFDIR }} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" - name: Setup db run: >- - docker-compose exec -T web bash -c + docker exec -T ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c "RAILS_ENV=test bundle exec rake db:schema:load db:migrate" + - name: Start containers + run: >- + docker run ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} + docker run ${{ env.REGISTRY }}/${{ github.repository }}/worker:${{ env.TAG }} - name: Run Specs env: # Specifies how many jobs you would like to run in parallel, @@ -87,7 +92,7 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} continue-on-error: true run: >- - docker-compose exec -T web bash -c + docker run exec -T ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" - name: Publish Test Report @@ -95,3 +100,4 @@ jobs: if: always() # always run even if the previous step fails with: report_paths: 'rspec*.xml' + fail_on_failure: true From e1128d2605bd35f38d5056fcfc6f0f5ff9cacce3 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 17:33:01 -0700 Subject: [PATCH 021/159] remover the confdir --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 55b245a..6714b02 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -38,7 +38,7 @@ jobs: ci_node_index: [0, 1, 2] env: ALLOW_ANONYMOUS_LOGIN: "yes" - CONFDIR: ${{ inputs.confdir }} + CONFDIR: /app/samvera/hyrax-webapp/solr/conf DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml From 3966f176d60033bedeffb8754f47b34693258cea Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 17:35:20 -0700 Subject: [PATCH 022/159] remover the confdir --- .github/workflows/test.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6714b02..245f893 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,9 +8,6 @@ on: tag: required: false type: string - confdir: - required: false - type: string workflow_call: inputs: From 224c6d6e204e3c889e4d2850950fc85ad19404ce Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 18:52:40 -0700 Subject: [PATCH 023/159] Updatung the retag to print out some information for troubleshooting, i forgot to take the other docker run exec to just docker run, and adding back the two commas after receiving feedback, thanks Rob, totally thought it was a typo --- .github/workflows/retag.yaml | 12 ++++++++++-- .github/workflows/test.yaml | 3 +-- setup-env/action.yaml | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index 6139dba..227b565 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -25,11 +25,19 @@ jobs: runs-on: self-hosted if: ${{ github.ref == 'refs/heads/main' }} steps: + - name: Tell me what's my TAG Before + run: >- + echo "My env.tag is: '${{ env.TAG }}'" + echo "My inputs.tag is '${{ inputs.tag }}'" - id: setup name: Setup uses: scientist-softserv/actions/setup-env@self-hosted-runner with: - tag: ${{ inputs.tag }} || ${{ env.TAG }} + tag: ${{ inputs.tag || env.TAG }} + - name: Tell me what's my TAG After the inputs.tag override + run: >- + echo "Don't forget our inputs.tag is ${{ inputs.tag }}" + echo "And now inputs.tag should override env.tag from the input in Stage 'Tell me what's my TAG Before' to ${{ env.TAG }}, which should be a different value only if their was a value in inputs.tag" - name: Retag web to latest uses: koraykoska/retag-docker-image@0.2.4 with: @@ -48,4 +56,4 @@ jobs: new_tag: latest username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - if: ${{ inputs.worker }} + if: ${{ inputs.worker }} \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 245f893..a5ec38c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,7 +8,6 @@ on: tag: required: false type: string - workflow_call: inputs: worker: @@ -89,7 +88,7 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} continue-on-error: true run: >- - docker run exec -T ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c + docker exec -T ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" - name: Publish Test Report diff --git a/setup-env/action.yaml b/setup-env/action.yaml index a047f1d..f24bc7f 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -17,7 +17,7 @@ runs: shell: bash - name: Downcase repo run: | - echo "REPO_LOWER=${REPO}" >> ${GITHUB_ENV} + echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} env: REPO: "${{ github.repository }}" shell: bash From 9c34970c25fdc5c100e7b89ae2bbc4ce9ed915c1 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 20:07:06 -0700 Subject: [PATCH 024/159] move the -T flag not seeing that tty is set by default in the docker exec command equivalent --- .github/workflows/test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a5ec38c..964a533 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -67,13 +67,13 @@ jobs: run: docker pull ${{ env.REGISTRY }}/${{ github.repository }}/worker:${{ env.TAG }} - name: Setup solr run: >- - docker exec -T ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c + docker exec ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c "solrcloud-upload-configset.sh ${{ env.CONFDIR }} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" - name: Setup db run: >- - docker exec -T ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c + docker exec ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c "RAILS_ENV=test bundle exec rake db:schema:load db:migrate" - name: Start containers run: >- @@ -88,7 +88,7 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} continue-on-error: true run: >- - docker exec -T ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c + docker exec ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" - name: Publish Test Report From 94f451cc1ca18639dfa037096609468cfb31a334 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 20:23:39 -0700 Subject: [PATCH 025/159] prepping to merge into main --- .github/workflows/build.yaml | 2 +- .github/workflows/deploy.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/retag.yaml | 2 +- .github/workflows/test.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 17e5b45..99a41ce 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -44,7 +44,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner + uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 2fedc81..bd6b8a2 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -43,7 +43,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner + uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index f66ef52..eb4073b 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -26,7 +26,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner + uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index 227b565..cd02784 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -31,7 +31,7 @@ jobs: echo "My inputs.tag is '${{ inputs.tag }}'" - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner + uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag || env.TAG }} - name: Tell me what's my TAG After the inputs.tag override diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 964a533..fe8a523 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -41,7 +41,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner + uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag || env.TAG }} - name: Checkout code From 65036266fd3d67c9064e606a1396058810ade1b4 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 21:01:34 -0700 Subject: [PATCH 026/159] Update readme, clean up syntax, add fall back env.tag, replace github.repository with REPO_LOWER, inputs.worker need to be evaluated since boolean, adds inputs.confdir so we can pass it from the caller, moves the block to be in order of operation to help for visualization --- .github/workflows/build.yaml | 9 ++++----- .github/workflows/deploy.yaml | 2 +- .github/workflows/lint.yaml | 10 ++++------ .github/workflows/retag.yaml | 4 ++-- .github/workflows/test.yaml | 24 +++++++++++++++--------- README.md | 2 +- setup-env/action.yaml | 9 ++++----- 7 files changed, 31 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 99a41ce..9ac81c9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -46,14 +46,13 @@ jobs: name: Setup uses: scientist-softserv/actions/setup-env@main with: - tag: ${{ inputs.tag }} + tag: ${{ inputs.tag || env.TAG }} - name: Checkout code uses: actions/checkout@v3 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Create Docker Context - run: >- - docker context create builders + run: docker context create builders - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: @@ -68,7 +67,7 @@ jobs: run: >- touch .env.development; touch .env; - TAG=latest docker-compose pull web || true + TAG=latest docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Build and push uses: docker/build-push-action@v3 with: @@ -82,6 +81,7 @@ jobs: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Build and push worker uses: docker/build-push-action@v3 + if: ${{ inputs.worker == 'true' }} with: context: . platforms: ${{ inputs.platforms }} @@ -91,4 +91,3 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - if: ${{ inputs.worker }} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index bd6b8a2..fe143c8 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -45,7 +45,7 @@ jobs: name: Setup uses: scientist-softserv/actions/setup-env@main with: - tag: ${{ inputs.tag }} + tag: ${{ inputs.tag || env.TAG }} - name: Checkout code uses: actions/checkout@v3 - name: Setup tmate session diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index eb4073b..25cd5b6 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -28,7 +28,7 @@ jobs: name: Setup uses: scientist-softserv/actions/setup-env@main with: - tag: ${{ inputs.tag }} + tag: ${{ inputs.tag || env.TAG }} - name: Checkout code uses: actions/checkout@v3 - name: Github Container Login @@ -41,14 +41,12 @@ jobs: run: >- touch .env.development; touch .env; - docker pull ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} + docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Pull worker image to prevent build if: ${{ inputs.worker == 'true' }} - run: >- - docker pull ${{ env.REGISTRY }}/${{ github.repository }}/worker:${{ env.TAG }} + run: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Run Rubocop - run: >- - docker run ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bundle exec rubocop --parallel --format progress + run: docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bundle exec rubocop --parallel --format progress - name: Publish Test Report uses: mikepenz/action-junit-report@v3 if: always() # always run even if the previous step fails diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index cd02784..de0881f 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -49,11 +49,11 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Retag worker to latest uses: koraykoska/retag-docker-image@0.2.4 + if: ${{ inputs.worker == 'true' }} with: registry: ${{ env.REGISTRY }} name: ${{ env.REPO_LOWER }}/worker old_tag: ${{ env.TAG }} new_tag: latest username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - if: ${{ inputs.worker }} \ No newline at end of file + password: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fe8a523..248b34d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,6 +8,9 @@ on: tag: required: false type: string + confdir: + required: false + type: string workflow_call: inputs: worker: @@ -16,6 +19,9 @@ on: tag: required: false type: string + confdir: + required: false + type: string env: REGISTRY: ghcr.io @@ -34,7 +40,7 @@ jobs: ci_node_index: [0, 1, 2] env: ALLOW_ANONYMOUS_LOGIN: "yes" - CONFDIR: /app/samvera/hyrax-webapp/solr/conf + CONFDIR: "/app/samvera/hyrax-webapp/solr/conf" DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml @@ -61,24 +67,24 @@ jobs: run: >- touch .env.development; touch .env; - docker pull ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} + docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Pull worker image to prevent build if: ${{ inputs.worker == 'true' }} - run: docker pull ${{ env.REGISTRY }}/${{ github.repository }}/worker:${{ env.TAG }} + run: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Setup solr run: >- - docker exec ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c - "solrcloud-upload-configset.sh ${{ env.CONFDIR }} && + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + "solrcloud-upload-configset.sh ${{ inputs.confdir }} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" - name: Setup db run: >- - docker exec ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c "RAILS_ENV=test bundle exec rake db:schema:load db:migrate" - name: Start containers run: >- - docker run ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} - docker run ${{ env.REGISTRY }}/${{ github.repository }}/worker:${{ env.TAG }} + docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Run Specs env: # Specifies how many jobs you would like to run in parallel, @@ -88,7 +94,7 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} continue-on-error: true run: >- - docker exec ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }} bash -c + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" - name: Publish Test Report diff --git a/README.md b/README.md index caac7ba..1700644 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # actions -Github action for Docker build, Rubocop linting, and running Rspec tests for Rails applications. +Github action for GitGuardian Scan, Docker build, Rubocop linting, and running Rspec tests for Rails applications configured to run on a self-hosted runner. diff --git a/setup-env/action.yaml b/setup-env/action.yaml index f24bc7f..096d30e 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -9,15 +9,14 @@ runs: using: "composite" steps: - name: Set env - run: >- - echo "TAG=${HEAD_TAG::8}" >> ${GITHUB_ENV}; - echo ${HEAD_TAG::8} env: HEAD_TAG: ${{ inputs.tag || github.event.pull_request.head.sha || github.sha }} shell: bash + run: >- + echo ${HEAD_TAG::8}; + echo "TAG=${HEAD_TAG::8}" >> ${GITHUB_ENV} - name: Downcase repo - run: | - echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} env: REPO: "${{ github.repository }}" shell: bash + run: echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} From d46a16c214219b23b578e70fd6ecd444e5b17cb1 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 5 Nov 2022 21:05:22 -0700 Subject: [PATCH 027/159] didn't save in order to push --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 248b34d..599ef56 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -40,7 +40,7 @@ jobs: ci_node_index: [0, 1, 2] env: ALLOW_ANONYMOUS_LOGIN: "yes" - CONFDIR: "/app/samvera/hyrax-webapp/solr/conf" + CONFDIR: ${{ inputs.confdir }} DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml From b1a9114957f8347ac7370b730988592f176be4f5 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 6 Nov 2022 11:03:53 -0800 Subject: [PATCH 028/159] remove the tag since we are tagging it latest. --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 599ef56..25f4a10 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -73,7 +73,7 @@ jobs: run: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Setup solr run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }} bash -c "solrcloud-upload-configset.sh ${{ inputs.confdir }} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" From d0654b1de2b9f69a9e0b58590cbc429914e4c6de Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 6 Nov 2022 11:07:49 -0800 Subject: [PATCH 029/159] latest tag, add it to the image versus declaring it as a variable bc it's not passed as latest --- .github/workflows/build.yaml | 14 +++++--------- .github/workflows/test.yaml | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9ac81c9..56b8202 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,18 +67,16 @@ jobs: run: >- touch .env.development; touch .env; - TAG=latest docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:latest - name: Build and push uses: docker/build-push-action@v3 with: context: . platforms: ${{ inputs.platforms }} target: ${{ inputs.target }} - cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + cache-from: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} push: true - tags: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Build and push worker uses: docker/build-push-action@v3 if: ${{ inputs.worker == 'true' }} @@ -86,8 +84,6 @@ jobs: context: . platforms: ${{ inputs.platforms }} target: ${{ inputs.workerTarget }} - cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + cache-from: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} push: true - tags: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 25f4a10..599ef56 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -73,7 +73,7 @@ jobs: run: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Setup solr run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }} bash -c + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c "solrcloud-upload-configset.sh ${{ inputs.confdir }} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" From 94209d0c6cd705b12fb861c21b74e82a4766b56d Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Sun, 6 Nov 2022 11:35:36 -0800 Subject: [PATCH 030/159] Update build.yaml inputs.worker is already a boolean no need to specify == 'true' --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 56b8202..7ae27d5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -79,7 +79,7 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Build and push worker uses: docker/build-push-action@v3 - if: ${{ inputs.worker == 'true' }} + if: ${{ inputs.worker }} with: context: . platforms: ${{ inputs.platforms }} From 4859b4efb603f442593af85fbbc93bbf41ef10e5 Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Sun, 6 Nov 2022 11:36:40 -0800 Subject: [PATCH 031/159] Update lint.yaml inputs.worker already == 'true' --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 25cd5b6..11c3af8 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -43,7 +43,7 @@ jobs: touch .env; docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Pull worker image to prevent build - if: ${{ inputs.worker == 'true' }} + if: ${{ inputs.worker }} run: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Run Rubocop run: docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bundle exec rubocop --parallel --format progress From 999f81dab0817d97b6d6a4b2aca14d15a3cf6130 Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Sun, 6 Nov 2022 11:37:21 -0800 Subject: [PATCH 032/159] Update retag.yaml inputs.worker already == 'true' --- .github/workflows/retag.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index de0881f..7173ca9 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -49,11 +49,11 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Retag worker to latest uses: koraykoska/retag-docker-image@0.2.4 - if: ${{ inputs.worker == 'true' }} + if: ${{ inputs.worker }} with: registry: ${{ env.REGISTRY }} name: ${{ env.REPO_LOWER }}/worker old_tag: ${{ env.TAG }} new_tag: latest username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + password: ${{ secrets.GITHUB_TOKEN }} From 8b6a7776f91446b0b66433ccd23e98d35b2f9a3f Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Sun, 6 Nov 2022 11:38:00 -0800 Subject: [PATCH 033/159] Update test.yaml inputs.worker already == 'true' --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 599ef56..3c634a3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,7 +69,7 @@ jobs: touch .env; docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Pull worker image to prevent build - if: ${{ inputs.worker == 'true' }} + if: ${{ inputs.worker }} run: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Setup solr run: >- From 27fc580b2f0549952d904ebcd31b8dd65ae5947f Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 6 Nov 2022 12:22:21 -0800 Subject: [PATCH 034/159] put things back - to a point where my need for order messed up the order of operations. things were there for a reason --- .github/workflows/build.yaml | 16 ++++++++++------ .github/workflows/deploy.yaml | 2 +- .github/workflows/lint.yaml | 7 ++++--- .github/workflows/retag.yaml | 10 +--------- .github/workflows/test.yaml | 14 +++++++------- setup-env/action.yaml | 7 ++++--- 6 files changed, 27 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7ae27d5..ce60e13 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -46,7 +46,7 @@ jobs: name: Setup uses: scientist-softserv/actions/setup-env@main with: - tag: ${{ inputs.tag || env.TAG }} + tag: ${{ inputs.tag }} - name: Checkout code uses: actions/checkout@v3 - name: Set up QEMU @@ -67,16 +67,18 @@ jobs: run: >- touch .env.development; touch .env; - docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:latest + TAG=latest pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} || true - name: Build and push uses: docker/build-push-action@v3 with: context: . platforms: ${{ inputs.platforms }} target: ${{ inputs.target }} - cache-from: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + cache-from: | + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} push: true - tags: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + tags: | + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Build and push worker uses: docker/build-push-action@v3 if: ${{ inputs.worker }} @@ -84,6 +86,8 @@ jobs: context: . platforms: ${{ inputs.platforms }} target: ${{ inputs.workerTarget }} - cache-from: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + cache-from: | + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} push: true - tags: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + tags: | + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} \ No newline at end of file diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index fe143c8..bd6b8a2 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -45,7 +45,7 @@ jobs: name: Setup uses: scientist-softserv/actions/setup-env@main with: - tag: ${{ inputs.tag || env.TAG }} + tag: ${{ inputs.tag }} - name: Checkout code uses: actions/checkout@v3 - name: Setup tmate session diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 11c3af8..69c7112 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -22,13 +22,13 @@ env: jobs: lint: - runs-on: self-hosted + runs-on: self-hosted steps: - id: setup name: Setup uses: scientist-softserv/actions/setup-env@main with: - tag: ${{ inputs.tag || env.TAG }} + tag: ${{ inputs.tag }} - name: Checkout code uses: actions/checkout@v3 - name: Github Container Login @@ -44,7 +44,8 @@ jobs: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Pull worker image to prevent build if: ${{ inputs.worker }} - run: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + run: >- + docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Run Rubocop run: docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bundle exec rubocop --parallel --format progress - name: Publish Test Report diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index 7173ca9..a538f91 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -25,19 +25,11 @@ jobs: runs-on: self-hosted if: ${{ github.ref == 'refs/heads/main' }} steps: - - name: Tell me what's my TAG Before - run: >- - echo "My env.tag is: '${{ env.TAG }}'" - echo "My inputs.tag is '${{ inputs.tag }}'" - id: setup name: Setup uses: scientist-softserv/actions/setup-env@main with: - tag: ${{ inputs.tag || env.TAG }} - - name: Tell me what's my TAG After the inputs.tag override - run: >- - echo "Don't forget our inputs.tag is ${{ inputs.tag }}" - echo "And now inputs.tag should override env.tag from the input in Stage 'Tell me what's my TAG Before' to ${{ env.TAG }}, which should be a different value only if their was a value in inputs.tag" + tag: ${{ inputs.tag }} - name: Retag web to latest uses: koraykoska/retag-docker-image@0.2.4 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3c634a3..27bbcea 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,7 +8,7 @@ on: tag: required: false type: string - confdir: + confdir: required: false type: string workflow_call: @@ -49,7 +49,7 @@ jobs: name: Setup uses: scientist-softserv/actions/setup-env@main with: - tag: ${{ inputs.tag || env.TAG }} + tag: ${{ inputs.tag }} - name: Checkout code uses: actions/checkout@v3 - name: Github Container Login @@ -70,7 +70,11 @@ jobs: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Pull worker image to prevent build if: ${{ inputs.worker }} - run: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + run: >- + docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + - name: Start containers + run: >- + docker run -d ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Setup solr run: >- docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c @@ -81,10 +85,6 @@ jobs: run: >- docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c "RAILS_ENV=test bundle exec rake db:schema:load db:migrate" - - name: Start containers - run: >- - docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Run Specs env: # Specifies how many jobs you would like to run in parallel, diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 096d30e..53df91d 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -13,10 +13,11 @@ runs: HEAD_TAG: ${{ inputs.tag || github.event.pull_request.head.sha || github.sha }} shell: bash run: >- - echo ${HEAD_TAG::8}; - echo "TAG=${HEAD_TAG::8}" >> ${GITHUB_ENV} + echo "TAG=${HEAD_TAG::8}" >> ${GITHUB_ENV}; + echo ${HEAD_TAG::8} - name: Downcase repo env: REPO: "${{ github.repository }}" shell: bash - run: echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} + run: | + echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} From 98df212214cf6725d7e496c5ff8237374aef8501 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 6 Nov 2022 12:27:48 -0800 Subject: [PATCH 035/159] yaml syntax really keeps you on your toes eh? --- .github/workflows/test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 27bbcea..afbc82b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,7 +8,7 @@ on: tag: required: false type: string - confdir: + confdir: required: false type: string workflow_call: @@ -102,4 +102,3 @@ jobs: if: always() # always run even if the previous step fails with: report_paths: 'rspec*.xml' - fail_on_failure: true From 30c060dc993446815648887d4491f9ed7d4dd277 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 6 Nov 2022 13:00:51 -0800 Subject: [PATCH 036/159] trying out a new config to troubleshoot why the docker exce image in the setup solr run doesnt have access or cant see the image with env.TAG --- .github/workflows/test.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index afbc82b..f52fb82 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -72,17 +72,16 @@ jobs: if: ${{ inputs.worker }} run: >- docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - - name: Start containers - run: >- - docker run -d ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Setup solr run: >- + docker run -d ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }}; docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c "solrcloud-upload-configset.sh ${{ inputs.confdir }} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" - name: Setup db run: >- + docker run -d ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }}; docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c "RAILS_ENV=test bundle exec rake db:schema:load db:migrate" - name: Run Specs From 9fdc9a73c50177238f8d469e4f4c4d9c83c0f6ff Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Sun, 6 Nov 2022 13:36:39 -0800 Subject: [PATCH 037/159] troubleshooting tools troubleshooting tools --- .github/workflows/test.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f52fb82..4405f1d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -72,16 +72,23 @@ jobs: if: ${{ inputs.worker }} run: >- docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - - name: Setup solr + - name: Start containers run: >- docker run -d ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }}; + docker ps; + ls -la; + docker logs ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} -f + - name: Sleep for 30 seconds + run: sleep 30s + shell: bash + - name: Setup solr + run: >- docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c "solrcloud-upload-configset.sh ${{ inputs.confdir }} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" - name: Setup db run: >- - docker run -d ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }}; docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c "RAILS_ENV=test bundle exec rake db:schema:load db:migrate" - name: Run Specs From d0c69f902ccdbd9118051ebbd48756f4f29872f5 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 6 Nov 2022 17:12:58 -0800 Subject: [PATCH 038/159] make it docker run like lint --- .github/workflows/build.yaml | 7 ++++--- .github/workflows/test.yaml | 19 ++++--------------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ce60e13..cfbcafe 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -51,13 +51,14 @@ jobs: uses: actions/checkout@v3 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - - name: Create Docker Context + - name: Set up Docker Context for Buildx run: docker context create builders - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v1 with: + version: latest endpoint: builders - - name: Github Container Login + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ${{ env.REGISTRY }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4405f1d..a8571c7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -58,11 +58,6 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - with: - limit-access-to-actor: true - name: Pull image to prevent build run: >- touch .env.development; @@ -74,22 +69,16 @@ jobs: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Start containers run: >- - docker run -d ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }}; - docker ps; - ls -la; - docker logs ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} -f - - name: Sleep for 30 seconds - run: sleep 30s - shell: bash + docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash; - name: Setup solr run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash; "solrcloud-upload-configset.sh ${{ inputs.confdir }} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" - name: Setup db run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash; "RAILS_ENV=test bundle exec rake db:schema:load db:migrate" - name: Run Specs env: @@ -100,7 +89,7 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} continue-on-error: true run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash; "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" - name: Publish Test Report From 2f43f7c58e26beed39860d2c7df23f001e0c676e Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 6 Nov 2022 17:14:41 -0800 Subject: [PATCH 039/159] testing out the fail_on_failure true flag --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a8571c7..ce28d5d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -97,3 +97,4 @@ jobs: if: always() # always run even if the previous step fails with: report_paths: 'rspec*.xml' + fail_on_failure: true From 40040ac465cabb3aeeffd62d0707f81372df2cb1 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 6 Nov 2022 20:59:10 -0800 Subject: [PATCH 040/159] tweaking for tests --- .github/workflows/test.yaml | 57 ++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ce28d5d..0bcd933 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -44,6 +44,48 @@ jobs: DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml + CHROME_HOSTNAME: chrome + DB_ADAPTER: postgresql + DB_HOST: db + DB_NAME: hyku + DB_URL: postgres://postgres:DatabaseFTW@db/hyku + DB_PASSWORD: DatabaseFTW + DB_TEST_NAME: hyku_test + DB_USER: postgres + DB_HOST: db + DB_PORT: 5432 + FCREPO_HOST: fcrepo + FCREPO_PORT: 8080 + FCREPO_URL: http://fcrepo:8080/rest + HYKU_BULKRAX_ENABLED: true + INITIAL_ADMIN_EMAIL: admin@example.com + INITIAL_ADMIN_PASSWORD: testing123 + IN_DOCKER: true + LD_LIBRARY_PATH: /opt/fits/tools/mediainfo/linux + PASSENGER_APP_ENV: development + RAILS_LOG_TO_STDOUT: true + REDIS_HOST: redis + SECRET_KEY_BASE: asdf + HYRAX_ACTIVE_JOB_QUEUE: sidekiq + HYRAX_FITS_PATH: /app/fits/fits.sh + SOLR_ADMIN_PASSWORD: SolrRocks + SOLR_ADMIN_USER: solr + SOLR_COLLECTION_NAME: hydra-development + SOLR_CONFIGSET_NAME: hyku + SOLR_HOST: solr + SOLR_PORT: 8983 + SOLR_URL: http://solr:SolrRocks@solr:8983/solr/ + HYKU_ADMIN_HOST: utk-hyku.test + HYKU_ADMIN_ONLY_TENANT_CREATION: false + HYKU_DEFAULT_HOST: %{tenant}.utk-hyku.test + HYKU_ROOT_HOST: utk-hyku.test + HYKU_MULTITENANT: true + GOOGLE_OAUTH_APP_NAME: UTK-Hyku + GOOGLE_OAUTHAPP_VERSION: 1.0 + GOOGLE_OAUTH_PRIVATE_KEY_PATH: cred.p12 + GOOGLE_OAUTH_PRIVATE_KEY_SECRET: notasecret + GOOGLE_OAUTH_CLIENT_EMAIL: hyku-analytics@hyku-analytics.iam.gserviceaccount.com + steps: - id: setup name: Setup @@ -69,17 +111,18 @@ jobs: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Start containers run: >- - docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash; + docker run -d ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Setup solr run: >- - docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash; - "solrcloud-upload-configset.sh ${{ inputs.confdir }} && + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c " + solrcloud-upload-configset.sh /app/samvera/hyrax-webapp/solr/conf && + solrcloud-assign-configset.sh && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" - name: Setup db run: >- - docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash; - "RAILS_ENV=test bundle exec rake db:schema:load db:migrate" + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c " + RAILS_ENV=test bundle exec rake db:schema:load db:migrate" - name: Run Specs env: # Specifies how many jobs you would like to run in parallel, @@ -89,8 +132,8 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} continue-on-error: true run: >- - docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash; - "gem install semaphore_test_boosters && + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c " + gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" - name: Publish Test Report uses: mikepenz/action-junit-report@v3 From 09510108090e6d56a88f3fdba63bfa161c593767 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 6 Nov 2022 21:02:16 -0800 Subject: [PATCH 041/159] syntax --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0bcd933..a4ec7f8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -77,7 +77,6 @@ jobs: SOLR_URL: http://solr:SolrRocks@solr:8983/solr/ HYKU_ADMIN_HOST: utk-hyku.test HYKU_ADMIN_ONLY_TENANT_CREATION: false - HYKU_DEFAULT_HOST: %{tenant}.utk-hyku.test HYKU_ROOT_HOST: utk-hyku.test HYKU_MULTITENANT: true GOOGLE_OAUTH_APP_NAME: UTK-Hyku From a69e592af4b0c74d85e9bf8e3ecd08b40e4d276c Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Sun, 6 Nov 2022 21:08:42 -0800 Subject: [PATCH 042/159] Update test.yaml --- .github/workflows/test.yaml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a4ec7f8..56769d4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -44,7 +44,6 @@ jobs: DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml - CHROME_HOSTNAME: chrome DB_ADAPTER: postgresql DB_HOST: db DB_NAME: hyku @@ -52,22 +51,11 @@ jobs: DB_PASSWORD: DatabaseFTW DB_TEST_NAME: hyku_test DB_USER: postgres - DB_HOST: db DB_PORT: 5432 FCREPO_HOST: fcrepo FCREPO_PORT: 8080 FCREPO_URL: http://fcrepo:8080/rest - HYKU_BULKRAX_ENABLED: true - INITIAL_ADMIN_EMAIL: admin@example.com - INITIAL_ADMIN_PASSWORD: testing123 - IN_DOCKER: true - LD_LIBRARY_PATH: /opt/fits/tools/mediainfo/linux - PASSENGER_APP_ENV: development - RAILS_LOG_TO_STDOUT: true REDIS_HOST: redis - SECRET_KEY_BASE: asdf - HYRAX_ACTIVE_JOB_QUEUE: sidekiq - HYRAX_FITS_PATH: /app/fits/fits.sh SOLR_ADMIN_PASSWORD: SolrRocks SOLR_ADMIN_USER: solr SOLR_COLLECTION_NAME: hydra-development @@ -75,15 +63,6 @@ jobs: SOLR_HOST: solr SOLR_PORT: 8983 SOLR_URL: http://solr:SolrRocks@solr:8983/solr/ - HYKU_ADMIN_HOST: utk-hyku.test - HYKU_ADMIN_ONLY_TENANT_CREATION: false - HYKU_ROOT_HOST: utk-hyku.test - HYKU_MULTITENANT: true - GOOGLE_OAUTH_APP_NAME: UTK-Hyku - GOOGLE_OAUTHAPP_VERSION: 1.0 - GOOGLE_OAUTH_PRIVATE_KEY_PATH: cred.p12 - GOOGLE_OAUTH_PRIVATE_KEY_SECRET: notasecret - GOOGLE_OAUTH_CLIENT_EMAIL: hyku-analytics@hyku-analytics.iam.gserviceaccount.com steps: - id: setup From b7165e8331cdddd7763d52b4c6c97d1bf4abe1ec Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 6 Nov 2022 23:07:02 -0800 Subject: [PATCH 043/159] tweaks for tests --- .github/workflows/build.yaml | 4 ++-- .github/workflows/test.yaml | 38 +++++++++++------------------------- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cfbcafe..a27edd6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -68,7 +68,7 @@ jobs: run: >- touch .env.development; touch .env; - TAG=latest pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} || true + TAG=latest docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} || true - name: Build and push uses: docker/build-push-action@v3 with: @@ -91,4 +91,4 @@ jobs: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} push: true tags: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} \ No newline at end of file + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 56769d4..48826ee 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -44,26 +44,6 @@ jobs: DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml - DB_ADAPTER: postgresql - DB_HOST: db - DB_NAME: hyku - DB_URL: postgres://postgres:DatabaseFTW@db/hyku - DB_PASSWORD: DatabaseFTW - DB_TEST_NAME: hyku_test - DB_USER: postgres - DB_PORT: 5432 - FCREPO_HOST: fcrepo - FCREPO_PORT: 8080 - FCREPO_URL: http://fcrepo:8080/rest - REDIS_HOST: redis - SOLR_ADMIN_PASSWORD: SolrRocks - SOLR_ADMIN_USER: solr - SOLR_COLLECTION_NAME: hydra-development - SOLR_CONFIGSET_NAME: hyku - SOLR_HOST: solr - SOLR_PORT: 8983 - SOLR_URL: http://solr:SolrRocks@solr:8983/solr/ - steps: - id: setup name: Setup @@ -78,6 +58,11 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + with: + limit-access-to-actor: true - name: Pull image to prevent build run: >- touch .env.development; @@ -92,15 +77,14 @@ jobs: docker run -d ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Setup solr run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c " - solrcloud-upload-configset.sh /app/samvera/hyrax-webapp/solr/conf && - solrcloud-assign-configset.sh && + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + "solrcloud-upload-configset.sh ${{ inputs.confdir }} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" - name: Setup db run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c " - RAILS_ENV=test bundle exec rake db:schema:load db:migrate" + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + "RAILS_ENV=test bundle exec rake db:schema:load db:migrate" - name: Run Specs env: # Specifies how many jobs you would like to run in parallel, @@ -110,8 +94,8 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} continue-on-error: true run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c " - gem install semaphore_test_boosters && + docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" - name: Publish Test Report uses: mikepenz/action-junit-report@v3 From 63c871bddf4f777788028b41fd74dee49695f571 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 13:44:30 -0800 Subject: [PATCH 044/159] Testing docker-compose commands on the runner again --- .github/workflows/test.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 48826ee..56820ca 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -74,16 +74,16 @@ jobs: docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - name: Start containers run: >- - docker run -d ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + docker-compose up -d web - name: Setup solr run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + docker-compose exec -T web bash -c "solrcloud-upload-configset.sh ${{ inputs.confdir }} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" - name: Setup db run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + docker-compose exec -T web bash -c "RAILS_ENV=test bundle exec rake db:schema:load db:migrate" - name: Run Specs env: @@ -94,7 +94,7 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} continue-on-error: true run: >- - docker exec ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bash -c + docker-compose exec -T web bash -c "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" - name: Publish Test Report @@ -102,4 +102,3 @@ jobs: if: always() # always run even if the previous step fails with: report_paths: 'rspec*.xml' - fail_on_failure: true From a922a104fcadd4a97251f9f8b1b5d409a53abba8 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 14:22:28 -0800 Subject: [PATCH 045/159] reverts to docker-compose commands --- .github/workflows/build.yaml | 2 +- .github/workflows/lint.yaml | 8 ++++---- .github/workflows/test.yaml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a27edd6..6d190d7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -68,7 +68,7 @@ jobs: run: >- touch .env.development; touch .env; - TAG=latest docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} || true + TAG=latest docker-compose pull web || true - name: Build and push uses: docker/build-push-action@v3 with: diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 69c7112..3e86d29 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -22,7 +22,7 @@ env: jobs: lint: - runs-on: self-hosted + runs-on: self-hosted steps: - id: setup name: Setup @@ -41,13 +41,13 @@ jobs: run: >- touch .env.development; touch .env; - docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + docker-compose pull web - name: Pull worker image to prevent build if: ${{ inputs.worker }} run: >- - docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + docker-compose pull worker - name: Run Rubocop - run: docker run ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} bundle exec rubocop --parallel --format progress + run: docker-compose run web bundle exec rubocop --parallel --format progress - name: Publish Test Report uses: mikepenz/action-junit-report@v3 if: always() # always run even if the previous step fails diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 56820ca..8d28eab 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -67,11 +67,11 @@ jobs: run: >- touch .env.development; touch .env; - docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + docker-compose pull web - name: Pull worker image to prevent build if: ${{ inputs.worker }} run: >- - docker pull ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + docker-compose pull worker - name: Start containers run: >- docker-compose up -d web From 4110ad0878b218796ab0671d72fa688e5a19663f Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Mon, 7 Nov 2022 16:47:19 -0800 Subject: [PATCH 046/159] try installing docker-compose on setup --- setup-env/action.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 53df91d..5a57396 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -21,3 +21,10 @@ runs: shell: bash run: | echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} + - name: Make sure docker-compose is installed + run: | + [ ! -f /home/runner/bin/docker-compose ] && { + mkdir /home/runner/bin; + curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; + chmod +x /home/runner/bin/docker-compose + } From 02747e49ee9b50d4b85f40fc30527d598e7aaa58 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 17:16:22 -0800 Subject: [PATCH 047/159] update syntax, update branch name, remove gitguardian and refrences --- .github/workflows/build.yaml | 2 +- .github/workflows/deploy.yaml | 2 +- .github/workflows/gitguardian.yaml | 25 ------------------------- .github/workflows/lint.yaml | 2 +- .github/workflows/retag.yaml | 2 +- .github/workflows/test.yaml | 2 +- README.md | 2 +- 7 files changed, 6 insertions(+), 31 deletions(-) delete mode 100644 .github/workflows/gitguardian.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6d190d7..92d5a74 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -44,7 +44,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index bd6b8a2..2fedc81 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -43,7 +43,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/gitguardian.yaml b/.github/workflows/gitguardian.yaml deleted file mode 100644 index 17417a8..0000000 --- a/.github/workflows/gitguardian.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: GitGuardian scan -on: - workflow_dispatch: - workflow_call: - -env: - REGISTRY: ghcr.io - -jobs: - scan: - name: GitGuardian scan - runs-on: self-hosted - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 # fetch all history so multiple commits can be scanned - - name: GitGuardian scan - uses: GitGuardian/ggshield-action@master - env: - GITHUB_PUSH_BEFORE_SHA: ${{ github.event.before }} - GITHUB_PUSH_BASE_SHA: ${{ github.event.base }} - GITHUB_PULL_BASE_SHA: ${{ github.event.pull_request.base.sha }} - GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 3e86d29..f2ae12c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -26,7 +26,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index a538f91..103ff96 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -27,7 +27,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - name: Retag web to latest diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8d28eab..98729ed 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,7 +47,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/README.md b/README.md index 1700644..eef2743 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # actions -Github action for GitGuardian Scan, Docker build, Rubocop linting, and running Rspec tests for Rails applications configured to run on a self-hosted runner. +Github action for Docker build, Rubocop linting, and running Rspec tests for Rails applications configured to run on a self-hosted runner. From 0c3446ad1df6d84e171cfff5dc894e16bbbe1d49 Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Mon, 7 Nov 2022 17:45:40 -0800 Subject: [PATCH 048/159] Update action.yaml --- setup-env/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 5a57396..e0c9f94 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -25,6 +25,6 @@ runs: run: | [ ! -f /home/runner/bin/docker-compose ] && { mkdir /home/runner/bin; - curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; - chmod +x /home/runner/bin/docker-compose + curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; + chmod +x /home/runner/bin/docker-compose } From 8b6140fcfa167e4698caf30c284231a0d23f95ce Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 17:50:05 -0800 Subject: [PATCH 049/159] pretty-please --- setup-env/action.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index e0c9f94..e1b6bb3 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -25,6 +25,5 @@ runs: run: | [ ! -f /home/runner/bin/docker-compose ] && { mkdir /home/runner/bin; - curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; - chmod +x /home/runner/bin/docker-compose - } + curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose; + chmod +x /home/runner/bin/docker-compose } From 595aa577c72e26e09c8827472c8e92ddb3ef3b17 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 17:53:03 -0800 Subject: [PATCH 050/159] ok now it wants accesss to the shell we are running the script with -- here ya go --- setup-env/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index e1b6bb3..134736f 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -22,6 +22,7 @@ runs: run: | echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} - name: Make sure docker-compose is installed + shell: bash run: | [ ! -f /home/runner/bin/docker-compose ] && { mkdir /home/runner/bin; From 7b08839f130bc28bf44b9fe2c328415fd913dbf5 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 19:48:37 -0800 Subject: [PATCH 051/159] Updating the script syntax take 1 --- setup-env/action.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 134736f..1c3433d 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -24,7 +24,9 @@ runs: - name: Make sure docker-compose is installed shell: bash run: | - [ ! -f /home/runner/bin/docker-compose ] && { - mkdir /home/runner/bin; - curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose; - chmod +x /home/runner/bin/docker-compose } + if [[ ! -f /home/runner/bin/docker-compose ]] + then + mkdir /home/runner/bin \ + curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose \ + chmod +x /home/runner/bin/docker-compose + fi From 9af2ed18c89f2cab301dd4578e1388f1b7867a99 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 19:58:46 -0800 Subject: [PATCH 052/159] trying this because I saw it somewhere and testing it here --- setup-env/action.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 1c3433d..d3d23bc 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -26,7 +26,8 @@ runs: run: | if [[ ! -f /home/runner/bin/docker-compose ]] then - mkdir /home/runner/bin \ - curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose \ + echo "The file docker-compose file does not exist yet!" + mkdir /home/runner/bin + curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose chmod +x /home/runner/bin/docker-compose fi From 77402a027a79c0fc3c491b7593a7b1f7d03228d7 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 20:07:15 -0800 Subject: [PATCH 053/159] This works in the runners bash --- setup-env/action.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index d3d23bc..f27187f 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -24,10 +24,9 @@ runs: - name: Make sure docker-compose is installed shell: bash run: | - if [[ ! -f /home/runner/bin/docker-compose ]] - then - echo "The file docker-compose file does not exist yet!" - mkdir /home/runner/bin - curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose + if [[ ! -f /home/runner/bin/docker-compose ]]; then + echo "The file docker-compose file does not exist yet!"; + mkdir /home/runner/bin; + curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; chmod +x /home/runner/bin/docker-compose fi From 0ab79f26b92baa9eb20ed869b8d49b3e9265d631 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 20:16:12 -0800 Subject: [PATCH 054/159] Shoot in the dark, im t o blame.... i give code, a bad name --- setup-env/action.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index f27187f..fc549b2 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -25,8 +25,7 @@ runs: shell: bash run: | if [[ ! -f /home/runner/bin/docker-compose ]]; then - echo "The file docker-compose file does not exist yet!"; - mkdir /home/runner/bin; - curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; + mkdir /home/runner/bin; \ + curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; \ chmod +x /home/runner/bin/docker-compose fi From 6b911ab280752a7eff2fa4dd15be366e98162186 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 20:30:11 -0800 Subject: [PATCH 055/159] tweaks for testing the script --- setup-env/action.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index fc549b2..c2a8178 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -25,7 +25,10 @@ runs: shell: bash run: | if [[ ! -f /home/runner/bin/docker-compose ]]; then - mkdir /home/runner/bin; \ - curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; \ - chmod +x /home/runner/bin/docker-compose + mkdir /home/runner/bin; + curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; + chmod +x /home/runner/bin/docker-compose; + sudo ls -la /home/runner/bin/ + else + echo "docker-compose is already installed" fi From 183ee139f412667413a3fc38b77391cd1075fc5a Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 23:14:01 -0800 Subject: [PATCH 056/159] Following documentation to find the path it needs for docker-compose https://github.com/docker/cli/issues/1534 --- setup-env/action.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index c2a8178..d4c202f 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -24,11 +24,11 @@ runs: - name: Make sure docker-compose is installed shell: bash run: | - if [[ ! -f /home/runner/bin/docker-compose ]]; then - mkdir /home/runner/bin; - curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; - chmod +x /home/runner/bin/docker-compose; - sudo ls -la /home/runner/bin/ + if [[ ! -f /usr/libexec/docker/cli-plugins/docker-compose ]]; then + mkdir /usr/libexec/docker/cli-plugins; + curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /usr/libexec/docker/cli-plugins/docker-compose ; + chmod +x /usr/libexec/docker/cli-plugins/docker-compose; + sudo ls -la /usr/libexec/docker/cli-plugins else echo "docker-compose is already installed" fi From 2a17977de2a4f3f34a0cf1de4522c61cbf602e14 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 23:39:04 -0800 Subject: [PATCH 057/159] tweaksz --- setup-env/action.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index d4c202f..8e730de 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -24,11 +24,12 @@ runs: - name: Make sure docker-compose is installed shell: bash run: | - if [[ ! -f /usr/libexec/docker/cli-plugins/docker-compose ]]; then - mkdir /usr/libexec/docker/cli-plugins; - curl -L "https://github.com/docker/compose/releases/download/v2.6.0/docker-compose-Linux-x86_64" -o /usr/libexec/docker/cli-plugins/docker-compose ; - chmod +x /usr/libexec/docker/cli-plugins/docker-compose; + if [[ ! -f $DOCKER_CONFIG/cli-plugins/docker-compose ]]; then + DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}; + mkdir -p $DOCKER_CONFIG/cli-plugins; + curl -SL https://github.com/docker/compose/releases/download/v2.12.2/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose ; sudo ls -la /usr/libexec/docker/cli-plugins else echo "docker-compose is already installed" fi + From 7c4a98844fbd74d9ef28f469db9d8dfe0a684be1 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 23:42:08 -0800 Subject: [PATCH 058/159] I am tired but so close --- setup-env/action.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 8e730de..b9161d7 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -28,8 +28,9 @@ runs: DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}; mkdir -p $DOCKER_CONFIG/cli-plugins; curl -SL https://github.com/docker/compose/releases/download/v2.12.2/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose ; - sudo ls -la /usr/libexec/docker/cli-plugins + sudo ls -la $DOCKER_CONFIG/cli-plugins else - echo "docker-compose is already installed" + echo "$DOCKER_CONFIG/cli-plugins/docker-compose is already installed and given the below permisisons:" + sudo ls -la $DOCKER_CONFIG/cli-plugins fi From 345f0966b1e7fdde769d66f178f89eec684fed22 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 7 Nov 2022 23:45:58 -0800 Subject: [PATCH 059/159] add the chmod +x --- setup-env/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index b9161d7..413dac0 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -28,6 +28,7 @@ runs: DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}; mkdir -p $DOCKER_CONFIG/cli-plugins; curl -SL https://github.com/docker/compose/releases/download/v2.12.2/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose ; + chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose; sudo ls -la $DOCKER_CONFIG/cli-plugins else echo "$DOCKER_CONFIG/cli-plugins/docker-compose is already installed and given the below permisisons:" From 8a11ea625ae252c6679f8678801d5735601e46af Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 8 Nov 2022 08:41:25 -0800 Subject: [PATCH 060/159] sudo? --- setup-env/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 413dac0..29d83a1 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -28,7 +28,7 @@ runs: DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}; mkdir -p $DOCKER_CONFIG/cli-plugins; curl -SL https://github.com/docker/compose/releases/download/v2.12.2/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose ; - chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose; + sudo chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose; sudo ls -la $DOCKER_CONFIG/cli-plugins else echo "$DOCKER_CONFIG/cli-plugins/docker-compose is already installed and given the below permisisons:" From 5b4f905083ea273f5274d620864c9e3e2ffecb8f Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 8 Nov 2022 08:43:27 -0800 Subject: [PATCH 061/159] print helpful info --- setup-env/action.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 29d83a1..5cd8196 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -29,9 +29,10 @@ runs: mkdir -p $DOCKER_CONFIG/cli-plugins; curl -SL https://github.com/docker/compose/releases/download/v2.12.2/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose ; sudo chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose; - sudo ls -la $DOCKER_CONFIG/cli-plugins + sudo ls -la $DOCKER_CONFIG/cli-plugins; + echo "docker compose version" else - echo "$DOCKER_CONFIG/cli-plugins/docker-compose is already installed and given the below permisisons:" + echo "$DOCKER_CONFIG/cli-plugins/docker-compose is already installed and given the below permisisons and version:" sudo ls -la $DOCKER_CONFIG/cli-plugins + echo "docker compose version" fi - From b8c3141eac6ca0cce701059dae3382d8c58f67f0 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 8 Nov 2022 12:47:58 -0800 Subject: [PATCH 062/159] update script --- .github/workflows/build.yaml | 2 +- .github/workflows/lint.yaml | 3 +-- setup-env/action.yaml | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 92d5a74..07de43b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -81,8 +81,8 @@ jobs: tags: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Build and push worker - uses: docker/build-push-action@v3 if: ${{ inputs.worker }} + uses: docker/build-push-action@v3 with: context: . platforms: ${{ inputs.platforms }} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index f2ae12c..c6f385f 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -40,12 +40,11 @@ jobs: - name: Pull image to prevent build run: >- touch .env.development; - touch .env; docker-compose pull web - name: Pull worker image to prevent build - if: ${{ inputs.worker }} run: >- docker-compose pull worker + if: ${{ inputs.worker }} - name: Run Rubocop run: docker-compose run web bundle exec rubocop --parallel --format progress - name: Publish Test Report diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 2a1f267..a6fa56c 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -25,6 +25,7 @@ runs: run: | if [ ! -f $DOCKER_CONFIG/cli-plugins/docker-compose ] then + echo "which docker" && DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} && mkdir -p $DOCKER_CONFIG/cli-plugins && curl -SL https://github.com/docker/compose/releases/download/v2.12.2/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose && From 41922ea62df83b36a25f4e2cb75e1c556adcf75f Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 8 Nov 2022 13:14:35 -0800 Subject: [PATCH 063/159] update to add docker-compose to the runer --- .github/workflows/test.yaml | 1 + setup-env/action.yaml | 20 ++++---------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fbfa79c..4a6b5a3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -75,6 +75,7 @@ jobs: - name: Start containers run: docker-compose up -d web - name: Check for and setup Solr if needed + shell: bash run: | if [ -d solr ] then diff --git a/setup-env/action.yaml b/setup-env/action.yaml index a6fa56c..2015b9b 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -21,19 +21,7 @@ runs: shell: bash run: | echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} - - name: Make sure docker-compose is installed - run: | - if [ ! -f $DOCKER_CONFIG/cli-plugins/docker-compose ] - then - echo "which docker" && - DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} && - mkdir -p $DOCKER_CONFIG/cli-plugins && - curl -SL https://github.com/docker/compose/releases/download/v2.12.2/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose && - sudo chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose && - sudo ls -la $DOCKER_CONFIG/cli-plugins && - echo "docker compose version" - else - echo "$DOCKER_CONFIG/cli-plugins/docker-compose is already installed and given the below permisisons and version:" && - sudo ls -la $DOCKER_CONFIG/cli-plugins && - echo "docker compose version" - fi + - name: Install docker-compose + uses: KengoTODA/actions-setup-docker-compose@main + with: + version: '1.29.2' \ No newline at end of file From bcba554349f6f2ab4d0eeb164566091f21468512 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 8 Nov 2022 14:14:25 -0800 Subject: [PATCH 064/159] be patient docker --- setup-env/action.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 2015b9b..2759601 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -24,4 +24,8 @@ runs: - name: Install docker-compose uses: KengoTODA/actions-setup-docker-compose@main with: - version: '1.29.2' \ No newline at end of file + version: '1.29.2' + shell: bash + run: >- + echo "export DOCKER_CLIENT_TIMEOUT=120" + echo "export COMPOSE_HTTP_TIMEOUT=120" From e9874ca4b3461664c5c367f7dc45412b111ba127 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 8 Nov 2022 14:21:58 -0800 Subject: [PATCH 065/159] move to env to the env setup section --- setup-env/action.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 2759601..2a4bb2d 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -14,7 +14,9 @@ runs: shell: bash run: >- echo "TAG=${HEAD_TAG::8}" >> ${GITHUB_ENV}; - echo ${HEAD_TAG::8} + echo ${HEAD_TAG::8}; + echo "export DOCKER_CLIENT_TIMEOUT=120"; + echo "export COMPOSE_HTTP_TIMEOUT=120" - name: Downcase repo env: REPO: "${{ github.repository }}" @@ -25,7 +27,3 @@ runs: uses: KengoTODA/actions-setup-docker-compose@main with: version: '1.29.2' - shell: bash - run: >- - echo "export DOCKER_CLIENT_TIMEOUT=120" - echo "export COMPOSE_HTTP_TIMEOUT=120" From df6db937df7a9ff193ac53c0b0510c73b328f8c8 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 8 Nov 2022 15:53:05 -0800 Subject: [PATCH 066/159] first you build it --- .github/workflows/build.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 25ac19b..dce5107 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,6 +5,9 @@ on: platforms: default: "linux/amd64,linux/arm64" type: string + subdir: + default: '.' + type: sting tag: required: false type: string @@ -22,6 +25,9 @@ on: platforms: default: "linux/amd64,linux/arm64" type: string + subdir: + default: '.' + type: sting tag: required: false type: string @@ -61,13 +67,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Pull from cache to speed up build run: >- + cd ${{ input.subdir }}; touch .env.development; touch .env; TAG=latest docker-compose pull web || true - name: Build and push uses: docker/build-push-action@v3 with: - context: . + context: ${{ input.subdir }} platforms: ${{ inputs.platforms }} target: ${{ inputs.target }} cache-from: | @@ -78,7 +85,7 @@ jobs: - name: Build and push worker uses: docker/build-push-action@v3 with: - context: . + context: ${{ input.subdir }} platforms: ${{ inputs.platforms }} target: ${{ inputs.workerTarget }} cache-from: | From 86cf19439756e53d8a0c98bda797349df16e4d28 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 8 Nov 2022 15:57:02 -0800 Subject: [PATCH 067/159] Roxaaannne --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index dce5107..ee0bdd6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,7 +7,7 @@ on: type: string subdir: default: '.' - type: sting + type: string tag: required: false type: string @@ -27,7 +27,7 @@ on: type: string subdir: default: '.' - type: sting + type: string tag: required: false type: string From f6002eddf5df04214effd843494cb3993b35de55 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 8 Nov 2022 16:02:06 -0800 Subject: [PATCH 068/159] SSSSSSS --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ee0bdd6..cf0eafc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,14 +67,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Pull from cache to speed up build run: >- - cd ${{ input.subdir }}; + cd ${{ inputs.subdir }}; touch .env.development; touch .env; TAG=latest docker-compose pull web || true - name: Build and push uses: docker/build-push-action@v3 with: - context: ${{ input.subdir }} + context: ${{ inputs.subdir }} platforms: ${{ inputs.platforms }} target: ${{ inputs.target }} cache-from: | @@ -85,7 +85,7 @@ jobs: - name: Build and push worker uses: docker/build-push-action@v3 with: - context: ${{ input.subdir }} + context: ${{ inputs.subdir }} platforms: ${{ inputs.platforms }} target: ${{ inputs.workerTarget }} cache-from: | From b15ccdfb55645a6da7c3a7f5e9aae048711a7e62 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 8 Nov 2022 16:09:03 -0800 Subject: [PATCH 069/159] jio uses root as context --- .github/workflows/build.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cf0eafc..ab13871 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,7 +74,8 @@ jobs: - name: Build and push uses: docker/build-push-action@v3 with: - context: ${{ inputs.subdir }} + context: . + file: ${{ inputs.subdir}}/Dockerfile platforms: ${{ inputs.platforms }} target: ${{ inputs.target }} cache-from: | @@ -85,7 +86,8 @@ jobs: - name: Build and push worker uses: docker/build-push-action@v3 with: - context: ${{ inputs.subdir }} + context: . + file: ${{ inputs.subdir}}/Dockerfile platforms: ${{ inputs.platforms }} target: ${{ inputs.workerTarget }} cache-from: | From b213875ff1cfbbe1680708a34858cb27c3477286 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 8 Nov 2022 16:16:49 -0800 Subject: [PATCH 070/159] support image names --- .github/workflows/build.yaml | 5 +++++ setup-env/action.yaml | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ab13871..9e80dd5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -2,6 +2,8 @@ name: "Build Docker Images" on: workflow_dispatch: inputs: + image_name: + type: string platforms: default: "linux/amd64,linux/arm64" type: string @@ -22,6 +24,8 @@ on: type: string workflow_call: inputs: + image_name: + type: string platforms: default: "linux/amd64,linux/arm64" type: string @@ -53,6 +57,7 @@ jobs: uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag }} + image_name: ${{ inputs.image_name }} - name: Checkout code uses: actions/checkout@v3 - name: Set up QEMU diff --git a/setup-env/action.yaml b/setup-env/action.yaml index f24bc7f..8b46d5c 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -4,6 +4,9 @@ inputs: tag: required: false type: string + image_name: + required: false + type: string runs: using: "composite" @@ -19,5 +22,5 @@ runs: run: | echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} env: - REPO: "${{ github.repository }}" + REPO: "${{ inputs.image_name || github.repository }}" shell: bash From 425d2beb588324e36dfbc305253af0bbcdab3613 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 9 Nov 2022 09:27:31 -0800 Subject: [PATCH 071/159] updates the build to include a boolean for solr since arce needs solr built and pushed to the registry --- .github/workflows/build.yaml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 07de43b..33f7e86 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,6 +5,9 @@ on: platforms: default: "linux/amd64,linux/arm64" type: string + solr: + required: false + type: boolean tag: required: false type: string @@ -22,6 +25,9 @@ on: platforms: default: "linux/amd64,linux/arm64" type: string + solr: + required: false + type: boolean tag: required: false type: string @@ -69,7 +75,7 @@ jobs: touch .env.development; touch .env; TAG=latest docker-compose pull web || true - - name: Build and push + - name: Build and push web uses: docker/build-push-action@v3 with: context: . @@ -92,3 +98,14 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + - name: Build and push solr + if: ${{ inputs.solr }} + uses: docker/build-push-action@v3 + with: + context: solr + platforms: ${{ inputs.platforms }} + cache-from: | + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} From c7184884b3c4fe38d686a60cf2f51ab883c3b866 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 9 Nov 2022 11:31:15 -0800 Subject: [PATCH 072/159] build it and tag as latest so I can get both architecture images built and tagged as latest and then remove the build step --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 33f7e86..dcec7da 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -108,4 +108,4 @@ jobs: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} push: true tags: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:latest From 1198558111f5680c1df4f6d914a5d8ba40f06a87 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 9 Nov 2022 11:34:47 -0800 Subject: [PATCH 073/159] we have a registry, why not use it for the cache --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index dcec7da..4fb0f6a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -94,7 +94,7 @@ jobs: platforms: ${{ inputs.platforms }} target: ${{ inputs.workerTarget }} cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} push: true tags: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} From 51083411651aad9831c5195d50e6a8b3a3b0c114 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 9 Nov 2022 12:40:58 -0800 Subject: [PATCH 074/159] providing solr build support for arces use-case --- .github/workflows/build.yaml | 3 ++- .github/workflows/deploy.yaml | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4fb0f6a..6159443 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -6,6 +6,7 @@ on: default: "linux/amd64,linux/arm64" type: string solr: + description: "Use if you need to build a specific Solr image and push to the registry" required: false type: boolean tag: @@ -108,4 +109,4 @@ jobs: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} push: true tags: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:latest + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 2fedc81..163b398 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -8,6 +8,10 @@ on: k8s-namespace: required: false type: string + solr: + description: "Add if your solr image is from the registry" + required: false + type: boolean tag: description: "Tag to override default of current branch sha" required: false @@ -53,8 +57,19 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} with: limit-access-to-actor: true + - name: Do deploy with solr registry image + if: ${{ inputs.solr }} + run: | + echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; + DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; + export DEPLOY_TAG=${TAG}; + export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; + export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; + export SOLR_IMAGE=ghcr.io/${REPO_LOWER}/solr; + ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} - name: Do deploy - run: >- + if: ${{ inputs.solr == 'false' }} + run: | echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; From 6c491840edfa8150d0fa2a1455c04a2276562735 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 9 Nov 2022 13:35:53 -0800 Subject: [PATCH 075/159] providing solr build support for arces use-case --- .github/workflows/deploy.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 163b398..adb43b8 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -68,7 +68,6 @@ jobs: export SOLR_IMAGE=ghcr.io/${REPO_LOWER}/solr; ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} - name: Do deploy - if: ${{ inputs.solr == 'false' }} run: | echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; From 2b31477510672dfedcac53c603b4d69ef99fc056 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Wed, 9 Nov 2022 14:03:32 -0800 Subject: [PATCH 076/159] subdirs for lint/test --- .github/workflows/lint.yaml | 22 ++++++++++++++++------ .github/workflows/test.yaml | 22 +++++++++++++++++----- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d2f5576..4aba6a4 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -2,20 +2,26 @@ name: "Lint for Rails Projects" on: workflow_dispatch: inputs: - worker: - required: false - type: boolean + subdir: + default: '.' + type: string tag: required: false type: string - workflow_call: - inputs: worker: required: false type: boolean + workflow_call: + inputs: + subdir: + default: '.' + type: string tag: required: false type: string + worker: + required: false + type: boolean env: REGISTRY: ghcr.io @@ -39,15 +45,19 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Pull image to prevent build run: >- + cd ${{ inputs.subdir }}; touch .env.development; touch .env; docker-compose pull web - name: Pull worker image to prevent build run: >- + cd ${{ inputs.subdir }}; docker-compose pull worker if: ${{ inputs.worker }} - name: Run Rubocop - run: docker-compose run web bundle exec rubocop --parallel --format progress + run: >- + cd ${{ inputs.subdir }}; + docker-compose run web bundle exec rubocop --parallel --format progress - name: Publish Test Report uses: mikepenz/action-junit-report@v3 if: always() # always run even if the previous step fails diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 54b3ea5..1245114 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,20 +2,27 @@ name: "Rspec for Rails Apps" on: workflow_dispatch: inputs: - worker: - required: false - type: boolean + subdir: + default: '.' + type: string tag: required: false type: string - workflow_call: - inputs: worker: required: false type: boolean + workflow_call: + inputs: + subdir: + default: '.' + type: string tag: required: false type: string + worker: + required: false + type: boolean + env: REGISTRY: ghcr.io @@ -59,17 +66,20 @@ jobs: limit-access-to-actor: true - name: Pull image to prevent build run: >- + cd ${{ inputs.subdir }}; touch .env.development; touch .env; docker-compose pull web - name: Pull worker image to prevent build run: >- + cd ${{ inputs.subdir }}; docker-compose pull worker if: ${{ inputs.worker }} - name: Start containers run: docker-compose up -d web - name: Check for and setup Solr if needed run: | + cd ${{ inputs.subdir }}; if [ -d solr ] then docker-compose exec -T web sh -c @@ -81,6 +91,7 @@ jobs: fi - name: Setup db run: >- + cd ${{ inputs.subdir }}; docker-compose exec -T web sh -c "RAILS_ENV=test bundle exec rake db:create db:schema:load db:migrate" - name: Run Specs @@ -92,6 +103,7 @@ jobs: CI_NODE_INDEX: ${{ matrix.ci_node_index }} continue-on-error: true run: >- + cd ${{ inputs.subdir }}; docker-compose exec -T web sh -c "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" From 49c2d118fda2ab9c937d7644710da6b2457bcfd8 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Wed, 9 Nov 2022 14:32:02 -0800 Subject: [PATCH 077/159] missed one --- .github/workflows/test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1245114..5219adc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -76,7 +76,9 @@ jobs: docker-compose pull worker if: ${{ inputs.worker }} - name: Start containers - run: docker-compose up -d web + run: >- + cd ${{ inputs.subdir }}; + docker-compose up -d web - name: Check for and setup Solr if needed run: | cd ${{ inputs.subdir }}; From 3228f853c7a7346051b73f806e612fde90d115c5 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Wed, 9 Nov 2022 16:18:37 -0800 Subject: [PATCH 078/159] submodule checkout --- .github/workflows/build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9e80dd5..a43c957 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -60,6 +60,8 @@ jobs: image_name: ${{ inputs.image_name }} - name: Checkout code uses: actions/checkout@v3 + with: + submodules: true - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx From 953e1c7db4dcad262d642dced9345c53e23d3281 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Thu, 10 Nov 2022 13:03:52 -0800 Subject: [PATCH 079/159] update for utk v 3.8 - if works need to set dynamically somehow --- setup-env/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 2a4bb2d..dee0858 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -26,4 +26,4 @@ runs: - name: Install docker-compose uses: KengoTODA/actions-setup-docker-compose@main with: - version: '1.29.2' + version: '3.8' From b835bd6b472a765385a929c5e71f5b1a13f2fa4b Mon Sep 17 00:00:00 2001 From: April Rieger Date: Thu, 10 Nov 2022 13:12:51 -0800 Subject: [PATCH 080/159] Ah, learned something that this is the release for Docker/Compose and it's version there: https://github.com/docker/compose/releases/tag/v2.12.2 --- setup-env/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index dee0858..78037ab 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -26,4 +26,4 @@ runs: - name: Install docker-compose uses: KengoTODA/actions-setup-docker-compose@main with: - version: '3.8' + version: '2.12.2' From 2e5dcf0abbd0d27ff70d36bacc7896230f13c262 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Thu, 10 Nov 2022 16:00:40 -0800 Subject: [PATCH 081/159] updates timeput to 360 --- setup-env/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 78037ab..e982e60 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -15,8 +15,8 @@ runs: run: >- echo "TAG=${HEAD_TAG::8}" >> ${GITHUB_ENV}; echo ${HEAD_TAG::8}; - echo "export DOCKER_CLIENT_TIMEOUT=120"; - echo "export COMPOSE_HTTP_TIMEOUT=120" + echo "export DOCKER_CLIENT_TIMEOUT=360"; + echo "export COMPOSE_HTTP_TIMEOUT=360" - name: Downcase repo env: REPO: "${{ github.repository }}" From 8d6ebc649c2720c0a0c805215f2e17efe91e127a Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 10 Nov 2022 18:50:35 -0800 Subject: [PATCH 082/159] another episode of Rob alphabetizes things. also support passing APP_PATH for subdir builds --- .github/workflows/build.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a43c957..81466ae 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -81,25 +81,29 @@ jobs: - name: Build and push uses: docker/build-push-action@v3 with: + build-args: | + APP_PATH=${{ inputs.subdir }} + cache-from: | + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} context: . file: ${{ inputs.subdir}}/Dockerfile platforms: ${{ inputs.platforms }} - target: ${{ inputs.target }} - cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} push: true tags: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + target: ${{ inputs.target }} - name: Build and push worker uses: docker/build-push-action@v3 with: + build-args: | + APP_PATH=${{ inputs.subdir }} + cache-from: | + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} context: . file: ${{ inputs.subdir}}/Dockerfile platforms: ${{ inputs.platforms }} - target: ${{ inputs.workerTarget }} - cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} push: true tags: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + target: ${{ inputs.workerTarget }} if: ${{ inputs.worker }} From 6cc75f455be29f5b5c488d2957846c2ff0b9964c Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 11 Nov 2022 18:10:59 -0800 Subject: [PATCH 083/159] ad the ability to specify a values file name before the -deploy.tmpl.yml --- .github/workflows/deploy.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index adb43b8..cae7e00 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -16,6 +16,10 @@ on: description: "Tag to override default of current branch sha" required: false type: string + helm-values-file-name: + required: false + type: string + env: REGISTRY: ghcr.io @@ -70,7 +74,7 @@ jobs: - name: Do deploy run: | echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; - DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; + DOLLAR=$ envsubst < ops/${{ inputs.helm-values-file-name || inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; From 9276b901241db7dd474e61afdb201007114ca7d0 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 11 Nov 2022 18:12:40 -0800 Subject: [PATCH 084/159] dis too --- .github/workflows/deploy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index cae7e00..c7c236a 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -17,6 +17,7 @@ on: required: false type: string helm-values-file-name: + description: Specify a values file name before the `-deploy.tmpl.yaml` required: false type: string From ecf50407b165585c79c936734af27e565d3d1e7e Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 11:05:31 -0800 Subject: [PATCH 085/159] Adds the addition of several ljk and hbk specific variables, Im not too happy with this solution but it allows really good ui for the team, imho. This also adds the addition of specifiy the helm values name before the -deploy.tmpl.yaml sincxe ljk brick and hbk brick had a different use case for their values file naming cinvension --- .github/workflows/deploy.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index c7c236a..3988880 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -17,11 +17,10 @@ on: required: false type: string helm-values-file-name: - description: Specify a values file name before the `-deploy.tmpl.yaml` + description: "Specify a values file name before the `-deploy.tmpl.yaml`" required: false type: string - env: REGISTRY: ghcr.io @@ -42,13 +41,22 @@ jobs: DB_PASSWORD: ${{ secrets.DB_PASSWORD }} GOOGLE_FONTS_KEY: ${{ secrets.GOOGLE_FONTS_KEY }} GOOGLE_OAUTH_PRIVATE_KEY_SECRET: ${{ secrets.GOOGLE_OAUTH_PRIVATE_KEY_SECRET }} + HBK_PAPERTRAIL_API_TOKEN: ${{ secrets.HBK_PAPERTRAIL_API_TOKEN }} + HBK_SENDGRID_PASSWORD: ${{ secrets.HBK_SENDGRID_PASSWORD }} + HBK_STRIPE_SECRET_KEY: ${{ secrets.HBK_STRIPE_SECRET_KEY }} IA_PASSWORD: ${{ secrets.IA_PASSWORD }} + LJK_SENDGRID_PASSWORD: ${{ secrets.LJK_SENDGRID_PASSWORD }} + LJK_SQUARE_ACCESS_TOKEN: ${{ secrets.LJK_SQUARE_ACCESS_TOKEN }} + LJK_SQUARE_WEBHOOK_SIGNATURE_KEY: ${{ secrets.LJK_SQUARE_WEBHOOK_SIGNATURE_KEY }} + LJK_STRIPE_SECRET_KEY: ${{ secrets.LJK_STRIPE_SECRET_KEY }} MAIL_PASS: ${{ secrets.MAIL_PASS }} POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }} SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} SOLR_ADMIN_PASSWORD: ${{ secrets.SOLR_ADMIN_PASSWORD }} + SQUARE_ACCESS_TOKEN: ${{ secrets.SQUARE_ACCESS_TOKEN }} + SQUARE_WEBHOOK_SIGNATURE_KEY: ${{ secrets.SQUARE_WEBHOOK_SIGNATURE_KEY }} steps: - id: setup name: Setup @@ -66,7 +74,7 @@ jobs: if: ${{ inputs.solr }} run: | echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; - DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; + DOLLAR=$ envsubst < ops/${{ inputs.helm-values-file-name || inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.helm-values-file-name || inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; @@ -75,7 +83,7 @@ jobs: - name: Do deploy run: | echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; - DOLLAR=$ envsubst < ops/${{ inputs.helm-values-file-name || inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; + DOLLAR=$ envsubst < ops/${{ inputs.helm-values-file-name || inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.helm-values-file-name || inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; From 312dbdf1c454c1339547bc875d36e22a5d7992bc Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 11:24:10 -0800 Subject: [PATCH 086/159] Read a closed and solved issue where the user added a login to the registry in order for the registry image to be available to the actions --- .github/workflows/retag.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index 103ff96..04bf298 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -30,6 +30,12 @@ jobs: uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Retag web to latest uses: koraykoska/retag-docker-image@0.2.4 with: From c8a23c8c5217b251f3c0141115765b7640c826af Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 13:11:08 -0800 Subject: [PATCH 087/159] Nope - revert --- .github/workflows/retag.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index 04bf298..5974306 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -30,12 +30,11 @@ jobs: uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + limit-access-to-actor: true - name: Retag web to latest uses: koraykoska/retag-docker-image@0.2.4 with: From 39e266f82a4571f0f7606c4a5697eab1b5bf6b9c Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 14:46:58 -0800 Subject: [PATCH 088/159] trying out a different way to retag main to latest from this marketplace action - https://github.com/marketplace/actions/docker-metadata-action --- .github/workflows/retag.yaml | 55 +++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index 5974306..62f0aed 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -1,4 +1,4 @@ -name: "Retag the item on main" +name: "Retag the image on main to latest" on: workflow_dispatch: inputs: @@ -8,6 +8,9 @@ on: tag: required: false type: string + solr: + required: false + type: string workflow_call: inputs: worker: @@ -16,6 +19,9 @@ on: tag: required: false type: string + solr: + required: false + type: string env: REGISTRY: ghcr.io @@ -30,27 +36,32 @@ jobs: uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - with: - limit-access-to-actor: true - - name: Retag web to latest - uses: koraykoska/retag-docker-image@0.2.4 + - name: Docker Metadata action for web + uses: docker/metadata-action@v4.1.1 with: - registry: ${{ env.REGISTRY }} - name: ${{ env.REPO_LOWER }} - old_tag: ${{ env.TAG }} - new_tag: latest - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Retag worker to latest - uses: koraykoska/retag-docker-image@0.2.4 + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + flavor: | + latest=auto + - name: Docker Metadata action for worker if: ${{ inputs.worker }} + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker + tags: | + type=raw,value=latest,enable={{is_default_branch}} + flavor: | + latest=auto + - name: Docker Metadata action for solr + if: ${{ inputs.solr }} + uses: docker/metadata-action@v4.1.1 with: - registry: ${{ env.REGISTRY }} - name: ${{ env.REPO_LOWER }}/worker - old_tag: ${{ env.TAG }} - new_tag: latest - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr + tags: | + type=raw,value=latest,enable={{is_default_branch}} + flavor: | + latest=auto \ No newline at end of file From d9532417335a0f596ecfc0f2fed2183d6234ac0f Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 15:39:38 -0800 Subject: [PATCH 089/159] ok reading baout placement and they have it before the build block --- .github/workflows/build.yaml | 10 +++++----- .github/workflows/retag.yaml | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6159443..f2a778e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -49,11 +49,6 @@ jobs: build: runs-on: self-hosted steps: - - id: setup - name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner - with: - tag: ${{ inputs.tag }} - name: Checkout code uses: actions/checkout@v3 - name: Set up QEMU @@ -65,6 +60,11 @@ jobs: with: version: latest endpoint: builders + - id: setup + name: Setup + uses: scientist-softserv/actions/setup-env@self-hosted-runner + with: + tag: ${{ inputs.tag }} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml index 62f0aed..5fc5033 100644 --- a/.github/workflows/retag.yaml +++ b/.github/workflows/retag.yaml @@ -42,7 +42,7 @@ jobs: images: | name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | - type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=latest flavor: | latest=auto - name: Docker Metadata action for worker @@ -52,7 +52,7 @@ jobs: images: | name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker tags: | - type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=latest flavor: | latest=auto - name: Docker Metadata action for solr @@ -62,6 +62,6 @@ jobs: images: | name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr tags: | - type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=latest flavor: | latest=auto \ No newline at end of file From 8f477954b3ec4e21cf056ea143d4da12313f3457 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 15:52:55 -0800 Subject: [PATCH 090/159] well, this needs to run inside the build so setting it up like the setup action to be called into the build workflow --- .github/workflows/build.yaml | 5 +++++ .github/workflows/retag.yaml => setup-retag/action.yaml | 0 2 files changed, 5 insertions(+) rename .github/workflows/retag.yaml => setup-retag/action.yaml (100%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f2a778e..95134fc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -65,6 +65,11 @@ jobs: uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} + - id: retag + name: Retag + uses: scientist-softserv/actions/setup-retag@self-hosted-runner + with: + tag: ${{ inputs.tag }} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: diff --git a/.github/workflows/retag.yaml b/setup-retag/action.yaml similarity index 100% rename from .github/workflows/retag.yaml rename to setup-retag/action.yaml From fdf236b5f8ac5b535b679c10b43185a102bf84c2 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 16:10:51 -0800 Subject: [PATCH 091/159] update the syntax for merge to main --- setup-retag/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-retag/action.yaml b/setup-retag/action.yaml index 5fc5033..d0e6f54 100644 --- a/setup-retag/action.yaml +++ b/setup-retag/action.yaml @@ -29,7 +29,7 @@ env: jobs: retag: runs-on: self-hosted - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ github.ref == format('refs/heads/{0}', 'main') }} steps: - id: setup name: Setup From 963da5c692324542e649ca2337d1712fc40fa57d Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 16:15:48 -0800 Subject: [PATCH 092/159] removing the need for it to be main for testing purposes --- setup-retag/action.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/setup-retag/action.yaml b/setup-retag/action.yaml index d0e6f54..cb56797 100644 --- a/setup-retag/action.yaml +++ b/setup-retag/action.yaml @@ -29,7 +29,6 @@ env: jobs: retag: runs-on: self-hosted - if: ${{ github.ref == format('refs/heads/{0}', 'main') }} steps: - id: setup name: Setup From fa4a76669934273ec76476bd2c4f7c56c2962893 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 16:26:29 -0800 Subject: [PATCH 093/159] read the documentation april --- .github/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 95134fc..66ef2c5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -92,6 +92,7 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + ${{ steps.meta.outputs.tags }} - name: Build and push worker if: ${{ inputs.worker }} uses: docker/build-push-action@v3 From 8885cbe9443274cce59091664da223a0adaa46bf Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 16:42:54 -0800 Subject: [PATCH 094/159] Updates the syntax for the tag to be latest and short sha --- .github/workflows/build.yaml | 50 ++++++++++++++++++--------- setup-retag/action.yaml | 66 ------------------------------------ 2 files changed, 34 insertions(+), 82 deletions(-) delete mode 100644 setup-retag/action.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 66ef2c5..d5ecd10 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -65,22 +65,44 @@ jobs: uses: scientist-softserv/actions/setup-env@self-hosted-runner with: tag: ${{ inputs.tag }} - - id: retag - name: Retag - uses: scientist-softserv/actions/setup-retag@self-hosted-runner + - name: Retag action for web + uses: docker/metadata-action@v4.1.1 with: - tag: ${{ inputs.tag }} + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} + tags: | + type=raw,value=latest + type=sha + flavor: | + latest=auto + - name: Retag action for worker + if: ${{ inputs.worker }} + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker + tags: | + type=raw,value=latest + type=sha + flavor: | + latest=auto + - name: Retag action for solr + if: ${{ inputs.solr }} + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr + tags: | + type=raw,value=latest + type=sha + flavor: | + latest=auto - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull from cache to speed up build - run: >- - touch .env.development; - touch .env; - TAG=latest docker-compose pull web || true - name: Build and push web uses: docker/build-push-action@v3 with: @@ -90,9 +112,7 @@ jobs: cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} push: true - tags: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - ${{ steps.meta.outputs.tags }} + tags: ${{ steps.meta.outputs.tags }} - name: Build and push worker if: ${{ inputs.worker }} uses: docker/build-push-action@v3 @@ -103,8 +123,7 @@ jobs: cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} push: true - tags: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + tags: ${{ steps.meta.outputs.tags }} - name: Build and push solr if: ${{ inputs.solr }} uses: docker/build-push-action@v3 @@ -114,5 +133,4 @@ jobs: cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} push: true - tags: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} + tags: ${{ steps.meta.outputs.tags }} diff --git a/setup-retag/action.yaml b/setup-retag/action.yaml deleted file mode 100644 index cb56797..0000000 --- a/setup-retag/action.yaml +++ /dev/null @@ -1,66 +0,0 @@ -name: "Retag the image on main to latest" -on: - workflow_dispatch: - inputs: - worker: - required: false - type: boolean - tag: - required: false - type: string - solr: - required: false - type: string - workflow_call: - inputs: - worker: - required: false - type: boolean - tag: - required: false - type: string - solr: - required: false - type: string - -env: - REGISTRY: ghcr.io - -jobs: - retag: - runs-on: self-hosted - steps: - - id: setup - name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner - with: - tag: ${{ inputs.tag }} - - name: Docker Metadata action for web - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} - tags: | - type=raw,value=latest - flavor: | - latest=auto - - name: Docker Metadata action for worker - if: ${{ inputs.worker }} - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker - tags: | - type=raw,value=latest - flavor: | - latest=auto - - name: Docker Metadata action for solr - if: ${{ inputs.solr }} - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr - tags: | - type=raw,value=latest - flavor: | - latest=auto \ No newline at end of file From aebf8c79781baecd1facb1674abc91ee64f3a08a Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 16:58:39 -0800 Subject: [PATCH 095/159] syntax --- .github/workflows/build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d5ecd10..834ca7e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -71,8 +71,8 @@ jobs: images: | name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | - type=raw,value=latest - type=sha + type=raw,value=latest,enable={{is_default_branch}} + type=sha,value=latest,enable={{is_default_branch}} flavor: | latest=auto - name: Retag action for worker @@ -82,8 +82,8 @@ jobs: images: | name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker tags: | - type=raw,value=latest - type=sha + type=raw,value=latest,enable={{is_default_branch}} + type=sha,value=latest,enable={{is_default_branch}} flavor: | latest=auto - name: Retag action for solr @@ -93,8 +93,8 @@ jobs: images: | name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr tags: | - type=raw,value=latest - type=sha + type=raw,value=latest,enable={{is_default_branch}} + type=sha,value=latest,enable={{is_default_branch}} flavor: | latest=auto - name: Login to GitHub Container Registry From 453fe440091432258ecbba29c489bd557fec9da6 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 17:08:40 -0800 Subject: [PATCH 096/159] updates more updates --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 834ca7e..a6fd6dd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -111,7 +111,7 @@ jobs: target: ${{ inputs.target }} cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - push: true + push: ${{ github.ref == 'refs/heads/main' }} tags: ${{ steps.meta.outputs.tags }} - name: Build and push worker if: ${{ inputs.worker }} @@ -122,7 +122,7 @@ jobs: target: ${{ inputs.workerTarget }} cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - push: true + push: ${{ github.ref == 'refs/heads/main' }} tags: ${{ steps.meta.outputs.tags }} - name: Build and push solr if: ${{ inputs.solr }} @@ -132,5 +132,5 @@ jobs: platforms: ${{ inputs.platforms }} cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} - push: true + push: ${{ github.ref == 'refs/heads/main' }} tags: ${{ steps.meta.outputs.tags }} From 8b2a5416b7dd9903d4a7fc1e766a780ddcafc107 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 17:12:27 -0800 Subject: [PATCH 097/159] put that back - didnt mean to delete it --- .github/workflows/build.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a6fd6dd..0286d3c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -49,6 +49,11 @@ jobs: build: runs-on: self-hosted steps: + - id: setup + name: Setup + uses: scientist-softserv/actions/setup-env@main + with: + tag: ${{ inputs.tag }} - name: Checkout code uses: actions/checkout@v3 - name: Set up QEMU From d33e82bb971e94fa6b18038e27e3c2512910530c Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 17:19:36 -0800 Subject: [PATCH 098/159] ah this is like css and js a bit --- .github/workflows/build.yaml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0286d3c..6606d92 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -65,12 +65,13 @@ jobs: with: version: latest endpoint: builders - - id: setup - name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner - with: - tag: ${{ inputs.tag }} + - name: Pull from cache to speed up build + run: >- + touch .env.development; + touch .env; + TAG=latest docker-compose pull web || true - name: Retag action for web + id: meta-web uses: docker/metadata-action@v4.1.1 with: images: | @@ -82,6 +83,7 @@ jobs: latest=auto - name: Retag action for worker if: ${{ inputs.worker }} + id: meta-worker uses: docker/metadata-action@v4.1.1 with: images: | @@ -93,6 +95,7 @@ jobs: latest=auto - name: Retag action for solr if: ${{ inputs.solr }} + id: meta-solr uses: docker/metadata-action@v4.1.1 with: images: | @@ -117,7 +120,7 @@ jobs: cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} push: ${{ github.ref == 'refs/heads/main' }} - tags: ${{ steps.meta.outputs.tags }} + tags: ${{ steps.meta-web.outputs.tags }} - name: Build and push worker if: ${{ inputs.worker }} uses: docker/build-push-action@v3 @@ -128,7 +131,7 @@ jobs: cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} push: ${{ github.ref == 'refs/heads/main' }} - tags: ${{ steps.meta.outputs.tags }} + tags: ${{ steps.meta-worker.outputs.tags }} - name: Build and push solr if: ${{ inputs.solr }} uses: docker/build-push-action@v3 @@ -138,4 +141,4 @@ jobs: cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} push: ${{ github.ref == 'refs/heads/main' }} - tags: ${{ steps.meta.outputs.tags }} + tags: ${{ steps.meta-solr.outputs.tags }} From baa0c317db8c7d08e9e4fd60a68b5e0487ba5560 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 17:41:24 -0800 Subject: [PATCH 099/159] updates --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6606d92..b8c1518 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -78,7 +78,7 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,value=latest,enable={{is_default_branch}} + type=sha,enable={{is_default_branch}},prefix= flavor: | latest=auto - name: Retag action for worker @@ -90,7 +90,7 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,value=latest,enable={{is_default_branch}} + type=sha,enable={{is_default_branch}},prefix= flavor: | latest=auto - name: Retag action for solr @@ -102,7 +102,7 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,value=latest,enable={{is_default_branch}} + type=sha,enable={{is_default_branch}},prefix= flavor: | latest=auto - name: Login to GitHub Container Registry From 3dd26e7c09819aa9052097c6879b96918e138842 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 17:46:49 -0800 Subject: [PATCH 100/159] more tweaks as I test and learn --- .github/workflows/build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b8c1518..d49f535 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -78,7 +78,7 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable={{is_default_branch}},prefix= + type=sha,enable=true,priority=100,prefix=,suffix=,format=short= flavor: | latest=auto - name: Retag action for worker @@ -90,7 +90,7 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable={{is_default_branch}},prefix= + type=sha,enable=true,priority=100,prefix=,suffix=,format=short flavor: | latest=auto - name: Retag action for solr @@ -102,7 +102,7 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable={{is_default_branch}},prefix= + type=sha,enable=true,priority=100,prefix=,suffix=,format=short flavor: | latest=auto - name: Login to GitHub Container Registry @@ -119,7 +119,7 @@ jobs: target: ${{ inputs.target }} cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - push: ${{ github.ref == 'refs/heads/main' }} + push: true tags: ${{ steps.meta-web.outputs.tags }} - name: Build and push worker if: ${{ inputs.worker }} @@ -130,7 +130,7 @@ jobs: target: ${{ inputs.workerTarget }} cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - push: ${{ github.ref == 'refs/heads/main' }} + push: true tags: ${{ steps.meta-worker.outputs.tags }} - name: Build and push solr if: ${{ inputs.solr }} @@ -140,5 +140,5 @@ jobs: platforms: ${{ inputs.platforms }} cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} - push: ${{ github.ref == 'refs/heads/main' }} + push: true tags: ${{ steps.meta-solr.outputs.tags }} From 05b4a237b9d5f1a95ffabc1799926f6cbe1d49e5 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 19:52:36 -0800 Subject: [PATCH 101/159] adding the ability to add a seperate solr build, push to the registry and pull from the registry not only in the build but in the deploy to accomodate seperate images needed --- .github/workflows/build.yaml | 20 +++++++++++++++++++- .github/workflows/deploy.yaml | 14 ++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 25ac19b..a3953e5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,6 +5,10 @@ on: platforms: default: "linux/amd64,linux/arm64" type: string + solr: + description: "Use if you need to build a specific Solr image and push to the registry" + required: false + type: boolean tag: required: false type: string @@ -22,6 +26,10 @@ on: platforms: default: "linux/amd64,linux/arm64" type: string + solr: + description: "Use if you need to build a specific Solr image and push to the registry" + required: false + type: boolean tag: required: false type: string @@ -76,6 +84,7 @@ jobs: tags: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - name: Build and push worker + if: ${{ inputs.worker }} uses: docker/build-push-action@v3 with: context: . @@ -86,4 +95,13 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - if: ${{ inputs.worker }} + - name: Build and push solr + if: ${{ inputs.solr }} + uses: docker/build-push-action@v3 + with: + context: solr + platforms: ${{ inputs.platforms }} + cache-from: | + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} + push: true + tags: ${{ steps.meta-solr.outputs.tags }} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 829e1fa..45a5405 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -8,6 +8,10 @@ on: k8s-namespace: required: false type: string + solr: + description: "Add if your solr image is from the registry" + required: false + type: boolean tag: description: "Tag to override default of current branch sha" required: false @@ -61,3 +65,13 @@ jobs: export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} + - name: Do deploy with solr registry image + if: ${{ inputs.solr }} + run: | + echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; + DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; + export DEPLOY_TAG=${TAG}; + export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; + export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; + export SOLR_IMAGE=ghcr.io/${REPO_LOWER}/solr; + ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} \ No newline at end of file From a0855f733f640479db5c6336b332b8cbaa7931dc Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 20:03:26 -0800 Subject: [PATCH 102/159] adds the ability to specify the confdir sinc eit varies by project --- .github/workflows/test.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ab09645..0b1b526 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,6 +2,9 @@ name: "Rspec for Rails Apps" on: workflow_dispatch: inputs: + confdir: + required: false + type: string worker: required: false type: boolean @@ -14,6 +17,9 @@ on: default: "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" workflow_call: inputs: + confdir: + required: false + type: string worker: required: false type: boolean @@ -42,7 +48,7 @@ jobs: ci_node_index: [0, 1, 2] env: ALLOW_ANONYMOUS_LOGIN: "yes" - CONFDIR: "/app/samvera/hyrax-webapp/solr/config" + CONFDIR: ${{ inputs.confdir }} DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml From 4dafeeb24e1af188c787144f09552715b348c9ab Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 20:05:37 -0800 Subject: [PATCH 103/159] adds the ability to specify the confdir sinc eit varies by project --- .github/workflows/test.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ab09645..0b1b526 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,6 +2,9 @@ name: "Rspec for Rails Apps" on: workflow_dispatch: inputs: + confdir: + required: false + type: string worker: required: false type: boolean @@ -14,6 +17,9 @@ on: default: "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" workflow_call: inputs: + confdir: + required: false + type: string worker: required: false type: boolean @@ -42,7 +48,7 @@ jobs: ci_node_index: [0, 1, 2] env: ALLOW_ANONYMOUS_LOGIN: "yes" - CONFDIR: "/app/samvera/hyrax-webapp/solr/config" + CONFDIR: ${{ inputs.confdir }} DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml From 0fae0426c3632fa5642cdf0c0767a2126be30f68 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 20:07:04 -0800 Subject: [PATCH 104/159] reverting and making pr --- .github/workflows/test.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0b1b526..ab09645 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,9 +2,6 @@ name: "Rspec for Rails Apps" on: workflow_dispatch: inputs: - confdir: - required: false - type: string worker: required: false type: boolean @@ -17,9 +14,6 @@ on: default: "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" workflow_call: inputs: - confdir: - required: false - type: string worker: required: false type: boolean @@ -48,7 +42,7 @@ jobs: ci_node_index: [0, 1, 2] env: ALLOW_ANONYMOUS_LOGIN: "yes" - CONFDIR: ${{ inputs.confdir }} + CONFDIR: "/app/samvera/hyrax-webapp/solr/config" DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml From 9564c3c2c7133032ef23e32c7b9e846ea93c5654 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 20:14:53 -0800 Subject: [PATCH 105/159] adding the retag action to the build stage and removing the retag workflow --- .github/workflows/build.yaml | 35 +++++++++++++++++++++++++ .github/workflows/retag.yaml | 51 ------------------------------------ 2 files changed, 35 insertions(+), 51 deletions(-) delete mode 100644 .github/workflows/retag.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 25ac19b..d436f47 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -64,6 +64,41 @@ jobs: touch .env.development; touch .env; TAG=latest docker-compose pull web || true + - name: Retag action for web + id: meta-web + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,enable=true,priority=100,prefix=,suffix=,format=short= + flavor: | + latest=auto + - name: Retag action for worker + if: ${{ inputs.worker }} + id: meta-worker + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,enable=true,priority=100,prefix=,suffix=,format=short + flavor: | + latest=auto + - name: Retag action for solr + if: ${{ inputs.solr }} + id: meta-solr + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,enable=true,priority=100,prefix=,suffix=,format=short + flavor: | + latest=auto - name: Build and push uses: docker/build-push-action@v3 with: diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml deleted file mode 100644 index f6b9e7e..0000000 --- a/.github/workflows/retag.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: "Retag the item on main" -on: - workflow_dispatch: - inputs: - worker: - required: false - type: boolean - tag: - required: false - type: string - workflow_call: - inputs: - worker: - required: false - type: boolean - tag: - required: false - type: string - -env: - REGISTRY: ghcr.io - -jobs: - retag: - runs-on: ubuntu-latest - if: ${{ github.ref == 'refs/heads/main' }} - steps: - - id: setup - name: Setup - uses: scientist-softserv/actions/setup-env@main - with: - tag: ${{ inputs.tag }} - - name: Retag web to latest - uses: koraykoska/retag-docker-image@0.2.4 - with: - registry: ${{ env.REGISTRY }} - name: ${{ env.REPO_LOWER }} - old_tag: ${{ env.TAG }} - new_tag: latest - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Retag worker to latest - uses: koraykoska/retag-docker-image@0.2.4 - with: - registry: ${{ env.REGISTRY }} - name: ${{ env.REPO_LOWER }}/worker - old_tag: ${{ env.TAG }} - new_tag: latest - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - if: ${{ inputs.worker }} From 23a904048da40669a5cf96ff2ef4394750a7c19d Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 20:16:25 -0800 Subject: [PATCH 106/159] adding the retag to the build" --- .github/workflows/build.yaml | 35 +++++++++++++++++++++++++ .github/workflows/retag.yaml | 51 ------------------------------------ 2 files changed, 35 insertions(+), 51 deletions(-) delete mode 100644 .github/workflows/retag.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 25ac19b..d436f47 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -64,6 +64,41 @@ jobs: touch .env.development; touch .env; TAG=latest docker-compose pull web || true + - name: Retag action for web + id: meta-web + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,enable=true,priority=100,prefix=,suffix=,format=short= + flavor: | + latest=auto + - name: Retag action for worker + if: ${{ inputs.worker }} + id: meta-worker + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,enable=true,priority=100,prefix=,suffix=,format=short + flavor: | + latest=auto + - name: Retag action for solr + if: ${{ inputs.solr }} + id: meta-solr + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,enable=true,priority=100,prefix=,suffix=,format=short + flavor: | + latest=auto - name: Build and push uses: docker/build-push-action@v3 with: diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml deleted file mode 100644 index f6b9e7e..0000000 --- a/.github/workflows/retag.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: "Retag the item on main" -on: - workflow_dispatch: - inputs: - worker: - required: false - type: boolean - tag: - required: false - type: string - workflow_call: - inputs: - worker: - required: false - type: boolean - tag: - required: false - type: string - -env: - REGISTRY: ghcr.io - -jobs: - retag: - runs-on: ubuntu-latest - if: ${{ github.ref == 'refs/heads/main' }} - steps: - - id: setup - name: Setup - uses: scientist-softserv/actions/setup-env@main - with: - tag: ${{ inputs.tag }} - - name: Retag web to latest - uses: koraykoska/retag-docker-image@0.2.4 - with: - registry: ${{ env.REGISTRY }} - name: ${{ env.REPO_LOWER }} - old_tag: ${{ env.TAG }} - new_tag: latest - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Retag worker to latest - uses: koraykoska/retag-docker-image@0.2.4 - with: - registry: ${{ env.REGISTRY }} - name: ${{ env.REPO_LOWER }}/worker - old_tag: ${{ env.TAG }} - new_tag: latest - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - if: ${{ inputs.worker }} From a8a3373f199610cee1ed20cbfb7c3e0e4e0c096c Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 20:19:53 -0800 Subject: [PATCH 107/159] Revert to make PR --- .github/workflows/build.yaml | 35 ------------------------- .github/workflows/retag.yaml | 51 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/retag.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d436f47..25ac19b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -64,41 +64,6 @@ jobs: touch .env.development; touch .env; TAG=latest docker-compose pull web || true - - name: Retag action for web - id: meta-web - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short= - flavor: | - latest=auto - - name: Retag action for worker - if: ${{ inputs.worker }} - id: meta-worker - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - flavor: | - latest=auto - - name: Retag action for solr - if: ${{ inputs.solr }} - id: meta-solr - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - flavor: | - latest=auto - name: Build and push uses: docker/build-push-action@v3 with: diff --git a/.github/workflows/retag.yaml b/.github/workflows/retag.yaml new file mode 100644 index 0000000..2215738 --- /dev/null +++ b/.github/workflows/retag.yaml @@ -0,0 +1,51 @@ +name: "Retag the item on main" +on: + workflow_dispatch: + inputs: + worker: + required: false + type: boolean + tag: + required: false + type: string + workflow_call: + inputs: + worker: + required: false + type: boolean + tag: + required: false + type: string + +env: + REGISTRY: ghcr.io + +jobs: + retag: + runs-on: ubuntu-latest + if: ${{ github.ref == 'refs/heads/main' }} + steps: + - id: setup + name: Setup + uses: scientist-softserv/actions/setup-env@main + with: + tag: ${{ inputs.tag }} + - name: Retag web to latest + uses: koraykoska/retag-docker-image@0.2.4 + with: + registry: ${{ env.REGISTRY }} + name: ${{ env.REPO_LOWER }} + old_tag: ${{ env.TAG }} + new_tag: latest + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Retag worker to latest + uses: koraykoska/retag-docker-image@0.2.4 + with: + registry: ${{ env.REGISTRY }} + name: ${{ env.REPO_LOWER }}/worker + old_tag: ${{ env.TAG }} + new_tag: latest + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + if: ${{ inputs.worker }} \ No newline at end of file From cbeca6224c04a731bd2c00ff2e5b65617f92bb4e Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 12 Nov 2022 20:21:36 -0800 Subject: [PATCH 108/159] add the solr retag --- .github/workflows/build.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a3953e5..87bdb94 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -72,6 +72,18 @@ jobs: touch .env.development; touch .env; TAG=latest docker-compose pull web || true + - name: Retag action for solr + if: ${{ inputs.solr }} + id: meta-solr + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,enable=true,priority=100,prefix=,suffix=,format=short + flavor: | + latest=auto - name: Build and push uses: docker/build-push-action@v3 with: From 6101f98b1ba96671f65de44105d0a8eb820ba03e Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 13 Nov 2022 14:15:08 -0800 Subject: [PATCH 109/159] Removes a typo --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d436f47..49c739e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -72,7 +72,7 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short= + type=sha,enable=true,priority=100,prefix=,suffix=,format=short flavor: | latest=auto - name: Retag action for worker From 3a5c4dd2927db96a1dd5bc9569d4e51615d9cde0 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 13 Nov 2022 17:28:05 -0800 Subject: [PATCH 110/159] wanting to see if the retag action is causing the tests to fail --- .github/workflows/build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d49f535..aca8560 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -65,11 +65,11 @@ jobs: with: version: latest endpoint: builders - - name: Pull from cache to speed up build - run: >- - touch .env.development; - touch .env; - TAG=latest docker-compose pull web || true + # - name: Pull from cache to speed up build + # run: >- + # touch .env.development; + # touch .env; + # TAG=latest docker-compose pull web || true - name: Retag action for web id: meta-web uses: docker/metadata-action@v4.1.1 @@ -78,7 +78,7 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short= + type=sha,enable=true,priority=100,prefix=,suffix=,format=short flavor: | latest=auto - name: Retag action for worker From 7748a6c6273e0bcd68360e29a0d4912aa49efcb7 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 13 Nov 2022 18:28:58 -0800 Subject: [PATCH 111/159] updating cache --- .github/workflows/build.yaml | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index aca8560..b813666 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -65,11 +65,6 @@ jobs: with: version: latest endpoint: builders - # - name: Pull from cache to speed up build - # run: >- - # touch .env.development; - # touch .env; - # TAG=latest docker-compose pull web || true - name: Retag action for web id: meta-web uses: docker/metadata-action@v4.1.1 @@ -78,9 +73,6 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - flavor: | - latest=auto - name: Retag action for worker if: ${{ inputs.worker }} id: meta-worker @@ -90,9 +82,6 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - flavor: | - latest=auto - name: Retag action for solr if: ${{ inputs.solr }} id: meta-solr @@ -102,9 +91,6 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - flavor: | - latest=auto - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -114,31 +100,32 @@ jobs: - name: Build and push web uses: docker/build-push-action@v3 with: + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:buildcache,mode=max context: . platforms: ${{ inputs.platforms }} - target: ${{ inputs.target }} - cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} push: true tags: ${{ steps.meta-web.outputs.tags }} + target: ${{ inputs.target }} - name: Build and push worker if: ${{ inputs.worker }} uses: docker/build-push-action@v3 with: + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:buildcache,mode=max context: . platforms: ${{ inputs.platforms }} - target: ${{ inputs.workerTarget }} - cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} push: true tags: ${{ steps.meta-worker.outputs.tags }} + target: ${{ inputs.target }} - name: Build and push solr if: ${{ inputs.solr }} uses: docker/build-push-action@v3 with: - context: solr + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:buildcache,mode=max + context: . platforms: ${{ inputs.platforms }} - cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} push: true tags: ${{ steps.meta-solr.outputs.tags }} + target: ${{ inputs.target }} From 25ae463813144188c9397a33f95a5e087939e7ee Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 13 Nov 2022 18:35:50 -0800 Subject: [PATCH 112/159] update to add tag for sha --- .github/workflows/build.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b813666..9a5460b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -105,7 +105,9 @@ jobs: context: . platforms: ${{ inputs.platforms }} push: true - tags: ${{ steps.meta-web.outputs.tags }} + tags: | + ${{ steps.meta-web.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} target: ${{ inputs.target }} - name: Build and push worker if: ${{ inputs.worker }} @@ -116,7 +118,9 @@ jobs: context: . platforms: ${{ inputs.platforms }} push: true - tags: ${{ steps.meta-worker.outputs.tags }} + tags: | + ${{ steps.meta-worker.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} target: ${{ inputs.target }} - name: Build and push solr if: ${{ inputs.solr }} @@ -127,5 +131,7 @@ jobs: context: . platforms: ${{ inputs.platforms }} push: true - tags: ${{ steps.meta-solr.outputs.tags }} + tags: | + ${{ steps.meta-solr.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} target: ${{ inputs.target }} From a12b541c05ad98331539d58b70ee5cbf5827051c Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 13 Nov 2022 18:42:35 -0800 Subject: [PATCH 113/159] syntax --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9a5460b..547f523 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -106,7 +106,7 @@ jobs: platforms: ${{ inputs.platforms }} push: true tags: | - ${{ steps.meta-web.outputs.tags }} + ${{ steps.meta-web.outputs.tags }}; ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} target: ${{ inputs.target }} - name: Build and push worker @@ -119,7 +119,7 @@ jobs: platforms: ${{ inputs.platforms }} push: true tags: | - ${{ steps.meta-worker.outputs.tags }} + ${{ steps.meta-worker.outputs.tags }}; ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} target: ${{ inputs.target }} - name: Build and push solr @@ -132,6 +132,6 @@ jobs: platforms: ${{ inputs.platforms }} push: true tags: | - ${{ steps.meta-solr.outputs.tags }} + ${{ steps.meta-solr.outputs.tags }}; ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} target: ${{ inputs.target }} From 67f25418df23d9ed792c511b7f8849f726618d13 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 13 Nov 2022 18:46:12 -0800 Subject: [PATCH 114/159] syntax --- .github/workflows/build.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 547f523..88908f8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -106,8 +106,8 @@ jobs: platforms: ${{ inputs.platforms }} push: true tags: | - ${{ steps.meta-web.outputs.tags }}; - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + ${{ steps.meta-web.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} target: ${{ inputs.target }} - name: Build and push worker if: ${{ inputs.worker }} @@ -119,7 +119,7 @@ jobs: platforms: ${{ inputs.platforms }} push: true tags: | - ${{ steps.meta-worker.outputs.tags }}; + ${{ steps.meta-worker.outputs.tags }} ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} target: ${{ inputs.target }} - name: Build and push solr @@ -132,6 +132,6 @@ jobs: platforms: ${{ inputs.platforms }} push: true tags: | - ${{ steps.meta-solr.outputs.tags }}; + ${{ steps.meta-solr.outputs.tags }} ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} target: ${{ inputs.target }} From 07c1c7967e5c016d2ae95da4e73d4d4b6b69e9f4 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 13 Nov 2022 19:03:26 -0800 Subject: [PATCH 115/159] rename buildcache to latest - see if it works --- .github/workflows/build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 88908f8..9995a62 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -100,8 +100,8 @@ jobs: - name: Build and push web uses: docker/build-push-action@v3 with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:buildcache - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:buildcache,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:latest + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:latest,mode=max context: . platforms: ${{ inputs.platforms }} push: true @@ -113,8 +113,8 @@ jobs: if: ${{ inputs.worker }} uses: docker/build-push-action@v3 with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:buildcache - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:buildcache,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:latest + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:latest,mode=max context: . platforms: ${{ inputs.platforms }} push: true @@ -126,8 +126,8 @@ jobs: if: ${{ inputs.solr }} uses: docker/build-push-action@v3 with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:buildcache - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:buildcache,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:latest + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:latest,mode=max context: . platforms: ${{ inputs.platforms }} push: true From d35b94253eb84194f05cb32ecb7bd42fe35452e0 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sun, 13 Nov 2022 19:21:16 -0800 Subject: [PATCH 116/159] put the name buildcache back since it will build on every ci build, not just main and that needs to be separate from latest --- .github/workflows/build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9995a62..88908f8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -100,8 +100,8 @@ jobs: - name: Build and push web uses: docker/build-push-action@v3 with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:latest - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:latest,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:buildcache,mode=max context: . platforms: ${{ inputs.platforms }} push: true @@ -113,8 +113,8 @@ jobs: if: ${{ inputs.worker }} uses: docker/build-push-action@v3 with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:latest - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:latest,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:buildcache,mode=max context: . platforms: ${{ inputs.platforms }} push: true @@ -126,8 +126,8 @@ jobs: if: ${{ inputs.solr }} uses: docker/build-push-action@v3 with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:latest - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:latest,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:buildcache,mode=max context: . platforms: ${{ inputs.platforms }} push: true From fc27f71d26f7edba5658e1be45b952db59df4fda Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Mon, 14 Nov 2022 09:44:51 -0800 Subject: [PATCH 117/159] add description to image_name --- .github/workflows/build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 81466ae..54b5337 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,6 +4,7 @@ on: inputs: image_name: type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" platforms: default: "linux/amd64,linux/arm64" type: string @@ -26,6 +27,7 @@ on: inputs: image_name: type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" platforms: default: "linux/amd64,linux/arm64" type: string From a64d003e3976d50164f62c7d97df903d3cb40385 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 14 Nov 2022 13:18:40 -0800 Subject: [PATCH 118/159] removes the values name specification and updates the docker/setup-buildx-action back to v2 --- .github/workflows/build.yaml | 2 +- .github/workflows/deploy.yaml | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 88908f8..b30a469 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -61,7 +61,7 @@ jobs: - name: Set up Docker Context for Buildx run: docker context create builders - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: version: latest endpoint: builders diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 3988880..6c467f5 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -16,10 +16,6 @@ on: description: "Tag to override default of current branch sha" required: false type: string - helm-values-file-name: - description: "Specify a values file name before the `-deploy.tmpl.yaml`" - required: false - type: string env: REGISTRY: ghcr.io @@ -41,22 +37,18 @@ jobs: DB_PASSWORD: ${{ secrets.DB_PASSWORD }} GOOGLE_FONTS_KEY: ${{ secrets.GOOGLE_FONTS_KEY }} GOOGLE_OAUTH_PRIVATE_KEY_SECRET: ${{ secrets.GOOGLE_OAUTH_PRIVATE_KEY_SECRET }} - HBK_PAPERTRAIL_API_TOKEN: ${{ secrets.HBK_PAPERTRAIL_API_TOKEN }} - HBK_SENDGRID_PASSWORD: ${{ secrets.HBK_SENDGRID_PASSWORD }} - HBK_STRIPE_SECRET_KEY: ${{ secrets.HBK_STRIPE_SECRET_KEY }} IA_PASSWORD: ${{ secrets.IA_PASSWORD }} - LJK_SENDGRID_PASSWORD: ${{ secrets.LJK_SENDGRID_PASSWORD }} - LJK_SQUARE_ACCESS_TOKEN: ${{ secrets.LJK_SQUARE_ACCESS_TOKEN }} - LJK_SQUARE_WEBHOOK_SIGNATURE_KEY: ${{ secrets.LJK_SQUARE_WEBHOOK_SIGNATURE_KEY }} - LJK_STRIPE_SECRET_KEY: ${{ secrets.LJK_STRIPE_SECRET_KEY }} MAIL_PASS: ${{ secrets.MAIL_PASS }} + PAPERTRAIL_API_TOKEN: ${{ secrets.PAPERTRAIL_API_TOKEN }} POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }} + SENDGRID_PASSWORD: ${{ secrets.SENDGRID_PASSWORD }} SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} SOLR_ADMIN_PASSWORD: ${{ secrets.SOLR_ADMIN_PASSWORD }} SQUARE_ACCESS_TOKEN: ${{ secrets.SQUARE_ACCESS_TOKEN }} SQUARE_WEBHOOK_SIGNATURE_KEY: ${{ secrets.SQUARE_WEBHOOK_SIGNATURE_KEY }} + STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }} steps: - id: setup name: Setup @@ -74,7 +66,7 @@ jobs: if: ${{ inputs.solr }} run: | echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; - DOLLAR=$ envsubst < ops/${{ inputs.helm-values-file-name || inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.helm-values-file-name || inputs.environment }}-deploy.yaml; + DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; @@ -83,7 +75,7 @@ jobs: - name: Do deploy run: | echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; - DOLLAR=$ envsubst < ops/${{ inputs.helm-values-file-name || inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.helm-values-file-name || inputs.environment }}-deploy.yaml; + DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; From c610d201e4482ec2899e3e9207c89a50dc1c5ce2 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 14 Nov 2022 13:30:35 -0800 Subject: [PATCH 119/159] updates to add a default confdir --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0b1b526..db48f35 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,6 +5,7 @@ on: confdir: required: false type: string + default: "/app/samvera/hyrax-webapp/solr/config" worker: required: false type: boolean @@ -20,6 +21,7 @@ on: confdir: required: false type: string + default: "/app/samvera/hyrax-webapp/solr/config" worker: required: false type: boolean From 924e87a186c7e9e368dd8288415cfb45d668feec Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 14 Nov 2022 16:44:47 -0800 Subject: [PATCH 120/159] ok here we go - had to straighten out some duplicates I saw --- .github/workflows/build.yaml | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c0a30f1..606033f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -124,52 +124,36 @@ jobs: with: build-args: | APP_PATH=${{ inputs.subdir }} - cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:latest context: . file: ${{ inputs.subdir}}/Dockerfile platforms: ${{ inputs.platforms }} push: true tags: | ${{ steps.meta-web.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} target: ${{ inputs.target }} - name: Build and push worker if: ${{ inputs.worker }} uses: docker/build-push-action@v3 with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:buildcache - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:buildcache,mode=max - context: . build-args: | APP_PATH=${{ inputs.subdir }} - cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:latest context: . file: ${{ inputs.subdir}}/Dockerfile platforms: ${{ inputs.platforms }} push: true tags: | ${{ steps.meta-worker.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - target: ${{ inputs.target }} target: ${{ inputs.workerTarget }} - name: Build and push solr if: ${{ inputs.solr }} uses: docker/build-push-action@v3 with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:buildcache - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:buildcache,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:latest context: . platforms: ${{ inputs.platforms }} push: true tags: | ${{ steps.meta-solr.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} target: ${{ inputs.target }} - context: solr - platforms: ${{ inputs.platforms }} - cache-from: | - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} - push: true - tags: ${{ steps.meta-solr.outputs.tags }} From efe685a55a18b4bfd46940b56bb46b71fb967a03 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Mon, 14 Nov 2022 22:18:19 -0800 Subject: [PATCH 121/159] add image name for deploy --- .github/workflows/deploy.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 45a5405..828e77c 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -2,6 +2,9 @@ name: "Deploy" on: workflow_call: inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" k8s-release-name: required: false type: string @@ -50,6 +53,7 @@ jobs: uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag }} + image_name: ${{ inputs.image_name }} - name: Checkout code uses: actions/checkout@v3 - name: Setup tmate session @@ -74,4 +78,4 @@ jobs: export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; export SOLR_IMAGE=ghcr.io/${REPO_LOWER}/solr; - ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} \ No newline at end of file + ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} From cdf42450a4ec2af54636d259be6938d8932d3edf Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 15 Nov 2022 09:33:01 -0800 Subject: [PATCH 122/159] adding back the metadata tags for our retag system --- .github/workflows/build.yaml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7bcd529..127ce9b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -96,9 +96,6 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - flavor: | - latest=auto - name: Retag action for worker if: ${{ inputs.worker }} id: meta-worker @@ -108,9 +105,6 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - flavor: | - latest=auto - name: Retag action for solr if: ${{ inputs.solr }} id: meta-solr @@ -120,10 +114,7 @@ jobs: name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr tags: | type=raw,value=latest,enable={{is_default_branch}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - flavor: | - latest=auto - - name: Build and push + - name: Build and push web uses: docker/build-push-action@v3 with: build-args: | @@ -135,6 +126,7 @@ jobs: platforms: ${{ inputs.platforms }} push: true tags: | + ${{ steps.meta-web.outputs.tags }} ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} target: ${{ inputs.target }} - name: Build and push worker @@ -150,6 +142,7 @@ jobs: platforms: ${{ inputs.platforms }} push: true tags: | + ${{ steps.meta-worker.outputs.tags }} ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} target: ${{ inputs.workerTarget }} - name: Build and push solr From 3c22e208e3092accdbd753bb4aa5cd43475cbfb8 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 15 Nov 2022 09:53:01 -0800 Subject: [PATCH 123/159] adding the solr tag for the short sha --- .github/workflows/build.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 127ce9b..3dfbdd7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -154,4 +154,6 @@ jobs: cache-from: | ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} push: true - tags: ${{ steps.meta-solr.outputs.tags }} + tags: | + ${{ steps.meta-solr.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} From 7eb92cca96075bf32dc30df9fd53a3ac0007bf3a Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 15 Nov 2022 10:32:56 -0800 Subject: [PATCH 124/159] There was a reason this was v1 --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 606033f..3b36ded 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -77,7 +77,7 @@ jobs: - name: Set up Docker Context for Buildx run: docker context create builders - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v1 with: version: latest endpoint: builders From 6af16e5872fb720e0d06494f3dd7d2e4a11c6ded Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 15 Nov 2022 10:42:45 -0800 Subject: [PATCH 125/159] Add the tags bacl --- .github/workflows/build.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3b36ded..9bdcbd5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -124,13 +124,14 @@ jobs: with: build-args: | APP_PATH=${{ inputs.subdir }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:latest + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} context: . file: ${{ inputs.subdir}}/Dockerfile platforms: ${{ inputs.platforms }} push: true tags: | ${{ steps.meta-web.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} target: ${{ inputs.target }} - name: Build and push worker if: ${{ inputs.worker }} @@ -138,22 +139,24 @@ jobs: with: build-args: | APP_PATH=${{ inputs.subdir }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:latest + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} context: . file: ${{ inputs.subdir}}/Dockerfile platforms: ${{ inputs.platforms }} push: true tags: | ${{ steps.meta-worker.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} target: ${{ inputs.workerTarget }} - name: Build and push solr if: ${{ inputs.solr }} uses: docker/build-push-action@v3 with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:latest + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} context: . platforms: ${{ inputs.platforms }} push: true tags: | ${{ steps.meta-solr.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} target: ${{ inputs.target }} From 1ba223f972faa9dcb6d7dc38c0f0708ba7d549cc Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 15 Nov 2022 10:57:50 -0800 Subject: [PATCH 126/159] use the confdir --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index db48f35..0736649 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -89,7 +89,7 @@ jobs: if [ -d solr ] then docker-compose exec -T web sh -c \ - "solrcloud-upload-configset.sh /app/samvera/hyrax-webapp/solr/config && + "solrcloud-upload-configset.sh ${CONFDIR} && SOLR_COLLECTION_NAME=hydra-test solrcloud-assign-configset.sh && solrcloud-assign-configset.sh" else From d6bc3ea0f41740a07ab469e221561f8b678dbe07 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 15 Nov 2022 11:00:48 -0800 Subject: [PATCH 127/159] add back v2 --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9bdcbd5..04d3fca 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -77,7 +77,7 @@ jobs: - name: Set up Docker Context for Buildx run: docker context create builders - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: version: latest endpoint: builders From 770e2284636de9d2b8bd88d135492e781d4946de Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 15 Nov 2022 11:12:20 -0800 Subject: [PATCH 128/159] the main is the thing --- .github/workflows/deploy.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 8e6d4a7..4e3cd7b 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -55,7 +55,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner + uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag }} image_name: ${{ inputs.image_name }} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 59d0ce1..2ef3687 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -32,7 +32,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner + uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag }} - name: Checkout code diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d6e1718..c330a94 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -56,7 +56,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@self-hosted-runner + uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag }} - name: Checkout code From 0faa8bbcf2a5a1cb4a5962829b7ef5fda526478f Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 15 Nov 2022 20:46:55 -0800 Subject: [PATCH 129/159] try a single arm builder --- .github/workflows/build.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 04d3fca..a311919 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -78,9 +78,15 @@ jobs: run: docker context create builders - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + id: builder with: version: latest endpoint: builders + - name: Append ARM buildx builder + uses: baschny/append-buildx-action@v1 + with: + builder: ${{ steps.builder.outputs.name }} + endpoint: "tcp://35.90.131.78:2375" - name: Pull from cache to speed up build run: >- cd ${{ inputs.subdir }}; From 71dab15311400669d47367752d779e1b849143f4 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 15 Nov 2022 20:49:56 -0800 Subject: [PATCH 130/159] use internal network --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a311919..317a51c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -86,7 +86,7 @@ jobs: uses: baschny/append-buildx-action@v1 with: builder: ${{ steps.builder.outputs.name }} - endpoint: "tcp://35.90.131.78:2375" + endpoint: "tcp://10.0.5.114:2375" - name: Pull from cache to speed up build run: >- cd ${{ inputs.subdir }}; From 30bd9418638d7b7e65a62645eb76072ddfebc27b Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 15 Nov 2022 21:12:26 -0800 Subject: [PATCH 131/159] try ssh auth --- .github/workflows/build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 317a51c..9219ec8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -86,7 +86,8 @@ jobs: uses: baschny/append-buildx-action@v1 with: builder: ${{ steps.builder.outputs.name }} - endpoint: "tcp://10.0.5.114:2375" + endpoint: "ssh://ec2-user@10.0.5.114" + ssh_private_key: ${{ secrets.SSH_BESTIE_BUILDER_PRIVATE_KEY }} - name: Pull from cache to speed up build run: >- cd ${{ inputs.subdir }}; From f04d729d79c5fa478b73ff34b7248b4f2aa117d5 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 15 Nov 2022 22:47:45 -0800 Subject: [PATCH 132/159] add env in case it is missing this keeps dockercompose from simply failing --- setup-env/action.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index e29cb06..07ee16e 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -30,3 +30,7 @@ runs: uses: KengoTODA/actions-setup-docker-compose@main with: version: '2.12.2' + - name: Confirm env exists + run: >- + touch .env; + touch .env.development From 7199756d91a270115c10c9548a7902fec4ea91f1 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 15 Nov 2022 22:49:04 -0800 Subject: [PATCH 133/159] fix missing shell --- setup-env/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 07ee16e..48394ff 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -31,6 +31,7 @@ runs: with: version: '2.12.2' - name: Confirm env exists + shell: bash run: >- touch .env; touch .env.development From 58c40fbc4ffb6d8d62ed0be5c0bb45878e6530bc Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 15 Nov 2022 22:50:58 -0800 Subject: [PATCH 134/159] nevermind - too soon in process --- setup-env/action.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 48394ff..e29cb06 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -30,8 +30,3 @@ runs: uses: KengoTODA/actions-setup-docker-compose@main with: version: '2.12.2' - - name: Confirm env exists - shell: bash - run: >- - touch .env; - touch .env.development From 26c65eb1ae07377db905f97b360df9882dd26aef Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Tue, 15 Nov 2022 22:51:35 -0800 Subject: [PATCH 135/159] make sure .env is present --- .github/workflows/lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 2ef3687..7899756 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -47,6 +47,7 @@ jobs: run: >- cd ${{ inputs.subdir }}; touch .env.development; + touch .env; docker-compose pull web - name: Pull worker image to prevent build run: >- From 258d2d891fac82cd8651e6fc8d3244e5c50dfc76 Mon Sep 17 00:00:00 2001 From: Benjamin Kiah Stroud Date: Thu, 17 Nov 2022 12:08:24 -0800 Subject: [PATCH 136/159] move duplicate confdir input from dispatch to call --- .github/workflows/test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8c48f56..aca0ef8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,15 +16,15 @@ on: required: false type: string default: "gem install semaphore_test_boosters && rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" - confdir: - required: false - type: string - default: "/app/samvera/hyrax-webapp/solr/config" worker: required: false type: boolean workflow_call: inputs: + confdir: + required: false + type: string + default: "/app/samvera/hyrax-webapp/solr/config" subdir: default: '.' type: string From 3c099f0bdd353763a89dce06bda936d46fd14d44 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Sun, 20 Nov 2022 01:27:42 -0800 Subject: [PATCH 137/159] update arm ip --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9219ec8..540482b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -86,7 +86,7 @@ jobs: uses: baschny/append-buildx-action@v1 with: builder: ${{ steps.builder.outputs.name }} - endpoint: "ssh://ec2-user@10.0.5.114" + endpoint: "ssh://ec2-user@10.0.6.48" ssh_private_key: ${{ secrets.SSH_BESTIE_BUILDER_PRIVATE_KEY }} - name: Pull from cache to speed up build run: >- From bb93018564496e49c89806437eddbb69db4dc1b6 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Sun, 20 Nov 2022 08:41:09 -0800 Subject: [PATCH 138/159] Make sure schema is writeable --- .github/workflows/lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 7899756..39629e5 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -57,6 +57,7 @@ jobs: - name: Run Rubocop run: >- cd ${{ inputs.subdir }}; + chmod 777 db/schema.rb; docker-compose run web bundle exec rubocop --parallel --format progress - name: Publish Test Report uses: mikepenz/action-junit-report@v3 From 93e4198c46df6a0e5a66f1e7374e7335154cc7a5 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Sun, 20 Nov 2022 13:53:54 -0800 Subject: [PATCH 139/159] make sure schema is writeable --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index aca0ef8..85a3ef7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -94,6 +94,7 @@ jobs: - name: Start containers run: >- cd ${{ inputs.subdir }}; + chmod 777 db/schema.rb; docker-compose up -d web - name: Check for and setup Solr if needed shell: bash From f3140ba77f3523e2c75e929de6afa01d5499f465 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Wed, 23 Nov 2022 13:55:40 -0800 Subject: [PATCH 140/159] solr cloud steps on solr cloud only --- .github/workflows/test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 85a3ef7..f579642 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -100,7 +100,8 @@ jobs: shell: bash run: | cd ${{ inputs.subdir }}; - if [ -d solr ] + solrcloud_path=$(which solrcloud-upload-configset.sh) + if [[ -d solr && -x "$solrcloud_path" ]] then docker-compose exec -T web sh -c \ "solrcloud-upload-configset.sh "${CONFDIR}" && From f0ea11555adcfbb03179544bc92a5d2599584951 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Wed, 23 Nov 2022 15:09:40 -0800 Subject: [PATCH 141/159] no bash, no problems --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f579642..e2435b0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -97,7 +97,7 @@ jobs: chmod 777 db/schema.rb; docker-compose up -d web - name: Check for and setup Solr if needed - shell: bash + shell: sh run: | cd ${{ inputs.subdir }}; solrcloud_path=$(which solrcloud-upload-configset.sh) From 099d4816aa9219317cefc786c49b88017de01930 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Wed, 23 Nov 2022 22:09:22 -0800 Subject: [PATCH 142/159] ok, some problems --- .github/workflows/test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e2435b0..1d8ba80 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -100,8 +100,7 @@ jobs: shell: sh run: | cd ${{ inputs.subdir }}; - solrcloud_path=$(which solrcloud-upload-configset.sh) - if [[ -d solr && -x "$solrcloud_path" ]] + if [ -d solr ] then docker-compose exec -T web sh -c \ "solrcloud-upload-configset.sh "${CONFDIR}" && From 45de035ee269451e409df662b9febdd605b4c1ed Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 08:55:22 -0800 Subject: [PATCH 143/159] updates to macos-11 github runner testing arm64 build --- .github/workflows/build.yaml | 93 +++++------------------------------- 1 file changed, 13 insertions(+), 80 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 540482b..1e35712 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -60,66 +60,25 @@ env: jobs: build: - runs-on: self-hosted + runs-on: macos-11 steps: - id: setup name: Setup uses: scientist-softserv/actions/setup-env@main - with: - tag: ${{ inputs.tag }} - image_name: ${{ inputs.image_name }} - name: Checkout code uses: actions/checkout@v3 - with: - submodules: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Context for Buildx - run: docker context create builders - name: Set up Docker Buildx + id: buildx uses: docker/setup-buildx-action@v2 - id: builder - with: - version: latest - endpoint: builders - - name: Append ARM buildx builder - uses: baschny/append-buildx-action@v1 - with: - builder: ${{ steps.builder.outputs.name }} - endpoint: "ssh://ec2-user@10.0.6.48" - ssh_private_key: ${{ secrets.SSH_BESTIE_BUILDER_PRIVATE_KEY }} - - name: Pull from cache to speed up build - run: >- - cd ${{ inputs.subdir }}; - touch .env.development; - touch .env; - TAG=latest docker-compose pull web || true - name: Retag action for web id: meta-web uses: docker/metadata-action@v4.1.1 with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - - name: Retag action for worker - if: ${{ inputs.worker }} - id: meta-worker - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker - tags: | - type=raw,value=latest,enable={{is_default_branch}} - - name: Retag action for solr - if: ${{ inputs.solr }} - id: meta-solr - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | - type=raw,value=latest,enable={{is_default_branch}} + type=raw + value=latest + enable={{is_default_branch}} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -129,41 +88,15 @@ jobs: - name: Build and push web uses: docker/build-push-action@v3 with: - build-args: | - APP_PATH=${{ inputs.subdir }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + builder: ${{ steps.buildx.outputs.name }} + cache-from: | + type=registry + ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} context: . - file: ${{ inputs.subdir}}/Dockerfile - platforms: ${{ inputs.platforms }} + labels: ${{ steps.meta-web.outputs.labels }} + platforms: linux/amd64,linux/arm64 push: true tags: | ${{ steps.meta-web.outputs.tags }} ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - target: ${{ inputs.target }} - - name: Build and push worker - if: ${{ inputs.worker }} - uses: docker/build-push-action@v3 - with: - build-args: | - APP_PATH=${{ inputs.subdir }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - context: . - file: ${{ inputs.subdir}}/Dockerfile - platforms: ${{ inputs.platforms }} - push: true - tags: | - ${{ steps.meta-worker.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - target: ${{ inputs.workerTarget }} - - name: Build and push solr - if: ${{ inputs.solr }} - uses: docker/build-push-action@v3 - with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} - context: . - platforms: ${{ inputs.platforms }} - push: true - tags: | - ${{ steps.meta-solr.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} - target: ${{ inputs.target }} + target: hyku-base From 02ef9a4fe77051b21edaa310c857689649ee0a12 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 09:16:49 -0800 Subject: [PATCH 144/159] Updates to add specific platform and services to split out builds to try and help speed up the build process for deployments --- .github/workflows/build-solr-arm64.yaml | 82 ++++++++++++++++++++ .github/workflows/build-solr.yaml | 82 ++++++++++++++++++++ .github/workflows/build-web-arm64.yaml | 84 ++++++++++++++++++++ .github/workflows/build-web.yaml | 84 ++++++++++++++++++++ .github/workflows/build-worker-arm64.yaml | 83 ++++++++++++++++++++ .github/workflows/build-worker.yaml | 82 ++++++++++++++++++++ .github/workflows/build.yaml | 93 +++++++++++++++++++---- 7 files changed, 577 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/build-solr-arm64.yaml create mode 100644 .github/workflows/build-solr.yaml create mode 100644 .github/workflows/build-web-arm64.yaml create mode 100644 .github/workflows/build-web.yaml create mode 100644 .github/workflows/build-worker-arm64.yaml create mode 100644 .github/workflows/build-worker.yaml diff --git a/.github/workflows/build-solr-arm64.yaml b/.github/workflows/build-solr-arm64.yaml new file mode 100644 index 0000000..7276565 --- /dev/null +++ b/.github/workflows/build-solr-arm64.yaml @@ -0,0 +1,82 @@ +name: "Build Docker Images" +on: + workflow_dispatch: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + workflow_call: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + +env: + REGISTRY: ghcr.io + +jobs: + build-solr-arm64: + runs-on: ubuntu-latest + steps: + - id: setup + name: Setup + uses: scientist-softserv/actions/setup-env@main + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - name: Retag action for solr + id: meta-solr + uses: docker/metadata-action@v4.1.1 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr + tags: | + type=raw + value=latest + enable={{is_default_branch}} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push solr + uses: docker/build-push-action@v3 + with: + builder: ${{ steps.buildx.outputs.name }} + cache-from: | + type=registry + ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + context: . + labels: ${{ steps.meta-solr.outputs.labels }} + platforms: linux/arm64 + push: true + tags: | + ${{ steps.meta-solr.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} + target: hyku-solr diff --git a/.github/workflows/build-solr.yaml b/.github/workflows/build-solr.yaml new file mode 100644 index 0000000..5506670 --- /dev/null +++ b/.github/workflows/build-solr.yaml @@ -0,0 +1,82 @@ +name: "Build Docker Images" +on: + workflow_dispatch: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + workflow_call: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + +env: + REGISTRY: ghcr.io + +jobs: + build-solr: + runs-on: ubuntu-latest + steps: + - id: setup + name: Setup + uses: scientist-softserv/actions/setup-env@main + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - name: Retag action for solr + id: meta-solr + uses: docker/metadata-action@v4.1.1 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr + tags: | + type=raw + value=latest + enable={{is_default_branch}} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push solr + uses: docker/build-push-action@v3 + with: + builder: ${{ steps.buildx.outputs.name }} + cache-from: | + type=registry + ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + context: . + labels: ${{ steps.meta-solr.outputs.labels }} + platforms: linux/amd64 + push: true + tags: | + ${{ steps.meta-solr.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} + target: hyku-solr diff --git a/.github/workflows/build-web-arm64.yaml b/.github/workflows/build-web-arm64.yaml new file mode 100644 index 0000000..10dee6f --- /dev/null +++ b/.github/workflows/build-web-arm64.yaml @@ -0,0 +1,84 @@ +name: "Build Docker Images" +on: + workflow_dispatch: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + workflow_call: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + +env: + REGISTRY: ghcr.io + +jobs: + build-web-arm64: + runs-on: macos-11 + steps: + - id: setup + name: Setup + uses: scientist-softserv/actions/setup-env@main + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - name: Retag action for web + id: meta-web + uses: docker/metadata-action@v4.1.1 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }} + tags: | + type=raw + value=latest + enable={{is_default_branch}} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push web + uses: docker/build-push-action@v3 + with: + builder: ${{ steps.buildx.outputs.name }} + cache-from: | + type=registry + ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + context: . + labels: ${{ steps.meta-web.outputs.labels }} + platforms: linux/arm64 + push: true + tags: | + ${{ steps.meta-web.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + target: hyku-base diff --git a/.github/workflows/build-web.yaml b/.github/workflows/build-web.yaml new file mode 100644 index 0000000..474e10d --- /dev/null +++ b/.github/workflows/build-web.yaml @@ -0,0 +1,84 @@ +name: "Build Docker Images" +on: + workflow_dispatch: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + workflow_call: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + +env: + REGISTRY: ghcr.io + +jobs: + build-web: + runs-on: ubuntu-latest + steps: + - id: setup + name: Setup + uses: scientist-softserv/actions/setup-env@main + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - name: Retag action for web + id: meta-web + uses: docker/metadata-action@v4.1.1 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }} + tags: | + type=raw + value=latest + enable={{is_default_branch}} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push web + uses: docker/build-push-action@v3 + with: + builder: ${{ steps.buildx.outputs.name }} + cache-from: | + type=registry + ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + context: . + labels: ${{ steps.meta-web.outputs.labels }} + platforms: linux/amd64 + push: true + tags: | + ${{ steps.meta-web.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + target: hyku-base diff --git a/.github/workflows/build-worker-arm64.yaml b/.github/workflows/build-worker-arm64.yaml new file mode 100644 index 0000000..b69ee33 --- /dev/null +++ b/.github/workflows/build-worker-arm64.yaml @@ -0,0 +1,83 @@ +on: +name: "Build Docker Images" +on: + workflow_dispatch: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + workflow_call: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + +env: + REGISTRY: ghcr.io + +jobs: + build-worker-arm64: + runs-on: ubuntu-latest + steps: + - id: setup + name: Setup + uses: scientist-softserv/actions/setup-env@main + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - name: Retag action for worker + id: meta-worker + uses: docker/metadata-action@v4.1.1 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker + tags: | + type=raw + value=latest + enable={{is_default_branch}} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push worker + uses: docker/build-push-action@v3 + with: + builder: ${{ steps.buildx.outputs.name }} + cache-from: | + type=registry + ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + context: . + labels: ${{ steps.meta-worker.outputs.labels }} + platforms: linux/arm64 + push: true + tags: | + ${{ steps.meta-worker.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + target: hyku-worker diff --git a/.github/workflows/build-worker.yaml b/.github/workflows/build-worker.yaml new file mode 100644 index 0000000..50b86d7 --- /dev/null +++ b/.github/workflows/build-worker.yaml @@ -0,0 +1,82 @@ +name: "Build Docker Images" +on: + workflow_dispatch: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + workflow_call: + inputs: + image_name: + type: string + description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" + subdir: + default: '.' + type: string + tag: + required: false + type: string + target: + required: false + type: string + workerTarget: + required: false + type: string + +env: + REGISTRY: ghcr.io + +jobs: + build-worker: + runs-on: ubuntu-latest + steps: + - id: setup + name: Setup + uses: scientist-softserv/actions/setup-env@main + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - name: Retag action for worker + id: meta-worker + uses: docker/metadata-action@v4.1.1 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker + tags: | + type=raw + value=latest + enable={{is_default_branch}} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push worker + uses: docker/build-push-action@v3 + with: + builder: ${{ steps.buildx.outputs.name }} + cache-from: | + type=registry + ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + context: . + labels: ${{ steps.meta-worker.outputs.labels }} + platforms: linux/amd64 + push: true + tags: | + ${{ steps.meta-worker.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + target: hyku-worker diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1e35712..540482b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -60,25 +60,66 @@ env: jobs: build: - runs-on: macos-11 + runs-on: self-hosted steps: - id: setup name: Setup uses: scientist-softserv/actions/setup-env@main + with: + tag: ${{ inputs.tag }} + image_name: ${{ inputs.image_name }} - name: Checkout code uses: actions/checkout@v3 + with: + submodules: true + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Context for Buildx + run: docker context create builders - name: Set up Docker Buildx - id: buildx uses: docker/setup-buildx-action@v2 + id: builder + with: + version: latest + endpoint: builders + - name: Append ARM buildx builder + uses: baschny/append-buildx-action@v1 + with: + builder: ${{ steps.builder.outputs.name }} + endpoint: "ssh://ec2-user@10.0.6.48" + ssh_private_key: ${{ secrets.SSH_BESTIE_BUILDER_PRIVATE_KEY }} + - name: Pull from cache to speed up build + run: >- + cd ${{ inputs.subdir }}; + touch .env.development; + touch .env; + TAG=latest docker-compose pull web || true - name: Retag action for web id: meta-web uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }} + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + - name: Retag action for worker + if: ${{ inputs.worker }} + id: meta-worker + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker + tags: | + type=raw,value=latest,enable={{is_default_branch}} + - name: Retag action for solr + if: ${{ inputs.solr }} + id: meta-solr + uses: docker/metadata-action@v4.1.1 + with: + images: | + name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr tags: | - type=raw - value=latest - enable={{is_default_branch}} + type=raw,value=latest,enable={{is_default_branch}} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -88,15 +129,41 @@ jobs: - name: Build and push web uses: docker/build-push-action@v3 with: - builder: ${{ steps.buildx.outputs.name }} - cache-from: | - type=registry - ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + build-args: | + APP_PATH=${{ inputs.subdir }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} context: . - labels: ${{ steps.meta-web.outputs.labels }} - platforms: linux/amd64,linux/arm64 + file: ${{ inputs.subdir}}/Dockerfile + platforms: ${{ inputs.platforms }} push: true tags: | ${{ steps.meta-web.outputs.tags }} ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - target: hyku-base + target: ${{ inputs.target }} + - name: Build and push worker + if: ${{ inputs.worker }} + uses: docker/build-push-action@v3 + with: + build-args: | + APP_PATH=${{ inputs.subdir }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + context: . + file: ${{ inputs.subdir}}/Dockerfile + platforms: ${{ inputs.platforms }} + push: true + tags: | + ${{ steps.meta-worker.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + target: ${{ inputs.workerTarget }} + - name: Build and push solr + if: ${{ inputs.solr }} + uses: docker/build-push-action@v3 + with: + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} + context: . + platforms: ${{ inputs.platforms }} + push: true + tags: | + ${{ steps.meta-solr.outputs.tags }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} + target: ${{ inputs.target }} From 077f0825811e24cd152e7be635008c3d6dfda2d2 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 10:09:41 -0800 Subject: [PATCH 145/159] updates syntax to clear error --- .github/workflows/build-worker-arm64.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-worker-arm64.yaml b/.github/workflows/build-worker-arm64.yaml index b69ee33..a64e321 100644 --- a/.github/workflows/build-worker-arm64.yaml +++ b/.github/workflows/build-worker-arm64.yaml @@ -1,4 +1,3 @@ -on: name: "Build Docker Images" on: workflow_dispatch: From 0f58c61c8d831efc05a3a741ce36786e652ebaa4 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 11:04:43 -0800 Subject: [PATCH 146/159] updates actions to split platforms and web/worker/solr --- .github/workflows/build-solr-arm64.yaml | 25 ++++++++--------------- .github/workflows/build-solr.yaml | 23 +++++++-------------- .github/workflows/build-web-arm64.yaml | 25 +++++++---------------- .github/workflows/build-web.yaml | 23 +++++++-------------- .github/workflows/build-worker-arm64.yaml | 23 ++++++++------------- .github/workflows/build-worker.yaml | 21 +++++++------------ .github/workflows/build.yaml | 5 ++++- .github/workflows/deploy.yaml | 10 ++++----- setup-env/action.yaml | 12 +++++------ 9 files changed, 58 insertions(+), 109 deletions(-) diff --git a/.github/workflows/build-solr-arm64.yaml b/.github/workflows/build-solr-arm64.yaml index 7276565..8938d9c 100644 --- a/.github/workflows/build-solr-arm64.yaml +++ b/.github/workflows/build-solr-arm64.yaml @@ -14,9 +14,6 @@ on: target: required: false type: string - workerTarget: - required: false - type: string workflow_call: inputs: image_name: @@ -31,20 +28,17 @@ on: target: required: false type: string - workerTarget: - required: false - type: string env: REGISTRY: ghcr.io jobs: build-solr-arm64: - runs-on: ubuntu-latest + runs-on: macos-11 steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@arm-wrestle - name: Checkout code uses: actions/checkout@v3 - name: Set up Docker Buildx @@ -54,11 +48,10 @@ jobs: id: meta-solr uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr + images: ${{ env.REGISTRY }}/${{ env.REPO }}/solr tags: | - type=raw - value=latest - enable={{is_default_branch}} + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -69,14 +62,12 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - cache-from: | - type=registry - ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + cache-from: type=registry,ref=${{ steps.repo_slug.outputs.result }}/solr:${{ env.TAG }} context: . labels: ${{ steps.meta-solr.outputs.labels }} platforms: linux/arm64 push: true tags: | ${{ steps.meta-solr.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} - target: hyku-solr + ${{ steps.repo_slug.outputs.result }}/solr:${{ env.TAG }} + target: ${{ inputs.target }} diff --git a/.github/workflows/build-solr.yaml b/.github/workflows/build-solr.yaml index 5506670..f1090c5 100644 --- a/.github/workflows/build-solr.yaml +++ b/.github/workflows/build-solr.yaml @@ -14,9 +14,6 @@ on: target: required: false type: string - workerTarget: - required: false - type: string workflow_call: inputs: image_name: @@ -31,9 +28,6 @@ on: target: required: false type: string - workerTarget: - required: false - type: string env: REGISTRY: ghcr.io @@ -44,7 +38,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@arm-wrestle - name: Checkout code uses: actions/checkout@v3 - name: Set up Docker Buildx @@ -54,11 +48,10 @@ jobs: id: meta-solr uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr + images: ${{ env.REGISTRY }}/${{ env.REPO }}/solr tags: | - type=raw - value=latest - enable={{is_default_branch}} + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -69,14 +62,12 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - cache-from: | - type=registry - ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + cache-from: type=registry,ref=${{ steps.repo_slug.outputs.result }}/solr:${{ env.TAG }} context: . labels: ${{ steps.meta-solr.outputs.labels }} platforms: linux/amd64 push: true tags: | ${{ steps.meta-solr.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} - target: hyku-solr + ${{ steps.repo_slug.outputs.result }}/solr:${{ env.TAG }} + target: ${{ inputs.target }} diff --git a/.github/workflows/build-web-arm64.yaml b/.github/workflows/build-web-arm64.yaml index 10dee6f..581840e 100644 --- a/.github/workflows/build-web-arm64.yaml +++ b/.github/workflows/build-web-arm64.yaml @@ -14,9 +14,6 @@ on: target: required: false type: string - workerTarget: - required: false - type: string workflow_call: inputs: image_name: @@ -31,9 +28,6 @@ on: target: required: false type: string - workerTarget: - required: false - type: string env: REGISTRY: ghcr.io @@ -44,11 +38,9 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@arm-wrestle - name: Checkout code uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 @@ -56,11 +48,10 @@ jobs: id: meta-web uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }} + images: ${{ env.REGISTRY }}/${{ env.REPO }} tags: | - type=raw - value=latest - enable={{is_default_branch}} + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -71,14 +62,12 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - cache-from: | - type=registry - ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + cache-from: type=registry,ref=${{ steps.repo_slug.outputs.result }}:${{ env.TAG }} context: . labels: ${{ steps.meta-web.outputs.labels }} platforms: linux/arm64 push: true tags: | ${{ steps.meta-web.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - target: hyku-base + ${{ steps.repo_slug.outputs.result }}:${{ env.TAG }} + target: ${{ inputs.target }} diff --git a/.github/workflows/build-web.yaml b/.github/workflows/build-web.yaml index 474e10d..5091652 100644 --- a/.github/workflows/build-web.yaml +++ b/.github/workflows/build-web.yaml @@ -14,9 +14,6 @@ on: target: required: false type: string - workerTarget: - required: false - type: string workflow_call: inputs: image_name: @@ -31,9 +28,6 @@ on: target: required: false type: string - workerTarget: - required: false - type: string env: REGISTRY: ghcr.io @@ -44,11 +38,9 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@arm-wrestle - name: Checkout code uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 @@ -56,11 +48,10 @@ jobs: id: meta-web uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }} + images: ${{ env.REGISTRY }}/${{ env.REPO }} tags: | - type=raw - value=latest - enable={{is_default_branch}} + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -73,12 +64,12 @@ jobs: builder: ${{ steps.buildx.outputs.name }} cache-from: | type=registry - ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + ref=${{ steps.repo_slug.outputs.result }}:${{ env.TAG }} context: . labels: ${{ steps.meta-web.outputs.labels }} platforms: linux/amd64 push: true tags: | ${{ steps.meta-web.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - target: hyku-base + ${{ steps.repo_slug.outputs.result }}:${{ env.TAG }} + target: ${{ inputs.target }} diff --git a/.github/workflows/build-worker-arm64.yaml b/.github/workflows/build-worker-arm64.yaml index a64e321..0c5ac8a 100644 --- a/.github/workflows/build-worker-arm64.yaml +++ b/.github/workflows/build-worker-arm64.yaml @@ -14,9 +14,6 @@ on: target: required: false type: string - workerTarget: - required: false - type: string workflow_call: inputs: image_name: @@ -31,20 +28,17 @@ on: target: required: false type: string - workerTarget: - required: false - type: string env: REGISTRY: ghcr.io jobs: build-worker-arm64: - runs-on: ubuntu-latest + runs-on: macos-11 steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@arm-wrestle - name: Checkout code uses: actions/checkout@v3 - name: Set up Docker Buildx @@ -54,11 +48,10 @@ jobs: id: meta-worker uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker + images: ${{ env.REGISTRY }}/${{ env.REPO }}/worker tags: | - type=raw - value=latest - enable={{is_default_branch}} + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -71,12 +64,12 @@ jobs: builder: ${{ steps.buildx.outputs.name }} cache-from: | type=registry - ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + ref=${{ steps.repo_slug.outputs.result }}/worker:${{ env.TAG }} context: . labels: ${{ steps.meta-worker.outputs.labels }} platforms: linux/arm64 push: true tags: | ${{ steps.meta-worker.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - target: hyku-worker + ${{ steps.repo_slug.outputs.result }}/worker:${{ env.TAG }} + target: ${{ inputs.target }} diff --git a/.github/workflows/build-worker.yaml b/.github/workflows/build-worker.yaml index 50b86d7..20dfc35 100644 --- a/.github/workflows/build-worker.yaml +++ b/.github/workflows/build-worker.yaml @@ -14,9 +14,6 @@ on: target: required: false type: string - workerTarget: - required: false - type: string workflow_call: inputs: image_name: @@ -31,9 +28,6 @@ on: target: required: false type: string - workerTarget: - required: false - type: string env: REGISTRY: ghcr.io @@ -44,7 +38,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@arm-wrestle - name: Checkout code uses: actions/checkout@v3 - name: Set up Docker Buildx @@ -54,11 +48,10 @@ jobs: id: meta-worker uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker + images: ${{ env.REGISTRY }}/${{ env.REPO }}/worker tags: | - type=raw - value=latest - enable={{is_default_branch}} + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -71,12 +64,12 @@ jobs: builder: ${{ steps.buildx.outputs.name }} cache-from: | type=registry - ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + ref=${{ steps.repo_slug.outputs.result }}/worker:${{ env.TAG }} context: . labels: ${{ steps.meta-worker.outputs.labels }} platforms: linux/amd64 push: true tags: | ${{ steps.meta-worker.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - target: hyku-worker + ${{ steps.repo_slug.outputs.result }}/worker:${{ env.TAG }} + target: ${{ inputs.target }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 540482b..aab33cf 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -101,6 +101,7 @@ jobs: images: | name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | + # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}} - name: Retag action for worker if: ${{ inputs.worker }} @@ -110,6 +111,7 @@ jobs: images: | name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker tags: | + # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}} - name: Retag action for solr if: ${{ inputs.solr }} @@ -119,6 +121,7 @@ jobs: images: | name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr tags: | + # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}} - name: Login to GitHub Container Registry uses: docker/login-action@v2 @@ -138,7 +141,7 @@ jobs: push: true tags: | ${{ steps.meta-web.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + ${{ steps.repo_slug.outputs.result }}:${{ env.TAG }} target: ${{ inputs.target }} - name: Build and push worker if: ${{ inputs.worker }} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 4e3cd7b..b708370 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -72,15 +72,15 @@ jobs: echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; - export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; - export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; - export SOLR_IMAGE=ghcr.io/${REPO_LOWER}/solr; + export DEPLOY_IMAGE=ghcr.io/${REPO}; + export WORKER_IMAGE=ghcr.io/${REPO}/worker; + export SOLR_IMAGE=ghcr.io/${REPO}/solr; ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} - name: Do deploy run: | echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; - export DEPLOY_IMAGE=ghcr.io/${REPO_LOWER}; - export WORKER_IMAGE=ghcr.io/${REPO_LOWER}/worker; + export DEPLOY_IMAGE=ghcr.io/${REPO}; + export WORKER_IMAGE=ghcr.io/${REPO}/worker; ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} diff --git a/setup-env/action.yaml b/setup-env/action.yaml index e29cb06..4f30047 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -20,13 +20,11 @@ runs: echo ${HEAD_TAG::8}; echo "export DOCKER_CLIENT_TIMEOUT=360"; echo "export COMPOSE_HTTP_TIMEOUT=360" - - name: Downcase repo + - name: Sanitize repo slug env: REPO: "${{ inputs.image_name || github.repository }}" - shell: bash - run: | - echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} - - name: Install docker-compose - uses: KengoTODA/actions-setup-docker-compose@main + uses: actions/github-script@v6 + id: repo_slug with: - version: '2.12.2' + result-encoding: string + script: return 'ghcr.io/${{ REPO }}'.toLowerCase() From 3e852fc5df0a5701982c74224d5fa47187ffacec Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 11:15:00 -0800 Subject: [PATCH 147/159] adds repo env variable --- .github/workflows/build-solr-arm64.yaml | 1 + .github/workflows/build-solr.yaml | 1 + .github/workflows/build-web-arm64.yaml | 1 + .github/workflows/build-web.yaml | 1 + .github/workflows/build-worker-arm64.yaml | 1 + .github/workflows/build-worker.yaml | 1 + .github/workflows/build.yaml | 172 ---------------------- .github/workflows/deploy.yaml | 13 +- setup-env/action.yaml | 4 +- 9 files changed, 14 insertions(+), 181 deletions(-) delete mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build-solr-arm64.yaml b/.github/workflows/build-solr-arm64.yaml index 8938d9c..8f53851 100644 --- a/.github/workflows/build-solr-arm64.yaml +++ b/.github/workflows/build-solr-arm64.yaml @@ -31,6 +31,7 @@ on: env: REGISTRY: ghcr.io + REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-solr-arm64: diff --git a/.github/workflows/build-solr.yaml b/.github/workflows/build-solr.yaml index f1090c5..d8e8309 100644 --- a/.github/workflows/build-solr.yaml +++ b/.github/workflows/build-solr.yaml @@ -31,6 +31,7 @@ on: env: REGISTRY: ghcr.io + REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-solr: diff --git a/.github/workflows/build-web-arm64.yaml b/.github/workflows/build-web-arm64.yaml index 581840e..173cca3 100644 --- a/.github/workflows/build-web-arm64.yaml +++ b/.github/workflows/build-web-arm64.yaml @@ -31,6 +31,7 @@ on: env: REGISTRY: ghcr.io + REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-web-arm64: diff --git a/.github/workflows/build-web.yaml b/.github/workflows/build-web.yaml index 5091652..866fb1d 100644 --- a/.github/workflows/build-web.yaml +++ b/.github/workflows/build-web.yaml @@ -31,6 +31,7 @@ on: env: REGISTRY: ghcr.io + REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-web: diff --git a/.github/workflows/build-worker-arm64.yaml b/.github/workflows/build-worker-arm64.yaml index 0c5ac8a..daaa11f 100644 --- a/.github/workflows/build-worker-arm64.yaml +++ b/.github/workflows/build-worker-arm64.yaml @@ -31,6 +31,7 @@ on: env: REGISTRY: ghcr.io + REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-worker-arm64: diff --git a/.github/workflows/build-worker.yaml b/.github/workflows/build-worker.yaml index 20dfc35..f159ba4 100644 --- a/.github/workflows/build-worker.yaml +++ b/.github/workflows/build-worker.yaml @@ -31,6 +31,7 @@ on: env: REGISTRY: ghcr.io + REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-worker: diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index aab33cf..0000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,172 +0,0 @@ -name: "Build Docker Images" -on: - workflow_dispatch: - inputs: - image_name: - type: string - description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" - platforms: - default: "linux/amd64,linux/arm64" - type: string - subdir: - default: '.' - type: string - solr: - description: "Use if you need to build a specific Solr image and push to the registry" - required: false - type: boolean - tag: - required: false - type: string - target: - required: false - type: string - worker: - required: false - type: boolean - workerTarget: - required: false - type: string - workflow_call: - inputs: - image_name: - type: string - description: "Docker image name part. Fills in ghcr.io/IMAGE_NAME. Typically this is the `repo_name`, but in some projects it might be `repo_name/sub_name`" - platforms: - default: "linux/amd64,linux/arm64" - type: string - subdir: - default: '.' - type: string - solr: - description: "Use if you need to build a specific Solr image and push to the registry" - required: false - type: boolean - tag: - required: false - type: string - target: - required: false - type: string - worker: - required: false - type: boolean - workerTarget: - required: false - type: string - -env: - REGISTRY: ghcr.io - -jobs: - build: - runs-on: self-hosted - steps: - - id: setup - name: Setup - uses: scientist-softserv/actions/setup-env@main - with: - tag: ${{ inputs.tag }} - image_name: ${{ inputs.image_name }} - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Context for Buildx - run: docker context create builders - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - id: builder - with: - version: latest - endpoint: builders - - name: Append ARM buildx builder - uses: baschny/append-buildx-action@v1 - with: - builder: ${{ steps.builder.outputs.name }} - endpoint: "ssh://ec2-user@10.0.6.48" - ssh_private_key: ${{ secrets.SSH_BESTIE_BUILDER_PRIVATE_KEY }} - - name: Pull from cache to speed up build - run: >- - cd ${{ inputs.subdir }}; - touch .env.development; - touch .env; - TAG=latest docker-compose pull web || true - - name: Retag action for web - id: meta-web - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }} - tags: | - # set latest tag for default branch - type=raw,value=latest,enable={{is_default_branch}} - - name: Retag action for worker - if: ${{ inputs.worker }} - id: meta-worker - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker - tags: | - # set latest tag for default branch - type=raw,value=latest,enable={{is_default_branch}} - - name: Retag action for solr - if: ${{ inputs.solr }} - id: meta-solr - uses: docker/metadata-action@v4.1.1 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr - tags: | - # set latest tag for default branch - type=raw,value=latest,enable={{is_default_branch}} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push web - uses: docker/build-push-action@v3 - with: - build-args: | - APP_PATH=${{ inputs.subdir }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} - context: . - file: ${{ inputs.subdir}}/Dockerfile - platforms: ${{ inputs.platforms }} - push: true - tags: | - ${{ steps.meta-web.outputs.tags }} - ${{ steps.repo_slug.outputs.result }}:${{ env.TAG }} - target: ${{ inputs.target }} - - name: Build and push worker - if: ${{ inputs.worker }} - uses: docker/build-push-action@v3 - with: - build-args: | - APP_PATH=${{ inputs.subdir }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - context: . - file: ${{ inputs.subdir}}/Dockerfile - platforms: ${{ inputs.platforms }} - push: true - tags: | - ${{ steps.meta-worker.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} - target: ${{ inputs.workerTarget }} - - name: Build and push solr - if: ${{ inputs.solr }} - uses: docker/build-push-action@v3 - with: - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} - context: . - platforms: ${{ inputs.platforms }} - push: true - tags: | - ${{ steps.meta-solr.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} - target: ${{ inputs.target }} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index b708370..c6fca16 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -22,6 +22,7 @@ on: env: REGISTRY: ghcr.io + REPO: ${{ steps.repo_slug.outputs.result }} jobs: deploy: @@ -55,7 +56,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@main + uses: scientist-softserv/actions/setup-env@arm-wrestle with: tag: ${{ inputs.tag }} image_name: ${{ inputs.image_name }} @@ -72,15 +73,15 @@ jobs: echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; - export DEPLOY_IMAGE=ghcr.io/${REPO}; - export WORKER_IMAGE=ghcr.io/${REPO}/worker; - export SOLR_IMAGE=ghcr.io/${REPO}/solr; + export DEPLOY_IMAGE=${REGISTRY}/${REPO}; + export WORKER_IMAGE=${REGISTRY}/${REPO}/worker; + export SOLR_IMAGE=${REGISTRY}/${REPO}/solr; ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} - name: Do deploy run: | echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; - export DEPLOY_IMAGE=ghcr.io/${REPO}; - export WORKER_IMAGE=ghcr.io/${REPO}/worker; + export DEPLOY_IMAGE=${REGISTRY}/${REPO}; + export WORKER_IMAGE=${REGISTRY}/${REPO}/worker; ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 4f30047..b2b060f 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -21,10 +21,8 @@ runs: echo "export DOCKER_CLIENT_TIMEOUT=360"; echo "export COMPOSE_HTTP_TIMEOUT=360" - name: Sanitize repo slug - env: - REPO: "${{ inputs.image_name || github.repository }}" uses: actions/github-script@v6 id: repo_slug with: result-encoding: string - script: return 'ghcr.io/${{ REPO }}'.toLowerCase() + script: return 'ghcr.io/${{ inputs.image_name || github.repository }}'.toLowerCase() From fd7057cfc082cf95eb7f8cb8b938549cc97ba2df Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 11:36:15 -0800 Subject: [PATCH 148/159] set env repo --- .github/workflows/build-solr-arm64.yaml | 1 - .github/workflows/build-solr.yaml | 1 - .github/workflows/build-web-arm64.yaml | 1 - .github/workflows/build-web.yaml | 1 - .github/workflows/build-worker-arm64.yaml | 1 - .github/workflows/build-worker.yaml | 1 - .github/workflows/deploy.yaml | 1 - setup-env/action.yaml | 6 ++++++ 8 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-solr-arm64.yaml b/.github/workflows/build-solr-arm64.yaml index 8f53851..8938d9c 100644 --- a/.github/workflows/build-solr-arm64.yaml +++ b/.github/workflows/build-solr-arm64.yaml @@ -31,7 +31,6 @@ on: env: REGISTRY: ghcr.io - REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-solr-arm64: diff --git a/.github/workflows/build-solr.yaml b/.github/workflows/build-solr.yaml index d8e8309..f1090c5 100644 --- a/.github/workflows/build-solr.yaml +++ b/.github/workflows/build-solr.yaml @@ -31,7 +31,6 @@ on: env: REGISTRY: ghcr.io - REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-solr: diff --git a/.github/workflows/build-web-arm64.yaml b/.github/workflows/build-web-arm64.yaml index 173cca3..581840e 100644 --- a/.github/workflows/build-web-arm64.yaml +++ b/.github/workflows/build-web-arm64.yaml @@ -31,7 +31,6 @@ on: env: REGISTRY: ghcr.io - REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-web-arm64: diff --git a/.github/workflows/build-web.yaml b/.github/workflows/build-web.yaml index 866fb1d..5091652 100644 --- a/.github/workflows/build-web.yaml +++ b/.github/workflows/build-web.yaml @@ -31,7 +31,6 @@ on: env: REGISTRY: ghcr.io - REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-web: diff --git a/.github/workflows/build-worker-arm64.yaml b/.github/workflows/build-worker-arm64.yaml index daaa11f..0c5ac8a 100644 --- a/.github/workflows/build-worker-arm64.yaml +++ b/.github/workflows/build-worker-arm64.yaml @@ -31,7 +31,6 @@ on: env: REGISTRY: ghcr.io - REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-worker-arm64: diff --git a/.github/workflows/build-worker.yaml b/.github/workflows/build-worker.yaml index f159ba4..20dfc35 100644 --- a/.github/workflows/build-worker.yaml +++ b/.github/workflows/build-worker.yaml @@ -31,7 +31,6 @@ on: env: REGISTRY: ghcr.io - REPO: ${{ steps.repo_slug.outputs.result }} jobs: build-worker: diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index c6fca16..73ab791 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -22,7 +22,6 @@ on: env: REGISTRY: ghcr.io - REPO: ${{ steps.repo_slug.outputs.result }} jobs: deploy: diff --git a/setup-env/action.yaml b/setup-env/action.yaml index b2b060f..b246cfd 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -26,3 +26,9 @@ runs: with: result-encoding: string script: return 'ghcr.io/${{ inputs.image_name || github.repository }}'.toLowerCase() + - name: Downcase repo + env: + REPO: "${{ steps.repo_slug.outputs.result }}" + shell: bash + run: | + echo "${REPO}" >> ${GITHUB_ENV} \ No newline at end of file From 62a6bfa4cbd1e7bc32fa1ac6e244c51c5690415d Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 11:42:03 -0800 Subject: [PATCH 149/159] syntax --- setup-env/action.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index b246cfd..78a33f1 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -27,8 +27,6 @@ runs: result-encoding: string script: return 'ghcr.io/${{ inputs.image_name || github.repository }}'.toLowerCase() - name: Downcase repo - env: - REPO: "${{ steps.repo_slug.outputs.result }}" shell: bash run: | - echo "${REPO}" >> ${GITHUB_ENV} \ No newline at end of file + echo "REPO=${ steps.repo_slug.outputs.result }" >> ${GITHUB_ENV} \ No newline at end of file From d4042667cffd93feca994c0c4ad3b10fd474c687 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 11:58:22 -0800 Subject: [PATCH 150/159] repo_lower --- .github/workflows/build-solr-arm64.yaml | 4 ++-- .github/workflows/build-solr.yaml | 4 ++-- .github/workflows/build-web-arm64.yaml | 4 ++-- .github/workflows/build-web.yaml | 6 +++--- .github/workflows/build-worker-arm64.yaml | 4 ++-- .github/workflows/build-worker.yaml | 6 +++--- setup-env/action.yaml | 4 +++- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-solr-arm64.yaml b/.github/workflows/build-solr-arm64.yaml index 8938d9c..7cc2817 100644 --- a/.github/workflows/build-solr-arm64.yaml +++ b/.github/workflows/build-solr-arm64.yaml @@ -62,12 +62,12 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - cache-from: type=registry,ref=${{ steps.repo_slug.outputs.result }}/solr:${{ env.TAG }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} context: . labels: ${{ steps.meta-solr.outputs.labels }} platforms: linux/arm64 push: true tags: | ${{ steps.meta-solr.outputs.tags }} - ${{ steps.repo_slug.outputs.result }}/solr:${{ env.TAG }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} target: ${{ inputs.target }} diff --git a/.github/workflows/build-solr.yaml b/.github/workflows/build-solr.yaml index f1090c5..79545ee 100644 --- a/.github/workflows/build-solr.yaml +++ b/.github/workflows/build-solr.yaml @@ -62,12 +62,12 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - cache-from: type=registry,ref=${{ steps.repo_slug.outputs.result }}/solr:${{ env.TAG }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} context: . labels: ${{ steps.meta-solr.outputs.labels }} platforms: linux/amd64 push: true tags: | ${{ steps.meta-solr.outputs.tags }} - ${{ steps.repo_slug.outputs.result }}/solr:${{ env.TAG }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr:${{ env.TAG }} target: ${{ inputs.target }} diff --git a/.github/workflows/build-web-arm64.yaml b/.github/workflows/build-web-arm64.yaml index 581840e..4acacb8 100644 --- a/.github/workflows/build-web-arm64.yaml +++ b/.github/workflows/build-web-arm64.yaml @@ -62,12 +62,12 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - cache-from: type=registry,ref=${{ steps.repo_slug.outputs.result }}:${{ env.TAG }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} context: . labels: ${{ steps.meta-web.outputs.labels }} platforms: linux/arm64 push: true tags: | ${{ steps.meta-web.outputs.tags }} - ${{ steps.repo_slug.outputs.result }}:${{ env.TAG }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} target: ${{ inputs.target }} diff --git a/.github/workflows/build-web.yaml b/.github/workflows/build-web.yaml index 5091652..63de77d 100644 --- a/.github/workflows/build-web.yaml +++ b/.github/workflows/build-web.yaml @@ -48,7 +48,7 @@ jobs: id: meta-web uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO }} + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} tags: | # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}} @@ -64,12 +64,12 @@ jobs: builder: ${{ steps.buildx.outputs.name }} cache-from: | type=registry - ref=${{ steps.repo_slug.outputs.result }}:${{ env.TAG }} + ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} context: . labels: ${{ steps.meta-web.outputs.labels }} platforms: linux/amd64 push: true tags: | ${{ steps.meta-web.outputs.tags }} - ${{ steps.repo_slug.outputs.result }}:${{ env.TAG }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} target: ${{ inputs.target }} diff --git a/.github/workflows/build-worker-arm64.yaml b/.github/workflows/build-worker-arm64.yaml index 0c5ac8a..8f262fa 100644 --- a/.github/workflows/build-worker-arm64.yaml +++ b/.github/workflows/build-worker-arm64.yaml @@ -64,12 +64,12 @@ jobs: builder: ${{ steps.buildx.outputs.name }} cache-from: | type=registry - ref=${{ steps.repo_slug.outputs.result }}/worker:${{ env.TAG }} + ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} context: . labels: ${{ steps.meta-worker.outputs.labels }} platforms: linux/arm64 push: true tags: | ${{ steps.meta-worker.outputs.tags }} - ${{ steps.repo_slug.outputs.result }}/worker:${{ env.TAG }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} target: ${{ inputs.target }} diff --git a/.github/workflows/build-worker.yaml b/.github/workflows/build-worker.yaml index 20dfc35..5b50e09 100644 --- a/.github/workflows/build-worker.yaml +++ b/.github/workflows/build-worker.yaml @@ -48,7 +48,7 @@ jobs: id: meta-worker uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO }}/worker + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker tags: | # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}} @@ -64,12 +64,12 @@ jobs: builder: ${{ steps.buildx.outputs.name }} cache-from: | type=registry - ref=${{ steps.repo_slug.outputs.result }}/worker:${{ env.TAG }} + ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} context: . labels: ${{ steps.meta-worker.outputs.labels }} platforms: linux/amd64 push: true tags: | ${{ steps.meta-worker.outputs.tags }} - ${{ steps.repo_slug.outputs.result }}/worker:${{ env.TAG }} + ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} target: ${{ inputs.target }} diff --git a/setup-env/action.yaml b/setup-env/action.yaml index 78a33f1..d1c72f1 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -27,6 +27,8 @@ runs: result-encoding: string script: return 'ghcr.io/${{ inputs.image_name || github.repository }}'.toLowerCase() - name: Downcase repo + env: + REPO_LOWER: "${{ steps.repo_slug.outputs.result }}" shell: bash run: | - echo "REPO=${ steps.repo_slug.outputs.result }" >> ${GITHUB_ENV} \ No newline at end of file + echo "REPO_LOWER" >> ${GITHUB_ENV} \ No newline at end of file From 4ea5dd57fdeda656b93f6d490cdf35fe12bcffa4 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 12:00:34 -0800 Subject: [PATCH 151/159] lowercase --- setup-env/action.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/setup-env/action.yaml b/setup-env/action.yaml index d1c72f1..e29cb06 100644 --- a/setup-env/action.yaml +++ b/setup-env/action.yaml @@ -20,15 +20,13 @@ runs: echo ${HEAD_TAG::8}; echo "export DOCKER_CLIENT_TIMEOUT=360"; echo "export COMPOSE_HTTP_TIMEOUT=360" - - name: Sanitize repo slug - uses: actions/github-script@v6 - id: repo_slug - with: - result-encoding: string - script: return 'ghcr.io/${{ inputs.image_name || github.repository }}'.toLowerCase() - name: Downcase repo env: - REPO_LOWER: "${{ steps.repo_slug.outputs.result }}" + REPO: "${{ inputs.image_name || github.repository }}" shell: bash run: | - echo "REPO_LOWER" >> ${GITHUB_ENV} \ No newline at end of file + echo "REPO_LOWER=${REPO,,}" >> ${GITHUB_ENV} + - name: Install docker-compose + uses: KengoTODA/actions-setup-docker-compose@main + with: + version: '2.12.2' From 9e8a12f9dfa1db4e03d9864a82d0e7faada09a8a Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 12:06:10 -0800 Subject: [PATCH 152/159] tweaking syntax --- .github/workflows/build-web.yaml | 4 +--- .github/workflows/build-worker-arm64.yaml | 4 +--- .github/workflows/build-worker.yaml | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-web.yaml b/.github/workflows/build-web.yaml index 63de77d..81468c9 100644 --- a/.github/workflows/build-web.yaml +++ b/.github/workflows/build-web.yaml @@ -62,9 +62,7 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - cache-from: | - type=registry - ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} context: . labels: ${{ steps.meta-web.outputs.labels }} platforms: linux/amd64 diff --git a/.github/workflows/build-worker-arm64.yaml b/.github/workflows/build-worker-arm64.yaml index 8f262fa..b333ee7 100644 --- a/.github/workflows/build-worker-arm64.yaml +++ b/.github/workflows/build-worker-arm64.yaml @@ -62,9 +62,7 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - cache-from: | - type=registry - ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} context: . labels: ${{ steps.meta-worker.outputs.labels }} platforms: linux/arm64 diff --git a/.github/workflows/build-worker.yaml b/.github/workflows/build-worker.yaml index 5b50e09..8c7b85d 100644 --- a/.github/workflows/build-worker.yaml +++ b/.github/workflows/build-worker.yaml @@ -62,9 +62,7 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - cache-from: | - type=registry - ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker:${{ env.TAG }} context: . labels: ${{ steps.meta-worker.outputs.labels }} platforms: linux/amd64 From 92f753dbdc19d3e315f27a998de2fb6980b8830e Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 12:09:18 -0800 Subject: [PATCH 153/159] macos-11 doesnt like doencase - swap back to ubuntu-latest --- .github/workflows/build-solr-arm64.yaml | 2 +- .github/workflows/build-web-arm64.yaml | 2 +- .github/workflows/build-worker-arm64.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-solr-arm64.yaml b/.github/workflows/build-solr-arm64.yaml index 7cc2817..73ba1d3 100644 --- a/.github/workflows/build-solr-arm64.yaml +++ b/.github/workflows/build-solr-arm64.yaml @@ -34,7 +34,7 @@ env: jobs: build-solr-arm64: - runs-on: macos-11 + runs-on: ubuntu-latest steps: - id: setup name: Setup diff --git a/.github/workflows/build-web-arm64.yaml b/.github/workflows/build-web-arm64.yaml index 4acacb8..137bf15 100644 --- a/.github/workflows/build-web-arm64.yaml +++ b/.github/workflows/build-web-arm64.yaml @@ -34,7 +34,7 @@ env: jobs: build-web-arm64: - runs-on: macos-11 + runs-on: ubuntu-latest steps: - id: setup name: Setup diff --git a/.github/workflows/build-worker-arm64.yaml b/.github/workflows/build-worker-arm64.yaml index b333ee7..8df7411 100644 --- a/.github/workflows/build-worker-arm64.yaml +++ b/.github/workflows/build-worker-arm64.yaml @@ -34,7 +34,7 @@ env: jobs: build-worker-arm64: - runs-on: macos-11 + runs-on: ubuntu-latest steps: - id: setup name: Setup From 370a68669ab676f9d438095af94f4eba91a04c64 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 13:12:53 -0800 Subject: [PATCH 154/159] add back qemu on ubuntu-latest --- .github/workflows/build-solr-arm64.yaml | 2 ++ .github/workflows/build-web-arm64.yaml | 2 ++ .github/workflows/build-worker-arm64.yaml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.github/workflows/build-solr-arm64.yaml b/.github/workflows/build-solr-arm64.yaml index 73ba1d3..cc8e922 100644 --- a/.github/workflows/build-solr-arm64.yaml +++ b/.github/workflows/build-solr-arm64.yaml @@ -41,6 +41,8 @@ jobs: uses: scientist-softserv/actions/setup-env@arm-wrestle - name: Checkout code uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 diff --git a/.github/workflows/build-web-arm64.yaml b/.github/workflows/build-web-arm64.yaml index 137bf15..d38dc91 100644 --- a/.github/workflows/build-web-arm64.yaml +++ b/.github/workflows/build-web-arm64.yaml @@ -41,6 +41,8 @@ jobs: uses: scientist-softserv/actions/setup-env@arm-wrestle - name: Checkout code uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 diff --git a/.github/workflows/build-worker-arm64.yaml b/.github/workflows/build-worker-arm64.yaml index 8df7411..8dc5656 100644 --- a/.github/workflows/build-worker-arm64.yaml +++ b/.github/workflows/build-worker-arm64.yaml @@ -41,6 +41,8 @@ jobs: uses: scientist-softserv/actions/setup-env@arm-wrestle - name: Checkout code uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 From 55fe375f340845e5197d456ed54a6fc52fa0d65c Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 14:09:10 -0800 Subject: [PATCH 155/159] add the right env back --- .github/workflows/build-web-arm64.yaml | 2 +- .github/workflows/build-worker-arm64.yaml | 2 +- .github/workflows/deploy.yaml | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-web-arm64.yaml b/.github/workflows/build-web-arm64.yaml index d38dc91..02541d6 100644 --- a/.github/workflows/build-web-arm64.yaml +++ b/.github/workflows/build-web-arm64.yaml @@ -50,7 +50,7 @@ jobs: id: meta-web uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO }} + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}} diff --git a/.github/workflows/build-worker-arm64.yaml b/.github/workflows/build-worker-arm64.yaml index 8dc5656..d9f9c41 100644 --- a/.github/workflows/build-worker-arm64.yaml +++ b/.github/workflows/build-worker-arm64.yaml @@ -50,7 +50,7 @@ jobs: id: meta-worker uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO }}/worker + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/worker tags: | # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 73ab791..5ad3ae2 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -72,15 +72,15 @@ jobs: echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; - export DEPLOY_IMAGE=${REGISTRY}/${REPO}; - export WORKER_IMAGE=${REGISTRY}/${REPO}/worker; - export SOLR_IMAGE=${REGISTRY}/${REPO}/solr; + export DEPLOY_IMAGE=${REGISTRY}/${REPO_LOWER}; + export WORKER_IMAGE=${REGISTRY}/${REPO_LOWER}/worker; + export SOLR_IMAGE=${REGISTRY}/${REPO_LOWER}/solr; ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} - name: Do deploy run: | echo $KUBECONFIG_FILE | base64 -d > $KUBECONFIG; DOLLAR=$ envsubst < ops/${{ inputs.environment }}-deploy.tmpl.yaml > ops/${{ inputs.environment }}-deploy.yaml; export DEPLOY_TAG=${TAG}; - export DEPLOY_IMAGE=${REGISTRY}/${REPO}; - export WORKER_IMAGE=${REGISTRY}/${REPO}/worker; + export DEPLOY_IMAGE=${REGISTRY}/${REPO_LOWER}; + export WORKER_IMAGE=${REGISTRY}/${REPO_LOWER}/worker; ./bin/helm_deploy ${{ inputs.k8s-release-name || format('{0}-{1}', github.event.repository.name, inputs.environment) }} ${{ inputs.k8s-namespace || format('{0}-{1}', github.event.repository.name, inputs.environment) }} From d9b20bef2ec164d824d1472faa94251cf5264567 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 14:10:03 -0800 Subject: [PATCH 156/159] add the right env back -- for realz --- .github/workflows/build-solr-arm64.yaml | 2 +- .github/workflows/build-solr.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-solr-arm64.yaml b/.github/workflows/build-solr-arm64.yaml index cc8e922..15df484 100644 --- a/.github/workflows/build-solr-arm64.yaml +++ b/.github/workflows/build-solr-arm64.yaml @@ -50,7 +50,7 @@ jobs: id: meta-solr uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO }}/solr + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr tags: | # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}} diff --git a/.github/workflows/build-solr.yaml b/.github/workflows/build-solr.yaml index 79545ee..d55273b 100644 --- a/.github/workflows/build-solr.yaml +++ b/.github/workflows/build-solr.yaml @@ -48,7 +48,7 @@ jobs: id: meta-solr uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO }}/solr + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}/solr tags: | # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}} From fce30d1721d759b8aa64c31c41f7f7b368da5244 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 15:52:29 -0800 Subject: [PATCH 157/159] setup goes back to main --- .github/workflows/deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 5ad3ae2..48ab70b 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -55,7 +55,7 @@ jobs: steps: - id: setup name: Setup - uses: scientist-softserv/actions/setup-env@arm-wrestle + uses: scientist-softserv/actions/setup-env@main with: tag: ${{ inputs.tag }} image_name: ${{ inputs.image_name }} From 242541d69919cf13577d0e97033360e1cf90f2c8 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 6 Dec 2022 16:10:56 -0800 Subject: [PATCH 158/159] updates to ensure chmod is added to the test file from main --- .github/workflows/lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 39629e5..f311534 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -52,6 +52,7 @@ jobs: - name: Pull worker image to prevent build run: >- cd ${{ inputs.subdir }}; + chmod 777 db/schema.rb; docker-compose pull worker if: ${{ inputs.worker }} - name: Run Rubocop From 8b4b50c137d68f73fe4f90c2f6c68887567907af Mon Sep 17 00:00:00 2001 From: April Rieger <63515648+aprilrieger@users.noreply.github.com> Date: Tue, 6 Dec 2022 21:00:46 -0800 Subject: [PATCH 159/159] dont tag image its set automatically to latest removes tag on end of image as the action defaults it to latest which is what we want --- .github/workflows/build-web.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-web.yaml b/.github/workflows/build-web.yaml index 81468c9..fcdb4f4 100644 --- a/.github/workflows/build-web.yaml +++ b/.github/workflows/build-web.yaml @@ -48,7 +48,7 @@ jobs: id: meta-web uses: docker/metadata-action@v4.1.1 with: - images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }}:${{ env.TAG }} + images: ${{ env.REGISTRY }}/${{ env.REPO_LOWER }} tags: | # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}}