From f625b5dce3bb75f91668d62732cab29a5447b754 Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Wed, 8 May 2024 16:34:13 -0300 Subject: [PATCH 01/19] testing pipeline --- .github/workflows/_unit-test.yml | 67 ++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/_unit-test.yml diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml new file mode 100644 index 000000000..5e2ceea5b --- /dev/null +++ b/.github/workflows/_unit-test.yml @@ -0,0 +1,67 @@ +name: Run unit tests + +on: + workflow_call: + inputs: + command: + required: true + type: string + image: + required: true + type: string + +jobs: + vars: + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + outputs: + token: ${{ steps.vars.outputs.token }} + environment: + name: ${{ inputs.environment }} + steps: + - name: GCP Auth + id: auth + uses: google-github-actions/auth@v2 + with: + token_format: 'access_token' + workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.SERVICE_ACCOUNT }} + - id: vars + run: | + echo "token=$(gcloud auth print-access-token)" >> $GITHUB_OUTPUT + unit-tests: + runs-on: ubuntu-latest + needs: [vars] + container: + image: ${{ inputs.image }} + credentials: + username: oauth2accesstoken + password: ${{ needs.vars.outputs.token }} + services: + postgis: + image: postgis/postgis:16-3.4-alpine + env: + POSTGRES_USER: testuser + POSTGRES_PASSWORD: fancypassword + POSTGRES_DB: testdb + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Setup + run: | + cd /cdip + export PYTHONPATH=/cdip/cdip_admin:$PYTHONPATH + python3 -m pip install -r /dependencies/requirements-dev.txt + pytest + env: + DB_HOST: postgis + DB_NAME: testdb + DB_USER: testuser + DB_PASSWORD: fancypassword + DB_PORT: 5432 + FQDN: testing.pamdas.org \ No newline at end of file From 6fabfe3e0ac3ba37b70996d2bb2d5aa8910a3601 Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 11:48:33 -0300 Subject: [PATCH 02/19] modifying main pipeline --- .github/workflows/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ebd38c9b5..2189d939b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,6 +26,15 @@ jobs: repository: ${{ needs.vars.outputs.repository }} tag: ${{ needs.vars.outputs.tag }} + unit-test: + uses: ./.github/workflows/_unit-tests.yml + needs: build + with: + image: "us-central1-docker.pkg.dev/cdip-78ca/gundi/admin-portal:${{ github.head_ref || github.ref_name }}-${{ github.run_number }}" + # image: "europe-west3-docker.pkg.dev/padas-app/er-mt/das-server:${{ github.head_ref || github.ref_name }}-${{ github.run_number }}" + command: | + pytest + deploy_dev: uses: PADAS/gundi-workflows/.github/workflows/deploy_k8s.yml@v2 if: startsWith(github.ref, 'refs/heads/main') From 887894d307bb09f60b67afddf1e4ece973ca42d1 Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 11:54:15 -0300 Subject: [PATCH 03/19] adding test branch --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2189d939b..4997d2b75 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,6 +3,7 @@ on: push: branches: - main + - gundi-3054 # remove this line before merge - 'release-**' jobs: From 7f177a5afc80946e925d10af67921d4ead6af38f Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 12:07:46 -0300 Subject: [PATCH 04/19] fix --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4997d2b75..d984c269a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: tag: ${{ needs.vars.outputs.tag }} unit-test: - uses: ./.github/workflows/_unit-tests.yml + uses: .github/workflows/_unit-tests.yml needs: build with: image: "us-central1-docker.pkg.dev/cdip-78ca/gundi/admin-portal:${{ github.head_ref || github.ref_name }}-${{ github.run_number }}" From dc539523666bfc5b24fe313897f5d964fe0cb1b0 Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 12:17:55 -0300 Subject: [PATCH 05/19] misspelling --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d984c269a..ec71eb4b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: tag: ${{ needs.vars.outputs.tag }} unit-test: - uses: .github/workflows/_unit-tests.yml + uses: .github/workflows/_unit-test.yml needs: build with: image: "us-central1-docker.pkg.dev/cdip-78ca/gundi/admin-portal:${{ github.head_ref || github.ref_name }}-${{ github.run_number }}" From d49b55b83b47870f4bf5d385ddc6ee838e3c5704 Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 12:19:47 -0300 Subject: [PATCH 06/19] one more try --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ec71eb4b0..1f18c9f23 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: tag: ${{ needs.vars.outputs.tag }} unit-test: - uses: .github/workflows/_unit-test.yml + uses: ./.github/workflows/_unit-test.yml needs: build with: image: "us-central1-docker.pkg.dev/cdip-78ca/gundi/admin-portal:${{ github.head_ref || github.ref_name }}-${{ github.run_number }}" From ad699a944cdf25a5ee2fc8dfa2e81de2c30ebf4e Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 13:46:47 -0300 Subject: [PATCH 07/19] set env in unit test step --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1f18c9f23..fefeb277c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,6 +31,7 @@ jobs: uses: ./.github/workflows/_unit-test.yml needs: build with: + environment: stage image: "us-central1-docker.pkg.dev/cdip-78ca/gundi/admin-portal:${{ github.head_ref || github.ref_name }}-${{ github.run_number }}" # image: "europe-west3-docker.pkg.dev/padas-app/er-mt/das-server:${{ github.head_ref || github.ref_name }}-${{ github.run_number }}" command: | From 4f352afaae6c053ae9b79768a451163bc18653cd Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 13:50:50 -0300 Subject: [PATCH 08/19] fix add env --- .github/workflows/_unit-test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml index 5e2ceea5b..f27e1ff64 100644 --- a/.github/workflows/_unit-test.yml +++ b/.github/workflows/_unit-test.yml @@ -9,6 +9,16 @@ on: image: required: true type: string + environment: + type: string + workload_identity_provider: + type: string + service_account: + type: string + +env: + WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.workload_identity_provider || vars.WORKLOAD_IDENTITY_PROVIDER }} + SERVICE_ACCOUNT: ${{ inputs.service_account || vars.SERVICE_ACCOUNT }} jobs: vars: From 4b7bb7edad005c980b7ef977fb0a267093add5d2 Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 14:35:59 -0300 Subject: [PATCH 09/19] fix --- .github/workflows/main.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fefeb277c..9c82fc11b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,11 +29,10 @@ jobs: unit-test: uses: ./.github/workflows/_unit-test.yml - needs: build + needs: [vars, build] with: environment: stage - image: "us-central1-docker.pkg.dev/cdip-78ca/gundi/admin-portal:${{ github.head_ref || github.ref_name }}-${{ github.run_number }}" - # image: "europe-west3-docker.pkg.dev/padas-app/er-mt/das-server:${{ github.head_ref || github.ref_name }}-${{ github.run_number }}" + image: "us-central1-docker.pkg.dev/cdip-78ca/gundi/admin-portal:"${{ needs.vars.outputs.repository }}:${{ needs.vars.outputs.tag }}"" command: | pytest From f83c3876a317b0fbd9bc41bbb52f9469b186ce00 Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 14:58:05 -0300 Subject: [PATCH 10/19] fix --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9c82fc11b..0f1dc8acf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,7 +32,7 @@ jobs: needs: [vars, build] with: environment: stage - image: "us-central1-docker.pkg.dev/cdip-78ca/gundi/admin-portal:"${{ needs.vars.outputs.repository }}:${{ needs.vars.outputs.tag }}"" + image: "${{ needs.vars.outputs.repository }}:${{ needs.vars.outputs.tag }}" command: | pytest From b194307bbcd4bb93f824cca2b30c29ad6b56567b Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 15:01:59 -0300 Subject: [PATCH 11/19] start testing ut step --- .github/workflows/_unit-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml index f27e1ff64..ba3a9deb2 100644 --- a/.github/workflows/_unit-test.yml +++ b/.github/workflows/_unit-test.yml @@ -64,8 +64,7 @@ jobs: steps: - name: Setup run: | - cd /cdip - export PYTHONPATH=/cdip/cdip_admin:$PYTHONPATH + export PYTHONPATH=/cdip_admin:$PYTHONPATH python3 -m pip install -r /dependencies/requirements-dev.txt pytest env: From 7a3e6b54fd061a3839547f3f49efb2c7a6a754e8 Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 9 May 2024 15:18:34 -0300 Subject: [PATCH 12/19] list directories --- .github/workflows/_unit-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml index ba3a9deb2..88ddd7eed 100644 --- a/.github/workflows/_unit-test.yml +++ b/.github/workflows/_unit-test.yml @@ -65,6 +65,7 @@ jobs: - name: Setup run: | export PYTHONPATH=/cdip_admin:$PYTHONPATH + ls python3 -m pip install -r /dependencies/requirements-dev.txt pytest env: From 40503ec021c9155806e512f82bfe909fa8517a04 Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 16 May 2024 15:22:26 -0300 Subject: [PATCH 13/19] test --- .github/workflows/_unit-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml index 88ddd7eed..0233c736f 100644 --- a/.github/workflows/_unit-test.yml +++ b/.github/workflows/_unit-test.yml @@ -64,8 +64,8 @@ jobs: steps: - name: Setup run: | + pwd export PYTHONPATH=/cdip_admin:$PYTHONPATH - ls python3 -m pip install -r /dependencies/requirements-dev.txt pytest env: From 14cad3b0d8d7df180f60d242059e656b5cee978e Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 16 May 2024 15:26:40 -0300 Subject: [PATCH 14/19] test --- .github/workflows/_unit-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml index 0233c736f..507bb15e8 100644 --- a/.github/workflows/_unit-test.yml +++ b/.github/workflows/_unit-test.yml @@ -64,7 +64,9 @@ jobs: steps: - name: Setup run: | - pwd + ls + cd /cdip + ls export PYTHONPATH=/cdip_admin:$PYTHONPATH python3 -m pip install -r /dependencies/requirements-dev.txt pytest From f0fcdc48193bd43ed6e18fa6ded2d72901f4dcae Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Thu, 16 May 2024 15:37:50 -0300 Subject: [PATCH 15/19] fix --- .github/workflows/_unit-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml index 507bb15e8..746401fdc 100644 --- a/.github/workflows/_unit-test.yml +++ b/.github/workflows/_unit-test.yml @@ -64,9 +64,7 @@ jobs: steps: - name: Setup run: | - ls - cd /cdip - ls + cd /workspace export PYTHONPATH=/cdip_admin:$PYTHONPATH python3 -m pip install -r /dependencies/requirements-dev.txt pytest From 25bc5aedfa30133cacaf8c9553d7b8eff90d10d3 Mon Sep 17 00:00:00 2001 From: Daniel Fernandez Date: Fri, 17 May 2024 14:37:21 -0600 Subject: [PATCH 16/19] add dev dependencies to dockerfile --- .github/workflows/_unit-test.yml | 20 ++++------ docker/Dockerfile | 66 +++++++++----------------------- 2 files changed, 27 insertions(+), 59 deletions(-) diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml index 746401fdc..2a8303d50 100644 --- a/.github/workflows/_unit-test.yml +++ b/.github/workflows/_unit-test.yml @@ -11,14 +11,6 @@ on: type: string environment: type: string - workload_identity_provider: - type: string - service_account: - type: string - -env: - WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.workload_identity_provider || vars.WORKLOAD_IDENTITY_PROVIDER }} - SERVICE_ACCOUNT: ${{ inputs.service_account || vars.SERVICE_ACCOUNT }} jobs: vars: @@ -64,10 +56,14 @@ jobs: steps: - name: Setup run: | - cd /workspace - export PYTHONPATH=/cdip_admin:$PYTHONPATH - python3 -m pip install -r /dependencies/requirements-dev.txt - pytest + cd /var/www/app + python3 -m pip install -r requirements-dev.txt + + - name: Run Tests + run: | + cd /var/www/app + ${{ inputs.command }} + # export PYTHONPATH=/var/www/app:$PYTHONPATH env: DB_HOST: postgis DB_NAME: testdb diff --git a/docker/Dockerfile b/docker/Dockerfile index 12550ca3c..c4cc76925 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,16 @@ FROM ubuntu:20.04 MAINTAINER chrisdo@vulcan.com -# use apt-get update and install in the same RUN command: http://lenguyenthedat.com/docker-cache/ +EXPOSE 8000 +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 +ENV LANGUAGE C.UTF-8 +ENV DJANGO_SETTINGS_MODULE=cdip_admin.settings +ENV DEBUG=False +ENV SERVICE_NAME=cdip_portal +ENV DB_HOST=host.docker.internal +ENV DJANGO_SILK_ENABLED=True + RUN apt-get update && \ apt-get -y upgrade && \ DEBIAN_FRONTEND=noninteractive \ @@ -18,54 +27,17 @@ RUN apt-get update && \ software-properties-common \ postgresql-client-12 \ vim \ - && apt-get autoremove - - -# Use ubuntugis/ppa for bionic since it's not available for 20.04 (focal) yet. -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 089EBE08314DF160 -RUN /bin/sh -c 'echo "deb http://ppa.launchpad.net/ubuntugis/ppa/ubuntu bionic main" >> /etc/apt/sources.list.d/ubuntugis.list' - -#RUN apt-get update -RUN apt-get update && \ - apt-get -y install gdal-bin libgdal-dev -RUN export CPLUS_INCLUDE_PATH=/usr/include/gdal -RUN export C_INCLUDE_PATH=/usr/include/gdal -RUN pip3 install GDAL==$(gdal-config --version) - -WORKDIR /workspace - -ADD dependencies/requirements.txt /workspace/requirements.txt - -RUN pip3 install -U "pip" - -RUN pip3 install -r /workspace/requirements.txt --no-cache-dir - -ARG built_version="" - -ADD ./cdip_admin /var/www/app -ADD ./swagger-fix/index.html /usr/local/lib/python3.8/dist-packages/rest_framework_swagger/templates/rest_framework_swagger/ - -EXPOSE 8000 - -ENV LC_ALL C.UTF-8 -ENV LANG C.UTF-8 -ENV LANGUAGE C.UTF-8 - -ENV BUILT_VERSION=${built_version} - -ENV DJANGO_SETTINGS_MODULE=cdip_admin.settings -ENV DEBUG=False - -# Default service name -ENV SERVICE_NAME=cdip_portal - -# if connecting to db running on localhost -ENV DB_HOST=host.docker.internal + gdal-bin libgdal-dev && \ + apt-get autoremove && \ + pip3 install -U "pip" WORKDIR /var/www/app +COPY dependencies . +RUN pip3 install -r requirements.txt --no-cache-dir -RUN mkdir -p /var/www/static -# Collect static files (using a stub database) -RUN python3 manage.py collectstatic --noinput --settings=cdip_admin.local_settings_nodb +COPY ./cdip_admin . +COPY ./swagger-fix/index.html /usr/local/lib/python3.8/dist-packages/rest_framework_swagger/templates/rest_framework_swagger/ +RUN mkdir -p /var/www/static && \ + python3 manage.py collectstatic --noinput --settings=cdip_admin.local_settings_nodb CMD ["/var/www/app/start_scripts/start.sh"] From 0a39812ce1cfea98908698e4eaa1994c6fc6d95a Mon Sep 17 00:00:00 2001 From: Daniel Fernandez Date: Fri, 17 May 2024 14:54:33 -0600 Subject: [PATCH 17/19] disable gcp env --- .github/workflows/_unit-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml index 2a8303d50..13e121a18 100644 --- a/.github/workflows/_unit-test.yml +++ b/.github/workflows/_unit-test.yml @@ -70,4 +70,5 @@ jobs: DB_USER: testuser DB_PASSWORD: fancypassword DB_PORT: 5432 - FQDN: testing.pamdas.org \ No newline at end of file + FQDN: testing.pamdas.org + GCP_ENVIRONMENT_ENABLED: false \ No newline at end of file From f4ddd7188a06b10cb472df792f5e8cb3550d4f46 Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Mon, 20 May 2024 10:01:05 -0300 Subject: [PATCH 18/19] set tracing to disabled --- .github/workflows/_unit-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml index 13e121a18..91bf9162f 100644 --- a/.github/workflows/_unit-test.yml +++ b/.github/workflows/_unit-test.yml @@ -71,4 +71,5 @@ jobs: DB_PASSWORD: fancypassword DB_PORT: 5432 FQDN: testing.pamdas.org - GCP_ENVIRONMENT_ENABLED: false \ No newline at end of file + GCP_ENVIRONMENT_ENABLED: false + TRACING_ENABLED: false \ No newline at end of file From ed2dd4d46642c5038e0f4948fd229b629390a0fe Mon Sep 17 00:00:00 2001 From: Marcos Davidovich Date: Mon, 20 May 2024 10:01:51 -0300 Subject: [PATCH 19/19] set celery tasks always eager --- .github/workflows/_unit-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_unit-test.yml b/.github/workflows/_unit-test.yml index 91bf9162f..c4c4a62c1 100644 --- a/.github/workflows/_unit-test.yml +++ b/.github/workflows/_unit-test.yml @@ -72,4 +72,5 @@ jobs: DB_PORT: 5432 FQDN: testing.pamdas.org GCP_ENVIRONMENT_ENABLED: false - TRACING_ENABLED: false \ No newline at end of file + TRACING_ENABLED: false + CELERY_TASK_ALWAYS_EAGER: true \ No newline at end of file