From 68870033649b8f5f0c53b056ddf25aaf624a9c59 Mon Sep 17 00:00:00 2001 From: gmodena Date: Wed, 8 Sep 2021 22:22:30 +0200 Subject: [PATCH 1/8] Convert the model ipython notebook to a script. Extract a python script from the notebook, and publish it as build artifact. --- .github/workflows/build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb0cf5e..9f54b4f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,3 +36,10 @@ jobs: export PYTHONPATH=${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.7-src.zip:${PYTHONPATH} export PATH=${PATH}:${SPARK_HOME}/bin:${SPARK_HOME}/sbin make test + - name: Convert notebook to python script + run: | + make py + - uses: actions/upload-artifact@v2 + with: + name: algorithm.py + path: ima/scripts/algorithm.py From 5feaaa4f7e3c9e91dc306a64755efe73982f9bb4 Mon Sep 17 00:00:00 2001 From: gmodena Date: Wed, 8 Sep 2021 22:26:02 +0200 Subject: [PATCH 2/8] Add nbconvert step to Makefile --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 8ec4459..62c6338 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ spark_version := 2.4.8 hadoop_version := 2.7 spark_home := spark-${spark_version}-bin-hadoop${hadoop_version} spark_tgz_url := https://downloads.apache.org/spark/spark-${spark_version}/${spark_home}.tgz +ima_notebook := algorithm.ipynb venv: requirements.txt test -d venv || python3 -m venv venv @@ -21,3 +22,7 @@ flake8: venv test: venv . venv/bin/activate; PYTHONPATH=${PYTHONPATH}:etl/ pytest --cov etl tests/ + +py: venv + # nbconvert output is saved as .py + . venv/bin/activate; jupyter nbconvert ${ima_notebook} --to script From 39c6d1205289af574e1094c1cc823e9bc5e5f576 Mon Sep 17 00:00:00 2001 From: gmodena Date: Wed, 8 Sep 2021 22:58:37 +0200 Subject: [PATCH 3/8] Add jupyter dep --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8de2ac0..b59b67e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pytest==6.2.2 pytest-spark==0.6.0 pytest-cov==2.10.1 flake8==3.8.4 +jupyter==1.0.0 From 88875a0aa618652c3de6c4584bbe7e8b8b2a5618 Mon Sep 17 00:00:00 2001 From: gmodena Date: Wed, 8 Sep 2021 23:04:01 +0200 Subject: [PATCH 4/8] Fix artifact path --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f54b4f..82c09b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,7 +39,9 @@ jobs: - name: Convert notebook to python script run: | make py + mkdir -p ima/scripts/ + mv algorithm.py ima/scripts - uses: actions/upload-artifact@v2 with: - name: algorithm.py + name: algorithm path: ima/scripts/algorithm.py From 5d3cb296f5ac8042f424557e6524ae0ad61c385b Mon Sep 17 00:00:00 2001 From: gmodena Date: Wed, 8 Sep 2021 23:23:52 +0200 Subject: [PATCH 5/8] Add workflow artifact pull request comment --- .github/workflows/build.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82c09b9..1db0a24 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,3 +45,13 @@ jobs: with: name: algorithm path: ima/scripts/algorithm.py + + - name: add artifact links to pull request and related issues step + uses: tonyhallett/artifacts-url-comments@v1.1.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + prefix: The build action generated the following artifacts + format: name + addTo: pullandissues + From 314d8cb3fa50ee03fbf9c4393e2e6c5ad9015943 Mon Sep 17 00:00:00 2001 From: gmodena Date: Wed, 8 Sep 2021 23:23:52 +0200 Subject: [PATCH 6/8] Add workflow artifact pull request comment --- .github/workflows/build.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82c09b9..1db0a24 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,3 +45,13 @@ jobs: with: name: algorithm path: ima/scripts/algorithm.py + + - name: add artifact links to pull request and related issues step + uses: tonyhallett/artifacts-url-comments@v1.1.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + prefix: The build action generated the following artifacts + format: name + addTo: pullandissues + From 7c00f9d0d86ae402ad20d96555295c735da60b32 Mon Sep 17 00:00:00 2001 From: gmodena Date: Wed, 8 Sep 2021 23:29:54 +0200 Subject: [PATCH 7/8] Fix. workflow file indentation --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1db0a24..60374f5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,11 +47,11 @@ jobs: path: ima/scripts/algorithm.py - name: add artifact links to pull request and related issues step - uses: tonyhallett/artifacts-url-comments@v1.1.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - prefix: The build action generated the following artifacts - format: name - addTo: pullandissues + uses: tonyhallett/artifacts-url-comments@v1.1.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + prefix: The build action generated the following artifacts + format: name + addTo: pullandissues From f6f477a8cee1d36e27cd8a08cffbec2929e7a5bb Mon Sep 17 00:00:00 2001 From: gmodena Date: Wed, 8 Sep 2021 23:42:10 +0200 Subject: [PATCH 8/8] Revert adding comments to a PR --- .github/workflows/build.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 60374f5..8673d3c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,12 +46,3 @@ jobs: name: algorithm path: ima/scripts/algorithm.py - - name: add artifact links to pull request and related issues step - uses: tonyhallett/artifacts-url-comments@v1.1.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - prefix: The build action generated the following artifacts - format: name - addTo: pullandissues -