diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb0cf5e..8673d3c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,3 +36,13 @@ jobs: export PYTHONPATH=${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.7-src.zip:${PYTHONPATH} export PATH=${PATH}:${SPARK_HOME}/bin:${SPARK_HOME}/sbin make test + - name: Convert notebook to python script + run: | + make py + mkdir -p ima/scripts/ + mv algorithm.py ima/scripts + - uses: actions/upload-artifact@v2 + with: + name: algorithm + path: ima/scripts/algorithm.py + diff --git a/Makefile b/Makefile index 8ec4459..62c6338 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ spark_version := 2.4.8 hadoop_version := 2.7 spark_home := spark-${spark_version}-bin-hadoop${hadoop_version} spark_tgz_url := https://downloads.apache.org/spark/spark-${spark_version}/${spark_home}.tgz +ima_notebook := algorithm.ipynb venv: requirements.txt test -d venv || python3 -m venv venv @@ -21,3 +22,7 @@ flake8: venv test: venv . venv/bin/activate; PYTHONPATH=${PYTHONPATH}:etl/ pytest --cov etl tests/ + +py: venv + # nbconvert output is saved as .py + . venv/bin/activate; jupyter nbconvert ${ima_notebook} --to script diff --git a/requirements.txt b/requirements.txt index 8de2ac0..b59b67e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pytest==6.2.2 pytest-spark==0.6.0 pytest-cov==2.10.1 flake8==3.8.4 +jupyter==1.0.0