pythonpackage.yml

name: Python package

on:
  pull_request:
    branches:
    - master
    - docs
  schedule:
  - cron: "0 2 * * 1-5"

# to make bash use ~/.bashrc, see https://github.com/marketplace/actions/setup-miniconda#example-5-custom-installer
defaults:
  run:
    shell: bash -l {0}

jobs:
  main:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      max-parallel: 10
      matrix:
        os: [ubuntu-latest, windows-latest, macOS-latest]
        python-version: [3.7, 3.8, 3.9]

    steps:
    - uses: maxim-lobanov/setup-xcode@v1
      # alternative would be to upgrade tapi to 1100.0.11, we can possibly remove this in the future
      if: matrix.os == 'macOS-latest'
      with:
        xcode-version: "11.7.0"
    - uses: actions/checkout@v1
      with:
        submodules: true
    - uses: actions/checkout@v2
      if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'linux-latest') }}
      with:
        repository: vaexio/vaex-enterprise
        token: ${{ secrets.PAT_PULL_ENTERPRISE }}
        path: vaex-enterprise
    - name: Install conda with Python ${{ matrix.python-version }}
      uses: conda-incubator/setup-miniconda@v2
      with:
        python-version: ${{ matrix.python-version }}
        mamba-version: "*"
        auto-activate-base: true
        channels: conda-forge
    - name: Conda init
      run: |
        conda init bash
    - name: Create env (with mamba)
      run: |
        ./ci/02-create-vaex-dev-env.sh ${{ matrix.python-version }} mamba
    - name: Extra non-windows installs
      if: matrix.os != 'windows-latest'
      run: |
        conda activate vaex-dev
        mamba install -c conda-forge py-xgboost tensorflow
        # xgboost is not available on windows on conda-forge
    - name: Extra installs
      if: matrix.python-version != '3.9' && matrix.python-version != '3.7'
      run: |
        conda activate vaex-dev
        mamba install -y -q --file ci/conda-env-extra.yml
    - name: Extra windows installs
      if: matrix.os == 'windows-latest'
      run: |
        conda activate vaex-dev
        pip install "ipython>=7" "dask>=1" xgboost
        pip uninstall -y xgboost  # seems to crash, tests will be skipped
        # not sure why, but the conda env is strange (old dask) if we put this constraint in the conda solver
    - name: free disk space
      if: matrix.os == 'ubuntu-latest'
      run: |
        sudo swapoff -a
        sudo rm -f /swapfile
        sudo apt clean
        docker rmi $(docker image ls -aq)
        df -h
    - name: Install OpenMP runtime (Mac-only)
      if: matrix.os == 'macOS-latest'
      run: |
        brew install libomp
    - name: Cache compiled binaries
      # this fails for this combination, leading to binaries filled with 0's
      if: matrix.python-version != '3.6' || matrix.os != 'macOS-latest'
      id: cache-compiled-binaries
      uses: actions/cache@v2
      with:
        path: packages/vaex-core/build/lib*
        key: ${{ runner.OS }}-${{ matrix.python-version }}-${{ hashFiles('packages/vaex-core/src/*') }}-v2
    - name: Fix cache timestamp
      if: steps.cache-compiled-binaries.outputs.cache-hit == 'true' && matrix.os != 'windows-latest'
      run: |
        find packages/vaex-core/build -type f -exec touch {} +
    - name: Install vaex
      run: |
        ./ci/03-install-vaex.sh
    - name: Install vaex-enterprise
      if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'linux-latest') }}
      run: |
        conda config --set always_yes yes --set changeps1 no
        source activate vaex-dev
        cd vaex-enterprise
        mamba install -c conda-forge distributed
        pip install ray
        pip install -e .
    - name: Lint with flake8
      run: |
        echo "soon"
        # pip install flake8
        # stop the build if there are Python syntax errors or undefined names
        # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
    - name: Test with pytest
      run: |
        ./ci/04-run-test-suite.sh
    - name: Test with pytest (with cache on)
      if: matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest'
      run: |
        VAEX_CACHE_RESULTS=1 ./ci/04-run-test-suite.sh
    - name: Check ml spec
      # no catboost for py39 and py37
      if: matrix.python-version != '3.9' && matrix.os != 'windows-latest' && matrix.python-version != '3.7'
      run: |
        conda activate vaex-dev
        python -m vaex.ml.spec packages/vaex-ml/vaex/ml/spec_new.json
        diff packages/vaex-ml/vaex/ml/spec_new.json packages/vaex-ml/vaex/ml/spec.json
    - name: Test with pytest (vaex-enterprise)
      if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'linux-latest') }}
      run: |
        conda activate vaex-dev
        py.test vaex-enterprise/tests --timeout=1000
    - name: Test notebooks
      if: matrix.os != 'windows-latest'
      run: |
        ./ci/05-run-notebooks.sh
    - name: Authenticate Google Cloud Platform
      if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') }}
      uses: google-github-actions/setup-gcloud@master
      with:
        project_id: ${{ secrets.GCP_PROJECT_ID_VAEX }}
        service_account_key: ${{ secrets.GCP_SA_KEY_VAEX }}
        export_default_credentials: true
    - name: Test vaex-contrib
      if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') }}
      env:
        PROJECT_ID: ${{ secrets.GCP_PROJECT_ID_VAEX }}
      run: |
        conda activate vaex-dev
        pip install -e 'packages/vaex-contrib[gcp]'
        ./ci/06-run-contrib-tests.sh
    - name: Copy dll
      if: (matrix.os == 'windows-latest')
      uses: ./ci/actions/windll
    - name: Build Python packages
      run: |
        conda activate vaex-dev
        mkdir dist
        (cd packages/vaex-core && python setup.py bdist_wheel); cp packages/vaex-core/dist/* dist
        (cd packages/vaex-graphql && python setup.py bdist_wheel); cp packages/vaex-graphql/dist/* dist
        (cd packages/vaex-jupyter && python setup.py bdist_wheel); cp packages/vaex-jupyter/dist/* dist
        (cd packages/vaex-ml && python setup.py bdist_wheel); cp packages/vaex-ml/dist/* dist
        (cd packages/vaex-viz && python setup.py bdist_wheel); cp packages/vaex-viz/dist/* dist
        (cd packages/vaex-astro && python setup.py bdist_wheel); cp packages/vaex-astro/dist/* dist
        (cd packages/vaex-hdf5 && python setup.py bdist_wheel); cp packages/vaex-hdf5/dist/* dist
        (cd packages/vaex-server && python setup.py bdist_wheel); cp packages/vaex-server/dist/* dist
        (cd packages/vaex-meta && python setup.py bdist_wheel); cp packages/vaex-server/dist/* dist

    - name: Upload builds
      uses: actions/upload-artifact@v2
      with:
        name: dist-${{ github.run_number }}-${{ matrix.python-version }}-${{ matrix.os }}
        path: ./dist


  install:
    runs-on: ${{ matrix.os }}
    needs: [main]

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macOS-latest]
        python-version: [3.7, 3.8, 3.9]

    steps:
    - uses: actions/download-artifact@v2
      with:
        name: dist-${{ github.run_number }}-${{ matrix.python-version }}-${{ matrix.os }}
        path: ./dist

    - name: Install conda with Python ${{ matrix.python-version }}
      uses: conda-incubator/setup-miniconda@v2
      with:
        python-version: ${{ matrix.python-version }}
        mamba-version: "*"
        auto-activate-base: true
        channels: conda-forge

    - name: Install the packages
      run: |
        pip install -vv dist/*

    - name: Test import
      run: |
        python -c "import vaex; df = vaex.example()"

    - name: Test server
      run: |
        vaex server --add-example --port 9999&

    - name: Wait for Vaex server
      uses: ifaxity/wait-on-action@v1
      with:
        resource: http-get://localhost:9999/hello
        timeout: 5000

    - name: Use the Vaex server
      run: |
        python -c "import vaex; df = vaex.open('ws://localhost:9999/example'); df.x.sum()"
        curl -i http://localhost:9999/histogram/example/x