pr-python-model-tests.yml

name: Model Server Tests

on:
  schedule:
    # This cron expression runs the job daily at 16:00 UTC (9am PT)
    - cron: "0 16 * * *"
  workflow_dispatch:

permissions:
  contents: read

env:
  # Bedrock
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_REGION_NAME: ${{ vars.AWS_REGION_NAME }}

  # API keys for testing
  COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
  LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY }}
  LITELLM_API_URL: ${{ secrets.LITELLM_API_URL }}
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
  AZURE_API_URL: ${{ vars.AZURE_API_URL }}

jobs:
  model-check:
    # See https://runs-on.com/runners/linux/
    runs-on:
      - runs-on
      - runner=4cpu-linux-arm64
      - "run-id=${{ github.run_id }}-model-check"
      - "extras=ecr-cache"
    timeout-minutes: 45

    env:
      PYTHONPATH: ./backend

    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Setup Python and Install Dependencies
        uses: ./.github/actions/setup-python-and-install-dependencies
        with:
          requirements: |
            backend/requirements/default.txt
            backend/requirements/dev.txt

      - name: Format branch name for cache
        id: format-branch
        env:
          PR_NUMBER: ${{ github.event.pull_request.number }}
          REF_NAME: ${{ github.ref_name }}
        run: |
          if [ -n "${PR_NUMBER}" ]; then
            CACHE_SUFFIX="${PR_NUMBER}"
          else
            # shellcheck disable=SC2001
            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
          fi
          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT

      - name: Login to Docker Hub
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f

      - name: Build and load
        uses: docker/bake-action@5be5f02ff8819ecd3092ea6b2e6261c31774f2b4 # ratchet:docker/bake-action@v6
        env:
          TAG: model-server-${{ github.run_id }}
        with:
          load: true
          targets: model-server
          set: |
            model-server.cache-from=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }}
            model-server.cache-from=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }}
            model-server.cache-from=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache
            model-server.cache-from=type=registry,ref=onyxdotapp/onyx-model-server:latest
            model-server.cache-to=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
            model-server.cache-to=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
            model-server.cache-to=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache,mode=max

      - name: Start Docker containers
        id: start_docker
        env:
          IMAGE_TAG: model-server-${{ github.run_id }}
        run: |
          cd deployment/docker_compose
          docker compose \
            -f docker-compose.yml \
            -f docker-compose.dev.yml \
            up -d --wait \
            inference_model_server

      - name: Run Tests
        run: |
          py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/llm
          py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/embedding

      - name: Alert on Failure
        if: failure() && github.event_name == 'schedule'
        uses: ./.github/actions/slack-notify
        with:
          webhook-url: ${{ secrets.SLACK_WEBHOOK }}
          failed-jobs: model-check
          title: "🚨 Scheduled Model Tests failed!"
          ref-name: ${{ github.ref_name }}

      - name: Dump all-container logs (optional)
        if: always()
        run: |
          cd deployment/docker_compose
          docker compose logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true

      - name: Upload logs
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
        with:
          name: docker-all-logs
          path: ${{ github.workspace }}/docker-compose.log