From 08f018e1fe0e2473ae235c4bbebc39d91d36b3ab Mon Sep 17 00:00:00 2001 From: Jack Luar Date: Tue, 17 Feb 2026 10:11:26 +0000 Subject: [PATCH] improve CI speed - use caching in `astral-sh/setup-uv` - use caching in HF dataset - parallelise init/init-dev targets in root - general CI to split and parallelise lint/unit tests before docker build - optimise dockerfile layers Signed-off-by: Jack Luar --- .github/workflows/ci-secret.yaml | 31 +++++++-- .github/workflows/ci.yaml | 110 ++++++++++++++++++++++--------- Makefile | 18 +++-- backend/Dockerfile | 22 +++++-- 4 files changed, 129 insertions(+), 52 deletions(-) diff --git a/.github/workflows/ci-secret.yaml b/.github/workflows/ci-secret.yaml index b8e94209..d9c82926 100644 --- a/.github/workflows/ci-secret.yaml +++ b/.github/workflows/ci-secret.yaml @@ -1,7 +1,7 @@ name: ORAssistant Secret CI run-name: ${{ github.actor }} started CI -on: +on: push: branches: - master @@ -20,25 +20,42 @@ jobs: build-backend-docker: runs-on: self-hosted steps: + - name: Checkout code + uses: actions/checkout@v4 - name: Setup python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Install uv - uses: astral-sh/setup-uv@v6 - - name: Checkout code - uses: actions/checkout@v4 + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: | + backend/uv.lock + frontend/uv.lock + evaluation/uv.lock - name: Setup prereqs run: | make init-dev - name: Run formatting checks run: | make check + - name: Cache HuggingFace dataset + uses: actions/cache@v4 + with: + path: backend/data/source_list.json + key: hf-source-list-v1 + - name: Download HuggingFace dataset + working-directory: backend + run: | + uv run huggingface-cli download \ + --repo-type dataset \ + The-OpenROAD-Project/ORAssistant_RAG_Dataset \ + --include source_list.json \ + --local-dir data/ - name: Run unit tests working-directory: backend run: | - uv pip install huggingface_hub[cli] - huggingface-cli download --repo-type dataset The-OpenROAD-Project/ORAssistant_RAG_Dataset --include source_list.json --local-dir data/ cp .env.test .env make test - name: Populate environment variables diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 57b0328c..6118bfa2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,7 @@ name: ORAssistant CI run-name: ${{ github.actor }} started CI -on: +on: pull_request: paths: - 'backend/**' @@ -15,36 +15,82 @@ defaults: shell: bash jobs: - build-backend-docker: + lint: runs-on: self-hosted steps: - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install uv - uses: astral-sh/setup-uv@v6 - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup prereqs - run: | - make init-dev - - - name: Run formatting checks - run: | - make check - - - name: Run unit tests - working-directory: backend - run: | - uv pip install huggingface_hub[cli] - huggingface-cli download --repo-type dataset The-OpenROAD-Project/ORAssistant_RAG_Dataset --include source_list.json --local-dir data/ - cp .env.test .env - make test - - - name: Build Docker images - run: | - docker compose build + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: | + backend/uv.lock + frontend/uv.lock + evaluation/uv.lock + + - name: Setup prereqs + run: make init-dev + + - name: Run formatting checks + run: make check + + test: + runs-on: self-hosted + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: | + backend/uv.lock + frontend/uv.lock + evaluation/uv.lock + + - name: Setup prereqs + run: make init-dev + + - name: Cache HuggingFace dataset + uses: actions/cache@v4 + with: + path: backend/data/source_list.json + key: hf-source-list-v1 + + - name: Download HuggingFace dataset + working-directory: backend + run: | + uv run huggingface-cli download \ + --repo-type dataset \ + The-OpenROAD-Project/ORAssistant_RAG_Dataset \ + --include source_list.json \ + --local-dir data/ + + - name: Run unit tests + working-directory: backend + run: | + cp .env.test .env + make test + + docker-build: + runs-on: self-hosted + needs: [lint, test] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Build Docker images + run: docker compose build diff --git a/Makefile b/Makefile index dfd05ae0..7cd2747d 100644 --- a/Makefile +++ b/Makefile @@ -3,21 +3,27 @@ GOOGLE_SECRET_JSON?=$(HOME)/secret.json .PHONY: init init: - @for folder in $(FOLDERS); do (cd $$folder && make init && cd ../); done + @pids=""; \ + for folder in $(FOLDERS); do (cd $$folder && make init) & pids="$$pids $$!"; done; \ + for pid in $$pids; do wait $$pid || exit 1; done .PHONY: init-dev init-dev: - @for folder in $(FOLDERS); do (cd $$folder && make init-dev && cd ../); done + @pids=""; \ + for folder in $(FOLDERS); do (cd $$folder && make init-dev) & pids="$$pids $$!"; done; \ + for pid in $$pids; do wait $$pid || exit 1; done .PHONY: format format: - @for folder in $(FOLDERS); do (cd $$folder && make format && cd ../); done + @pids=""; \ + for folder in $(FOLDERS); do (cd $$folder && make format) & pids="$$pids $$!"; done; \ + for pid in $$pids; do wait $$pid || exit 1; done .PHONY: check check: - @for folder in $(FOLDERS); do \ - (cd $$folder && make check && cd ../) || exit 1; \ - done + @pids=""; \ + for folder in $(FOLDERS); do (cd $$folder && make check) & pids="$$pids $$!"; done; \ + for pid in $$pids; do wait $$pid || exit 1; done @. ./backend/.venv/bin/activate && \ pre-commit run --all-files diff --git a/backend/Dockerfile b/backend/Dockerfile index b8cbb94f..5eccbb60 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -2,6 +2,7 @@ FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim WORKDIR /ORAssistant-backend +# Layer 1: system deps RUN apt-get update && apt-get install -y \ build-essential \ curl \ @@ -15,18 +16,25 @@ RUN apt-get update && apt-get install -y \ git lfs install && \ rm -rf /var/lib/apt/lists/* -RUN pip install uv - -COPY ./pyproject.toml /ORAssistant-backend/pyproject.toml -COPY . . - -RUN uv venv .venv && uv sync --dev && uv run /ORAssistant-backend/src/post_install.py - +# Layer 2: HF dataset RUN git clone https://huggingface.co/datasets/The-OpenROAD-Project/ORAssistant_RAG_Dataset && \ mkdir -p data && \ mv ORAssistant_RAG_Dataset/* data/ && \ rm -rf ORAssistant_RAG_Dataset +# Layer 3: Python dep manifests +COPY ./pyproject.toml /ORAssistant-backend/pyproject.toml +COPY ./uv.lock /ORAssistant-backend/uv.lock + +# Layer 4: Install Python deps +RUN uv venv .venv && uv sync --dev + +# Layer 5: Source code +COPY . . + +# Layer 6: Post install +RUN uv run /ORAssistant-backend/src/post_install.py + EXPOSE 8000 CMD ["uv", "run", "uvicorn", "src.api.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]