From d84f4bc6287a68ddf23f12f353f429858a0f0745 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Fri, 13 Feb 2026 12:40:17 +0530 Subject: [PATCH 1/2] Add fast post-merge gateway integration check with matrix and cache --- .../gateway-post-merge-integration-check.yml | 167 ++++++++++++++++++ gateway/gateway-builder/Makefile | 3 + gateway/gateway-controller/Makefile | 4 + gateway/gateway-runtime/Makefile | 7 +- 4 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/gateway-post-merge-integration-check.yml diff --git a/.github/workflows/gateway-post-merge-integration-check.yml b/.github/workflows/gateway-post-merge-integration-check.yml new file mode 100644 index 000000000..a128fbba4 --- /dev/null +++ b/.github/workflows/gateway-post-merge-integration-check.yml @@ -0,0 +1,167 @@ +name: Gateway Post-Merge Integration Check + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - "gateway/**" + - "tests/mock-servers/**" + - "samples/sample-service/**" + - ".github/workflows/gateway-post-merge-integration-check.yml" + +permissions: + contents: read + +concurrency: + group: gateway-post-merge-it-${{ github.ref }} + cancel-in-progress: true + +jobs: + integration-test: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - mode: normal + compose_file: docker-compose.test.yaml + suffix: normal + - mode: postgres + compose_file: docker-compose.test.postgres.yaml + suffix: postgres + env: + BUILDX_CACHE_ARGS: >- + --cache-from type=gha,scope=gateway-it-${{ matrix.mode }} + --cache-to type=gha,mode=max,scope=gateway-it-${{ matrix.mode }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.25" + cache: true + cache-dependency-path: | + go.work.sum + gateway/gateway-controller/go.sum + gateway/gateway-builder/go.sum + gateway/gateway-runtime/policy-engine/go.sum + gateway/it/go.sum + gateway/system-policies/analytics/go.sum + gateway/sample-policies/count-letters/go.sum + gateway/sample-policies/uppercase-body/go.sum + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build coverage-instrumented images + run: | + cd gateway + make build-coverage VERSION=test + + - name: Build mock server images + run: | + for mock in mock-jwks mock-azure-content-safety mock-aws-bedrock-guardrail mock-embedding-provider mock-analytics-collector; do + echo "Building $mock..." + docker build -t ghcr.io/wso2/api-platform/$mock:latest tests/mock-servers/$mock + done + + - name: Build sample-service + run: | + cd samples/sample-service + make build + + - name: Record test start time + run: echo "TEST_START_TS=$(date +%s)" >> "$GITHUB_ENV" + + - name: Verify gateway-controller uses PostgreSQL + if: matrix.mode == 'postgres' + run: | + set -euo pipefail + cd gateway/it + + PROJECT="gateway-it-postgres-verify" + cleanup() { + docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml down -v --remove-orphans || true + } + trap cleanup EXIT + + docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml up -d postgres gateway-controller + + timeout 90 bash -c 'until curl -fsS http://localhost:9090/health >/dev/null; do sleep 2; done' + + docker logs it-gateway-controller > /tmp/gateway-controller.log 2>&1 || true + grep -Eq "Initializing PostgreSQL storage|PostgreSQL schema up to date" /tmp/gateway-controller.log + + schema_count="$(docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml exec -T postgres \ + psql -U gateway -d gateway_test -tAc "SELECT COUNT(*) FROM schema_migrations WHERE id = 1 AND version >= 1;")" + [ "$schema_count" = "1" ] + + conn_count="$(docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml exec -T postgres \ + psql -U gateway -d gateway_test -tAc "SELECT COUNT(*) FROM pg_stat_activity WHERE application_name = 'gateway-controller';")" + [ "${conn_count:-0}" -ge 1 ] + + - name: Run integration tests (normal) + if: matrix.mode == 'normal' + run: | + cd gateway + COMPOSE_FILE=${{ matrix.compose_file }} make test-integration + + - name: Run integration tests (postgres) + if: matrix.mode == 'postgres' + run: | + cd gateway + COMPOSE_FILE=${{ matrix.compose_file }} make test-integration + + - name: Publish test duration metrics + if: always() + run: | + end_ts="$(date +%s)" + start_ts="${TEST_START_TS:-$end_ts}" + duration="$((end_ts - start_ts))" + minutes="$((duration / 60))" + seconds="$((duration % 60))" + echo "### Gateway integration timing" >> "$GITHUB_STEP_SUMMARY" + echo "- Mode: \`${{ matrix.mode }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "- Duration: ${minutes}m ${seconds}s (${duration}s)" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload coverage report + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-report-${{ matrix.suffix }} + path: gateway/it/coverage/output + retention-days: 7 + + - name: Upload test reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-reports-${{ matrix.suffix }} + path: gateway/it/reports/ + retention-days: 7 + + - name: Debug on failure - Dump logs + if: failure() + run: | + echo "=== Docker Containers ===" + docker ps -a + + echo "" + echo "=== gateway/it/logs Directory Contents ===" + if [ -d gateway/it/logs ]; then + if [ "$(ls -A gateway/it/logs)" ]; then + for f in gateway/it/logs/*; do + echo "" + echo "--- Contents of $f ---" + cat "$f" + done + else + echo "No log files found in gateway/it/logs." + fi + else + echo "Directory gateway/it/logs does not exist." + fi diff --git a/gateway/gateway-builder/Makefile b/gateway/gateway-builder/Makefile index 7a4912a2d..31009a052 100644 --- a/gateway/gateway-builder/Makefile +++ b/gateway/gateway-builder/Makefile @@ -30,6 +30,7 @@ DOCKER_REGISTRY ?= ghcr.io/wso2/api-platform IMAGE_NAME := $(DOCKER_REGISTRY)/gateway-builder IMAGE_TAG := $(VERSION) FULL_IMAGE := $(IMAGE_NAME):$(IMAGE_TAG) +BUILDX_CACHE_ARGS ?= .DEFAULT_GOAL := help @@ -58,6 +59,7 @@ help: ## Show this help message build: ## Build Docker image using buildx @echo "Building Docker image ($(IMAGE_NAME):$(VERSION))..." @docker buildx build -f Dockerfile \ + $(BUILDX_CACHE_ARGS) \ --build-context policy-engine=../gateway-runtime/policy-engine \ --build-context system-policies=../system-policies \ --build-context sdk=../../sdk \ @@ -93,6 +95,7 @@ push: ## Push Docker image to registry build-and-push-multiarch: ## Build and push multi-architecture Docker image (linux/amd64, linux/arm64) @echo "Building and pushing multi-arch Docker image: $(IMAGE_NAME):$(VERSION)" docker buildx build -f Dockerfile \ + $(BUILDX_CACHE_ARGS) \ --build-context policy-engine=../gateway-runtime/policy-engine \ --build-context system-policies=../system-policies \ --build-context sdk=../../sdk \ diff --git a/gateway/gateway-controller/Makefile b/gateway/gateway-controller/Makefile index 1a36821c1..d7c06b41b 100644 --- a/gateway/gateway-controller/Makefile +++ b/gateway/gateway-controller/Makefile @@ -27,6 +27,7 @@ LDFLAGS := -X main.Version=$(VERSION) -X main.GitCommit=$(GIT_COMMIT) -X main.Bu # Docker image configuration DOCKER_REGISTRY ?= ghcr.io/wso2/api-platform IMAGE_NAME := $(DOCKER_REGISTRY)/gateway-controller +BUILDX_CACHE_ARGS ?= .PHONY: help generate build build-local test generate-listener-certs push build-coverage-image @@ -48,6 +49,7 @@ test: ## Run unit and integration tests build: generate test ## Build Docker image using buildx @echo "Building Docker image ($(IMAGE_NAME):$(VERSION))..." docker buildx build -f Dockerfile \ + $(BUILDX_CACHE_ARGS) \ --build-context sdk=../../sdk \ --build-context common=../../common \ --build-arg VERSION=$(VERSION) \ @@ -76,6 +78,7 @@ push: ## Push Docker image to registry build-and-push-multiarch: ## Build and push multi-architecture Docker image (linux/amd64, linux/arm64) @echo "Building and pushing multi-arch Docker image: $(IMAGE_NAME):$(VERSION)" docker buildx build -f Dockerfile \ + $(BUILDX_CACHE_ARGS) \ --build-context sdk=../../sdk \ --build-context common=../../common \ --platform linux/amd64,linux/arm64 \ @@ -88,6 +91,7 @@ build-and-push-multiarch: ## Build and push multi-architecture Docker image (lin build-coverage-image: test ## Build coverage-instrumented gateway-controller image docker buildx build -f Dockerfile \ + $(BUILDX_CACHE_ARGS) \ --build-context sdk=../../sdk \ --build-context common=../../common \ --build-arg VERSION=$(VERSION) \ diff --git a/gateway/gateway-runtime/Makefile b/gateway/gateway-runtime/Makefile index e2ea59198..22ad92d08 100644 --- a/gateway/gateway-runtime/Makefile +++ b/gateway/gateway-runtime/Makefile @@ -25,6 +25,7 @@ GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") # Docker image configuration DOCKER_REGISTRY ?= ghcr.io/wso2/api-platform IMAGE_NAME := $(DOCKER_REGISTRY)/gateway-runtime +BUILDX_CACHE_ARGS ?= .PHONY: help build build-local build-coverage-image push clean build-and-push-multiarch test @@ -38,6 +39,7 @@ build: ## Build Gateway Runtime Docker image using buildx @echo "Building Gateway Runtime Docker image ($(IMAGE_NAME):$(VERSION))..." @mkdir -p target && cp ../build.yaml target/ docker buildx build -f Dockerfile \ + $(BUILDX_CACHE_ARGS) \ --build-context sdk=../../sdk \ --build-context common=../../common \ --build-context gateway-builder=../gateway-builder \ @@ -74,7 +76,8 @@ build-local: ## Build Gateway Runtime Docker image locally (faster, no buildx) build-coverage-image: test ## Build Gateway Runtime Docker image with coverage instrumentation @echo "Building Gateway Runtime coverage image ($(IMAGE_NAME)-coverage:$(VERSION))..." @mkdir -p target && cp ../build.yaml target/ - DOCKER_BUILDKIT=1 docker build -f Dockerfile \ + docker buildx build -f Dockerfile \ + $(BUILDX_CACHE_ARGS) \ --build-context sdk=../../sdk \ --build-context common=../../common \ --build-context gateway-builder=../gateway-builder \ @@ -85,6 +88,7 @@ build-coverage-image: test ## Build Gateway Runtime Docker image with coverage i --build-arg GIT_COMMIT=$(GIT_COMMIT) \ --build-arg ENABLE_COVERAGE=true \ -t $(IMAGE_NAME)-coverage:$(VERSION) \ + --load \ . @rm -rf target @echo "Coverage image built successfully: $(IMAGE_NAME)-coverage:$(VERSION)" @@ -98,6 +102,7 @@ build-and-push-multiarch: ## Build and push Gateway Runtime Docker image for mul @echo "Building and pushing multi-arch Gateway Runtime Docker image ($(VERSION))..." @mkdir -p target && cp ../build.yaml target/ docker buildx build -f Dockerfile \ + $(BUILDX_CACHE_ARGS) \ --build-context sdk=../../sdk \ --build-context common=../../common \ --build-context gateway-builder=../gateway-builder \ From c252c2a72621cb85056f868415c7ee933984af54 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Fri, 13 Feb 2026 13:36:50 +0530 Subject: [PATCH 2/2] Merge gateway PR integration workflows into matrix pipeline --- .../gateway-integration-test-postgres.yml | 106 -------------- .../workflows/gateway-integration-test.yml | 132 +++++++++++++++++- 2 files changed, 125 insertions(+), 113 deletions(-) delete mode 100644 .github/workflows/gateway-integration-test-postgres.yml diff --git a/.github/workflows/gateway-integration-test-postgres.yml b/.github/workflows/gateway-integration-test-postgres.yml deleted file mode 100644 index 0327b915d..000000000 --- a/.github/workflows/gateway-integration-test-postgres.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Gateway Integration Test (Postgres) - -on: - workflow_dispatch: - pull_request: - branches: - - main - paths: - - 'gateway/**' - - '.github/workflows/gateway-integration-test-postgres.yml' - -jobs: - integration-test: - runs-on: ubuntu-24.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.25' - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build coverage-instrumented images - run: | - cd gateway - make build-coverage VERSION=test - - - name: Build mock server images - run: | - for mock in mock-jwks mock-azure-content-safety mock-aws-bedrock-guardrail mock-embedding-provider mock-analytics-collector; do - echo "Building $mock..." - docker build -t ghcr.io/wso2/api-platform/$mock:latest tests/mock-servers/$mock - done - - - name: Verify gateway-controller uses PostgreSQL - run: | - set -euo pipefail - cd gateway/it - - PROJECT="gateway-it-postgres-verify" - cleanup() { - docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml down -v --remove-orphans || true - } - trap cleanup EXIT - - docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml up -d postgres gateway-controller - - timeout 90 bash -c 'until curl -fsS http://localhost:9090/health >/dev/null; do sleep 2; done' - - docker logs it-gateway-controller > /tmp/gateway-controller.log 2>&1 || true - grep -Eq "Initializing PostgreSQL storage|PostgreSQL schema up to date" /tmp/gateway-controller.log - - schema_count="$(docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml exec -T postgres \ - psql -U gateway -d gateway_test -tAc "SELECT COUNT(*) FROM schema_migrations WHERE id = 1 AND version >= 1;")" - [ "$schema_count" = "1" ] - - conn_count="$(docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml exec -T postgres \ - psql -U gateway -d gateway_test -tAc "SELECT COUNT(*) FROM pg_stat_activity WHERE application_name = 'gateway-controller';")" - [ "${conn_count:-0}" -ge 1 ] - - - name: Run integration tests - run: | - cd gateway - COMPOSE_FILE=docker-compose.test.postgres.yaml make test-integration - - - name: Upload coverage report - uses: actions/upload-artifact@v4 - if: always() - with: - name: coverage-report-postgres - path: gateway/it/coverage/output - retention-days: 7 - - - name: Upload test reports - uses: actions/upload-artifact@v4 - if: always() - with: - name: test-reports-postgres - path: gateway/it/reports/ - retention-days: 7 - - - name: Debug on failure - Dump logs - if: failure() - run: | - echo "=== Docker Containers ===" - docker ps -a - - echo "" - echo "=== gateway/it/logs Directory Contents ===" - if [ -d gateway/it/logs ]; then - if [ "$(ls -A gateway/it/logs)" ]; then - for f in gateway/it/logs/*; do - echo "" - echo "--- Contents of $f ---" - cat "$f" - done - else - echo "No log files found in gateway/it/logs." - fi - else - echo "Directory gateway/it/logs does not exist." - fi diff --git a/.github/workflows/gateway-integration-test.yml b/.github/workflows/gateway-integration-test.yml index e3d1cd36b..22b584c56 100644 --- a/.github/workflows/gateway-integration-test.yml +++ b/.github/workflows/gateway-integration-test.yml @@ -6,12 +6,22 @@ on: branches: - main paths: - - 'gateway/**' - - '.github/workflows/gateway-integration-test.yml' + - "gateway/**" + - "tests/mock-servers/**" + - "samples/sample-service/**" + - ".github/workflows/gateway-integration-test.yml" + - ".github/workflows/gateway-integration-test-postgres.yml" + +permissions: + contents: read jobs: - integration-test: + build-images: runs-on: ubuntu-24.04 + env: + BUILDX_CACHE_ARGS: >- + --cache-from type=gha,scope=gateway-it-pr-build + --cache-to type=gha,mode=max,scope=gateway-it-pr-build steps: - name: Checkout code uses: actions/checkout@v4 @@ -19,7 +29,17 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.25' + go-version: "1.25" + cache: true + cache-dependency-path: | + go.work.sum + gateway/gateway-controller/go.sum + gateway/gateway-builder/go.sum + gateway/gateway-runtime/policy-engine/go.sum + gateway/it/go.sum + gateway/system-policies/analytics/go.sum + gateway/sample-policies/count-letters/go.sum + gateway/sample-policies/uppercase-body/go.sum - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -41,16 +61,114 @@ jobs: cd samples/sample-service make build + - name: Export reusable integration images + run: | + mkdir -p /tmp/ci-images + docker save \ + ghcr.io/wso2/api-platform/gateway-controller-coverage:test \ + ghcr.io/wso2/api-platform/gateway-runtime-coverage:test \ + ghcr.io/wso2/api-platform/sample-service:latest \ + ghcr.io/wso2/api-platform/mock-jwks:latest \ + ghcr.io/wso2/api-platform/mock-azure-content-safety:latest \ + ghcr.io/wso2/api-platform/mock-aws-bedrock-guardrail:latest \ + ghcr.io/wso2/api-platform/mock-embedding-provider:latest \ + ghcr.io/wso2/api-platform/mock-analytics-collector:latest | gzip > /tmp/ci-images/gateway-it-images.tar.gz + + - name: Upload reusable integration images + uses: actions/upload-artifact@v4 + with: + name: gateway-it-images + path: /tmp/ci-images/gateway-it-images.tar.gz + retention-days: 1 + + integration-test: + runs-on: ubuntu-24.04 + needs: build-images + strategy: + fail-fast: false + matrix: + include: + - mode: sqlite + compose_file: docker-compose.test.yaml + suffix: normal + - mode: postgres + compose_file: docker-compose.test.postgres.yaml + suffix: postgres + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.25" + cache: true + cache-dependency-path: | + go.work.sum + gateway/it/go.sum + + - name: Download reusable integration images + uses: actions/download-artifact@v4 + with: + name: gateway-it-images + path: /tmp/ci-images + + - name: Load reusable integration images + run: | + gzip -dc /tmp/ci-images/gateway-it-images.tar.gz | docker load + + - name: Record test start time + run: echo "TEST_START_TS=$(date +%s)" >> "$GITHUB_ENV" + + - name: Verify gateway-controller uses PostgreSQL + if: matrix.mode == 'postgres' + run: | + set -euo pipefail + cd gateway/it + + PROJECT="gateway-it-postgres-verify" + cleanup() { + docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml down -v --remove-orphans || true + } + trap cleanup EXIT + + docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml up -d postgres gateway-controller + + timeout 90 bash -c 'until curl -fsS http://localhost:9090/health >/dev/null; do sleep 2; done' + + docker logs it-gateway-controller > /tmp/gateway-controller.log 2>&1 || true + grep -Eq "Initializing PostgreSQL storage|PostgreSQL schema up to date" /tmp/gateway-controller.log + + schema_count="$(docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml exec -T postgres \ + psql -U gateway -d gateway_test -tAc "SELECT COUNT(*) FROM schema_migrations WHERE id = 1 AND version >= 1;")" + [ "$schema_count" = "1" ] + + conn_count="$(docker compose -p "$PROJECT" -f docker-compose.test.postgres.yaml exec -T postgres \ + psql -U gateway -d gateway_test -tAc "SELECT COUNT(*) FROM pg_stat_activity WHERE application_name = 'gateway-controller';")" + [ "${conn_count:-0}" -ge 1 ] + - name: Run integration tests run: | cd gateway - make test-integration + COMPOSE_FILE=${{ matrix.compose_file }} make test-integration + + - name: Publish test duration metrics + if: always() + run: | + end_ts="$(date +%s)" + start_ts="${TEST_START_TS:-$end_ts}" + duration="$((end_ts - start_ts))" + minutes="$((duration / 60))" + seconds="$((duration % 60))" + echo "### Gateway integration timing" >> "$GITHUB_STEP_SUMMARY" + echo "- Mode: \`${{ matrix.mode }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "- Duration: ${minutes}m ${seconds}s (${duration}s)" >> "$GITHUB_STEP_SUMMARY" - name: Upload coverage report uses: actions/upload-artifact@v4 if: always() with: - name: coverage-report + name: coverage-report-${{ matrix.suffix }} path: gateway/it/coverage/output retention-days: 7 @@ -58,7 +176,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-reports + name: test-reports-${{ matrix.suffix }} path: gateway/it/reports/ retention-days: 7