From 7a9cd16846871afaae42e4ae3637ce10194369a3 Mon Sep 17 00:00:00 2001 From: Jesse Turner Date: Fri, 20 Feb 2026 19:11:06 +0000 Subject: [PATCH 1/2] chore: split e2e workflow into PR-focused and weekly full suite Run only 4 focused tests (strands-bedrock, strands-openai, langgraph-bedrock, googleadk-gemini) on PRs for fast feedback. Add a weekly workflow that runs the full e2e suite every Monday at 9 AM EST. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/e2e-tests-weekly.yml | 56 ++++++++++++++++++++++++++ .github/workflows/e2e-tests.yml | 2 +- 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/e2e-tests-weekly.yml diff --git a/.github/workflows/e2e-tests-weekly.yml b/.github/workflows/e2e-tests-weekly.yml new file mode 100644 index 00000000..d0a12939 --- /dev/null +++ b/.github/workflows/e2e-tests-weekly.yml @@ -0,0 +1,56 @@ +name: E2E Tests (Weekly Full Suite) +on: + workflow_dispatch: + inputs: + aws_region: + description: 'AWS region for deployment' + default: 'us-east-1' + schedule: + - cron: '0 14 * * 1' # Every Monday at 9 AM EST (14:00 UTC) + +permissions: + id-token: write # OIDC — lets GitHub assume an AWS IAM role via short-lived token (no stored keys) + contents: read + +jobs: + e2e: + runs-on: ubuntu-latest + environment: e2e-testing + timeout-minutes: 60 + steps: + - uses: actions/checkout@v6 + with: + ref: main + - uses: actions/setup-node@v6 + with: + node-version: '20.x' + cache: 'npm' + - name: Configure git + run: | + git config --global user.email "ci@amazon.com" + git config --global user.name "CI" + - uses: astral-sh/setup-uv@v5 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.E2E_AWS_ROLE_ARN }} + aws-region: ${{ inputs.aws_region || 'us-east-1' }} + - name: Get AWS Account ID + id: aws + run: echo "account_id=$(aws sts get-caller-identity --query Account --output text)" >> "$GITHUB_OUTPUT" + - name: Get API keys from Secrets Manager + uses: aws-actions/aws-secretsmanager-get-secrets@v2 + with: + secret-ids: | + E2E,${{ secrets.E2E_SECRET_ARN }} + parse-json-secrets: true + - run: npm ci + - run: npm run build + - name: Run E2E tests + env: + AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} + AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} + ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} + run: npm run test:e2e diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 344391d0..edde6245 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -53,4 +53,4 @@ jobs: ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} - run: npm run test:e2e + run: npx vitest run --project e2e strands-bedrock strands-openai langgraph-bedrock googleadk-gemini From a7030a094ed5c9d91cd88d62bd282dea2a3ef2b6 Mon Sep 17 00:00:00 2001 From: Jesse Turner Date: Fri, 20 Feb 2026 20:05:56 +0000 Subject: [PATCH 2/2] fix: clean up API key credential providers in e2e test teardown The e2e afterAll was only tearing down CDK stacks but not deleting the API key credential providers created as a pre-deploy step. This caused providers to accumulate and hit the account limit, failing all non-Bedrock tests. Co-Authored-By: Claude Opus 4.6 --- e2e-tests/e2e-helper.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/e2e-tests/e2e-helper.ts b/e2e-tests/e2e-helper.ts index 764f9d0f..ee05454b 100644 --- a/e2e-tests/e2e-helper.ts +++ b/e2e-tests/e2e-helper.ts @@ -1,4 +1,8 @@ import { hasAwsCredentials, parseJsonOutput, prereqs, runCLI } from '../src/test-utils/index.js'; +import { + BedrockAgentCoreControlClient, + DeleteApiKeyCredentialProviderCommand, +} from '@aws-sdk/client-bedrock-agentcore-control'; import { execSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { mkdir, rm, writeFile } from 'node:fs/promises'; @@ -94,6 +98,20 @@ export function createE2ESuite(cfg: E2EConfig) { console.log('Teardown stdout:', result.stdout); console.log('Teardown stderr:', result.stderr); } + + // Delete the API key credential provider from the account. + // These are created as a pre-deploy step outside CDK and are not + // cleaned up by stack teardown, so we must delete them explicitly. + if (cfg.modelProvider !== 'Bedrock' && agentName) { + const providerName = `${agentName}${cfg.modelProvider}`; + const region = process.env.AWS_REGION ?? 'us-east-1'; + try { + const client = new BedrockAgentCoreControlClient({ region }); + await client.send(new DeleteApiKeyCredentialProviderCommand({ name: providerName })); + } catch { + // Best-effort cleanup — don't fail the test if deletion fails + } + } } if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); }, 600000);