ISL-174: benchmarks core e2e consolidation

env.example → .env.example

@@ -29,4 +29,4 @@ BL_WORKSPACE=your_bl_workspace
 
 ######### RENDER ########
 RENDER_API_KEY=your_render_api_key
-RENDER_OWNER_ID=your_render_owner_id
+RENDER_OWNER_ID=your_render_owner_id

.github/workflows/ci.yml

@@ -0,0 +1,24 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches: [master]
+
+permissions:
+  contents: read
+
+jobs:
+  ci:
+    name: Typecheck
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: npm
+      - run: npm ci
+      - run: npm run ci

.github/workflows/devops-hygiene.yml

@@ -0,0 +1,33 @@
+name: DevOps Hygiene
+
+on:
+  pull_request:
+    branches: [main, master]
+  push:
+    branches: [main, master]
+
+permissions:
+  contents: read
+
+jobs:
+  hygiene:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Validate env file conventions
+        run: |
+          set -euo pipefail
+          tracked_env="$(git ls-files | awk '($0==".env" || $0 ~ /\/\.env$/){print}')"
+          legacy_example="$(git ls-files | awk '($0==".example.env" || $0 ~ /\/\.example\.env$/){print}')"
+
+          if [[ -n "$tracked_env" ]]; then
+            echo "Tracked .env files are not allowed:"
+            echo "$tracked_env"
+            exit 1
+          fi
+
+          if [[ -n "$legacy_example" ]]; then
+            echo "Legacy .example.env file names found. Use .env.example instead:"
+            echo "$legacy_example"
+            exit 1
+          fi

.github/workflows/validation-matrix.yml

@@ -0,0 +1,58 @@
+name: Validation Matrix
+
+on:
+  workflow_dispatch:
+    inputs:
+      iterations_list:
+        description: "Comma-separated iterations list (e.g. 5,10,25)"
+        required: false
+        default: "5,10,25"
+      provider:
+        description: "Optional single provider filter"
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+
+jobs:
+  matrix:
+    name: Run validation matrix
+    runs-on: namespace-profile-default
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: npm
+      - run: npm ci
+      - name: Run matrix benchmarks
+        env:
+          COMPUTESDK_API_KEY: ${{ secrets.COMPUTESDK_API_KEY }}
+          E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
+          VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
+          VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }}
+          VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
+          BL_API_KEY: ${{ secrets.BL_API_KEY }}
+          BL_WORKSPACE: ${{ secrets.BL_WORKSPACE }}
+          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
+          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
+          DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
+          RAILWAY_API_KEY: ${{ secrets.RAILWAY_API_KEY }}
+          RAILWAY_PROJECT_ID: ${{ secrets.RAILWAY_PROJECT_ID }}
+          RAILWAY_ENVIRONMENT_ID: ${{ secrets.RAILWAY_ENVIRONMENT_ID }}
+          NSC_TOKEN: ${{ secrets.NSC_TOKEN }}
+          RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }}
+          RENDER_OWNER_ID: ${{ secrets.RENDER_OWNER_ID }}
+        run: |
+          provider_arg=""
+          if [[ -n "${{ github.event.inputs.provider }}" ]]; then
+            provider_arg="--provider ${{ github.event.inputs.provider }}"
+          fi
+          npm run bench:matrix -- --iterations-list "${{ github.event.inputs.iterations_list }}" $provider_arg
+      - name: Upload matrix artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: validation-matrix-results
+          path: results/*.json

CONTRIBUTING.md

@@ -35,7 +35,7 @@ export const yourProvider: DirectBenchmarkConfig = {
 
 2. Add to the providers array in `src/direct-run.ts`
 
-3. Update `env.example` with required environment variables
+3. Update `.env.example` with required environment variables
 
 4. Submit a PR with:
    - The code changes
@@ -76,7 +76,7 @@ Documentation improvements are always welcome. No issue required for typos, clar
 git clone https://github.com/computesdk/benchmarks.git
 cd benchmarks
 npm install
-cp env.example .env
+cp .env.example .env
 ```
 
 ### Running Tests Locally
@@ -90,6 +90,9 @@ npm run bench:direct:e2b
 
 # Run with custom iterations
 npm run bench:direct -- --iterations 5
+
+# Run validation matrix across multiple iteration sets
+npm run bench:matrix -- --iterations-list 5,10,25
 ```
 
 ### Code Style

METHODOLOGY.md

@@ -215,10 +215,13 @@ Reproduce our results:
 git clone https://github.com/computesdk/benchmarks.git
 cd benchmarks
 npm install
-cp env.example .env  # Add your API keys
+cp .env.example .env  # Add your API keys
 
 # Run with same settings as CI
 npm run bench:direct -- --iterations 10
+
+# Run a validation matrix (multiple iteration sets)
+npm run bench:matrix -- --iterations-list 5,10,25
 ```
 
 **Note**: Your results will differ based on your network location and conditions.

package.json

@@ -13,8 +13,11 @@
     "bench:railway": "tsx src/run.ts --provider railway",
     "bench:namespace": "tsx src/run.ts --provider namespace",
     "bench:render": "tsx src/run.ts --provider render",
+    "bench:matrix": "tsx src/run-matrix.ts",
     "update-readme": "tsx src/update-readme.ts",
-    "generate-svg": "tsx src/generate-svg.ts"
+    "generate-svg": "tsx src/generate-svg.ts",
+    "typecheck": "tsc --noEmit",
+    "ci": "npm run typecheck"
   },
   "dependencies": {
     "@computesdk/blaxel": "^1.5.7",

src/benchmark.ts

@@ -1,5 +1,10 @@
 import type { ProviderConfig, BenchmarkResult, TimingResult, Stats } from './types.js';
 
+function buildIterationRequestId(runId: string | undefined, provider: string, iteration: number): string {
+  const seed = runId ?? 'run';
+  return `${seed}-${provider}-${iteration + 1}`;
+}
+
 function computeStats(values: number[]): Stats {
   if (values.length === 0) return { min: 0, max: 0, median: 0, avg: 0 };
 
@@ -18,13 +23,14 @@ function computeStats(values: number[]): Stats {
 }
 
 export async function runBenchmark(config: ProviderConfig): Promise<BenchmarkResult> {
-  const { name, iterations = 10, timeout = 120_000, requiredEnvVars } = config;
+  const { name, runId, iterations = 10, timeout = 120_000, requiredEnvVars } = config;
 
   // Check if all required credentials are available
   const missingVars = requiredEnvVars.filter(v => !process.env[v]);
   if (missingVars.length > 0) {
     return {
       provider: name,
+      runId,
       iterations: [],
       summary: { ttiMs: { min: 0, max: 0, median: 0, avg: 0 } },
       skipped: true,
@@ -39,15 +45,17 @@ export async function runBenchmark(config: ProviderConfig): Promise<BenchmarkRes
 
   for (let i = 0; i < iterations; i++) {
     console.log(`  Iteration ${i + 1}/${iterations}...`);
+    const iterationRequestId = buildIterationRequestId(runId, name, i);
+    const startedAt = new Date().toISOString();
 
     try {
-      const iterationResult = await runIteration(compute, timeout);
+      const iterationResult = await runIteration(compute, timeout, iterationRequestId);
       results.push(iterationResult);
       console.log(`    TTI: ${(iterationResult.ttiMs / 1000).toFixed(2)}s`);
     } catch (err) {
       const error = err instanceof Error ? err.message : String(err);
       console.log(`    FAILED: ${error}`);
-      results.push({ ttiMs: 0, error });
+      results.push({ ttiMs: 0, requestId: iterationRequestId, startedAt, error });
     }
   }
 
@@ -57,6 +65,7 @@ export async function runBenchmark(config: ProviderConfig): Promise<BenchmarkRes
   if (successful.length === 0) {
     return {
       provider: name,
+      runId,
       iterations: results,
       summary: { ttiMs: { min: 0, max: 0, median: 0, avg: 0 } },
       skipped: true,
@@ -66,15 +75,17 @@ export async function runBenchmark(config: ProviderConfig): Promise<BenchmarkRes
 
   return {
     provider: name,
+    runId,
     iterations: results,
     summary: {
       ttiMs: computeStats(successful.map(r => r.ttiMs)),
     },
   };
 }
 
-async function runIteration(compute: any, timeout: number): Promise<TimingResult> {
+async function runIteration(compute: any, timeout: number, requestId: string): Promise<TimingResult> {
   let sandbox: any = null;
+  const startedAt = new Date().toISOString();
 
   try {
     const start = performance.now();
@@ -89,7 +100,7 @@ async function runIteration(compute: any, timeout: number): Promise<TimingResult
 
     const ttiMs = performance.now() - start;
 
-    return { ttiMs };
+    return { ttiMs, requestId, startedAt };
   } finally {
     if (sandbox) {
       try {

src/run-matrix.ts

@@ -0,0 +1,115 @@
+import { config } from 'dotenv';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { runBenchmark } from './benchmark.js';
+import { printResultsTable, writeResultsJson } from './table.js';
+import { providers } from './providers.js';
+import type { BenchmarkResult, RunMetadata } from './types.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+config({ path: path.resolve(__dirname, '../.env') });
+
+const args = process.argv.slice(2);
+const providerFilter = getArgValue(args, '--provider');
+const iterationsListArg = getArgValue(args, '--iterations-list') || '1,5,10,25';
+const timeoutMs = parseInt(getArgValue(args, '--timeout-ms') || '120000', 10);
+const matrixId = getArgValue(args, '--matrix-id') || `matrix-${Date.now()}`;
+
+function getArgValue(argv: string[], flag: string): string | undefined {
+  const idx = argv.indexOf(flag);
+  return idx !== -1 && idx + 1 < argv.length ? argv[idx + 1] : undefined;
+}
+
+function parseIterationsList(value: string): number[] {
+  const parsed = value
+    .split(',')
+    .map(v => parseInt(v.trim(), 10))
+    .filter(v => Number.isFinite(v) && v > 0);
+
+  const dedup = Array.from(new Set(parsed));
+  if (dedup.length === 0) {
+    throw new Error(`Invalid --iterations-list value: ${value}`);
+  }
+  return dedup;
+}
+
+function buildMedianMatrix(runs: Array<{ iterations: number; results: BenchmarkResult[] }>) {
+  const matrix: Record<string, Record<string, number | null>> = {};
+  for (const run of runs) {
+    for (const result of run.results) {
+      if (!matrix[result.provider]) {
+        matrix[result.provider] = {};
+      }
+      matrix[result.provider][String(run.iterations)] = result.skipped ? null : result.summary.ttiMs.median;
+    }
+  }
+  return matrix;
+}
+
+async function main() {
+  const iterationsList = parseIterationsList(iterationsListArg);
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+
+  const toRun = providerFilter
+    ? providers.filter(p => p.name === providerFilter)
+    : providers;
+
+  if (toRun.length === 0) {
+    console.error(`Unknown provider: ${providerFilter}`);
+    console.error(`Available: ${providers.map(p => p.name).join(', ')}`);
+    process.exit(1);
+  }
+
+  console.log('ComputeSDK Benchmark Validation Matrix');
+  console.log(`Matrix ID: ${matrixId}`);
+  console.log(`Iterations sets: ${iterationsList.join(', ')}`);
+  console.log(`Date: ${new Date().toISOString()}\n`);
+
+  const matrixRuns: Array<{ iterations: number; runId: string; results: BenchmarkResult[]; file: string }> = [];
+
+  for (const iterations of iterationsList) {
+    const runId = `${matrixId}-i${iterations}`;
+    console.log(`\n=== Matrix run: ${runId} ===`);
+    const results: BenchmarkResult[] = [];
+
+    for (const providerConfig of toRun) {
+      const result = await runBenchmark({ ...providerConfig, iterations, timeout: timeoutMs, runId });
+      results.push(result);
+    }
+
+    printResultsTable(results);
+
+    const runFile = `${timestamp}-${runId}.json`;
+    const runPath = path.resolve(__dirname, `../results/${runFile}`);
+    const metadata: RunMetadata = {
+      runId,
+      mode: 'matrix',
+      providerFilter,
+      iterations,
+      timeoutMs,
+    };
+    await writeResultsJson(results, runPath, metadata);
+    matrixRuns.push({ iterations, runId, results, file: runFile });
+  }
+
+  const summary = {
+    version: '1.0',
+    matrixId,
+    timestamp: new Date().toISOString(),
+    providerFilter: providerFilter || null,
+    timeoutMs,
+    iterationsList,
+    runFiles: matrixRuns.map(r => ({ runId: r.runId, iterations: r.iterations, file: r.file })),
+    medianMatrix: buildMedianMatrix(matrixRuns),
+  };
+
+  const matrixOut = path.resolve(__dirname, `../results/matrix-${timestamp}.json`);
+  fs.writeFileSync(matrixOut, JSON.stringify(summary, null, 2));
+  console.log(`\nMatrix summary written to ${matrixOut}`);
+}
+
+main().catch(err => {
+  console.error('Matrix benchmark failed:', err);
+  process.exit(1);
+});