feat(integ-runner): detect bootstrap errors and retry tests in valid regions

.projenrc.ts

@@ -1456,6 +1456,7 @@ const integRunner = configureProject(
       '@types/yargs',
       'constructs@^10',
       '@aws-cdk/integ-tests-alpha@2.184.1-alpha.0',
+      'fast-check@^3.23.2',
     ],
     allowPrivateDeps: true,
     tsconfig: {

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -8,7 +8,7 @@ import type { IntegTest, IntegTestInfo } from './runner/integration-tests';
 import { IntegrationTests } from './runner/integration-tests';
 import { processUnstableFeatures, availableFeaturesDescription } from './unstable-features';
 import type { IntegRunnerMetrics, IntegTestWorkerConfig, DestructiveChange } from './workers';
-import { runSnapshotTests, runIntegrationTests } from './workers';
+import { runSnapshotTests, runIntegrationTests, printEnvironmentsSummary } from './workers';
 import { watchIntegrationTest } from './workers/integ-watch-worker';
 
 // https://github.com/yargs/yargs/issues/1929
@@ -184,7 +184,7 @@ async function run(options: ReturnType<typeof parseCliArgs>) {
 
     // run integration tests if `--update-on-failed` OR `--force` is used
     if (options.runUpdateOnFailed || options.force) {
-      const { success, metrics } = await runIntegrationTests({
+      const { success, metrics, testEnvironments } = await runIntegrationTests({
         pool,
         tests: testsToRun,
         regions: options.testRegions,
@@ -197,6 +197,9 @@ async function run(options: ReturnType<typeof parseCliArgs>) {
       });
       testsSucceeded = success;
 
+      // Print summary of removed environments due to bootstrap errors
+      printEnvironmentsSummary(testEnvironments);
+
       if (options.clean === false) {
         logger.warning('Not cleaning up stacks since "--no-clean" was used');
       }

packages/@aws-cdk/integ-runner/lib/workers/common.ts

@@ -2,6 +2,7 @@ import { format } from 'util';
 import type { ResourceImpact } from '@aws-cdk/cloudformation-diff';
 import * as chalk from 'chalk';
 import * as logger from '../logger';
+import type { TestEnvironment, EnvironmentSummary } from './environment-pool';
 import type { IntegTestInfo } from '../runner/integration-tests';
 
 /**
@@ -118,6 +119,11 @@ export interface IntegBatchResponse {
    * list represents metrics from a single worker (account + region).
    */
   readonly metrics: IntegRunnerMetrics[];
+
+  /**
+   * Summary of the environments involed in the test run.
+   */
+  readonly testEnvironments: EnvironmentSummary;
 }
 
 /**
@@ -220,6 +226,11 @@ export enum DiagnosticReason {
    * The assertion failed
    */
   ASSERTION_FAILED = 'ASSERTION_FAILED',
+
+  /**
+   * The environment is not bootstrapped - test can be retried in a different environment
+   */
+  NOT_BOOTSTRAPPED = 'NOT_BOOTSTRAPPED',
 }
 
 /**
@@ -235,7 +246,7 @@ export interface Diagnostic {
   /**
    * The name of the stack
    */
-  readonly stackName: string;
+  readonly stackName?: string;
 
   /**
    * The diagnostic message
@@ -261,6 +272,12 @@ export interface Diagnostic {
    * Relevant config options that were used for the integ test
    */
   readonly config?: Record<string, any>;
+
+  /**
+   * The environment where the diagnostic occurred.
+   * Used for NOT_BOOTSTRAPPED diagnostics to track which environment failed.
+   */
+  readonly environment?: TestEnvironment;
 }
 
 export function printSummary(total: number, failed: number): void {
@@ -281,35 +298,45 @@ export function formatAssertionResults(results: AssertionResults): string {
     .join('\n      ');
 }
 
+/**
+ * Formats a status keyword with 2 spaces prefix and right-padded to 12 characters total.
+ */
+function formatStatus(keyword: string): string {
+  return `  ${keyword}`.padEnd(12);
+}
+
 /**
  * Print out the results from tests
  */
 export function printResults(diagnostic: Diagnostic): void {
   switch (diagnostic.reason) {
     case DiagnosticReason.SNAPSHOT_SUCCESS:
-      logger.success('  UNCHANGED  %s %s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`));
+      logger.success('%s %s %s', formatStatus('UNCHANGED'), diagnostic.testName, chalk.gray(`${diagnostic.duration}s`));
       break;
     case DiagnosticReason.TEST_SUCCESS:
-      logger.success('  SUCCESS    %s %s\n      ', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`));
+      logger.success('%s %s %s\n      ', formatStatus('SUCCESS'), diagnostic.testName, chalk.gray(`${diagnostic.duration}s`));
       break;
     case DiagnosticReason.NO_SNAPSHOT:
-      logger.error('  NEW        %s %s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`));
+      logger.error('%s %s %s', formatStatus('NEW'), diagnostic.testName, chalk.gray(`${diagnostic.duration}s`));
       break;
     case DiagnosticReason.SNAPSHOT_FAILED:
-      logger.error('  CHANGED    %s %s\n%s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
+      logger.error('%s %s %s\n%s', formatStatus('CHANGED'), diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
       break;
     case DiagnosticReason.TEST_WARNING:
-      logger.warning('  WARN       %s %s\n%s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
+      logger.warning('%s %s %s\n%s', formatStatus('WARN'), diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
       break;
     case DiagnosticReason.SNAPSHOT_ERROR:
     case DiagnosticReason.TEST_ERROR:
-      logger.error('  ERROR      %s %s\n%s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
+      logger.error('%s %s %s\n%s', formatStatus('ERROR'), diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
       break;
     case DiagnosticReason.TEST_FAILED:
-      logger.error('  FAILED     %s %s\n%s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
+      logger.error('%s %s %s\n%s', formatStatus('FAILED'), diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
       break;
     case DiagnosticReason.ASSERTION_FAILED:
-      logger.error('  ASSERT     %s %s\n%s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
+      logger.error('%s %s %s\n%s', formatStatus('ASSERT'), diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
+      break;
+    case DiagnosticReason.NOT_BOOTSTRAPPED:
+      logger.warning('%s %s %s\n%s', formatStatus('ENV'), diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), indentLines(diagnostic.message, 6));
       break;
   }
   for (const addl of diagnostic.additionalMessages ?? []) {
@@ -344,3 +371,24 @@ export function formatError(error: any): string {
 
   return `${name}: ${message}`;
 }
+
+/**
+ * Prints a summary of environments that were removed due to bootstrap errors
+ */
+export function printEnvironmentsSummary(summary: EnvironmentSummary): void {
+  if (summary.removed.length === 0) {
+    return;
+  }
+
+  logger.warning('\n%s', chalk.bold('Environments removed due to bootstrap errors:'));
+
+  for (const env of summary.removed) {
+    const profileStr = env.profile ? `${env.profile}/` : '';
+    const accountStr = env.account ? `aws://${env.account}/${env.region}` : env.region;
+
+    logger.warning('  • %s%s', profileStr, env.region);
+    logger.warning('    Run: %s', chalk.blue(`cdk bootstrap ${accountStr}`));
+  }
+
+  logger.warning('');
+}

packages/@aws-cdk/integ-runner/lib/workers/environment-pool.ts

@@ -0,0 +1,101 @@
+/**
+ * Identifies a specific profile+region combination (an "environment" for test execution)
+ */
+export interface TestEnvironment {
+  readonly profile?: string;
+  readonly region: string;
+  readonly account?: string;
+}
+
+export interface EnvironmentSummary {
+  /**
+   * Enviornments that got removed from the pool during the test run.s
+   */
+  readonly removed: RemovedEnvironment[];
+}
+
+/**
+ * Information about why an environment was removed
+ */
+export interface RemovedEnvironment extends TestEnvironment {
+  readonly reason: string;
+  readonly removedAt: Date;
+}
+
+/**
+ * Manages a pool of test environments for integration test workers.
+ *
+ * This class serves as a centralized pool for test environments, handling:
+ * - Tracking which environments are available vs removed
+ * - Recording removal reasons for reporting
+ *
+ * Future extensions could include:
+ * - Load balancing across environments
+ * - Rate limiting per environment
+ * - Environment health scoring
+ * - Automatic environment recovery
+ */
+export class EnvironmentPool {
+  private readonly availableEnvironments: Set<string>;
+  private readonly removedEnvironments: Map<string, RemovedEnvironment> = new Map();
+
+  constructor(environments: TestEnvironment[]) {
+    this.availableEnvironments = new Set(environments.map(e => this.makeKey(e)));
+  }
+
+  /**
+   * Creates a unique key for a profile+region combination
+   */
+  private makeKey(env: TestEnvironment): string {
+    return `${env.profile ?? 'default'}:${env.region}`;
+  }
+
+  /**
+   * Parses a key back into a TestEnvironment
+   */
+  private parseKey(key: string): TestEnvironment {
+    const [profile, region] = key.split(':');
+    return {
+      profile: profile === 'default' ? undefined : profile,
+      region,
+    };
+  }
+
+  /**
+   * Marks an environment as removed (unavailable for future tests)
+   */
+  public removeEnvironment(env: TestEnvironment, reason: string): void {
+    const key = this.makeKey(env);
+    if (this.availableEnvironments.has(key)) {
+      this.availableEnvironments.delete(key);
+      this.removedEnvironments.set(key, {
+        ...env,
+        reason,
+        removedAt: new Date(),
+      });
+    }
+  }
+
+  /**
+   * Checks if an environment is still available
+   */
+  public isAvailable(env: TestEnvironment): boolean {
+    return this.availableEnvironments.has(this.makeKey(env));
+  }
+
+  /**
+   * Gets all available environments
+   */
+  public getAvailableEnvironments(): TestEnvironment[] {
+    return Array.from(this.availableEnvironments).map(key => this.parseKey(key));
+  }
+
+  /**
+   * Gets all removed environments with their removal info
+   */
+  public summary(): EnvironmentSummary {
+    return {
+      removed: Array.from(this.removedEnvironments.values()),
+    };
+  }
+}

packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts

@@ -1,3 +1,4 @@
+import { ToolkitError } from '@aws-cdk/toolkit-lib';
 import * as workerpool from 'workerpool';
 import { IntegSnapshotRunner, IntegTestRunner } from '../../runner';
 import type { IntegTestInfo } from '../../runner/integration-tests';
@@ -58,23 +59,41 @@ export async function integTestWorker(request: IntegTestBatchRequest): Promise<I
               testName: `${runner.testName}-${testCaseName} (${request.profile}/${request.region})`,
               message: formatAssertionResults(results),
               duration: (Date.now() - start) / 1000,
-            });
+            } as Diagnostic);
           } else {
             workerpool.workerEmit({
               reason: DiagnosticReason.TEST_SUCCESS,
               testName: `${runner.testName}-${testCaseName}`,
               message: results ? formatAssertionResults(results) : 'NO ASSERTIONS',
               duration: (Date.now() - start) / 1000,
-            });
+            } as Diagnostic);
           }
         } catch (e) {
-          failures.push(testInfo);
-          workerpool.workerEmit({
+          const diagnostic: Diagnostic = {
             reason: DiagnosticReason.TEST_FAILED,
             testName: `${runner.testName}-${testCaseName} (${request.profile}/${request.region})`,
             message: `Integration test failed: ${formatError(e)}`,
             duration: (Date.now() - start) / 1000,
-          });
+          };
+
+          // Check if this is a bootstrap error
+          if (ToolkitError.isBootstrapError(e)) {
+            // Emit NOT_BOOTSTRAPPED diagnostic with environment
+            workerpool.workerEmit({
+              ...diagnostic,
+              reason: DiagnosticReason.NOT_BOOTSTRAPPED,
+              message: formatError(e),
+              environment: {
+                profile: request.profile,
+                region: request.region,
+                account: e.environment.account,
+              },
+            } as Diagnostic);
+          } else {
+            // Non-bootstrap error - record as failure
+            failures.push(testInfo);
+            workerpool.workerEmit(diagnostic);
+          }
         }
       }
     } catch (e) {

packages/@aws-cdk/integ-runner/lib/workers/index.ts

@@ -1,3 +1,4 @@
 export * from './common';
+export * from './environment-pool';
 export * from './integ-test-worker';
 export * from './integ-snapshot-worker';