test_runner: support test order randomization

Author: pmarchini

doc/api/cli.md

@@ -2762,6 +2762,34 @@ changes:
 Configures the test runner to only execute top level tests that have the `only`
 option set. This flag is not necessary when test isolation is disabled.
 
+### `--test-random-seed`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Set the seed used to randomize the order in which test files are executed.
+Providing this flag enables randomization implicitly, even without
+`--test-randomize`.
+
+The value must be an integer between `0` and `4294967295`.
+
+This flag cannot be used with `--watch`.
+
+### `--test-randomize`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Randomize the order in which test files are executed. This can help detect
+tests that rely on shared state or execution order.
+
+The seed used for randomization is printed in the test summary and can be
+reused with `--test-random-seed`.
+
+This flag cannot be used with `--watch`.
+
 ### `--test-reporter`
 
 <!-- YAML
@@ -3679,6 +3707,8 @@ one is included in the list below.
 * `--test-isolation`
 * `--test-name-pattern`
 * `--test-only`
+* `--test-random-seed`
+* `--test-randomize`
 * `--test-reporter-destination`
 * `--test-reporter`
 * `--test-rerun-failures`

doc/api/test.md

@@ -585,6 +585,38 @@ prevent shell expansion, which can reduce portability across systems.
 node --test "**/*.test.js" "**/*.spec.js"
 ```
 
+### Randomizing test file execution order
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1.0 - Early development
+
+The test runner can randomize the order of discovered test files to help detect
+order-dependent tests. Use `--test-randomize` to enable this mode.
+
+```bash
+node --test --test-randomize
+```
+
+When randomization is enabled, the test runner prints the seed used for the run
+as a diagnostic message:
+
+```text
+Randomized test order seed: 12345
+```
+
+Use `--test-random-seed=<number>` to replay the same order in a deterministic
+way. Supplying `--test-random-seed` also enables randomization, so
+`--test-randomize` is optional when a seed is provided:
+
+```bash
+node --test --test-randomize --test-random-seed=12345
+```
+
+`--test-randomize` and `--test-random-seed` are not supported with `--watch` mode.
+
 Matching files are executed as test files.
 More information on the test file execution can be found
 in the [test runner execution model][] section.
@@ -625,6 +657,8 @@ test runner functionality:
 * `--test-reporter` - Reporting is managed by the parent process
 * `--test-reporter-destination` - Output destinations are controlled by the parent
 * `--experimental-config-file` - Config file paths are managed by the parent
+* `--test-randomize` - File randomization is managed by the parent process
+* `--test-random-seed` - File randomization seed is managed by the parent process
 
 All other Node.js options from command line arguments, environment variables,
 and configuration files are inherited by the child processes.
@@ -1531,6 +1565,13 @@ changes:
       that specifies the index of the shard to run. This option is _required_.
     * `total` {number} is a positive integer that specifies the total number
       of shards to split the test files to. This option is _required_.
+  * `randomize` {boolean} Randomize the execution order of test files.
+    This option is not supported with `watch: true`.
+    **Default:** `false`.
+  * `randomSeed` {number} Seed used when randomizing test file order. If this
+    option is set, runs can replay the same randomized file order
+    deterministically, and setting this option also enables randomization.
+    **Default:** `undefined`.
   * `rerunFailuresFilePath` {string} A file path where the test runner will
     store the state of the tests to allow rerunning only the failed tests on a next run.
     see \[Rerunning failed tests]\[] for more information.

doc/node-config-schema.json

@@ -910,6 +910,14 @@
           "type": "boolean",
           "description": "run tests with 'only' option set"
         },
+        "test-random-seed": {
+          "type": "number",
+          "description": "seed used to randomize test file execution order"
+        },
+        "test-randomize": {
+          "type": "boolean",
+          "description": "run test files in a random order"
+        },
         "test-reporter": {
           "oneOf": [
             {

doc/node.1

@@ -1357,6 +1357,20 @@ tests must satisfy \fBboth\fR requirements in order to be executed.
 Configures the test runner to only execute top level tests that have the \fBonly\fR
 option set. This flag is not necessary when test isolation is disabled.
 .
+.It Fl -test-random-seed
+Set the seed used to randomize the order in which test files are executed.
+Providing this flag enables randomization implicitly, even without
+\fB--test-randomize\fR.
+The value must be an integer between 0 and 4294967295.
+This flag cannot be used with \fB--watch\fR.
+.
+.It Fl -test-randomize
+Randomize the order in which test files are executed.
+This can help detect tests that rely on shared state or execution order.
+The seed used for randomization is printed in the test summary and can be
+reused with \fB--test-random-seed\fR.
+This flag cannot be used with \fB--watch\fR.
+.
 .It Fl -test-reporter
 A test reporter to use when running tests. See the documentation on
 test reporters for more details.
@@ -2034,6 +2048,10 @@ one is included in the list below.
 .It
 \fB--test-reporter-destination\fR
 .It
+\fB--test-randomize\fR
+.It
+\fB--test-random-seed\fR
+.It
 \fB--test-reporter\fR
 .It
 \fB--test-rerun-failures\fR

lib/internal/test_runner/runner.js

@@ -56,6 +56,7 @@ const {
   validateObject,
   validateOneOf,
   validateInteger,
+  validateUint32,
   validateString,
   validateStringArray,
 } = require('internal/validators');
@@ -81,10 +82,12 @@ const {
 const { FastBuffer } = require('internal/buffer');
 
 const {
+  createRandomSeed,
   convertStringToRegExp,
   countCompletedTest,
   kDefaultPattern,
   parseCommandLine,
+  shuffleArrayWithSeed,
 } = require('internal/test_runner/utils');
 const { Glob } = require('internal/fs/glob');
 const { once } = require('events');
@@ -102,12 +105,14 @@ const kIsolatedProcessName = Symbol('kIsolatedProcessName');
 const kFilterArgs = [
   '--test',
   '--experimental-test-coverage',
+  '--test-randomize',
   '--watch',
   '--experimental-default-config-file',
 ];
 const kFilterArgValues = [
   '--test-reporter',
   '--test-reporter-destination',
+  '--test-random-seed',
   '--experimental-config-file',
 ];
 const kDiagnosticsFilterArgs = ['tests', 'suites', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms'];
@@ -619,6 +624,8 @@ function run(options = kEmptyObject) {
     lineCoverage = 0,
     branchCoverage = 0,
     functionCoverage = 0,
+    randomize: suppliedRandomize,
+    randomSeed: suppliedRandomSeed,
     execArgv = [],
     argv = [],
     cwd = process.cwd(),
@@ -647,6 +654,37 @@ function run(options = kEmptyObject) {
   if (globPatterns != null) {
     validateArray(globPatterns, 'options.globPatterns');
   }
+  if (suppliedRandomize != null) {
+    validateBoolean(suppliedRandomize, 'options.randomize');
+  }
+  if (suppliedRandomSeed != null) {
+    validateUint32(suppliedRandomSeed, 'options.randomSeed');
+  }
+  let randomize = suppliedRandomize;
+  let randomSeed = suppliedRandomSeed;
+
+  if (randomSeed != null) {
+    randomize = true;
+  }
+  if (watch) {
+    if (randomSeed != null) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.randomSeed',
+        randomSeed,
+        'is not supported with watch mode',
+      );
+    }
+    if (randomize) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.randomize',
+        randomize,
+        'is not supported with watch mode',
+      );
+    }
+  }
+  if (randomize) {
+    randomSeed ??= createRandomSeed();
+  }
 
   validateString(cwd, 'options.cwd');
 
@@ -757,10 +795,16 @@ function run(options = kEmptyObject) {
     cwd,
     globalSetupPath,
   };
+
   const root = createTestTree(rootTestOptions, globalOptions);
   let testFiles = files ?? createTestFileList(globPatterns, cwd);
   const { isTestRunner } = globalOptions;
 
+  if (randomize) {
+    testFiles = shuffleArrayWithSeed(testFiles, randomSeed);
+    root.diagnostic(`Randomized test order seed: ${randomSeed}`);
+  }
+
   if (shard) {
     testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1);
   }
@@ -786,6 +830,8 @@ function run(options = kEmptyObject) {
     execArgv,
     rerunFailuresFilePath,
     env,
+    randomize,
+    randomSeed,
   };
 
   if (isolation === 'process') {

lib/internal/test_runner/utils.js

@@ -7,11 +7,14 @@ const {
   ArrayPrototypePop,
   ArrayPrototypePush,
   ArrayPrototypeReduce,
+  ArrayPrototypeSlice,
   ArrayPrototypeSome,
   JSONParse,
   MathFloor,
+  MathImul,
   MathMax,
   MathMin,
+  MathRandom,
   NumberParseInt,
   NumberPrototypeToFixed,
   ObjectGetOwnPropertyDescriptor,
@@ -45,6 +48,7 @@ const { compose } = require('stream');
 const {
   validateInteger,
   validateFunction,
+  validateUint32,
 } = require('internal/validators');
 const { validatePath } = require('internal/fs/utils');
 const { kEmptyObject } = require('internal/util');
@@ -58,6 +62,7 @@ const coverageColors = {
 
 const kMultipleCallbackInvocations = 'multipleCallbackInvocations';
 const kRegExpPattern = /^\/(.*)\/([a-z]*)$/;
+const kMaxRandomSeed = 0xFFFF_FFFF;
 
 const kPatterns = ['test', 'test/**/*', 'test-*', '*[._-]test'];
 const kFileExtensions = ['js', 'mjs', 'cjs'];
@@ -133,6 +138,54 @@ const kBuiltinReporters = new SafeMap([
 const kDefaultReporter = 'spec';
 const kDefaultDestination = 'stdout';
 
+/**
+ * Create a random uint32 seed.
+ * @returns {number}
+ */
+function createRandomSeed() {
+  return MathFloor(MathRandom() * (kMaxRandomSeed + 1));
+}
+
+/**
+ * Create a Mulberry32 pseudo-random number generator from a uint32 seed.
+ * @param {number} seed
+ * @returns {() => number}
+ */
+function createSeededGenerator(seed) {
+  let state = seed >>> 0;
+  return () => {
+    state = (state + 0x6D2B79F5) | 0;
+    let value = MathImul(state ^ state >>> 15, 1 | state);
+    value ^= value + MathImul(value ^ value >>> 7, 61 | value);
+    return ((value ^ value >>> 14) >>> 0) / 4_294_967_296;
+  };
+}
+
+/**
+ * Return a deterministically shuffled copy of an array.
+ * @template T
+ * @param {T[]} values
+ * @param {number} seed
+ * @returns {T[]}
+ */
+function shuffleArrayWithSeed(values, seed) {
+  if (values.length < 2) {
+    return values;
+  }
+
+  const randomized = ArrayPrototypeSlice(values);
+  const random = createSeededGenerator(seed);
+
+  for (let i = randomized.length - 1; i > 0; i--) {
+    const j = MathFloor(random() * (i + 1));
+    const tmp = randomized[i];
+    randomized[i] = randomized[j];
+    randomized[j] = tmp;
+  }
+
+  return randomized;
+}
+
 function tryBuiltinReporter(name) {
   const builtinPath = kBuiltinReporters.get(name);
 
@@ -217,6 +270,10 @@ function parseCommandLine() {
   const updateSnapshots = getOptionValue('--test-update-snapshots');
   const watch = getOptionValue('--watch');
   const timeout = getOptionValue('--test-timeout') || Infinity;
+  let randomize = getOptionValue('--test-randomize');
+  const hasRandomSeedOption = getOptionValue('[has_test_random_seed]');
+  const randomSeedOption = getOptionValue('--test-random-seed');
+  let randomSeed;
   const rerunFailuresFilePath = getOptionValue('--test-rerun-failures');
   const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
   const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
@@ -328,6 +385,12 @@ function parseCommandLine() {
     validatePath(rerunFailuresFilePath, '--test-rerun-failures');
   }
 
+  if (hasRandomSeedOption) {
+    validateUint32(randomSeedOption, '--test-random-seed');
+    randomSeed = randomSeedOption;
+    randomize = true;
+  }
+
   const setup = reporterScope.bind(async (rootReporter) => {
     const reportersMap = await getReportersMap(reporters, destinations);
     for (let i = 0; i < reportersMap.length; i++) {
@@ -362,6 +425,8 @@ function parseCommandLine() {
     timeout,
     updateSnapshots,
     watch,
+    randomize,
+    randomSeed,
     rerunFailuresFilePath,
   };
 
@@ -649,10 +714,14 @@ async function setupGlobalSetupTeardownFunctions(globalSetupPath, cwd) {
 module.exports = {
   convertStringToRegExp,
   countCompletedTest,
+  createRandomSeed,
+  createSeededGenerator,
   createDeferredCallback,
   isTestFailureError,
   kDefaultPattern,
+  kMaxRandomSeed,
   parseCommandLine,
+  shuffleArrayWithSeed,
   reporterScope,
   shouldColorizeTestFiles,
   getCoverageReport,