From ea051effacbe725d42d430c30f24689e5118bc38 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 25 Mar 2025 10:22:00 +0100 Subject: [PATCH 01/18] Add s8s plugin --- .github/CODEOWNERS | 3 + README.md | 24 +++++++ global.d.ts | 14 ++++ packages/core/src/types.ts | 3 + packages/factory/package.json | 1 + packages/factory/src/index.ts | 3 + packages/plugins/synthetics/README.md | 21 ++++++ packages/plugins/synthetics/package.json | 25 +++++++ packages/plugins/synthetics/src/constants.ts | 8 +++ packages/plugins/synthetics/src/index.test.ts | 30 ++++++++ packages/plugins/synthetics/src/index.ts | 71 +++++++++++++++++++ packages/plugins/synthetics/src/types.ts | 9 +++ packages/plugins/synthetics/tsconfig.json | 10 +++ .../published/esbuild-plugin/src/index.ts | 2 + packages/published/rollup-plugin/src/index.ts | 2 + packages/published/rspack-plugin/src/index.ts | 2 + packages/published/vite-plugin/src/index.ts | 2 + .../published/webpack-plugin/src/index.ts | 2 + yarn.lock | 10 +++ 19 files changed, 242 insertions(+) create mode 100644 packages/plugins/synthetics/README.md create mode 100644 packages/plugins/synthetics/package.json create mode 100644 packages/plugins/synthetics/src/constants.ts create mode 100644 packages/plugins/synthetics/src/index.test.ts create mode 100644 packages/plugins/synthetics/src/index.ts create mode 100644 packages/plugins/synthetics/src/types.ts create mode 100644 packages/plugins/synthetics/tsconfig.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bdae1550f..147f03e89 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -29,3 +29,6 @@ packages/plugins/analytics @yoannmoin # Custom Hooks packages/plugins/custom-hooks @yoannmoinet + +# Synthetics +packages/plugins/synthetics @yoannmoinet @etnbrd diff --git a/README.md b/README.md index de30e57ab..725f6ae4b 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ To interact with Datadog directly from your builds. - [`customPlugins`](#customplugins) - [Features](#features) - [Error Tracking](#error-tracking-----) + - [Synthetics](#synthetics-----) - [Telemetry](#telemetry-----) - [Contributing](#contributing) - [License](#license) @@ -103,6 +104,9 @@ Follow the specific documentation for each bundler: service: string; }; }; + synthetics?: { + disabled?: boolean; + }; telemetry?: { disabled?: boolean; enableTracing?: boolean; @@ -246,6 +250,26 @@ datadogWebpackPlugin({ +### Synthetics ESBuild Rollup Rspack Vite Webpack + +> Interact with Synthetics at build time. + +#### [📝 Full documentation ➡️](/packages/plugins/synthetics#readme) + +
+ +Configuration + +```typescript +datadogWebpackPlugin({ + synthetics?: { + disabled?: boolean, + } +}); +``` + +
+ ### Telemetry ESBuild Rollup Rspack Vite Webpack > Display and send telemetry data as metrics to Datadog. diff --git a/global.d.ts b/global.d.ts index 1be630767..64b9284f1 100644 --- a/global.d.ts +++ b/global.d.ts @@ -19,6 +19,20 @@ declare global { * For instance, we only submit logs to Datadog when the environment is `production`. */ BUILD_PLUGINS_ENV?: Env; + /** + * Enable the dev server of our synthetics plugin. + * + * This is only used by datadog-ci, that will trigger a build, + * using the customer's build command, which, if it includes our plugin, + * will launch a dev-server over the outdir of the build so datadog-ci + * can trigger a tunnel and a test batch over the branch's code. + * + */ + BUILD_PLUGINS_S8S_LOCAL?: '1'; + /** + * The port of the dev server of our synthetics plugin. + */ + BUILD_PLUGINS_S8S_PORT?: string; /** * Defined in github actions when running in CI. */ diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 1cd1d9d80..b555e1186 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -13,6 +13,8 @@ import type { ErrorTrackingOptions } from '@dd/error-tracking-plugin/types'; import type * as errorTracking from '@dd/error-tracking-plugin'; import type { RumOptions } from '@dd/rum-plugin/types'; import type * as rum from '@dd/rum-plugin'; +import type { SyntheticsOptions } from '@dd/synthetics-plugin/types'; +import type * as synthetics from '@dd/synthetics-plugin'; import type { TelemetryOptions } from '@dd/telemetry-plugin/types'; import type * as telemetry from '@dd/telemetry-plugin'; // #imports-injection-marker @@ -197,6 +199,7 @@ export interface Options extends BaseOptions { // #types-injection-marker [errorTracking.CONFIG_KEY]?: ErrorTrackingOptions; [rum.CONFIG_KEY]?: RumOptions; + [synthetics.CONFIG_KEY]?: SyntheticsOptions; [telemetry.CONFIG_KEY]?: TelemetryOptions; // #types-injection-marker customPlugins?: GetCustomPlugins; diff --git a/packages/factory/package.json b/packages/factory/package.json index a6c97eea5..a3270ce94 100644 --- a/packages/factory/package.json +++ b/packages/factory/package.json @@ -25,6 +25,7 @@ "@dd/internal-git-plugin": "workspace:*", "@dd/internal-injection-plugin": "workspace:*", "@dd/rum-plugin": "workspace:*", + "@dd/synthetics-plugin": "workspace:*", "@dd/telemetry-plugin": "workspace:*", "chalk": "2.3.1", "unplugin": "1.16.0" diff --git a/packages/factory/src/index.ts b/packages/factory/src/index.ts index 9c203c823..2b3d348ec 100644 --- a/packages/factory/src/index.ts +++ b/packages/factory/src/index.ts @@ -28,6 +28,7 @@ import { getContext, validateOptions } from './helpers'; // #imports-injection-marker import * as errorTracking from '@dd/error-tracking-plugin'; import * as rum from '@dd/rum-plugin'; +import * as synthetics from '@dd/synthetics-plugin'; import * as telemetry from '@dd/telemetry-plugin'; import { getAnalyticsPlugins } from '@dd/internal-analytics-plugin'; import { getBuildReportPlugins } from '@dd/internal-build-report-plugin'; @@ -39,6 +40,7 @@ import { getInjectionPlugins } from '@dd/internal-injection-plugin'; // #types-export-injection-marker export type { types as ErrorTrackingTypes } from '@dd/error-tracking-plugin'; export type { types as RumTypes } from '@dd/rum-plugin'; +export type { types as SyntheticsTypes } from '@dd/synthetics-plugin'; export type { types as TelemetryTypes } from '@dd/telemetry-plugin'; // #types-export-injection-marker @@ -102,6 +104,7 @@ export const buildPluginFactory = ({ // #configs-injection-marker errorTracking, rum, + synthetics, telemetry, // #configs-injection-marker ]; diff --git a/packages/plugins/synthetics/README.md b/packages/plugins/synthetics/README.md new file mode 100644 index 000000000..3f0586505 --- /dev/null +++ b/packages/plugins/synthetics/README.md @@ -0,0 +1,21 @@ +# Synthetics Plugin + +Interact with Synthetics at build time. + + + +## Table of content + + + + +- [Configuration](#configuration) + + +## Configuration + +```ts +synthetics?: { + disabled?: boolean; +} +``` \ No newline at end of file diff --git a/packages/plugins/synthetics/package.json b/packages/plugins/synthetics/package.json new file mode 100644 index 000000000..ed02baa12 --- /dev/null +++ b/packages/plugins/synthetics/package.json @@ -0,0 +1,25 @@ +{ + "name": "@dd/synthetics-plugin", + "packageManager": "yarn@4.0.2", + "license": "MIT", + "private": true, + "author": "Datadog", + "description": "Interact with Synthetics at build time.", + "homepage": "https://github.com/DataDog/build-plugins/tree/main/packages/plugins/synthetics#readme", + "repository": { + "type": "git", + "url": "https://github.com/DataDog/build-plugins", + "directory": "packages/plugins/synthetics" + }, + "exports": { + ".": "./src/index.ts", + "./*": "./src/*.ts" + }, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@dd/core": "workspace:*", + "chalk": "2.3.1" + } +} diff --git a/packages/plugins/synthetics/src/constants.ts b/packages/plugins/synthetics/src/constants.ts new file mode 100644 index 000000000..019604b7d --- /dev/null +++ b/packages/plugins/synthetics/src/constants.ts @@ -0,0 +1,8 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { PluginName } from '@dd/core/types'; + +export const CONFIG_KEY = 'synthetics' as const; +export const PLUGIN_NAME: PluginName = 'datadog-synthetics-plugin' as const; diff --git a/packages/plugins/synthetics/src/index.test.ts b/packages/plugins/synthetics/src/index.test.ts new file mode 100644 index 000000000..2603c4c61 --- /dev/null +++ b/packages/plugins/synthetics/src/index.test.ts @@ -0,0 +1,30 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { getPlugins } from '@dd/synthetics-plugin'; +import { getContextMock } from '@dd/tests/_jest/helpers/mocks'; +import { runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; + +describe('Synthetics Plugin', () => { + describe('getPlugins', () => { + test('Should not initialize the plugin if disabled', async () => { + expect(getPlugins({ synthetics: { disabled: true } }, getContextMock())).toHaveLength( + 0, + ); + }); + + test('Should initialize the plugin if enabled and not configured', async () => { + expect( + getPlugins({ synthetics: { disabled: false } }, getContextMock()).length, + ).toBeGreaterThan(0); + expect(getPlugins({}, getContextMock()).length).toBeGreaterThan(0); + }); + }); + + test('Should run the server at the end of the build.', async () => { + await runBundlers({ + logLevel: 'debug', + }); + }); +}); diff --git a/packages/plugins/synthetics/src/index.ts b/packages/plugins/synthetics/src/index.ts new file mode 100644 index 000000000..4e9653354 --- /dev/null +++ b/packages/plugins/synthetics/src/index.ts @@ -0,0 +1,71 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { GlobalContext, GetPlugins, Options } from '@dd/core/types'; +import chalk from 'chalk'; + +import { CONFIG_KEY, PLUGIN_NAME } from './constants'; +import type { SyntheticsOptions, SyntheticsOptionsWithDefaults } from './types'; + +export { CONFIG_KEY, PLUGIN_NAME }; + +export const helpers = { + // Add the helpers you'd like to expose here. +}; + +export type types = { + // Add the types you'd like to expose here. + SyntheticsOptions: SyntheticsOptions; +}; + +// Deal with validation and defaults here. +export const validateOptions = (config: Options): SyntheticsOptionsWithDefaults => { + const validatedOptions: SyntheticsOptionsWithDefaults = { + // We don't want to disable it by default. + disabled: false, + ...config[CONFIG_KEY], + }; + return validatedOptions; +}; + +export const getPlugins: GetPlugins = (opts: Options, context: GlobalContext) => { + const log = context.getLogger(PLUGIN_NAME); + // Verify configuration. + const options = validateOptions(opts); + + if (options.disabled) { + return []; + } + + return [ + { + name: PLUGIN_NAME, + async writeBundle() { + // Execute code after the bundle is written. + // https://rollupjs.org/plugin-development/#writebundle + const { BUILD_PLUGINS_S8S_LOCAL, BUILD_PLUGINS_S8S_PORT } = process.env; + const runServer = + !options.disabled && BUILD_PLUGINS_S8S_LOCAL === '1' && BUILD_PLUGINS_S8S_PORT; + + if (BUILD_PLUGINS_S8S_LOCAL && !BUILD_PLUGINS_S8S_PORT) { + log.warn( + `Synthetics local server port is not set, please use ${chalk.bold.yellow('$BUILD_PLUGINS_S8S_PORT=1234')}.`, + ); + } + + if (!BUILD_PLUGINS_S8S_LOCAL && BUILD_PLUGINS_S8S_PORT) { + log.warn( + `Got server port but Synthetics local server is disabled, please use ${chalk.bold.yellow('$BUILD_PLUGINS_S8S_LOCAL=1')}.`, + ); + } + if (runServer) { + const port = +BUILD_PLUGINS_S8S_PORT; + log.info( + `Starting Synthetics local server on ${chalk.bold.cyan(`http://127.0.0.1:${port}`)}.`, + ); + } + }, + }, + ]; +}; diff --git a/packages/plugins/synthetics/src/types.ts b/packages/plugins/synthetics/src/types.ts new file mode 100644 index 000000000..a5f9b33c2 --- /dev/null +++ b/packages/plugins/synthetics/src/types.ts @@ -0,0 +1,9 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +export type SyntheticsOptions = { + disabled?: boolean; +}; + +export type SyntheticsOptionsWithDefaults = Required; diff --git a/packages/plugins/synthetics/tsconfig.json b/packages/plugins/synthetics/tsconfig.json new file mode 100644 index 000000000..6c1d3065e --- /dev/null +++ b/packages/plugins/synthetics/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "baseUrl": "./", + "rootDir": "./", + "outDir": "./dist" + }, + "include": ["**/*"], + "exclude": ["dist", "node_modules"] +} \ No newline at end of file diff --git a/packages/published/esbuild-plugin/src/index.ts b/packages/published/esbuild-plugin/src/index.ts index 6ff523630..0444ce184 100644 --- a/packages/published/esbuild-plugin/src/index.ts +++ b/packages/published/esbuild-plugin/src/index.ts @@ -11,6 +11,7 @@ import type { // #types-export-injection-marker ErrorTrackingTypes, RumTypes, + SyntheticsTypes, TelemetryTypes, // #types-export-injection-marker } from '@dd/factory'; @@ -24,6 +25,7 @@ export type { // #types-export-injection-marker ErrorTrackingTypes, RumTypes, + SyntheticsTypes, TelemetryTypes, // #types-export-injection-marker }; diff --git a/packages/published/rollup-plugin/src/index.ts b/packages/published/rollup-plugin/src/index.ts index 59c46cba7..55b7cb292 100644 --- a/packages/published/rollup-plugin/src/index.ts +++ b/packages/published/rollup-plugin/src/index.ts @@ -11,6 +11,7 @@ import type { // #types-export-injection-marker ErrorTrackingTypes, RumTypes, + SyntheticsTypes, TelemetryTypes, // #types-export-injection-marker } from '@dd/factory'; @@ -24,6 +25,7 @@ export type { // #types-export-injection-marker ErrorTrackingTypes, RumTypes, + SyntheticsTypes, TelemetryTypes, // #types-export-injection-marker }; diff --git a/packages/published/rspack-plugin/src/index.ts b/packages/published/rspack-plugin/src/index.ts index aec0805e7..c36b2f570 100644 --- a/packages/published/rspack-plugin/src/index.ts +++ b/packages/published/rspack-plugin/src/index.ts @@ -11,6 +11,7 @@ import type { // #types-export-injection-marker ErrorTrackingTypes, RumTypes, + SyntheticsTypes, TelemetryTypes, // #types-export-injection-marker } from '@dd/factory'; @@ -24,6 +25,7 @@ export type { // #types-export-injection-marker ErrorTrackingTypes, RumTypes, + SyntheticsTypes, TelemetryTypes, // #types-export-injection-marker }; diff --git a/packages/published/vite-plugin/src/index.ts b/packages/published/vite-plugin/src/index.ts index 7ea5cfbf6..492cbadd9 100644 --- a/packages/published/vite-plugin/src/index.ts +++ b/packages/published/vite-plugin/src/index.ts @@ -11,6 +11,7 @@ import type { // #types-export-injection-marker ErrorTrackingTypes, RumTypes, + SyntheticsTypes, TelemetryTypes, // #types-export-injection-marker } from '@dd/factory'; @@ -24,6 +25,7 @@ export type { // #types-export-injection-marker ErrorTrackingTypes, RumTypes, + SyntheticsTypes, TelemetryTypes, // #types-export-injection-marker }; diff --git a/packages/published/webpack-plugin/src/index.ts b/packages/published/webpack-plugin/src/index.ts index 24a29bbfd..e7e847877 100644 --- a/packages/published/webpack-plugin/src/index.ts +++ b/packages/published/webpack-plugin/src/index.ts @@ -11,6 +11,7 @@ import type { // #types-export-injection-marker ErrorTrackingTypes, RumTypes, + SyntheticsTypes, TelemetryTypes, // #types-export-injection-marker } from '@dd/factory'; @@ -24,6 +25,7 @@ export type { // #types-export-injection-marker ErrorTrackingTypes, RumTypes, + SyntheticsTypes, TelemetryTypes, // #types-export-injection-marker }; diff --git a/yarn.lock b/yarn.lock index 5466520b7..a0f9d9d04 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1690,6 +1690,7 @@ __metadata: "@dd/internal-git-plugin": "workspace:*" "@dd/internal-injection-plugin": "workspace:*" "@dd/rum-plugin": "workspace:*" + "@dd/synthetics-plugin": "workspace:*" "@dd/telemetry-plugin": "workspace:*" chalk: "npm:2.3.1" unplugin: "npm:1.16.0" @@ -1756,6 +1757,15 @@ __metadata: languageName: unknown linkType: soft +"@dd/synthetics-plugin@workspace:*, @dd/synthetics-plugin@workspace:packages/plugins/synthetics": + version: 0.0.0-use.local + resolution: "@dd/synthetics-plugin@workspace:packages/plugins/synthetics" + dependencies: + "@dd/core": "workspace:*" + chalk: "npm:2.3.1" + languageName: unknown + linkType: soft + "@dd/telemetry-plugin@workspace:*, @dd/telemetry-plugin@workspace:packages/plugins/telemetry": version: 0.0.0-use.local resolution: "@dd/telemetry-plugin@workspace:packages/plugins/telemetry" From 92843b897621140adfe83bacad9ad0f3ece80810 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 26 Mar 2025 10:54:07 +0100 Subject: [PATCH 02/18] Update/add utility types --- packages/core/src/types.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index b555e1186..4121672c1 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -24,8 +24,11 @@ import type { UnpluginOptions } from 'unplugin'; import type { ALL_ENVS, FULL_NAME_BUNDLERS, SUPPORTED_BUNDLERS } from './constants'; +// Re-assign B into A. export type Assign = Omit & B; -export type WithRequired = T & { [P in K]-?: T[P] }; +// Type object with specified keys required. +export type Ensure = Assign>>; +// Target one item from an iterable. export type IterableElement> = IterableType extends Iterable ? ElementType : never; From cad4d4a636b95f1c63d91a2cb47bbce465af57af Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 26 Mar 2025 10:55:32 +0100 Subject: [PATCH 03/18] Actually run the server and update configurations --- packages/plugins/synthetics/package.json | 1 + packages/plugins/synthetics/src/constants.ts | 3 + packages/plugins/synthetics/src/index.ts | 94 +++++++++++--------- packages/plugins/synthetics/src/types.ts | 18 +++- packages/plugins/synthetics/src/validate.ts | 45 ++++++++++ yarn.lock | 1 + 6 files changed, 120 insertions(+), 42 deletions(-) create mode 100644 packages/plugins/synthetics/src/validate.ts diff --git a/packages/plugins/synthetics/package.json b/packages/plugins/synthetics/package.json index ed02baa12..52997ed3b 100644 --- a/packages/plugins/synthetics/package.json +++ b/packages/plugins/synthetics/package.json @@ -20,6 +20,7 @@ }, "dependencies": { "@dd/core": "workspace:*", + "@dd/error-tracking-plugin": "workspace:*", "chalk": "2.3.1" } } diff --git a/packages/plugins/synthetics/src/constants.ts b/packages/plugins/synthetics/src/constants.ts index 019604b7d..c927c98d2 100644 --- a/packages/plugins/synthetics/src/constants.ts +++ b/packages/plugins/synthetics/src/constants.ts @@ -6,3 +6,6 @@ import type { PluginName } from '@dd/core/types'; export const CONFIG_KEY = 'synthetics' as const; export const PLUGIN_NAME: PluginName = 'datadog-synthetics-plugin' as const; + +export const API_PREFIX = '_datadog-ci_'; +export const DEFAULT_PORT = 1234; diff --git a/packages/plugins/synthetics/src/index.ts b/packages/plugins/synthetics/src/index.ts index 4e9653354..cb4493212 100644 --- a/packages/plugins/synthetics/src/index.ts +++ b/packages/plugins/synthetics/src/index.ts @@ -2,70 +2,82 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { runServer } from '@dd/core/helpers/server'; import type { GlobalContext, GetPlugins, Options } from '@dd/core/types'; +import { CONFIG_KEY as ERROR_TRACKING } from '@dd/error-tracking-plugin'; +import { API_PREFIX, CONFIG_KEY, PLUGIN_NAME } from '@dd/synthetics-plugin/constants'; +import type { BuildStatus, SyntheticsOptions } from '@dd/synthetics-plugin/types'; +import { validateOptions } from '@dd/synthetics-plugin/validate'; import chalk from 'chalk'; -import { CONFIG_KEY, PLUGIN_NAME } from './constants'; -import type { SyntheticsOptions, SyntheticsOptionsWithDefaults } from './types'; - export { CONFIG_KEY, PLUGIN_NAME }; -export const helpers = { - // Add the helpers you'd like to expose here. -}; - export type types = { // Add the types you'd like to expose here. SyntheticsOptions: SyntheticsOptions; }; -// Deal with validation and defaults here. -export const validateOptions = (config: Options): SyntheticsOptionsWithDefaults => { - const validatedOptions: SyntheticsOptionsWithDefaults = { - // We don't want to disable it by default. - disabled: false, - ...config[CONFIG_KEY], - }; - return validatedOptions; -}; - export const getPlugins: GetPlugins = (opts: Options, context: GlobalContext) => { const log = context.getLogger(PLUGIN_NAME); // Verify configuration. - const options = validateOptions(opts); + const options = validateOptions(opts, context, log); if (options.disabled) { return []; } + const response: { outDir?: string; publicPath?: string; status: BuildStatus } = { + publicPath: opts[ERROR_TRACKING]?.sourcemaps?.minifiedPathPrefix, + status: 'running', + }; + const getServerResponse = () => { + return response; + }; + + if (options.server.run) { + const port = options.server.port; + log.info( + `Starting Synthetics local server on ${chalk.bold.cyan(`http://127.0.0.1:${port}`)}.`, + ); + + const server = runServer({ + port, + root: context.bundler.outDir, + routes: { + [`/${API_PREFIX}/build-status`]: { + get: (req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(getServerResponse())); + }, + }, + [`/${API_PREFIX}/kill`]: { + get: (req, res) => { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('ok'); + // kill kill kill. + server.close(); + server.closeAllConnections(); + server.closeIdleConnections(); + }, + }, + }, + }); + } + return [ { name: PLUGIN_NAME, - async writeBundle() { - // Execute code after the bundle is written. - // https://rollupjs.org/plugin-development/#writebundle - const { BUILD_PLUGINS_S8S_LOCAL, BUILD_PLUGINS_S8S_PORT } = process.env; - const runServer = - !options.disabled && BUILD_PLUGINS_S8S_LOCAL === '1' && BUILD_PLUGINS_S8S_PORT; - - if (BUILD_PLUGINS_S8S_LOCAL && !BUILD_PLUGINS_S8S_PORT) { - log.warn( - `Synthetics local server port is not set, please use ${chalk.bold.yellow('$BUILD_PLUGINS_S8S_PORT=1234')}.`, - ); - } - - if (!BUILD_PLUGINS_S8S_LOCAL && BUILD_PLUGINS_S8S_PORT) { - log.warn( - `Got server port but Synthetics local server is disabled, please use ${chalk.bold.yellow('$BUILD_PLUGINS_S8S_LOCAL=1')}.`, - ); - } - if (runServer) { - const port = +BUILD_PLUGINS_S8S_PORT; - log.info( - `Starting Synthetics local server on ${chalk.bold.cyan(`http://127.0.0.1:${port}`)}.`, - ); + bundlerReport(bundlerReport) { + response.outDir = bundlerReport.outDir; + }, + buildReport(buildReport) { + if (buildReport.errors.length) { + response.status = 'fail'; } }, + writeBundle() { + response.status = 'success'; + }, }, ]; }; diff --git a/packages/plugins/synthetics/src/types.ts b/packages/plugins/synthetics/src/types.ts index a5f9b33c2..644fcbefd 100644 --- a/packages/plugins/synthetics/src/types.ts +++ b/packages/plugins/synthetics/src/types.ts @@ -2,8 +2,24 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import type { Assign, Ensure } from '@dd/core/types'; + +export type ServerOptions = { + port?: number; + root?: string; + run?: boolean; +}; + export type SyntheticsOptions = { disabled?: boolean; + server?: ServerOptions; }; -export type SyntheticsOptionsWithDefaults = Required; +export type SyntheticsOptionsWithDefaults = Assign< + Ensure, + { + server: Ensure; + } +>; + +export type BuildStatus = 'running' | 'success' | 'fail'; diff --git a/packages/plugins/synthetics/src/validate.ts b/packages/plugins/synthetics/src/validate.ts new file mode 100644 index 000000000..e991a4ed5 --- /dev/null +++ b/packages/plugins/synthetics/src/validate.ts @@ -0,0 +1,45 @@ +import type { Options, Logger, GlobalContext } from '@dd/core/types'; +import { CONFIG_KEY, DEFAULT_PORT } from '@dd/synthetics-plugin/constants'; +import type { SyntheticsOptionsWithDefaults } from '@dd/synthetics-plugin/types'; +import chalk from 'chalk'; + +export const validateOptions = ( + config: Options, + context: GlobalContext, + log: Logger, +): SyntheticsOptionsWithDefaults => { + const validatedOptions: SyntheticsOptionsWithDefaults = { + // We don't want to disable it by default. + disabled: false, + ...config[CONFIG_KEY], + server: { + run: false, + port: DEFAULT_PORT, + root: context.bundler.outDir, + ...config[CONFIG_KEY]?.server, + }, + }; + + const { BUILD_PLUGINS_S8S_LOCAL, BUILD_PLUGINS_S8S_PORT } = process.env; + + if (BUILD_PLUGINS_S8S_LOCAL && !BUILD_PLUGINS_S8S_PORT) { + log.warn( + `Synthetics local server port is not set, please use ${chalk.bold.yellow('$BUILD_PLUGINS_S8S_PORT=1234')}.`, + ); + } + + if (!BUILD_PLUGINS_S8S_LOCAL && BUILD_PLUGINS_S8S_PORT) { + log.warn( + `Got server port but Synthetics local server is disabled, please use ${chalk.bold.yellow('$BUILD_PLUGINS_S8S_LOCAL=1')}.`, + ); + } + + validatedOptions.server.run = + !validatedOptions.disabled && BUILD_PLUGINS_S8S_LOCAL === '1' && !!BUILD_PLUGINS_S8S_PORT; + + if (BUILD_PLUGINS_S8S_PORT) { + validatedOptions.server.port = +BUILD_PLUGINS_S8S_PORT; + } + + return validatedOptions; +}; diff --git a/yarn.lock b/yarn.lock index a0f9d9d04..700d17de1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1762,6 +1762,7 @@ __metadata: resolution: "@dd/synthetics-plugin@workspace:packages/plugins/synthetics" dependencies: "@dd/core": "workspace:*" + "@dd/error-tracking-plugin": "workspace:*" chalk: "npm:2.3.1" languageName: unknown linkType: soft From 0ab5804f56c743e37e5abea58d87b45af4c3d13a Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 26 Mar 2025 10:56:00 +0100 Subject: [PATCH 04/18] Add tests --- packages/plugins/synthetics/src/index.test.ts | 88 ++++++++++++++++++- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/packages/plugins/synthetics/src/index.test.ts b/packages/plugins/synthetics/src/index.test.ts index 2603c4c61..5363286eb 100644 --- a/packages/plugins/synthetics/src/index.test.ts +++ b/packages/plugins/synthetics/src/index.test.ts @@ -2,9 +2,21 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { runServer } from '@dd/core/helpers/server'; +import { API_PREFIX, DEFAULT_PORT } from '@dd/synthetics-plugin/constants'; import { getPlugins } from '@dd/synthetics-plugin'; import { getContextMock } from '@dd/tests/_jest/helpers/mocks'; -import { runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; +import nock from 'nock'; + +jest.mock('@dd/core/helpers/server', () => { + const original = jest.requireActual('@dd/core/helpers/server'); + return { + ...original, + runServer: jest.fn(original.runServer), + }; +}); + +const runServerMocked = jest.mocked(runServer); describe('Synthetics Plugin', () => { describe('getPlugins', () => { @@ -22,9 +34,77 @@ describe('Synthetics Plugin', () => { }); }); - test('Should run the server at the end of the build.', async () => { - await runBundlers({ - logLevel: 'debug', + describe('Server', () => { + beforeAll(() => { + // Allow local server. + nock.enableNetConnect('127.0.0.1'); + }); + + afterEach(async () => { + // Kill the server. + try { + await fetch(`http://127.0.0.1:${DEFAULT_PORT}/${API_PREFIX}/kill`); + } catch (e) { + // Do nothing. + } + }); + + afterAll(() => { + nock.cleanAll(); + nock.disableNetConnect(); + }); + + describe('to run or not to run', () => { + afterEach(() => { + // Remove the variables we've set. + delete process.env.BUILD_PLUGINS_S8S_LOCAL; + delete process.env.BUILD_PLUGINS_S8S_PORT; + }); + const expectations = [ + { + description: 'not run with no variables', + env: {}, + shouldRun: false, + }, + { + description: 'not run with missing port', + env: { + BUILD_PLUGINS_S8S_LOCAL: '1', + }, + shouldRun: false, + }, + { + description: 'not run with missing local', + env: { + BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), + }, + shouldRun: false, + }, + { + description: 'run with both variables', + env: { + BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), + BUILD_PLUGINS_S8S_LOCAL: '1', + }, + shouldRun: true, + }, + ]; + + test.each(expectations)( + 'Should $description.', + async ({ description, env, shouldRun }) => { + // Set the variables. + Object.assign(process.env, env); + // Run the plugin. + getPlugins({}, getContextMock()); + // Check the server. + if (shouldRun) { + expect(runServerMocked).toHaveBeenCalled(); + } else { + expect(runServerMocked).not.toHaveBeenCalled(); + } + }, + ); }); }); }); From 9986a72a883709f197283b4cff0c885200c6d411 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 26 Mar 2025 16:29:05 +0100 Subject: [PATCH 05/18] Finalize feature and testing --- global.d.ts | 10 +- packages/plugins/synthetics/src/index.test.ts | 229 +++++++++++++++--- packages/plugins/synthetics/src/index.ts | 65 ++--- packages/plugins/synthetics/src/types.ts | 3 +- packages/plugins/synthetics/src/validate.ts | 39 ++- 5 files changed, 251 insertions(+), 95 deletions(-) diff --git a/global.d.ts b/global.d.ts index 64b9284f1..8d504755d 100644 --- a/global.d.ts +++ b/global.d.ts @@ -20,18 +20,14 @@ declare global { */ BUILD_PLUGINS_ENV?: Env; /** - * Enable the dev server of our synthetics plugin. + * The port of the dev server of our synthetics plugin. * - * This is only used by datadog-ci, that will trigger a build, - * using the customer's build command, which, if it includes our plugin, + * This is only used by datadog-ci, in its build'n test workflow, + * using the customer's build command, if it includes our plugin, * will launch a dev-server over the outdir of the build so datadog-ci * can trigger a tunnel and a test batch over the branch's code. * */ - BUILD_PLUGINS_S8S_LOCAL?: '1'; - /** - * The port of the dev server of our synthetics plugin. - */ BUILD_PLUGINS_S8S_PORT?: string; /** * Defined in github actions when running in CI. diff --git a/packages/plugins/synthetics/src/index.test.ts b/packages/plugins/synthetics/src/index.test.ts index 5363286eb..c3c364db6 100644 --- a/packages/plugins/synthetics/src/index.test.ts +++ b/packages/plugins/synthetics/src/index.test.ts @@ -4,9 +4,13 @@ import { runServer } from '@dd/core/helpers/server'; import { API_PREFIX, DEFAULT_PORT } from '@dd/synthetics-plugin/constants'; +import type { ServerResponse } from '@dd/synthetics-plugin/types'; import { getPlugins } from '@dd/synthetics-plugin'; import { getContextMock } from '@dd/tests/_jest/helpers/mocks'; +import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; +import fs from 'fs'; import nock from 'nock'; +import path from 'path'; jest.mock('@dd/core/helpers/server', () => { const original = jest.requireActual('@dd/core/helpers/server'); @@ -18,6 +22,48 @@ jest.mock('@dd/core/helpers/server', () => { const runServerMocked = jest.mocked(runServer); +const getApiUrl = (port: number = DEFAULT_PORT) => `http://127.0.0.1:${port}`; +const getInternalApiUrl = (port: number = DEFAULT_PORT) => `${getApiUrl(port)}/${API_PREFIX}`; +const safeFetch = async (route: string, port: number) => { + try { + return await fetch(`${getInternalApiUrl(port)}${route}`); + } catch (e) { + // Do nothing. + } +}; + +// Wait for the local server to tell us that the build is complete. +const waitingForBuild = (port: number, cb: (resp: ServerResponse) => void) => { + return new Promise((resolve, reject) => { + // Stop the polling after 10 seconds. + const timeout = setTimeout(() => { + clearInterval(interval); + reject(new Error('Timeout.')); + }, 10000); + + // Poll all the local servers until we get the build status. + const interval = setInterval(async () => { + const res = await safeFetch('/build-status', port); + if (res?.ok) { + const data = (await res.json()) as ServerResponse; + cb(data); + + if (['success', 'fail'].includes(data.status)) { + clearInterval(interval); + clearTimeout(timeout); + + if (data.status === 'success') { + resolve(); + } + if (data.status === 'fail') { + reject(new Error('Build failed.')); + } + } + } + }, 100); + }); +}; + describe('Synthetics Plugin', () => { describe('getPlugins', () => { test('Should not initialize the plugin if disabled', async () => { @@ -40,71 +86,184 @@ describe('Synthetics Plugin', () => { nock.enableNetConnect('127.0.0.1'); }); - afterEach(async () => { - // Kill the server. - try { - await fetch(`http://127.0.0.1:${DEFAULT_PORT}/${API_PREFIX}/kill`); - } catch (e) { - // Do nothing. - } - }); - afterAll(() => { nock.cleanAll(); nock.disableNetConnect(); }); describe('to run or not to run', () => { - afterEach(() => { + afterEach(async () => { // Remove the variables we've set. delete process.env.BUILD_PLUGINS_S8S_LOCAL; delete process.env.BUILD_PLUGINS_S8S_PORT; + + // Kill the server. + await safeFetch('/kill', DEFAULT_PORT); }); + const expectations = [ { - description: 'not run with no variables', + description: 'not run with no env and no config', env: {}, + config: {}, shouldRun: false, }, { - description: 'not run with missing port', + description: 'run with port in env and no config', env: { - BUILD_PLUGINS_S8S_LOCAL: '1', + BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), }, - shouldRun: false, + config: {}, + shouldRun: true, + }, + { + description: 'run with no variables and full config', + env: {}, + config: { + synthetics: { + server: { + run: true, + port: DEFAULT_PORT, + }, + }, + }, + shouldRun: true, + }, + { + description: 'run with no variables and just config.run', + env: {}, + config: { + synthetics: { + server: { + run: true, + }, + }, + }, + shouldRun: true, }, { - description: 'not run with missing local', + description: 'not run with config.run false', env: { BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), }, + config: { + synthetics: { + server: { + run: false, + }, + }, + }, shouldRun: false, }, { - description: 'run with both variables', - env: { - BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), - BUILD_PLUGINS_S8S_LOCAL: '1', + description: 'not run with disabled and config.run', + env: {}, + config: { + synthetics: { + disabled: true, + server: { + run: true, + }, + }, }, - shouldRun: true, + shouldRun: false, }, ]; - test.each(expectations)( - 'Should $description.', - async ({ description, env, shouldRun }) => { - // Set the variables. - Object.assign(process.env, env); - // Run the plugin. - getPlugins({}, getContextMock()); - // Check the server. - if (shouldRun) { - expect(runServerMocked).toHaveBeenCalled(); - } else { - expect(runServerMocked).not.toHaveBeenCalled(); - } - }, - ); + test.each(expectations)('Should $description.', async ({ config, env, shouldRun }) => { + // Set the variables. + Object.assign(process.env, env); + // Run the plugin. + const [plugin] = getPlugins(config, getContextMock()); + if (plugin?.bundlerReport) { + // Trigger the bundlerReport hook where the server starts. + plugin.bundlerReport(getContextMock().bundler); + } + // Check the server. + if (shouldRun) { + expect(runServerMocked).toHaveBeenCalled(); + } else { + expect(runServerMocked).not.toHaveBeenCalled(); + } + }); + }); + + // We need to loop over bundlers because we'll use a different port for each one of them + // to avoid port conflicts. + describe.each(BUNDLERS)('$name', (bundler) => { + // Get an incremental port to prevent conflicts. + const port = DEFAULT_PORT + BUNDLERS.indexOf(bundler); + + let buildProm: Promise; + let outDir: string; + const serverResponses: Set = new Set(); + + beforeAll(async () => { + // Run the builds. + // Do not await the promise as the server will be running. + buildProm = runBundlers( + { + synthetics: { + server: { + run: true, + port, + }, + }, + // Use a custom plugin to get the cwd and outdir of the build. + customPlugins: () => [ + { + name: 'get-outdirs', + bundlerReport: (report) => { + outDir = report.outDir; + }, + }, + ], + }, + undefined, + [bundler.name], + ); + + // Instead, wait for the server to tell us that the build is complete. + await waitingForBuild(port, (resp) => { + serverResponses.add(resp); + }); + }); + + afterAll(async () => { + await safeFetch('/kill', port); + // Wait for the build to finish now that the server is killed. + if (buildProm) { + await buildProm; + } + }); + + test('Should report the build status.', async () => { + // Verify that we have the running and success statuses. + const reportedStatus = Array.from(serverResponses).filter((resp) => + ['fail', 'success', 'running'].includes(resp.status), + ); + expect(reportedStatus.length).toBeGreaterThan(0); + }); + + test('Should report the outDir.', async () => { + // Verify that we have the running and success statuses. + const reportedOutDirs = new Set( + Array.from(serverResponses).map((resp) => resp.outDir), + ); + // We should have only one outDir. + expect(reportedOutDirs.size).toBe(1); + // It should be the same as the one we reported from the build. + expect(reportedOutDirs.values().next().value).toEqual(outDir); + }); + + test('Should actually serve the built files.', async () => { + // Query a file from the server. + const res = await fetch(`${getApiUrl(port)}/main.js`); + expect(res.ok).toBe(true); + const text = await res.text(); + // Confirm that the file served by the server is the same as the one on disk. + expect(text).toEqual(fs.readFileSync(path.join(outDir, 'main.js'), 'utf-8')); + }); }); }); }); diff --git a/packages/plugins/synthetics/src/index.ts b/packages/plugins/synthetics/src/index.ts index cb4493212..fb258d1aa 100644 --- a/packages/plugins/synthetics/src/index.ts +++ b/packages/plugins/synthetics/src/index.ts @@ -6,7 +6,7 @@ import { runServer } from '@dd/core/helpers/server'; import type { GlobalContext, GetPlugins, Options } from '@dd/core/types'; import { CONFIG_KEY as ERROR_TRACKING } from '@dd/error-tracking-plugin'; import { API_PREFIX, CONFIG_KEY, PLUGIN_NAME } from '@dd/synthetics-plugin/constants'; -import type { BuildStatus, SyntheticsOptions } from '@dd/synthetics-plugin/types'; +import type { ServerResponse, SyntheticsOptions } from '@dd/synthetics-plugin/types'; import { validateOptions } from '@dd/synthetics-plugin/validate'; import chalk from 'chalk'; @@ -26,49 +26,50 @@ export const getPlugins: GetPlugins = (opts: Options, context: GlobalContext) => return []; } - const response: { outDir?: string; publicPath?: string; status: BuildStatus } = { + const response: ServerResponse = { publicPath: opts[ERROR_TRACKING]?.sourcemaps?.minifiedPathPrefix, status: 'running', }; + const getServerResponse = () => { return response; }; - if (options.server.run) { - const port = options.server.port; - log.info( - `Starting Synthetics local server on ${chalk.bold.cyan(`http://127.0.0.1:${port}`)}.`, - ); - - const server = runServer({ - port, - root: context.bundler.outDir, - routes: { - [`/${API_PREFIX}/build-status`]: { - get: (req, res) => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(getServerResponse())); - }, - }, - [`/${API_PREFIX}/kill`]: { - get: (req, res) => { - res.writeHead(200, { 'Content-Type': 'text/html' }); - res.end('ok'); - // kill kill kill. - server.close(); - server.closeAllConnections(); - server.closeIdleConnections(); - }, - }, - }, - }); - } - return [ { name: PLUGIN_NAME, + // Wait for us to have the bundler report to start the server over the outDir. bundlerReport(bundlerReport) { response.outDir = bundlerReport.outDir; + if (options.server.run) { + const port = options.server.port; + log.debug( + `Starting Synthetics local server on ${chalk.bold.cyan(`http://127.0.0.1:${port}`)}.`, + ); + + const server = runServer({ + port, + root: options.server.root || response.outDir, + routes: { + [`/${API_PREFIX}/build-status`]: { + get: (req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(getServerResponse())); + }, + }, + [`/${API_PREFIX}/kill`]: { + get: (req, res) => { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('ok'); + // kill kill kill. + server.close(); + server.closeAllConnections(); + server.closeIdleConnections(); + }, + }, + }, + }); + } }, buildReport(buildReport) { if (buildReport.errors.length) { diff --git a/packages/plugins/synthetics/src/types.ts b/packages/plugins/synthetics/src/types.ts index 644fcbefd..7dc740cf2 100644 --- a/packages/plugins/synthetics/src/types.ts +++ b/packages/plugins/synthetics/src/types.ts @@ -18,8 +18,9 @@ export type SyntheticsOptions = { export type SyntheticsOptionsWithDefaults = Assign< Ensure, { - server: Ensure; + server: Ensure; } >; export type BuildStatus = 'running' | 'success' | 'fail'; +export type ServerResponse = { outDir?: string; publicPath?: string; status: BuildStatus }; diff --git a/packages/plugins/synthetics/src/validate.ts b/packages/plugins/synthetics/src/validate.ts index e991a4ed5..f8ea2a425 100644 --- a/packages/plugins/synthetics/src/validate.ts +++ b/packages/plugins/synthetics/src/validate.ts @@ -8,38 +8,37 @@ export const validateOptions = ( context: GlobalContext, log: Logger, ): SyntheticsOptionsWithDefaults => { + // Get values from environment. + const { BUILD_PLUGINS_S8S_PORT } = process.env; + + // Which port has been requested? + // This can either be enabled via env var or configuration. + const askedPort = BUILD_PLUGINS_S8S_PORT || config[CONFIG_KEY]?.server?.port; + + // Define defaults. const validatedOptions: SyntheticsOptionsWithDefaults = { // We don't want to disable it by default. disabled: false, ...config[CONFIG_KEY], server: { - run: false, - port: DEFAULT_PORT, - root: context.bundler.outDir, + run: !!BUILD_PLUGINS_S8S_PORT, + port: BUILD_PLUGINS_S8S_PORT ? +BUILD_PLUGINS_S8S_PORT : DEFAULT_PORT, ...config[CONFIG_KEY]?.server, }, }; - const { BUILD_PLUGINS_S8S_LOCAL, BUILD_PLUGINS_S8S_PORT } = process.env; - - if (BUILD_PLUGINS_S8S_LOCAL && !BUILD_PLUGINS_S8S_PORT) { - log.warn( - `Synthetics local server port is not set, please use ${chalk.bold.yellow('$BUILD_PLUGINS_S8S_PORT=1234')}.`, - ); - } + // If we've been asked to run, but no port was given, + // we'll use the default port, so warn the user. + if (validatedOptions.server.run && !askedPort) { + log.info( + `Synthetics local server port is not set, you can use either : + - ${chalk.bold.yellow('export BUILD_PLUGINS_S8S_PORT=1234')} before running your build. + - ${chalk.bold.yellow('config.synthetics.server.port: 1234')} in your configuration. - if (!BUILD_PLUGINS_S8S_LOCAL && BUILD_PLUGINS_S8S_PORT) { - log.warn( - `Got server port but Synthetics local server is disabled, please use ${chalk.bold.yellow('$BUILD_PLUGINS_S8S_LOCAL=1')}.`, +Server will still run with the default port ${chalk.bold.cyan(DEFAULT_PORT.toString())}. +`, ); } - validatedOptions.server.run = - !validatedOptions.disabled && BUILD_PLUGINS_S8S_LOCAL === '1' && !!BUILD_PLUGINS_S8S_PORT; - - if (BUILD_PLUGINS_S8S_PORT) { - validatedOptions.server.port = +BUILD_PLUGINS_S8S_PORT; - } - return validatedOptions; }; From d0b449089e0d8b5c88cc7b1582443fc70e8f2284 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 26 Mar 2025 17:59:28 +0100 Subject: [PATCH 06/18] Add missing OSS header --- packages/plugins/synthetics/src/validate.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/plugins/synthetics/src/validate.ts b/packages/plugins/synthetics/src/validate.ts index f8ea2a425..7a1dda1ae 100644 --- a/packages/plugins/synthetics/src/validate.ts +++ b/packages/plugins/synthetics/src/validate.ts @@ -1,3 +1,7 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + import type { Options, Logger, GlobalContext } from '@dd/core/types'; import { CONFIG_KEY, DEFAULT_PORT } from '@dd/synthetics-plugin/constants'; import type { SyntheticsOptionsWithDefaults } from '@dd/synthetics-plugin/types'; From 79144189ec1c12fadce887aa4a9302123e2ab044 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Thu, 27 Mar 2025 10:34:24 +0100 Subject: [PATCH 07/18] Use local relative imports --- packages/plugins/synthetics/src/constants.ts | 4 ++-- packages/plugins/synthetics/src/index.ts | 7 ++++--- packages/plugins/synthetics/src/validate.ts | 5 +++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/plugins/synthetics/src/constants.ts b/packages/plugins/synthetics/src/constants.ts index c927c98d2..5ccdaa5e1 100644 --- a/packages/plugins/synthetics/src/constants.ts +++ b/packages/plugins/synthetics/src/constants.ts @@ -7,5 +7,5 @@ import type { PluginName } from '@dd/core/types'; export const CONFIG_KEY = 'synthetics' as const; export const PLUGIN_NAME: PluginName = 'datadog-synthetics-plugin' as const; -export const API_PREFIX = '_datadog-ci_'; -export const DEFAULT_PORT = 1234; +export const API_PREFIX = '_datadog-ci_' as const; +export const DEFAULT_PORT = 1234 as const; diff --git a/packages/plugins/synthetics/src/index.ts b/packages/plugins/synthetics/src/index.ts index fb258d1aa..59640db24 100644 --- a/packages/plugins/synthetics/src/index.ts +++ b/packages/plugins/synthetics/src/index.ts @@ -5,11 +5,12 @@ import { runServer } from '@dd/core/helpers/server'; import type { GlobalContext, GetPlugins, Options } from '@dd/core/types'; import { CONFIG_KEY as ERROR_TRACKING } from '@dd/error-tracking-plugin'; -import { API_PREFIX, CONFIG_KEY, PLUGIN_NAME } from '@dd/synthetics-plugin/constants'; -import type { ServerResponse, SyntheticsOptions } from '@dd/synthetics-plugin/types'; -import { validateOptions } from '@dd/synthetics-plugin/validate'; import chalk from 'chalk'; +import { API_PREFIX, CONFIG_KEY, PLUGIN_NAME } from './constants'; +import type { ServerResponse, SyntheticsOptions } from './types'; +import { validateOptions } from './validate'; + export { CONFIG_KEY, PLUGIN_NAME }; export type types = { diff --git a/packages/plugins/synthetics/src/validate.ts b/packages/plugins/synthetics/src/validate.ts index 7a1dda1ae..3a15f4cf0 100644 --- a/packages/plugins/synthetics/src/validate.ts +++ b/packages/plugins/synthetics/src/validate.ts @@ -3,10 +3,11 @@ // Copyright 2019-Present Datadog, Inc. import type { Options, Logger, GlobalContext } from '@dd/core/types'; -import { CONFIG_KEY, DEFAULT_PORT } from '@dd/synthetics-plugin/constants'; -import type { SyntheticsOptionsWithDefaults } from '@dd/synthetics-plugin/types'; import chalk from 'chalk'; +import { CONFIG_KEY, DEFAULT_PORT } from './constants'; +import type { SyntheticsOptionsWithDefaults } from './types'; + export const validateOptions = ( config: Options, context: GlobalContext, From 9267855dc57476ad83bfe431879cf83de5d2b2c6 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Thu, 27 Mar 2025 11:44:49 +0100 Subject: [PATCH 08/18] Remove unused env var --- packages/plugins/synthetics/src/index.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/plugins/synthetics/src/index.test.ts b/packages/plugins/synthetics/src/index.test.ts index c3c364db6..634a1479a 100644 --- a/packages/plugins/synthetics/src/index.test.ts +++ b/packages/plugins/synthetics/src/index.test.ts @@ -93,8 +93,7 @@ describe('Synthetics Plugin', () => { describe('to run or not to run', () => { afterEach(async () => { - // Remove the variables we've set. - delete process.env.BUILD_PLUGINS_S8S_LOCAL; + // Remove the variable we've set. delete process.env.BUILD_PLUGINS_S8S_PORT; // Kill the server. From 627e847ff64bd5d10e0c29a60dc032335798b1e9 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 31 Mar 2025 10:08:55 +0200 Subject: [PATCH 09/18] Update doc --- global.d.ts | 6 +++--- packages/plugins/synthetics/README.md | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/global.d.ts b/global.d.ts index 8d504755d..4a289433c 100644 --- a/global.d.ts +++ b/global.d.ts @@ -22,10 +22,10 @@ declare global { /** * The port of the dev server of our synthetics plugin. * - * This is only used by datadog-ci, in its build'n test workflow, + * This is only used by datadog-ci, in its build-and-test command, * using the customer's build command, if it includes our plugin, - * will launch a dev-server over the outdir of the build so datadog-ci - * can trigger a tunnel and a test batch over the branch's code. + * will launch a dev server to serve the outdir of the build so datadog-ci + * can trigger a CI batch over the branch's code using a tunnel. * */ BUILD_PLUGINS_S8S_PORT?: string; diff --git a/packages/plugins/synthetics/README.md b/packages/plugins/synthetics/README.md index 3f0586505..1655a40af 100644 --- a/packages/plugins/synthetics/README.md +++ b/packages/plugins/synthetics/README.md @@ -18,4 +18,9 @@ Interact with Synthetics at build time. synthetics?: { disabled?: boolean; } -``` \ No newline at end of file +``` + +## Build and test + +Using [`datadog-ci`'s `synthetics build-and-test` command](https://github.com/DataDog/datadog-ci/tree/master/src/commands/synthetics#run-tests-command), +you can have the build spin a dev server to serve the outdir of the build in order [to trigger a CI batch](https://docs.datadoghq.com/continuous_testing/) over the branch's code. From 6ff010cf0664401b5c9a8347a10e11a433f537b7 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 31 Mar 2025 10:14:04 +0200 Subject: [PATCH 10/18] Simplify setup --- packages/plugins/synthetics/src/constants.ts | 1 - packages/plugins/synthetics/src/index.test.ts | 64 ++++--------------- packages/plugins/synthetics/src/index.ts | 12 ++-- packages/plugins/synthetics/src/types.ts | 8 +-- packages/plugins/synthetics/src/validate.ts | 36 +++-------- 5 files changed, 29 insertions(+), 92 deletions(-) diff --git a/packages/plugins/synthetics/src/constants.ts b/packages/plugins/synthetics/src/constants.ts index 5ccdaa5e1..541a2d603 100644 --- a/packages/plugins/synthetics/src/constants.ts +++ b/packages/plugins/synthetics/src/constants.ts @@ -8,4 +8,3 @@ export const CONFIG_KEY = 'synthetics' as const; export const PLUGIN_NAME: PluginName = 'datadog-synthetics-plugin' as const; export const API_PREFIX = '_datadog-ci_' as const; -export const DEFAULT_PORT = 1234 as const; diff --git a/packages/plugins/synthetics/src/index.test.ts b/packages/plugins/synthetics/src/index.test.ts index 634a1479a..4ab2a4cee 100644 --- a/packages/plugins/synthetics/src/index.test.ts +++ b/packages/plugins/synthetics/src/index.test.ts @@ -3,7 +3,7 @@ // Copyright 2019-Present Datadog, Inc. import { runServer } from '@dd/core/helpers/server'; -import { API_PREFIX, DEFAULT_PORT } from '@dd/synthetics-plugin/constants'; +import { API_PREFIX } from '@dd/synthetics-plugin/constants'; import type { ServerResponse } from '@dd/synthetics-plugin/types'; import { getPlugins } from '@dd/synthetics-plugin'; import { getContextMock } from '@dd/tests/_jest/helpers/mocks'; @@ -22,6 +22,8 @@ jest.mock('@dd/core/helpers/server', () => { const runServerMocked = jest.mocked(runServer); +const DEFAULT_PORT = 1234; + const getApiUrl = (port: number = DEFAULT_PORT) => `http://127.0.0.1:${port}`; const getInternalApiUrl = (port: number = DEFAULT_PORT) => `${getApiUrl(port)}/${API_PREFIX}`; const safeFetch = async (route: string, port: number) => { @@ -93,7 +95,7 @@ describe('Synthetics Plugin', () => { describe('to run or not to run', () => { afterEach(async () => { - // Remove the variable we've set. + // Remove the variable we may have set. delete process.env.BUILD_PLUGINS_S8S_PORT; // Kill the server. @@ -108,7 +110,7 @@ describe('Synthetics Plugin', () => { shouldRun: false, }, { - description: 'run with port in env and no config', + description: 'run with port in env', env: { BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), }, @@ -116,53 +118,13 @@ describe('Synthetics Plugin', () => { shouldRun: true, }, { - description: 'run with no variables and full config', - env: {}, - config: { - synthetics: { - server: { - run: true, - port: DEFAULT_PORT, - }, - }, - }, - shouldRun: true, - }, - { - description: 'run with no variables and just config.run', - env: {}, - config: { - synthetics: { - server: { - run: true, - }, - }, - }, - shouldRun: true, - }, - { - description: 'not run with config.run false', + description: 'not run with disabled and config.run', env: { BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), }, - config: { - synthetics: { - server: { - run: false, - }, - }, - }, - shouldRun: false, - }, - { - description: 'not run with disabled and config.run', - env: {}, config: { synthetics: { disabled: true, - server: { - run: true, - }, }, }, shouldRun: false, @@ -198,16 +160,15 @@ describe('Synthetics Plugin', () => { const serverResponses: Set = new Set(); beforeAll(async () => { + // Set the variables. + Object.assign(process.env, { + BUILD_PLUGINS_S8S_PORT: JSON.stringify(port), + }); + // Run the builds. // Do not await the promise as the server will be running. buildProm = runBundlers( { - synthetics: { - server: { - run: true, - port, - }, - }, // Use a custom plugin to get the cwd and outdir of the build. customPlugins: () => [ { @@ -229,6 +190,9 @@ describe('Synthetics Plugin', () => { }); afterAll(async () => { + // Remove the variable we may have set. + delete process.env.BUILD_PLUGINS_S8S_PORT; + // Kill the server. await safeFetch('/kill', port); // Wait for the build to finish now that the server is killed. if (buildProm) { diff --git a/packages/plugins/synthetics/src/index.ts b/packages/plugins/synthetics/src/index.ts index 59640db24..032f6d126 100644 --- a/packages/plugins/synthetics/src/index.ts +++ b/packages/plugins/synthetics/src/index.ts @@ -21,7 +21,7 @@ export type types = { export const getPlugins: GetPlugins = (opts: Options, context: GlobalContext) => { const log = context.getLogger(PLUGIN_NAME); // Verify configuration. - const options = validateOptions(opts, context, log); + const options = validateOptions(opts, log); if (options.disabled) { return []; @@ -32,17 +32,13 @@ export const getPlugins: GetPlugins = (opts: Options, context: GlobalContext) => status: 'running', }; - const getServerResponse = () => { - return response; - }; - return [ { name: PLUGIN_NAME, // Wait for us to have the bundler report to start the server over the outDir. bundlerReport(bundlerReport) { response.outDir = bundlerReport.outDir; - if (options.server.run) { + if (options.server?.run) { const port = options.server.port; log.debug( `Starting Synthetics local server on ${chalk.bold.cyan(`http://127.0.0.1:${port}`)}.`, @@ -50,12 +46,12 @@ export const getPlugins: GetPlugins = (opts: Options, context: GlobalContext) => const server = runServer({ port, - root: options.server.root || response.outDir, + root: response.outDir, routes: { [`/${API_PREFIX}/build-status`]: { get: (req, res) => { res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(getServerResponse())); + res.end(JSON.stringify(response)); }, }, [`/${API_PREFIX}/kill`]: { diff --git a/packages/plugins/synthetics/src/types.ts b/packages/plugins/synthetics/src/types.ts index 7dc740cf2..7d80aecfa 100644 --- a/packages/plugins/synthetics/src/types.ts +++ b/packages/plugins/synthetics/src/types.ts @@ -5,20 +5,18 @@ import type { Assign, Ensure } from '@dd/core/types'; export type ServerOptions = { - port?: number; - root?: string; - run?: boolean; + port: number; + run: boolean; }; export type SyntheticsOptions = { disabled?: boolean; - server?: ServerOptions; }; export type SyntheticsOptionsWithDefaults = Assign< Ensure, { - server: Ensure; + server?: ServerOptions; } >; diff --git a/packages/plugins/synthetics/src/validate.ts b/packages/plugins/synthetics/src/validate.ts index 3a15f4cf0..7f43b4ee0 100644 --- a/packages/plugins/synthetics/src/validate.ts +++ b/packages/plugins/synthetics/src/validate.ts @@ -2,47 +2,27 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import type { Options, Logger, GlobalContext } from '@dd/core/types'; -import chalk from 'chalk'; +import type { Options, Logger } from '@dd/core/types'; -import { CONFIG_KEY, DEFAULT_PORT } from './constants'; +import { CONFIG_KEY } from './constants'; import type { SyntheticsOptionsWithDefaults } from './types'; -export const validateOptions = ( - config: Options, - context: GlobalContext, - log: Logger, -): SyntheticsOptionsWithDefaults => { +export const validateOptions = (config: Options, log: Logger): SyntheticsOptionsWithDefaults => { // Get values from environment. const { BUILD_PLUGINS_S8S_PORT } = process.env; - // Which port has been requested? - // This can either be enabled via env var or configuration. - const askedPort = BUILD_PLUGINS_S8S_PORT || config[CONFIG_KEY]?.server?.port; - // Define defaults. const validatedOptions: SyntheticsOptionsWithDefaults = { // We don't want to disable it by default. disabled: false, ...config[CONFIG_KEY], - server: { - run: !!BUILD_PLUGINS_S8S_PORT, - port: BUILD_PLUGINS_S8S_PORT ? +BUILD_PLUGINS_S8S_PORT : DEFAULT_PORT, - ...config[CONFIG_KEY]?.server, - }, }; - // If we've been asked to run, but no port was given, - // we'll use the default port, so warn the user. - if (validatedOptions.server.run && !askedPort) { - log.info( - `Synthetics local server port is not set, you can use either : - - ${chalk.bold.yellow('export BUILD_PLUGINS_S8S_PORT=1234')} before running your build. - - ${chalk.bold.yellow('config.synthetics.server.port: 1234')} in your configuration. - -Server will still run with the default port ${chalk.bold.cyan(DEFAULT_PORT.toString())}. -`, - ); + if (BUILD_PLUGINS_S8S_PORT) { + validatedOptions.server = { + run: true, + port: +BUILD_PLUGINS_S8S_PORT, + }; } return validatedOptions; From f5d2cb2ea0714015b2ae22473ecee5948bff269a Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 31 Mar 2025 10:55:24 +0200 Subject: [PATCH 11/18] Update TOC --- packages/plugins/synthetics/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/plugins/synthetics/README.md b/packages/plugins/synthetics/README.md index 1655a40af..82b7d2858 100644 --- a/packages/plugins/synthetics/README.md +++ b/packages/plugins/synthetics/README.md @@ -10,6 +10,7 @@ Interact with Synthetics at build time. - [Configuration](#configuration) +- [Build and test](#build-and-test) ## Configuration From 0c38e127a88cf68fe42e084da192be02f5e15009 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 4 Apr 2025 10:46:51 +0200 Subject: [PATCH 12/18] Update codeowners --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 147f03e89..f01d53b81 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -31,4 +31,4 @@ packages/plugins/analytics @yoannmoin packages/plugins/custom-hooks @yoannmoinet # Synthetics -packages/plugins/synthetics @yoannmoinet @etnbrd +packages/plugins/synthetics @yoannmoinet @DataDog/synthetics-ct From ed64d1cefa63549f58acef677ec583f6dc6b2ec1 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 4 Apr 2025 10:51:30 +0200 Subject: [PATCH 13/18] Do not run multiple servers --- packages/plugins/synthetics/src/index.test.ts | 57 ++++++++++++------- packages/plugins/synthetics/src/index.ts | 8 ++- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/packages/plugins/synthetics/src/index.test.ts b/packages/plugins/synthetics/src/index.test.ts index 4ab2a4cee..826dbf4b7 100644 --- a/packages/plugins/synthetics/src/index.test.ts +++ b/packages/plugins/synthetics/src/index.test.ts @@ -104,10 +104,11 @@ describe('Synthetics Plugin', () => { const expectations = [ { - description: 'not run with no env and no config', + description: 'not run with no env', env: {}, config: {}, - shouldRun: false, + triggers: 1, + expectedNumberOfCalls: 0, }, { description: 'run with port in env', @@ -115,10 +116,11 @@ describe('Synthetics Plugin', () => { BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), }, config: {}, - shouldRun: true, + triggers: 1, + expectedNumberOfCalls: 1, }, { - description: 'not run with disabled and config.run', + description: 'not run with disabled and port in env', env: { BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), }, @@ -127,26 +129,39 @@ describe('Synthetics Plugin', () => { disabled: true, }, }, - shouldRun: false, + triggers: 1, + expectedNumberOfCalls: 0, + }, + { + description: 'not run more than once', + env: { + BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), + }, + config: {}, + triggers: 3, + expectedNumberOfCalls: 1, }, ]; - test.each(expectations)('Should $description.', async ({ config, env, shouldRun }) => { - // Set the variables. - Object.assign(process.env, env); - // Run the plugin. - const [plugin] = getPlugins(config, getContextMock()); - if (plugin?.bundlerReport) { - // Trigger the bundlerReport hook where the server starts. - plugin.bundlerReport(getContextMock().bundler); - } - // Check the server. - if (shouldRun) { - expect(runServerMocked).toHaveBeenCalled(); - } else { - expect(runServerMocked).not.toHaveBeenCalled(); - } - }); + test.each(expectations)( + 'Should $description.', + async ({ config, env, expectedNumberOfCalls, triggers }) => { + // Set the variables. + Object.assign(process.env, env); + + // Run the plugin. + const [plugin] = getPlugins(config, getContextMock()); + if (plugin?.bundlerReport) { + // Trigger the bundlerReport hook where the server starts. + for (let i = 0; i < triggers; i++) { + plugin.bundlerReport(getContextMock().bundler); + } + } + + // Check the server. + expect(runServerMocked).toHaveBeenCalledTimes(expectedNumberOfCalls); + }, + ); }); // We need to loop over bundlers because we'll use a different port for each one of them diff --git a/packages/plugins/synthetics/src/index.ts b/packages/plugins/synthetics/src/index.ts index 032f6d126..13e975ea9 100644 --- a/packages/plugins/synthetics/src/index.ts +++ b/packages/plugins/synthetics/src/index.ts @@ -6,6 +6,7 @@ import { runServer } from '@dd/core/helpers/server'; import type { GlobalContext, GetPlugins, Options } from '@dd/core/types'; import { CONFIG_KEY as ERROR_TRACKING } from '@dd/error-tracking-plugin'; import chalk from 'chalk'; +import type { Server } from 'http'; import { API_PREFIX, CONFIG_KEY, PLUGIN_NAME } from './constants'; import type { ServerResponse, SyntheticsOptions } from './types'; @@ -32,19 +33,22 @@ export const getPlugins: GetPlugins = (opts: Options, context: GlobalContext) => status: 'running', }; + // Keep it global to avoid creating a new server on each run. + let server: Server; + return [ { name: PLUGIN_NAME, // Wait for us to have the bundler report to start the server over the outDir. bundlerReport(bundlerReport) { response.outDir = bundlerReport.outDir; - if (options.server?.run) { + if (options.server?.run && !server) { const port = options.server.port; log.debug( `Starting Synthetics local server on ${chalk.bold.cyan(`http://127.0.0.1:${port}`)}.`, ); - const server = runServer({ + server = runServer({ port, root: response.outDir, routes: { From 9430eb913245b926408cb496c2b105569b689363 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 4 Apr 2025 10:59:36 +0200 Subject: [PATCH 14/18] Do not make the server fail the build --- packages/plugins/synthetics/src/index.test.ts | 27 +++++++++++- packages/plugins/synthetics/src/index.ts | 42 ++++++++++--------- 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/packages/plugins/synthetics/src/index.test.ts b/packages/plugins/synthetics/src/index.test.ts index 826dbf4b7..049e2076b 100644 --- a/packages/plugins/synthetics/src/index.test.ts +++ b/packages/plugins/synthetics/src/index.test.ts @@ -1,12 +1,11 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. - import { runServer } from '@dd/core/helpers/server'; import { API_PREFIX } from '@dd/synthetics-plugin/constants'; import type { ServerResponse } from '@dd/synthetics-plugin/types'; import { getPlugins } from '@dd/synthetics-plugin'; -import { getContextMock } from '@dd/tests/_jest/helpers/mocks'; +import { getContextMock, mockLogFn } from '@dd/tests/_jest/helpers/mocks'; import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; import fs from 'fs'; import nock from 'nock'; @@ -162,6 +161,30 @@ describe('Synthetics Plugin', () => { expect(runServerMocked).toHaveBeenCalledTimes(expectedNumberOfCalls); }, ); + + test('Should not throw if the server fails to start.', async () => { + // Set the variables. + Object.assign(process.env, { + BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), + }); + + // Make `runServer` throw an error. + runServerMocked.mockImplementationOnce(() => { + throw new Error('Not available.'); + }); + + // Run the plugin. + const [plugin] = getPlugins({ synthetics: { disabled: false } }, getContextMock()); + if (plugin?.bundlerReport) { + plugin.bundlerReport(getContextMock().bundler); + } + + expect(runServerMocked).toHaveBeenCalledTimes(1); + expect(mockLogFn).toHaveBeenCalledWith( + expect.stringContaining('Not available'), + 'error', + ); + }); }); // We need to loop over bundlers because we'll use a different port for each one of them diff --git a/packages/plugins/synthetics/src/index.ts b/packages/plugins/synthetics/src/index.ts index 13e975ea9..ed9752944 100644 --- a/packages/plugins/synthetics/src/index.ts +++ b/packages/plugins/synthetics/src/index.ts @@ -48,28 +48,32 @@ export const getPlugins: GetPlugins = (opts: Options, context: GlobalContext) => `Starting Synthetics local server on ${chalk.bold.cyan(`http://127.0.0.1:${port}`)}.`, ); - server = runServer({ - port, - root: response.outDir, - routes: { - [`/${API_PREFIX}/build-status`]: { - get: (req, res) => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(response)); + try { + server = runServer({ + port, + root: response.outDir, + routes: { + [`/${API_PREFIX}/build-status`]: { + get: (req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }, }, - }, - [`/${API_PREFIX}/kill`]: { - get: (req, res) => { - res.writeHead(200, { 'Content-Type': 'text/html' }); - res.end('ok'); - // kill kill kill. - server.close(); - server.closeAllConnections(); - server.closeIdleConnections(); + [`/${API_PREFIX}/kill`]: { + get: (req, res) => { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('ok'); + // kill kill kill. + server.close(); + server.closeAllConnections(); + server.closeIdleConnections(); + }, }, }, - }, - }); + }); + } catch (e) { + log.error(`Error starting Synthetics local server: ${e}`); + } } }, buildReport(buildReport) { From fc29fed872b98010bd47e37e44ffb40c92327a6a Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 4 Apr 2025 11:08:35 +0200 Subject: [PATCH 15/18] Add comment --- packages/plugins/synthetics/src/validate.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/plugins/synthetics/src/validate.ts b/packages/plugins/synthetics/src/validate.ts index 7f43b4ee0..44f84c3a6 100644 --- a/packages/plugins/synthetics/src/validate.ts +++ b/packages/plugins/synthetics/src/validate.ts @@ -18,6 +18,7 @@ export const validateOptions = (config: Options, log: Logger): SyntheticsOptions ...config[CONFIG_KEY], }; + // We will only run the server if we have a port defined in the environment. if (BUILD_PLUGINS_S8S_PORT) { validatedOptions.server = { run: true, From 0cc8f29fe32e70a3e83a96a23d938841d756ca31 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 4 Apr 2025 17:18:05 +0200 Subject: [PATCH 16/18] Missing new line --- packages/plugins/synthetics/src/index.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/plugins/synthetics/src/index.test.ts b/packages/plugins/synthetics/src/index.test.ts index 049e2076b..5badb9293 100644 --- a/packages/plugins/synthetics/src/index.test.ts +++ b/packages/plugins/synthetics/src/index.test.ts @@ -1,6 +1,7 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. + import { runServer } from '@dd/core/helpers/server'; import { API_PREFIX } from '@dd/synthetics-plugin/constants'; import type { ServerResponse } from '@dd/synthetics-plugin/types'; From 0c1e9877b19bab5f241bef9125ec6dc4d1803571 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 7 Apr 2025 10:28:01 +0200 Subject: [PATCH 17/18] Update tests to better catch multiple server runs --- packages/plugins/synthetics/src/index.test.ts | 85 ++++++++++++++----- 1 file changed, 64 insertions(+), 21 deletions(-) diff --git a/packages/plugins/synthetics/src/index.test.ts b/packages/plugins/synthetics/src/index.test.ts index 5badb9293..5f3acd51a 100644 --- a/packages/plugins/synthetics/src/index.test.ts +++ b/packages/plugins/synthetics/src/index.test.ts @@ -3,10 +3,11 @@ // Copyright 2019-Present Datadog, Inc. import { runServer } from '@dd/core/helpers/server'; +import { getContext } from '@dd/factory/helpers'; import { API_PREFIX } from '@dd/synthetics-plugin/constants'; import type { ServerResponse } from '@dd/synthetics-plugin/types'; import { getPlugins } from '@dd/synthetics-plugin'; -import { getContextMock, mockLogFn } from '@dd/tests/_jest/helpers/mocks'; +import { getContextMock, mockLogFn, mockLogger } from '@dd/tests/_jest/helpers/mocks'; import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; import fs from 'fs'; import nock from 'nock'; @@ -20,12 +21,22 @@ jest.mock('@dd/core/helpers/server', () => { }; }); -const runServerMocked = jest.mocked(runServer); +jest.mock('@dd/factory/helpers', () => { + const original = jest.requireActual('@dd/factory/helpers'); + return { + ...original, + getContext: jest.fn(original.getContext), + }; +}); -const DEFAULT_PORT = 1234; +const runServerMocked = jest.mocked(runServer); +const getContextMocked = jest.mocked(getContext); -const getApiUrl = (port: number = DEFAULT_PORT) => `http://127.0.0.1:${port}`; -const getInternalApiUrl = (port: number = DEFAULT_PORT) => `${getApiUrl(port)}/${API_PREFIX}`; +// Get an incremental port to prevent conflicts. +let PORT = 1234; +const getPort = () => PORT++; +const getApiUrl = (port: number = getPort()) => `http://127.0.0.1:${port}`; +const getInternalApiUrl = (port: number = getPort()) => `${getApiUrl(port)}/${API_PREFIX}`; const safeFetch = async (route: string, port: number) => { try { return await fetch(`${getInternalApiUrl(port)}${route}`); @@ -97,9 +108,6 @@ describe('Synthetics Plugin', () => { afterEach(async () => { // Remove the variable we may have set. delete process.env.BUILD_PLUGINS_S8S_PORT; - - // Kill the server. - await safeFetch('/kill', DEFAULT_PORT); }); const expectations = [ @@ -113,7 +121,7 @@ describe('Synthetics Plugin', () => { { description: 'run with port in env', env: { - BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), + BUILD_PLUGINS_S8S_PORT: JSON.stringify(getPort()), }, config: {}, triggers: 1, @@ -122,7 +130,7 @@ describe('Synthetics Plugin', () => { { description: 'not run with disabled and port in env', env: { - BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), + BUILD_PLUGINS_S8S_PORT: JSON.stringify(getPort()), }, config: { synthetics: { @@ -135,7 +143,7 @@ describe('Synthetics Plugin', () => { { description: 'not run more than once', env: { - BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), + BUILD_PLUGINS_S8S_PORT: JSON.stringify(getPort()), }, config: {}, triggers: 3, @@ -160,13 +168,24 @@ describe('Synthetics Plugin', () => { // Check the server. expect(runServerMocked).toHaveBeenCalledTimes(expectedNumberOfCalls); + + // No error should have been logged. + expect(mockLogFn).not.toHaveBeenCalledWith( + expect.stringContaining('Error starting Synthetics local server'), + 'error', + ); + + if (env.BUILD_PLUGINS_S8S_PORT) { + // Kill the server. + await safeFetch('/kill', Number(env.BUILD_PLUGINS_S8S_PORT)); + } }, ); test('Should not throw if the server fails to start.', async () => { // Set the variables. Object.assign(process.env, { - BUILD_PLUGINS_S8S_PORT: JSON.stringify(DEFAULT_PORT), + BUILD_PLUGINS_S8S_PORT: JSON.stringify(getPort()), }); // Make `runServer` throw an error. @@ -191,19 +210,26 @@ describe('Synthetics Plugin', () => { // We need to loop over bundlers because we'll use a different port for each one of them // to avoid port conflicts. describe.each(BUNDLERS)('$name', (bundler) => { - // Get an incremental port to prevent conflicts. - const port = DEFAULT_PORT + BUNDLERS.indexOf(bundler); + const port = getPort(); let buildProm: Promise; let outDir: string; const serverResponses: Set = new Set(); - beforeAll(async () => { + beforeEach(async () => { // Set the variables. Object.assign(process.env, { BUILD_PLUGINS_S8S_PORT: JSON.stringify(port), }); + // Mock the getPlugins function to mock the context. + getContextMocked.mockImplementation((opts) => { + const original = jest.requireActual('@dd/factory/helpers'); + const mockContext = original.getContext(opts); + mockContext.getLogger = jest.fn(() => mockLogger); + return mockContext; + }); + // Run the builds. // Do not await the promise as the server will be running. buildProm = runBundlers( @@ -228,7 +254,7 @@ describe('Synthetics Plugin', () => { }); }); - afterAll(async () => { + afterEach(async () => { // Remove the variable we may have set. delete process.env.BUILD_PLUGINS_S8S_PORT; // Kill the server. @@ -239,15 +265,16 @@ describe('Synthetics Plugin', () => { } }); - test('Should report the build status.', async () => { + // NOTE: Due to how mocks reset and the beforeEach/afterEach, we need to have only one test. + test('Should respond as expected.', async () => { + // Build status. // Verify that we have the running and success statuses. const reportedStatus = Array.from(serverResponses).filter((resp) => ['fail', 'success', 'running'].includes(resp.status), ); expect(reportedStatus.length).toBeGreaterThan(0); - }); - test('Should report the outDir.', async () => { + // Outdir. // Verify that we have the running and success statuses. const reportedOutDirs = new Set( Array.from(serverResponses).map((resp) => resp.outDir), @@ -256,15 +283,31 @@ describe('Synthetics Plugin', () => { expect(reportedOutDirs.size).toBe(1); // It should be the same as the one we reported from the build. expect(reportedOutDirs.values().next().value).toEqual(outDir); - }); - test('Should actually serve the built files.', async () => { + // Built files. // Query a file from the server. const res = await fetch(`${getApiUrl(port)}/main.js`); expect(res.ok).toBe(true); const text = await res.text(); // Confirm that the file served by the server is the same as the one on disk. expect(text).toEqual(fs.readFileSync(path.join(outDir, 'main.js'), 'utf-8')); + + // Only one log should have been called. + expect(mockLogFn).toHaveBeenCalledWith( + expect.stringContaining('Starting Synthetics local server'), + 'debug', + ); + const nbCalls = mockLogFn.mock.calls.filter( + ([message, level]) => + message.includes('Starting Synthetics local server') && level === 'debug', + ).length; + expect(nbCalls).toBe(1); + + // No error should have been logged. + expect(mockLogFn).not.toHaveBeenCalledWith( + expect.stringContaining('Error starting Synthetics local server'), + 'error', + ); }); }); }); From 2706a9166427b522f1c9c7a291d7c0bc6ca100bf Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Apr 2025 10:36:06 +0200 Subject: [PATCH 18/18] wip --- packages/core/src/helpers/server.ts | 13 +- .../plugins/synthetics/src/helpers/server.ts | 209 ++++++++++++++++++ packages/plugins/synthetics/src/index.ts | 70 +----- packages/plugins/synthetics/src/types.ts | 10 +- packages/plugins/synthetics/src/validate.ts | 4 +- 5 files changed, 233 insertions(+), 73 deletions(-) create mode 100644 packages/plugins/synthetics/src/helpers/server.ts diff --git a/packages/core/src/helpers/server.ts b/packages/core/src/helpers/server.ts index 8a897aa99..aae3275d8 100644 --- a/packages/core/src/helpers/server.ts +++ b/packages/core/src/helpers/server.ts @@ -24,7 +24,7 @@ type File = { content: string; }; type RouteVerb = 'get' | 'post' | 'put' | 'patch' | 'delete'; -type Routes = Record< +export type Routes = Record< string, Partial< Record< @@ -33,13 +33,13 @@ type Routes = Record< > > >; -type Response = { +export type Response = { statusCode: number; headers: Record; body: string; error?: Error; }; -type RunServerOptions = { +export type RunServerOptions = { port: number; root: string; routes?: Routes; @@ -76,8 +76,8 @@ export const prepareFile = async (root: string, requestUrl: string): Promise { - const server = http.createServer(async (req, res) => { +export const getServer = ({ root, middleware, routes }: Omit) => { + return http.createServer(async (req, res) => { const response: Response = { statusCode: 200, headers: {}, @@ -125,7 +125,10 @@ export const runServer = ({ port, root, middleware, routes }: RunServerOptions) res.writeHead(response.statusCode, response.headers); res.end(response.body); }); +}; +export const runServer = ({ port, root, middleware, routes }: RunServerOptions) => { + const server = getServer({ root, middleware, routes }); server.listen(port); return server; }; diff --git a/packages/plugins/synthetics/src/helpers/server.ts b/packages/plugins/synthetics/src/helpers/server.ts new file mode 100644 index 000000000..abffdf2b6 --- /dev/null +++ b/packages/plugins/synthetics/src/helpers/server.ts @@ -0,0 +1,209 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { doRequest } from '@dd/core/helpers/request'; +import type { Routes } from '@dd/core/helpers/server'; +import { getServer } from '@dd/core/helpers/server'; +import type { Options, PluginOptions, Logger } from '@dd/core/types'; +import { CONFIG_KEY as ERROR_TRACKING } from '@dd/error-tracking-plugin'; +import type { Server } from 'http'; +import url from 'url'; + +import { API_PREFIX, PLUGIN_NAME } from '../constants'; +import type { ServerResponse, SyntheticsOptionsWithDefaults } from '../types'; + +const identifier = 'Server Running from the Datadog Synthetics Plugin'; +const triedPorts: number[] = []; +export const getPort = (): number => { + const MIN_PORT = 49152; + const MAX_PORT = 65535; + const port = Math.floor(Math.random() * (MAX_PORT - MIN_PORT + 1)) + MIN_PORT; + if (triedPorts.includes(port)) { + return getPort(); + } + triedPorts.push(port); + return port; +}; + +const killServer = (server?: Server) => { + if (!server) { + return; + } + server.close(); + server.closeAllConnections(); + server.closeIdleConnections(); +}; + +// Keep it global to avoid creating a new server on each run. +let SERVER: Server | undefined; +const RUNNING_SERVERS: number[] = []; + +const verifyServer = async ( + server: Server, + port: number, + cb: (portUsed: number) => Promise | void, +): Promise => { + // Listen for errors. + const errorListener = async (e: any) => { + server.removeListener('listening', successListener); + // If the port is already in use. + if (e.code === 'EADDRINUSE') { + // Verify if another instance of the plugin is running on this port. + const resp = await doRequest({ + url: `http://127.0.0.1:${port}/${API_PREFIX}/build-status`, + retries: 0, + type: 'json', + }); + + if (resp.identifier === identifier) { + // Another instance of the plugin is running on this port. + // Instrument the other server so we can piggyback on it. + return verifyServer(server, getPort(), cb); + } else { + // We have something else running here. + // Throw an error as the feature can't work. + throw new Error(`Something else is running on port ${port}.`); + } + } else { + // Something else happened. + // Forward the error. + throw e; + } + }; + + const successListener = async () => { + // Remove the error listener. + server.removeListener('error', errorListener); + // Callback to communicate the port used. + await cb(port); + }; + + server.once('error', errorListener); + server.once('listening', successListener); + server.listen(port); +}; + +const setupMasterServer = (routes: Routes, log: Logger, runningServers: number[]) => { + // The master server should forward file request to its sub servers. + // Could use middleware to do this. + // It should also have a register route. + routes[`/${API_PREFIX}/register`] = { + get: (req, res) => { + const sendError = (message: string) => { + res.writeHead(400, { 'Content-Type': 'text/html' }); + res.end(message); + }; + + if (!req.url) { + return sendError('Missing URL.'); + } + + const query = url.parse(req.url, true).query; + if (!query) { + return sendError('Missing query.'); + } + + if (!query.port) { + return sendError('Missing port.'); + } + + log.debug(`Registering port ${query.port} to the master server.`); + const portsToRegister = Array.isArray(query.port) ? query.port : [query.port]; + runningServers.push(...portsToRegister.map(Number)); + + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('ok'); + }, + }; +}; + +const setupSubServer = async (masterPort: number, port: number) => { + // The sub server should register to the master server. + await doRequest({ + url: `http://127.0.0.1:${masterPort}/${API_PREFIX}/register?port=${port}`, + }); +}; + +export const getServerPlugin = ( + opts: Options, + options: SyntheticsOptionsWithDefaults, + log: Logger, +): PluginOptions => { + // This is the mutable response the server will use to report the build's status. + const response: ServerResponse = { + publicPath: opts[ERROR_TRACKING]?.sourcemaps?.minifiedPathPrefix, + status: 'running', + identifier, + }; + + const routes: Routes = { + // TODO: The master server should forward file request to its sub servers. + // Could use a special __catch_all__, "/*" or "/" route. + // Route to get the build status. + [`/${API_PREFIX}/build-status`]: { + get: (req, res) => { + // TODO: Status should be based on all the running servers. + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }, + }, + // Route to kill the server. + [`/${API_PREFIX}/kill`]: { + get: (req, res) => { + // TODO: Kill all the sub-servers if we're the master server. + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('ok'); + // kill kill kill. + killServer(SERVER); + }, + }, + }; + + return { + name: PLUGIN_NAME, + // Wait for us to have the bundler report to start the server over the outDir. + bundlerReport(bundlerReport) { + response.outDir = bundlerReport.outDir; + if (options.server?.run && !SERVER) { + const port = options.server.port; + + // Only create the server first. + SERVER = getServer({ + root: response.outDir, + routes, + }); + + try { + verifyServer(SERVER, port, async (portUsed) => { + if (portUsed === port) { + log.debug(`Setting up master server on port ${portUsed}.`); + setupMasterServer(routes, log, RUNNING_SERVERS); + } else { + log.debug(`Setting up sub server on port ${portUsed}.`); + await setupSubServer(port, portUsed); + } + }); + } catch (e) { + log.error(`Error starting Synthetics local server: ${e}`); + } + } + }, + buildReport(buildReport) { + if (buildReport.errors.length) { + response.status = 'fail'; + } + }, + writeBundle() { + response.status = 'success'; + }, + esbuild: { + setup(build) { + build.onDispose(() => { + // We kill the plugin when the build is disposed. + killServer(SERVER); + }); + }, + }, + }; +}; diff --git a/packages/plugins/synthetics/src/index.ts b/packages/plugins/synthetics/src/index.ts index ed9752944..98a3d73d7 100644 --- a/packages/plugins/synthetics/src/index.ts +++ b/packages/plugins/synthetics/src/index.ts @@ -2,14 +2,11 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { runServer } from '@dd/core/helpers/server'; import type { GlobalContext, GetPlugins, Options } from '@dd/core/types'; -import { CONFIG_KEY as ERROR_TRACKING } from '@dd/error-tracking-plugin'; -import chalk from 'chalk'; -import type { Server } from 'http'; -import { API_PREFIX, CONFIG_KEY, PLUGIN_NAME } from './constants'; -import type { ServerResponse, SyntheticsOptions } from './types'; +import { CONFIG_KEY, PLUGIN_NAME } from './constants'; +import { getServerPlugin } from './helpers/server'; +import type { SyntheticsOptions } from './types'; import { validateOptions } from './validate'; export { CONFIG_KEY, PLUGIN_NAME }; @@ -22,68 +19,11 @@ export type types = { export const getPlugins: GetPlugins = (opts: Options, context: GlobalContext) => { const log = context.getLogger(PLUGIN_NAME); // Verify configuration. - const options = validateOptions(opts, log); + const options = validateOptions(opts); if (options.disabled) { return []; } - const response: ServerResponse = { - publicPath: opts[ERROR_TRACKING]?.sourcemaps?.minifiedPathPrefix, - status: 'running', - }; - - // Keep it global to avoid creating a new server on each run. - let server: Server; - - return [ - { - name: PLUGIN_NAME, - // Wait for us to have the bundler report to start the server over the outDir. - bundlerReport(bundlerReport) { - response.outDir = bundlerReport.outDir; - if (options.server?.run && !server) { - const port = options.server.port; - log.debug( - `Starting Synthetics local server on ${chalk.bold.cyan(`http://127.0.0.1:${port}`)}.`, - ); - - try { - server = runServer({ - port, - root: response.outDir, - routes: { - [`/${API_PREFIX}/build-status`]: { - get: (req, res) => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(response)); - }, - }, - [`/${API_PREFIX}/kill`]: { - get: (req, res) => { - res.writeHead(200, { 'Content-Type': 'text/html' }); - res.end('ok'); - // kill kill kill. - server.close(); - server.closeAllConnections(); - server.closeIdleConnections(); - }, - }, - }, - }); - } catch (e) { - log.error(`Error starting Synthetics local server: ${e}`); - } - } - }, - buildReport(buildReport) { - if (buildReport.errors.length) { - response.status = 'fail'; - } - }, - writeBundle() { - response.status = 'success'; - }, - }, - ]; + return [getServerPlugin(opts, options, log)]; }; diff --git a/packages/plugins/synthetics/src/types.ts b/packages/plugins/synthetics/src/types.ts index 7d80aecfa..a7dbb554d 100644 --- a/packages/plugins/synthetics/src/types.ts +++ b/packages/plugins/synthetics/src/types.ts @@ -21,4 +21,12 @@ export type SyntheticsOptionsWithDefaults = Assign< >; export type BuildStatus = 'running' | 'success' | 'fail'; -export type ServerResponse = { outDir?: string; publicPath?: string; status: BuildStatus }; +export type ServerResponse = { + outDir?: string; + publicPath?: string; + status: BuildStatus; + /** + * Used to identify the server in case of multiple instances. + */ + identifier: string; +}; diff --git a/packages/plugins/synthetics/src/validate.ts b/packages/plugins/synthetics/src/validate.ts index 44f84c3a6..196179714 100644 --- a/packages/plugins/synthetics/src/validate.ts +++ b/packages/plugins/synthetics/src/validate.ts @@ -2,12 +2,12 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import type { Options, Logger } from '@dd/core/types'; +import type { Options } from '@dd/core/types'; import { CONFIG_KEY } from './constants'; import type { SyntheticsOptionsWithDefaults } from './types'; -export const validateOptions = (config: Options, log: Logger): SyntheticsOptionsWithDefaults => { +export const validateOptions = (config: Options): SyntheticsOptionsWithDefaults => { // Get values from environment. const { BUILD_PLUGINS_S8S_PORT } = process.env;