From 80551ed4638a73503426b58c14def10d5203de7a Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 2 May 2025 19:14:09 +0100 Subject: [PATCH 01/19] update testcontainers packages --- internal-packages/testcontainers/package.json | 6 +- pnpm-lock.yaml | 185 +++++++++++++----- 2 files changed, 143 insertions(+), 48 deletions(-) diff --git a/internal-packages/testcontainers/package.json b/internal-packages/testcontainers/package.json index 33a4e43870e..5edce5836be 100644 --- a/internal-packages/testcontainers/package.json +++ b/internal-packages/testcontainers/package.json @@ -10,10 +10,10 @@ "ioredis": "^5.3.2" }, "devDependencies": { - "@testcontainers/postgresql": "^10.13.1", - "@testcontainers/redis": "^10.13.1", + "@testcontainers/postgresql": "^10.25.0", + "@testcontainers/redis": "^10.25.0", "@trigger.dev/core": "workspace:*", - "testcontainers": "^10.13.1", + "testcontainers": "^10.25.0", "tinyexec": "^0.3.0", "vitest": "^1.4.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 94bac59ab05..a2fb3e1f6ce 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1006,17 +1006,17 @@ importers: version: 5.3.2 devDependencies: '@testcontainers/postgresql': - specifier: ^10.13.1 - version: 10.13.1 + specifier: ^10.25.0 + version: 10.25.0 '@testcontainers/redis': - specifier: ^10.13.1 - version: 10.13.1 + specifier: ^10.25.0 + version: 10.25.0 '@trigger.dev/core': specifier: workspace:* version: link:../../packages/core testcontainers: - specifier: ^10.13.1 - version: 10.13.1 + specifier: ^10.25.0 + version: 10.25.0 tinyexec: specifier: ^0.3.0 version: 0.3.0 @@ -8075,7 +8075,6 @@ packages: dependencies: '@grpc/proto-loader': 0.7.13 '@js-sdsl/ordered-map': 4.4.2 - dev: false /@grpc/grpc-js@1.8.17: resolution: {integrity: sha512-DGuSbtMFbaRsyffMf+VEkVu8HkSXEUfO3UyGJNtqxW9ABdtTIA+2UXAJpwbJS+xfQxuwqLUeELmL6FuZkOqPxw==} @@ -8093,7 +8092,6 @@ packages: long: 5.2.3 protobufjs: 7.3.2 yargs: 17.7.2 - dev: false /@grpc/proto-loader@0.7.7: resolution: {integrity: sha512-1TIeXOi8TuSCQprPItwoMymZXxWT0CPxUhkrkeCUH+D8U7QDwQ6b7SUz2MaLuWM2llT+J/TVFLmQI5KtML3BhQ==} @@ -8697,7 +8695,6 @@ packages: /@js-sdsl/ordered-map@4.4.2: resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} - dev: false /@jsep-plugin/assignment@1.3.0(jsep@1.4.0): resolution: {integrity: sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ==} @@ -17796,19 +17793,21 @@ packages: zod: 3.23.8 dev: false - /@testcontainers/postgresql@10.13.1: - resolution: {integrity: sha512-HAh/3uLAzAhOmzXsOE6hVxkvetczPnX/Zoyt+SgK7QotW98Npr1MDx8OKiaLGTJ8XkIvVvS4Ch6bl+frt4pnkQ==} + /@testcontainers/postgresql@10.25.0: + resolution: {integrity: sha512-VkpqpX9YZ8aq4wfk6sJRopGTmlBdE1kErzAFWJ/1pY/XrEZ7nxdfFBG+En2icQnbv3BIFQYysEKxEFMNB+hQVw==} dependencies: - testcontainers: 10.13.1 + testcontainers: 10.25.0 transitivePeerDependencies: + - bare-buffer - supports-color dev: true - /@testcontainers/redis@10.13.1: - resolution: {integrity: sha512-pXg15o4oTRaEyb5xryQZUdePtoRId/+3TeU7vnUgDpqOmRacF8/7zL7jqs13uPh1uea6M7a8MDgHQM8j8kXZUg==} + /@testcontainers/redis@10.25.0: + resolution: {integrity: sha512-ALNrrnYnB59kV5c/EjiUkzn0roCtcnOu2KfHHF8xBi3vq3dYSqzADL8rL2BExeoFhyaEtlUT9P4ZecRB60O+/Q==} dependencies: - testcontainers: 10.13.1 + testcontainers: 10.25.0 transitivePeerDependencies: + - bare-buffer - supports-color dev: true @@ -20361,6 +20360,12 @@ packages: requiresBuild: true optional: true + /bare-events@2.5.4: + resolution: {integrity: sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==} + requiresBuild: true + dev: true + optional: true + /bare-fs@2.3.5: resolution: {integrity: sha512-SlE9eTxifPDJrT6YgemQ1WGFleevzwY+XAP1Xqgl56HtcrisC2CHCZ2tq6dBpcH2TnNxwUEUGhweo+lrQtYuiw==} requiresBuild: true @@ -20368,11 +20373,36 @@ packages: bare-events: 2.4.2 bare-path: 2.1.3 bare-stream: 2.3.0 + dev: false + optional: true + + /bare-fs@4.1.4: + resolution: {integrity: sha512-r8+26Voz8dGX3AYpJdFb1ZPaUSM8XOLCZvy+YGpRTmwPHIxA7Z3Jov/oMPtV7hfRQbOnH8qGlLTzQAbgtdNN0Q==} + engines: {bare: '>=1.16.0'} + requiresBuild: true + peerDependencies: + bare-buffer: '*' + peerDependenciesMeta: + bare-buffer: + optional: true + dependencies: + bare-events: 2.5.4 + bare-path: 3.0.0 + bare-stream: 2.6.5(bare-events@2.5.4) + dev: true optional: true /bare-os@2.4.4: resolution: {integrity: sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==} requiresBuild: true + dev: false + optional: true + + /bare-os@3.6.1: + resolution: {integrity: sha512-uaIjxokhFidJP+bmmvKSgiMzj2sV5GPHaZVAIktcxcpCyBFFWO+YlikVAdhmUo2vYFvFhOXIAlldqV29L8126g==} + engines: {bare: '>=1.14.0'} + requiresBuild: true + dev: true optional: true /bare-path@2.1.3: @@ -20380,6 +20410,15 @@ packages: requiresBuild: true dependencies: bare-os: 2.4.4 + dev: false + optional: true + + /bare-path@3.0.0: + resolution: {integrity: sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==} + requiresBuild: true + dependencies: + bare-os: 3.6.1 + dev: true optional: true /bare-stream@2.3.0: @@ -20388,6 +20427,24 @@ packages: dependencies: b4a: 1.6.6 streamx: 2.20.1 + dev: false + optional: true + + /bare-stream@2.6.5(bare-events@2.5.4): + resolution: {integrity: sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==} + requiresBuild: true + peerDependencies: + bare-buffer: '*' + bare-events: '*' + peerDependenciesMeta: + bare-buffer: + optional: true + bare-events: + optional: true + dependencies: + bare-events: 2.5.4 + streamx: 2.22.0 + dev: true optional: true /base64-js@1.5.1: @@ -22192,19 +22249,7 @@ packages: resolution: {integrity: sha512-plizRs/Vf15H+GCVxq2EUvyPK7ei9b/cVesHvjnX4xaXjM9spHe2Ytq0BitndFgvTJ3E3NljPNUEl7BAN43iZw==} engines: {node: '>= 6.0.0'} dependencies: - yaml: 2.3.1 - dev: true - - /docker-modem@3.0.8: - resolution: {integrity: sha512-f0ReSURdM3pcKPNS30mxOHSbaFLcknGmQjwSfmbcdOw1XWKXVhukM3NJHhr7NpY9BIyyWQb0EBo3KQvvuU5egQ==} - engines: {node: '>= 8.0'} - dependencies: - debug: 4.4.0(supports-color@10.0.0) - readable-stream: 3.6.0 - split-ca: 1.0.1 - ssh2: 1.16.0 - transitivePeerDependencies: - - supports-color + yaml: 2.7.1 dev: true /docker-modem@5.0.6: @@ -22217,21 +22262,24 @@ packages: ssh2: 1.16.0 transitivePeerDependencies: - supports-color - dev: false - /dockerode@3.3.5: - resolution: {integrity: sha512-/0YNa3ZDNeLr/tSckmD69+Gq+qVNhvKfAHNeZJBnp7EOP6RGKV8ORrJHkUn20So5wU+xxT7+1n5u8PjHbfjbSA==} + /dockerode@4.0.4: + resolution: {integrity: sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==} engines: {node: '>= 8.0'} dependencies: '@balena/dockerignore': 1.0.2 - docker-modem: 3.0.8 + '@grpc/grpc-js': 1.12.6 + '@grpc/proto-loader': 0.7.13 + docker-modem: 5.0.6 + protobufjs: 7.3.2 tar-fs: 2.0.1 + uuid: 10.0.0 transitivePeerDependencies: - supports-color - dev: true + dev: false - /dockerode@4.0.4: - resolution: {integrity: sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==} + /dockerode@4.0.6: + resolution: {integrity: sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==} engines: {node: '>= 8.0'} dependencies: '@balena/dockerignore': 1.0.2 @@ -22239,11 +22287,11 @@ packages: '@grpc/proto-loader': 0.7.13 docker-modem: 5.0.6 protobufjs: 7.3.2 - tar-fs: 2.0.1 + tar-fs: 2.1.2 uuid: 10.0.0 transitivePeerDependencies: - supports-color - dev: false + dev: true /doctrine@2.1.0: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} @@ -24603,6 +24651,11 @@ packages: resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} engines: {node: '>=8'} + /get-port@7.1.0: + resolution: {integrity: sha512-QB9NKEeDg3xxVwCCwJQ9+xycaz6pBB6iQ76wiWMl1927n0Kir6alPiP+yuiICLLU4jpMe08dXfpebuQppFA2zw==} + engines: {node: '>=16'} + dev: true + /get-proto@1.0.1: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} @@ -32607,6 +32660,17 @@ packages: optionalDependencies: bare-events: 2.4.2 + /streamx@2.22.0: + resolution: {integrity: sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==} + requiresBuild: true + dependencies: + fast-fifo: 1.3.2 + text-decoder: 1.2.0 + optionalDependencies: + bare-events: 2.5.4 + dev: true + optional: true + /strict-event-emitter@0.5.1: resolution: {integrity: sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==} dev: false @@ -33203,6 +33267,7 @@ packages: mkdirp-classic: 0.5.3 pump: 3.0.0 tar-stream: 2.2.0 + dev: false /tar-fs@2.1.1: resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} @@ -33213,6 +33278,15 @@ packages: tar-stream: 2.2.0 dev: true + /tar-fs@2.1.2: + resolution: {integrity: sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==} + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.0 + tar-stream: 2.2.0 + dev: true + /tar-fs@3.0.6: resolution: {integrity: sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==} dependencies: @@ -33221,6 +33295,19 @@ packages: optionalDependencies: bare-fs: 2.3.5 bare-path: 2.1.3 + dev: false + + /tar-fs@3.0.8: + resolution: {integrity: sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==} + dependencies: + pump: 3.0.0 + tar-stream: 3.1.7 + optionalDependencies: + bare-fs: 4.1.4 + bare-path: 3.0.0 + transitivePeerDependencies: + - bare-buffer + dev: true /tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} @@ -33380,25 +33467,26 @@ packages: minimatch: 9.0.5 dev: true - /testcontainers@10.13.1: - resolution: {integrity: sha512-JBbOhxmygj/ouH/47GnoVNt+c55Telh/45IjVxEbDoswsLchVmJiuKiw/eF6lE5i7LN+/99xsrSCttI3YRtirg==} + /testcontainers@10.25.0: + resolution: {integrity: sha512-X3x6cjorEMgei1vVx3M7dnTMzWoWOTi4krpUf3C2iOvOcwsaMUHbca9J4yzpN65ieiWhcK2dA5dxpZyUonwC2Q==} dependencies: '@balena/dockerignore': 1.0.2 '@types/dockerode': 3.3.35 archiver: 7.0.1 async-lock: 1.4.1 byline: 5.0.0 - debug: 4.3.7(supports-color@10.0.0) + debug: 4.4.0(supports-color@10.0.0) docker-compose: 0.24.8 - dockerode: 3.3.5 - get-port: 5.1.1 + dockerode: 4.0.6 + get-port: 7.1.0 proper-lockfile: 4.1.2 properties-reader: 2.3.0 ssh-remote-port-forward: 1.0.4 - tar-fs: 3.0.6 + tar-fs: 3.0.8 tmp: 0.2.3 - undici: 5.28.4 + undici: 5.29.0 transitivePeerDependencies: + - bare-buffer - supports-color dev: true @@ -34341,6 +34429,14 @@ packages: engines: {node: '>=14.0'} dependencies: '@fastify/busboy': 2.0.0 + dev: false + + /undici@5.29.0: + resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} + engines: {node: '>=14.0'} + dependencies: + '@fastify/busboy': 2.0.0 + dev: true /unenv-nightly@1.10.0-1717606461.a117952: resolution: {integrity: sha512-u3TfBX02WzbHTpaEfWEKwDijDSFAHcgXkayUZ+MVDrjhLFvgAJzFGTSTmwlEhwWi2exyRQey23ah9wELMM6etg==} @@ -34758,7 +34854,6 @@ packages: /uuid@10.0.0: resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} hasBin: true - dev: false /uuid@3.4.0: resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} From 68eb755f0848500402968f967346929e563d1c37 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 2 May 2025 19:14:39 +0100 Subject: [PATCH 02/19] increase cleanup timeout and add better logs --- internal-packages/testcontainers/src/index.ts | 63 ++++++++++++++++--- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index ae5dcc76d6d..5a254086b85 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -31,17 +31,54 @@ type ContainerWithElectricContext = NetworkContext & PostgresContext & ElectricC type Use = (value: T) => Promise; +let cleanupOrder = 0; +let activeCleanups = 0; + +/** + * Logs the cleanup of a resource. + * @param resource - The resource that is being cleaned up. + * @param fn - The cleanup function. + */ +async function logCleanup(resource: string, fn: () => Promise) { + const start = new Date(); + const order = cleanupOrder++; + const activeAtStart = ++activeCleanups; + + let error: unknown = null; + try { + await fn(); + } catch (err) { + error = err instanceof Error ? err.message : String(err); + } + + const end = new Date(); + const activeAtEnd = --activeCleanups; + const parallel = activeAtStart > 1 || activeAtEnd > 0; + + console.log( + JSON.stringify({ + order, + start: start.toISOString(), + end: end.toISOString(), + parallel, + resource, + durationMs: end.getTime() - start.getTime(), + error, + activeAtStart, + activeAtEnd, + }) + ); +} + const network = async ({}, use: Use) => { const network = await new Network().start(); try { await use(network); } finally { - try { - await network.stop(); - } catch (error) { - console.warn("Network stop error (ignored):", error); - } // Make sure to stop the network after use + await logCleanup("network", async () => { + await network.stop(); + }); } }; @@ -55,7 +92,9 @@ const postgresContainer = async ( } finally { // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await container.stop({ timeout: 10 }); + await logCleanup("postgresContainer", async () => { + await container.stop({ timeout: 30 }); + }); } }; @@ -77,7 +116,9 @@ const prisma = async ( try { await use(prisma); } finally { - await prisma.$disconnect(); + await logCleanup("prisma", async () => { + await prisma.$disconnect(); + }); } }; @@ -96,7 +137,9 @@ const redisContainer = async ( } finally { // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await container.stop({ timeout: 10 }); + await logCleanup("redisContainer", async () => { + await container.stop({ timeout: 30 }); + }); } }; @@ -148,7 +191,9 @@ const electricOrigin = async ( } finally { // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await container.stop({ timeout: 10 }); + await logCleanup("electricContainer", async () => { + await container.stop({ timeout: 30 }); + }); } }; From c6fa8b7ebad30617f8dd884f56d2948223d242d2 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 2 May 2025 23:50:37 +0100 Subject: [PATCH 03/19] small tweaks --- internal-packages/testcontainers/src/index.ts | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index 5a254086b85..1027f133ed9 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -37,16 +37,16 @@ let activeCleanups = 0; /** * Logs the cleanup of a resource. * @param resource - The resource that is being cleaned up. - * @param fn - The cleanup function. + * @param promise - The cleanup promise to await.. */ -async function logCleanup(resource: string, fn: () => Promise) { +async function logCleanup(resource: string, promise: Promise) { const start = new Date(); const order = cleanupOrder++; const activeAtStart = ++activeCleanups; let error: unknown = null; try { - await fn(); + await promise; } catch (err) { error = err instanceof Error ? err.message : String(err); } @@ -58,11 +58,11 @@ async function logCleanup(resource: string, fn: () => Promise) { console.log( JSON.stringify({ order, + resource, + durationMs: end.getTime() - start.getTime(), start: start.toISOString(), end: end.toISOString(), parallel, - resource, - durationMs: end.getTime() - start.getTime(), error, activeAtStart, activeAtEnd, @@ -76,9 +76,7 @@ const network = async ({}, use: Use) => { await use(network); } finally { // Make sure to stop the network after use - await logCleanup("network", async () => { - await network.stop(); - }); + await logCleanup("network", network.stop()); } }; @@ -92,9 +90,7 @@ const postgresContainer = async ( } finally { // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await logCleanup("postgresContainer", async () => { - await container.stop({ timeout: 30 }); - }); + await logCleanup("postgresContainer", container.stop({ timeout: 30 })); } }; @@ -116,9 +112,7 @@ const prisma = async ( try { await use(prisma); } finally { - await logCleanup("prisma", async () => { - await prisma.$disconnect(); - }); + await logCleanup("prisma", prisma.$disconnect()); } }; @@ -137,9 +131,7 @@ const redisContainer = async ( } finally { // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await logCleanup("redisContainer", async () => { - await container.stop({ timeout: 30 }); - }); + await logCleanup("redisContainer", container.stop({ timeout: 30 })); } }; @@ -191,9 +183,7 @@ const electricOrigin = async ( } finally { // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await logCleanup("electricContainer", async () => { - await container.stop({ timeout: 30 }); - }); + await logCleanup("electricContainer", container.stop({ timeout: 30 })); } }; From 8f4ebdd1f2118d1c8ca618585c334924635fa37f Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 2 May 2025 23:51:46 +0100 Subject: [PATCH 04/19] decrease docker network size so we can have more of them --- .github/workflows/unit-tests.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e638ac78746..f71e8216b28 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -8,6 +8,24 @@ jobs: name: "๐Ÿงช Unit Tests" runs-on: ubuntu-latest steps: + - name: ๐Ÿ”ง Configure docker address pool + run: | + CONFIG='{ + "default-address-pools" : [ + { + "base" : "172.17.0.0/12", + "size" : 20 + }, + { + "base" : "192.168.0.0/16", + "size" : 24 + } + ] + }' + mkdir -p /etc/docker + echo "$CONFIG" | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: โฌ‡๏ธ Checkout repo uses: actions/checkout@v4 with: From 5595852d2027bf43f0e2fa2e0ee68acd51ed2560 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Sat, 3 May 2025 00:01:07 +0100 Subject: [PATCH 05/19] add a test flow to check this all works --- .github/workflows/testcontainers.yml | 77 ++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/workflows/testcontainers.yml diff --git a/.github/workflows/testcontainers.yml b/.github/workflows/testcontainers.yml new file mode 100644 index 00000000000..53d8b2f6170 --- /dev/null +++ b/.github/workflows/testcontainers.yml @@ -0,0 +1,77 @@ +name: "๐Ÿณ Testcontainers" + +on: + workflow_call: + workflow_dispatch: + push: + +jobs: + unitTests: + name: "๐Ÿงช Unit Tests (run ${{ matrix.run }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] + steps: + - name: ๐Ÿ”ง Configure docker address pool + run: | + CONFIG='{ + "default-address-pools" : [ + { + "base" : "172.17.0.0/12", + "size" : 20 + }, + { + "base" : "192.168.0.0/16", + "size" : 24 + } + ] + }' + mkdir -p /etc/docker + echo "$CONFIG" | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 8.15.5 + + - name: โŽ” Setup node + uses: buildjet/setup-node@v4 + with: + node-version: 20.11.1 + cache: "pnpm" + + # ..to avoid rate limits when pulling images + - name: ๐Ÿณ Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ“€ Generate Prisma Client + run: pnpm run generate + + - name: ๐Ÿงช Run Webapp Unit Tests + run: pnpm run test:webapp + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres + SESSION_SECRET: "secret" + MAGIC_LINK_SECRET: "secret" + ENCRYPTION_KEY: "secret" + + - name: ๐Ÿงช Run Package Unit Tests + run: pnpm run test:packages + + - name: ๐Ÿงช Run Internal Unit Tests + run: pnpm run test:internal From 9bf7a19dd582f0ad5adc570feaaf766423e0580b Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Sat, 3 May 2025 19:12:23 +0100 Subject: [PATCH 06/19] await all engine.quit calls --- .../src/engine/tests/attemptFailures.test.ts | 12 ++++++------ .../src/engine/tests/batchTrigger.test.ts | 2 +- .../src/engine/tests/batchTriggerAndWait.test.ts | 4 ++-- .../src/engine/tests/cancelling.test.ts | 4 ++-- .../src/engine/tests/checkpoints.test.ts | 2 +- .../run-engine/src/engine/tests/delays.test.ts | 8 ++++---- .../src/engine/tests/dequeuing.test.ts | 4 ++-- .../src/engine/tests/pendingVersion.test.ts | 4 ++-- .../run-engine/src/engine/tests/priority.test.ts | 4 ++-- .../run-engine/src/engine/tests/trigger.test.ts | 4 ++-- .../src/engine/tests/triggerAndWait.test.ts | 4 ++-- .../run-engine/src/engine/tests/ttl.test.ts | 2 +- .../src/engine/tests/waitpoints.test.ts | 16 ++++++++-------- 13 files changed, 35 insertions(+), 35 deletions(-) diff --git a/internal-packages/run-engine/src/engine/tests/attemptFailures.test.ts b/internal-packages/run-engine/src/engine/tests/attemptFailures.test.ts index 7381a6d908b..7bc5122f7e4 100644 --- a/internal-packages/run-engine/src/engine/tests/attemptFailures.test.ts +++ b/internal-packages/run-engine/src/engine/tests/attemptFailures.test.ts @@ -155,7 +155,7 @@ describe("RunEngine attempt failures", () => { expect(executionData4.run.attemptNumber).toBe(2); expect(executionData4.run.status).toBe("COMPLETED_SUCCESSFULLY"); } finally { - engine.quit(); + await engine.quit(); } }); @@ -266,7 +266,7 @@ describe("RunEngine attempt failures", () => { expect(executionData3.run.attemptNumber).toBe(1); expect(executionData3.run.status).toBe("COMPLETED_WITH_ERRORS"); } finally { - engine.quit(); + await engine.quit(); } }); @@ -375,7 +375,7 @@ describe("RunEngine attempt failures", () => { expect(executionData3.run.attemptNumber).toBe(1); expect(executionData3.run.status).toBe("CRASHED"); } finally { - engine.quit(); + await engine.quit(); } }); @@ -482,7 +482,7 @@ describe("RunEngine attempt failures", () => { expect(executionData.run.attemptNumber).toBe(1); expect(executionData.run.status).toBe("CRASHED"); } finally { - engine.quit(); + await engine.quit(); } }); @@ -639,7 +639,7 @@ describe("RunEngine attempt failures", () => { expect(executionData4.run.attemptNumber).toBe(2); expect(executionData4.run.status).toBe("COMPLETED_SUCCESSFULLY"); } finally { - engine.quit(); + await engine.quit(); } }); @@ -803,7 +803,7 @@ describe("RunEngine attempt failures", () => { expect(finalExecutionData.run.attemptNumber).toBe(2); expect(finalExecutionData.run.status).toBe("CRASHED"); } finally { - engine.quit(); + await engine.quit(); } }); }); diff --git a/internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts b/internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts index bcaa3a59f69..7b6626bcf93 100644 --- a/internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts +++ b/internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts @@ -177,7 +177,7 @@ describe("RunEngine batchTrigger", () => { }); expect(batchAfter2?.status).toBe("COMPLETED"); } finally { - engine.quit(); + await engine.quit(); } }); }); diff --git a/internal-packages/run-engine/src/engine/tests/batchTriggerAndWait.test.ts b/internal-packages/run-engine/src/engine/tests/batchTriggerAndWait.test.ts index 36deab46987..58ea7244ab0 100644 --- a/internal-packages/run-engine/src/engine/tests/batchTriggerAndWait.test.ts +++ b/internal-packages/run-engine/src/engine/tests/batchTriggerAndWait.test.ts @@ -352,7 +352,7 @@ describe("RunEngine batchTriggerAndWait", () => { }); expect(batchAfter?.status === "COMPLETED"); } finally { - engine.quit(); + await engine.quit(); } }); @@ -570,7 +570,7 @@ describe("RunEngine batchTriggerAndWait", () => { ); expect(parentAfterTriggerAndWait.batch).toBeUndefined(); } finally { - engine.quit(); + await engine.quit(); } } ); diff --git a/internal-packages/run-engine/src/engine/tests/cancelling.test.ts b/internal-packages/run-engine/src/engine/tests/cancelling.test.ts index d4792d0b0cf..91702faba77 100644 --- a/internal-packages/run-engine/src/engine/tests/cancelling.test.ts +++ b/internal-packages/run-engine/src/engine/tests/cancelling.test.ts @@ -220,7 +220,7 @@ describe("RunEngine cancelling", () => { ); expect(envConcurrencyCompleted).toBe(0); } finally { - engine.quit(); + await engine.quit(); } } ); @@ -321,7 +321,7 @@ describe("RunEngine cancelling", () => { ); expect(envConcurrencyCompleted).toBe(0); } finally { - engine.quit(); + await engine.quit(); } }); diff --git a/internal-packages/run-engine/src/engine/tests/checkpoints.test.ts b/internal-packages/run-engine/src/engine/tests/checkpoints.test.ts index 88d116b0b20..d9fcd5da8cb 100644 --- a/internal-packages/run-engine/src/engine/tests/checkpoints.test.ts +++ b/internal-packages/run-engine/src/engine/tests/checkpoints.test.ts @@ -1375,7 +1375,7 @@ describe("RunEngine checkpoints", () => { }); expect(batchAfter?.status === "COMPLETED"); } finally { - engine.quit(); + await engine.quit(); } }); }); diff --git a/internal-packages/run-engine/src/engine/tests/delays.test.ts b/internal-packages/run-engine/src/engine/tests/delays.test.ts index 7b48859b559..cf131f55ad4 100644 --- a/internal-packages/run-engine/src/engine/tests/delays.test.ts +++ b/internal-packages/run-engine/src/engine/tests/delays.test.ts @@ -86,7 +86,7 @@ describe("RunEngine delays", () => { assertNonNullable(executionData2); expect(executionData2.snapshot.executionStatus).toBe("QUEUED"); } finally { - engine.quit(); + await engine.quit(); } }); @@ -183,7 +183,7 @@ describe("RunEngine delays", () => { assertNonNullable(executionData3); expect(executionData3.snapshot.executionStatus).toBe("QUEUED"); } finally { - engine.quit(); + await engine.quit(); } }); @@ -287,7 +287,7 @@ describe("RunEngine delays", () => { expect(run3.status).toBe("EXPIRED"); } finally { - engine.quit(); + await engine.quit(); } }); @@ -398,7 +398,7 @@ describe("RunEngine delays", () => { expect(executionData4.snapshot.executionStatus).toBe("FINISHED"); expect(executionData4.run.status).toBe("CANCELED"); } finally { - engine.quit(); + await engine.quit(); } }); }); diff --git a/internal-packages/run-engine/src/engine/tests/dequeuing.test.ts b/internal-packages/run-engine/src/engine/tests/dequeuing.test.ts index 6d2f79053fd..c0d269017fe 100644 --- a/internal-packages/run-engine/src/engine/tests/dequeuing.test.ts +++ b/internal-packages/run-engine/src/engine/tests/dequeuing.test.ts @@ -77,7 +77,7 @@ describe("RunEngine dequeuing", () => { expect(dequeued.length).toBe(5); } finally { - engine.quit(); + await engine.quit(); } }); @@ -169,7 +169,7 @@ describe("RunEngine dequeuing", () => { const queueLength3 = await engine.runQueue.lengthOfEnvQueue(authenticatedEnvironment); expect(queueLength3).toBe(12); } finally { - engine.quit(); + await engine.quit(); } } ); diff --git a/internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts b/internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts index 2164920cfc9..c3fa33eba18 100644 --- a/internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts +++ b/internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts @@ -158,7 +158,7 @@ describe("RunEngine pending version", () => { ); expect(queueLength2).toBe(2); } finally { - engine.quit(); + await engine.quit(); } } ); @@ -319,7 +319,7 @@ describe("RunEngine pending version", () => { ); expect(queueLength3).toBe(1); } finally { - engine.quit(); + await engine.quit(); } } ); diff --git a/internal-packages/run-engine/src/engine/tests/priority.test.ts b/internal-packages/run-engine/src/engine/tests/priority.test.ts index c5bb40788ea..6f31f9df7d4 100644 --- a/internal-packages/run-engine/src/engine/tests/priority.test.ts +++ b/internal-packages/run-engine/src/engine/tests/priority.test.ts @@ -103,7 +103,7 @@ describe("RunEngine priority", () => { expect(dequeue2.length).toBe(1); expect(dequeue2[0].run.friendlyId).toBe(runs[2].friendlyId); } finally { - engine.quit(); + await engine.quit(); } } ); @@ -197,7 +197,7 @@ describe("RunEngine priority", () => { expect(dequeue[3].run.friendlyId).toBe(runs[4].friendlyId); expect(dequeue[4].run.friendlyId).toBe(runs[0].friendlyId); } finally { - engine.quit(); + await engine.quit(); } } ); diff --git a/internal-packages/run-engine/src/engine/tests/trigger.test.ts b/internal-packages/run-engine/src/engine/tests/trigger.test.ts index 2736b27b0af..2716cf3df12 100644 --- a/internal-packages/run-engine/src/engine/tests/trigger.test.ts +++ b/internal-packages/run-engine/src/engine/tests/trigger.test.ts @@ -198,7 +198,7 @@ describe("RunEngine trigger()", () => { expect(runWaitpointAfter[0].type).toBe("RUN"); expect(runWaitpointAfter[0].output).toBe(`{"foo":"bar"}`); } finally { - engine.quit(); + await engine.quit(); } }); @@ -325,7 +325,7 @@ describe("RunEngine trigger()", () => { expect(output.type).toBe(error.type); expect(runWaitpointAfter[0].outputIsError).toBe(true); } finally { - engine.quit(); + await engine.quit(); } }); }); diff --git a/internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts b/internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts index 911b1416224..fe806168bb0 100644 --- a/internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts +++ b/internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts @@ -189,7 +189,7 @@ describe("RunEngine triggerAndWait", () => { ); expect(parentExecutionDataAfter.completedWaitpoints![0].output).toBe('{"foo":"bar"}'); } finally { - engine.quit(); + await engine.quit(); } }); @@ -445,7 +445,7 @@ describe("RunEngine triggerAndWait", () => { ); expect(parent2ExecutionDataAfter.completedWaitpoints![0].output).toBe('{"foo":"bar"}'); } finally { - engine.quit(); + await engine.quit(); } } ); diff --git a/internal-packages/run-engine/src/engine/tests/ttl.test.ts b/internal-packages/run-engine/src/engine/tests/ttl.test.ts index 2643f5cae72..0ede60fbfde 100644 --- a/internal-packages/run-engine/src/engine/tests/ttl.test.ts +++ b/internal-packages/run-engine/src/engine/tests/ttl.test.ts @@ -102,7 +102,7 @@ describe("RunEngine ttl", () => { ); expect(envConcurrencyCompleted).toBe(0); } finally { - engine.quit(); + await engine.quit(); } }); }); diff --git a/internal-packages/run-engine/src/engine/tests/waitpoints.test.ts b/internal-packages/run-engine/src/engine/tests/waitpoints.test.ts index 3a8446edea7..3e4ae20afaa 100644 --- a/internal-packages/run-engine/src/engine/tests/waitpoints.test.ts +++ b/internal-packages/run-engine/src/engine/tests/waitpoints.test.ts @@ -121,7 +121,7 @@ describe("RunEngine Waitpoints", () => { const executionDataAfter = await engine.getRunExecutionData({ runId: run.id }); expect(executionDataAfter?.snapshot.executionStatus).toBe("EXECUTING"); } finally { - engine.quit(); + await engine.quit(); } }); @@ -261,7 +261,7 @@ describe("RunEngine Waitpoints", () => { }); expect(runWaitpoint).toBeNull(); } finally { - engine.quit(); + await engine.quit(); } }); @@ -400,7 +400,7 @@ describe("RunEngine Waitpoints", () => { }); expect(runWaitpoint).toBeNull(); } finally { - engine.quit(); + await engine.quit(); } } ); @@ -516,7 +516,7 @@ describe("RunEngine Waitpoints", () => { }); expect(runWaitpoint).toBeNull(); } finally { - engine.quit(); + await engine.quit(); } }); @@ -664,7 +664,7 @@ describe("RunEngine Waitpoints", () => { expect(runWaitpoints.length).toBe(0); } } finally { - engine.quit(); + await engine.quit(); } } ); @@ -814,7 +814,7 @@ describe("RunEngine Waitpoints", () => { const isTimeout = isWaitpointOutputTimeout(waitpoint2.output); expect(isTimeout).toBe(true); } finally { - engine.quit(); + await engine.quit(); } } ); @@ -966,7 +966,7 @@ describe("RunEngine Waitpoints", () => { expect(waitpoint2.status).toBe("COMPLETED"); expect(waitpoint2.outputIsError).toBe(false); } finally { - engine.quit(); + await engine.quit(); } }); @@ -1126,7 +1126,7 @@ describe("RunEngine Waitpoints", () => { expect(waitpoint2.status).toBe("COMPLETED"); expect(waitpoint2.outputIsError).toBe(false); } finally { - engine.quit(); + await engine.quit(); } }); From cc62dbd1fa36eaf856bf67ecbd9dd2ce5a3fa47d Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Sat, 3 May 2025 23:15:50 +0100 Subject: [PATCH 07/19] add docker diagnostics --- internal-packages/testcontainers/package.json | 1 + internal-packages/testcontainers/src/index.ts | 150 +++++++++++++++++- pnpm-lock.yaml | 13 +- 3 files changed, 160 insertions(+), 4 deletions(-) diff --git a/internal-packages/testcontainers/package.json b/internal-packages/testcontainers/package.json index 5edce5836be..ab41c7c4a37 100644 --- a/internal-packages/testcontainers/package.json +++ b/internal-packages/testcontainers/package.json @@ -13,6 +13,7 @@ "@testcontainers/postgresql": "^10.25.0", "@testcontainers/redis": "^10.25.0", "@trigger.dev/core": "workspace:*", + "std-env": "^3.9.0", "testcontainers": "^10.25.0", "tinyexec": "^0.3.0", "vitest": "^1.4.0" diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index 1027f133ed9..f6db5bd37b6 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -5,6 +5,8 @@ import { RedisOptions } from "ioredis"; import { Network, type StartedNetwork } from "testcontainers"; import { test } from "vitest"; import { createElectricContainer, createPostgresContainer, createRedisContainer } from "./utils"; +import { x } from "tinyexec"; +import { isCI } from "std-env"; export { assertNonNullable } from "./utils"; export { StartedRedisContainer }; @@ -45,6 +47,7 @@ async function logCleanup(resource: string, promise: Promise) { const activeAtStart = ++activeCleanups; let error: unknown = null; + try { await promise; } catch (err) { @@ -52,24 +55,169 @@ async function logCleanup(resource: string, promise: Promise) { } const end = new Date(); + const durationMs = end.getTime() - start.getTime(); const activeAtEnd = --activeCleanups; const parallel = activeAtStart > 1 || activeAtEnd > 0; + if (!isCI) { + return; + } + + let dockerDiagnostics: DockerDiagnostics = {}; + + // Only run docker diagnostics if there was an error or cleanup took longer than 5s + if (error || durationMs > 5000) { + try { + dockerDiagnostics = await getDockerDiagnostics(); + } catch (diagnosticErr) { + console.error("Failed to get docker diagnostics:", diagnosticErr); + } + } + console.log( JSON.stringify({ order, resource, - durationMs: end.getTime() - start.getTime(), + durationMs, start: start.toISOString(), end: end.toISOString(), parallel, error, activeAtStart, activeAtEnd, + ...dockerDiagnostics, }) ); } +function stringToLines(str: string): string[] { + return str.split("\n").filter(Boolean); +} + +async function getDockerNetworks(): Promise { + try { + const result = await x("docker", ["network", "ls" /* , "--no-trunc" */]); + return stringToLines(result.stdout); + } catch (error) { + console.error(error); + return ["error: check additional logs for more details"]; + } +} + +async function getDockerContainers(): Promise { + try { + const result = await x("docker", ["ps", "-a" /* , "--no-trunc" */]); + return stringToLines(result.stdout); + } catch (error) { + console.error(error); + return ["error: check additional logs for more details"]; + } +} + +type DockerNetworkAttachment = { + networkId: string; + networkName: string; + containers: string[]; +}; + +export async function getDockerNetworkAttachments(): Promise { + let attachments: DockerNetworkAttachment[] = []; + let networkIds: string[] = []; + + try { + const result = await x("docker", ["network", "ls", "-q"]); + networkIds = stringToLines(result.stdout); + } catch (err) { + console.error("Failed to list docker networks:", err); + } + + for (const networkId of networkIds) { + try { + const inspectResult = await x("docker", [ + "network", + "inspect", + "--format", + '{{ .Name }}{{ range $k, $v := .Containers }} {{ printf "%.12s %s" $k .Name }}{{ end }}', + networkId, + ]); + + const [networkName, ...containers] = inspectResult.stdout.trim().split(/\s+/); + attachments.push({ networkId, networkName, containers }); + } catch (err) { + console.error(`Failed to inspect network ${networkId}:`, err); + attachments.push({ networkId, networkName: String(err), containers: [] }); + } + } + + return attachments; +} + +type DockerContainerNetwork = { + containerId: string; + containerName: string; + networks: string[]; +}; + +export async function getDockerContainerNetworks(): Promise { + let results: DockerContainerNetwork[] = []; + let containers: string[] = []; + + try { + const result = await x("docker", [ + "ps", + "-a", + "--format", + '{{.ID | printf "%.12s"}} {{.Names}}', + ]); + containers = stringToLines(result.stdout); + } catch (err) { + console.error("Failed to list docker containers:", err); + } + + for (const [containerId, containerName] of containers.map((c) => c.trim().split(/\s+/))) { + try { + const inspectResult = await x("docker", [ + "inspect", + "--format", + "{{ range $k, $v := .NetworkSettings.Networks }}{{ $k }}{{ end }}", + containerId, + ]); + + const networks = inspectResult.stdout.trim().split(/\s+/); + + results.push({ containerId, containerName, networks }); + } catch (err) { + console.error(`Failed to inspect container ${containerId}:`, err); + results.push({ containerId, containerName: String(err), networks: [] }); + } + } + + return results; +} + +type DockerDiagnostics = { + containers?: string[]; + networks?: string[]; + containerNetworks?: DockerContainerNetwork[]; + networkAttachments?: DockerNetworkAttachment[]; +}; + +async function getDockerDiagnostics(): Promise { + const [containers, networks, networkAttachments, containerNetworks] = await Promise.all([ + getDockerContainers(), + getDockerNetworks(), + getDockerNetworkAttachments(), + getDockerContainerNetworks(), + ]); + + return { + containers, + networks, + containerNetworks, + networkAttachments, + }; +} + const network = async ({}, use: Use) => { const network = await new Network().start(); try { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a2fb3e1f6ce..ce52e7c6c83 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1014,6 +1014,9 @@ importers: '@trigger.dev/core': specifier: workspace:* version: link:../../packages/core + std-env: + specifier: ^3.9.0 + version: 3.9.0 testcontainers: specifier: ^10.25.0 version: 10.25.0 @@ -20358,12 +20361,12 @@ packages: /bare-events@2.4.2: resolution: {integrity: sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==} requiresBuild: true + dev: false optional: true /bare-events@2.5.4: resolution: {integrity: sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==} requiresBuild: true - dev: true optional: true /bare-fs@2.3.5: @@ -32623,6 +32626,10 @@ packages: /std-env@3.8.1: resolution: {integrity: sha512-vj5lIj3Mwf9D79hBkltk5qmkFI+biIKWS2IBxEyEU3AX1tUf7AoL8nSazCOiiqQsGKIq01SClsKEzweu34uwvA==} + /std-env@3.9.0: + resolution: {integrity: sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==} + dev: true + /stoppable@1.1.0: resolution: {integrity: sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==} engines: {node: '>=4', npm: '>=6'} @@ -32658,7 +32665,7 @@ packages: queue-tick: 1.0.1 text-decoder: 1.2.0 optionalDependencies: - bare-events: 2.4.2 + bare-events: 2.5.4 /streamx@2.22.0: resolution: {integrity: sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==} @@ -35369,7 +35376,7 @@ packages: magic-string: 0.30.17 pathe: 1.1.2 picocolors: 1.1.1 - std-env: 3.8.1 + std-env: 3.9.0 strip-literal: 2.1.0 tinybench: 2.9.0 tinypool: 0.8.3 From 9a53fedd520409cc89dfb643c976f7ccc3b3ec1e Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Sat, 3 May 2025 23:16:11 +0100 Subject: [PATCH 08/19] reduce number of test runs --- .github/workflows/testcontainers.yml | 2 +- .github/workflows/unit-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testcontainers.yml b/.github/workflows/testcontainers.yml index 53d8b2f6170..a8ccbd7e2b4 100644 --- a/.github/workflows/testcontainers.yml +++ b/.github/workflows/testcontainers.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] + run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: - name: ๐Ÿ”ง Configure docker address pool run: | diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index f71e8216b28..df4dce01820 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -8,7 +8,7 @@ jobs: name: "๐Ÿงช Unit Tests" runs-on: ubuntu-latest steps: - - name: ๐Ÿ”ง Configure docker address pool + - name: ๐Ÿ”ง Configure docker run: | CONFIG='{ "default-address-pools" : [ From c2a5ea381ebaba55dc87e24743d274b9c6ad6052 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 00:29:28 +0100 Subject: [PATCH 09/19] improve network attachment output --- internal-packages/testcontainers/src/index.ts | 75 ++++++++++++------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index f6db5bd37b6..a924548bb54 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -94,6 +94,10 @@ function stringToLines(str: string): string[] { return str.split("\n").filter(Boolean); } +function lineToWords(line: string): string[] { + return line.trim().split(/\s+/); +} + async function getDockerNetworks(): Promise { try { const result = await x("docker", ["network", "ls" /* , "--no-trunc" */]); @@ -114,53 +118,68 @@ async function getDockerContainers(): Promise { } } -type DockerNetworkAttachment = { - networkId: string; - networkName: string; - containers: string[]; +type DockerResource = { id: string; name: string }; + +type DockerNetworkAttachment = DockerResource & { + containers: DockerResource[]; }; export async function getDockerNetworkAttachments(): Promise { let attachments: DockerNetworkAttachment[] = []; - let networkIds: string[] = []; + let networks: DockerResource[] = []; try { - const result = await x("docker", ["network", "ls", "-q"]); - networkIds = stringToLines(result.stdout); + const result = await x("docker", [ + "network", + "ls", + "--format", + '{{.ID | printf "%.12s"}} {{.Name}}', + ]); + + const lines = stringToLines(result.stdout); + + networks = lines.map((line) => { + const [id, name] = lineToWords(line); + return { id, name }; + }); } catch (err) { console.error("Failed to list docker networks:", err); } - for (const networkId of networkIds) { + for (const { id, name } of networks) { try { - const inspectResult = await x("docker", [ + // Get containers, one per line: id name\n + const containersResult = await x("docker", [ "network", "inspect", "--format", - '{{ .Name }}{{ range $k, $v := .Containers }} {{ printf "%.12s %s" $k .Name }}{{ end }}', - networkId, + "{{range $k, $v := .Containers}}{{$k}} {{$v.Name}}\n{{end}}", + id, ]); + const lines = stringToLines(containersResult.stdout); - const [networkName, ...containers] = inspectResult.stdout.trim().split(/\s+/); - attachments.push({ networkId, networkName, containers }); + const containers: DockerResource[] = lines.map((line) => { + const [id, name] = lineToWords(line); + return { id, name }; + }); + + attachments.push({ id, name, containers }); } catch (err) { - console.error(`Failed to inspect network ${networkId}:`, err); - attachments.push({ networkId, networkName: String(err), containers: [] }); + console.error(`Failed to inspect network ${id}:`, err); + attachments.push({ id, name, containers: [] }); } } return attachments; } -type DockerContainerNetwork = { - containerId: string; - containerName: string; +type DockerContainerNetwork = DockerResource & { networks: string[]; }; export async function getDockerContainerNetworks(): Promise { let results: DockerContainerNetwork[] = []; - let containers: string[] = []; + let containers: DockerResource[] = []; try { const result = await x("docker", [ @@ -169,26 +188,32 @@ export async function getDockerContainerNetworks(): Promise { + const [id, name] = lineToWords(line); + return { id, name }; + }); } catch (err) { console.error("Failed to list docker containers:", err); } - for (const [containerId, containerName] of containers.map((c) => c.trim().split(/\s+/))) { + for (const { id, name } of containers) { try { const inspectResult = await x("docker", [ "inspect", "--format", "{{ range $k, $v := .NetworkSettings.Networks }}{{ $k }}{{ end }}", - containerId, + id, ]); const networks = inspectResult.stdout.trim().split(/\s+/); - results.push({ containerId, containerName, networks }); + results.push({ id, name, networks }); } catch (err) { - console.error(`Failed to inspect container ${containerId}:`, err); - results.push({ containerId, containerName: String(err), networks: [] }); + console.error(`Failed to inspect container ${id}:`, err); + results.push({ id, name: String(err), networks: [] }); } } From a55c49eaa147d2f7ab568a54adf862dd3d6366b3 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 00:30:04 +0100 Subject: [PATCH 10/19] add setup logs --- internal-packages/testcontainers/src/index.ts | 137 +++++++++++++++--- 1 file changed, 114 insertions(+), 23 deletions(-) diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index a924548bb54..debbd1eec18 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -3,10 +3,10 @@ import { StartedRedisContainer } from "@testcontainers/redis"; import { PrismaClient } from "@trigger.dev/database"; import { RedisOptions } from "ioredis"; import { Network, type StartedNetwork } from "testcontainers"; -import { test } from "vitest"; +import { TaskContext, test } from "vitest"; import { createElectricContainer, createPostgresContainer, createRedisContainer } from "./utils"; import { x } from "tinyexec"; -import { isCI } from "std-env"; +import { isCI, env } from "std-env"; export { assertNonNullable } from "./utils"; export { StartedRedisContainer }; @@ -41,7 +41,11 @@ let activeCleanups = 0; * @param resource - The resource that is being cleaned up. * @param promise - The cleanup promise to await.. */ -async function logCleanup(resource: string, promise: Promise) { +async function logCleanup( + resource: string, + promise: Promise, + metadata: Record = {} +) { const start = new Date(); const order = cleanupOrder++; const activeAtStart = ++activeCleanups; @@ -66,7 +70,7 @@ async function logCleanup(resource: string, promise: Promise) { let dockerDiagnostics: DockerDiagnostics = {}; // Only run docker diagnostics if there was an error or cleanup took longer than 5s - if (error || durationMs > 5000) { + if (error || durationMs > 5000 || env.DOCKER_DIAGNOSTICS) { try { dockerDiagnostics = await getDockerDiagnostics(); } catch (diagnosticErr) { @@ -85,6 +89,7 @@ async function logCleanup(resource: string, promise: Promise) { error, activeAtStart, activeAtEnd, + ...metadata, ...dockerDiagnostics, }) ); @@ -121,7 +126,7 @@ async function getDockerContainers(): Promise { type DockerResource = { id: string; name: string }; type DockerNetworkAttachment = DockerResource & { - containers: DockerResource[]; + containers: string[]; }; export async function getDockerNetworkAttachments(): Promise { @@ -153,15 +158,11 @@ export async function getDockerNetworkAttachments(): Promise { - const [id, name] = lineToWords(line); - return { id, name }; - }); + const containers = stringToLines(containersResult.stdout); attachments.push({ id, name, containers }); } catch (err) { @@ -204,11 +205,11 @@ export async function getDockerContainerNetworks(): Promise { }; } -const network = async ({}, use: Use) => { +const network = async ({ task }: TaskContext, use: Use) => { + const testName = task.name; + + logSetup("network: starting", { testName }); + + const start = Date.now(); const network = await new Network().start(); + const startDurationMs = Date.now() - start; + + const metadata = { + testName, + networkId: network.getId().slice(0, 12), + networkName: network.getName(), + startDurationMs, + }; + + logSetup("network: started", metadata); + try { await use(network); } finally { // Make sure to stop the network after use - await logCleanup("network", network.stop()); + await logCleanup("network", network.stop(), metadata); } }; const postgresContainer = async ( - { network }: { network: StartedNetwork }, + { network, task }: { network: StartedNetwork } & TaskContext, use: Use ) => { + const testName = task.name; + + logSetup("postgresContainer: starting", { testName }); + + const start = Date.now(); const { container } = await createPostgresContainer(network); + const startDurationMs = Date.now() - start; + + const metadata = { + testName, + containerId: container.getId().slice(0, 12), + containerName: container.getName(), + containerNetworkNames: container.getNetworkNames(), + startDurationMs, + }; + + logSetup("postgresContainer: started", metadata); + try { await use(container); } finally { // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await logCleanup("postgresContainer", container.stop({ timeout: 30 })); + await logCleanup("postgresContainer", container.stop({ timeout: 30 }), metadata); } }; const prisma = async ( - { postgresContainer }: { postgresContainer: StartedPostgreSqlContainer }, + { postgresContainer, task }: { postgresContainer: StartedPostgreSqlContainer } & TaskContext, use: Use ) => { + const testName = task.name; const url = postgresContainer.getConnectionUri(); console.log("Initializing Prisma with URL:", url); @@ -285,26 +320,65 @@ const prisma = async ( try { await use(prisma); } finally { - await logCleanup("prisma", prisma.$disconnect()); + await logCleanup("prisma", prisma.$disconnect(), { testName }); } }; export const postgresTest = test.extend({ network, postgresContainer, prisma }); +let setupOrder = 0; + +function logSetup(resource: string, metadata: Record) { + const order = setupOrder++; + + if (!isCI) { + return; + } + + console.log( + JSON.stringify({ + type: "setup", + order, + resource, + timestamp: new Date().toISOString(), + ...metadata, + }) + ); +} + const redisContainer = async ( - { network }: { network: StartedNetwork }, + { network, task }: { network: StartedNetwork } & TaskContext, use: Use ) => { + const testName = task.name; + + logSetup("redisContainer: starting", { testName }); + + const start = Date.now(); + const { container } = await createRedisContainer({ port: 6379, network, }); + + const startDurationMs = Date.now() - start; + + const metadata = { + containerName: container.getName(), + containerId: container.getId().slice(0, 12), + containerNetworkNames: container.getNetworkNames(), + startDurationMs, + testName, + }; + + logSetup("redisContainer: started", metadata); + try { await use(container); } finally { // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await logCleanup("redisContainer", container.stop({ timeout: 30 })); + await logCleanup("redisContainer", container.stop({ timeout: 30 }), metadata); } }; @@ -347,16 +421,33 @@ const electricOrigin = async ( { postgresContainer, network, - }: { postgresContainer: StartedPostgreSqlContainer; network: StartedNetwork }, + task, + }: { postgresContainer: StartedPostgreSqlContainer; network: StartedNetwork } & TaskContext, use: Use ) => { + const testName = task.name; + + logSetup("electricOrigin: starting", { testName }); + + const start = Date.now(); const { origin, container } = await createElectricContainer(postgresContainer, network); + const startDurationMs = Date.now() - start; + + const metadata = { + testName, + containerId: container.getId().slice(0, 12), + containerName: container.getName(), + startDurationMs, + }; + + logSetup("electricOrigin: started", metadata); + try { await use(origin); } finally { // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await logCleanup("electricContainer", container.stop({ timeout: 30 })); + await logCleanup("electricContainer", container.stop({ timeout: 30 }), metadata); } }; From f033f82a4b155c80624c5176c1d90a746dc555f7 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 00:30:32 +0100 Subject: [PATCH 11/19] log redis setup errors --- internal-packages/testcontainers/src/utils.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts index b770c7fceec..c47db301ea4 100644 --- a/internal-packages/testcontainers/src/utils.ts +++ b/internal-packages/testcontainers/src/utils.ts @@ -87,8 +87,14 @@ async function verifyRedisConnection(container: StartedRedisContainer) { }, }); + const containerMetadata = { + containerId: container.getId().slice(0, 12), + containerName: container.getName(), + containerNetworkNames: container.getNetworkNames(), + }; + redis.on("error", (error) => { - // swallow the error + console.log("verifyRedisConnection error", error, containerMetadata); }); try { From c66a3025841404f7dbca5adfc30bf7eb8bbc674a Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 00:32:52 +0100 Subject: [PATCH 12/19] add cleanup log type --- internal-packages/testcontainers/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index debbd1eec18..7e8f94a3557 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -80,6 +80,7 @@ async function logCleanup( console.log( JSON.stringify({ + type: "cleanup", order, resource, durationMs, From 2e4795546382b0f263d8d5dfde5ee0d50693834f Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 11:20:31 +0100 Subject: [PATCH 13/19] stop redis container if setup fails --- internal-packages/testcontainers/src/utils.ts | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts index c47db301ea4..5f71fefbae4 100644 --- a/internal-packages/testcontainers/src/utils.ts +++ b/internal-packages/testcontainers/src/utils.ts @@ -1,7 +1,9 @@ import { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql"; import { RedisContainer, StartedRedisContainer } from "@testcontainers/redis"; +import { tryCatch } from "@trigger.dev/core"; import Redis from "ioredis"; import path from "path"; +import { isDebug } from "std-env"; import { GenericContainer, StartedNetwork, Wait } from "testcontainers"; import { x } from "tinyexec"; import { expect } from "vitest"; @@ -67,7 +69,12 @@ export async function createRedisContainer({ .start(); // Add a verification step - await verifyRedisConnection(startedContainer); + const [error] = await tryCatch(verifyRedisConnection(startedContainer)); + + if (error) { + await startedContainer.stop({ timeout: 30 }); + throw new Error("verifyRedisConnection error", { cause: error }); + } return { container: startedContainer, @@ -94,11 +101,21 @@ async function verifyRedisConnection(container: StartedRedisContainer) { }; redis.on("error", (error) => { - console.log("verifyRedisConnection error", error, containerMetadata); + if (isDebug) { + console.log("verifyRedisConnection: client error", error, containerMetadata); + } + + // Don't throw here, we'll do that below if the ping fails }); try { await redis.ping(); + } catch (error) { + if (isDebug) { + console.log("verifyRedisConnection: ping error", error, containerMetadata); + } + + throw new Error("verifyRedisConnection: ping error", { cause: error }); } finally { await redis.quit(); } From f5672ab833f67ba4d11fcacd8d4860a68615c581 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 11:22:19 +0100 Subject: [PATCH 14/19] disable ipv6 --- .github/workflows/testcontainers.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testcontainers.yml b/.github/workflows/testcontainers.yml index a8ccbd7e2b4..9fb7df4f4c1 100644 --- a/.github/workflows/testcontainers.yml +++ b/.github/workflows/testcontainers.yml @@ -14,6 +14,12 @@ jobs: matrix: run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: + - name: ๐Ÿ”ง Disable IPv6 + run: | + sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1 + - name: ๐Ÿ”ง Configure docker address pool run: | CONFIG='{ @@ -30,7 +36,9 @@ jobs: }' mkdir -p /etc/docker echo "$CONFIG" | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker + + - name: ๐Ÿ”ง Restart docker + run: sudo systemctl restart docker - name: โฌ‡๏ธ Checkout repo uses: actions/checkout@v4 From 19b749d0906cb78b594967556700c8105b7da88c Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 12:22:00 +0100 Subject: [PATCH 15/19] tidy things up a bit --- .../testcontainers/src/docker.ts | 150 ++++++++ internal-packages/testcontainers/src/index.ts | 340 ++---------------- internal-packages/testcontainers/src/logs.ts | 101 ++++++ internal-packages/testcontainers/src/utils.ts | 59 ++- 4 files changed, 337 insertions(+), 313 deletions(-) create mode 100644 internal-packages/testcontainers/src/docker.ts create mode 100644 internal-packages/testcontainers/src/logs.ts diff --git a/internal-packages/testcontainers/src/docker.ts b/internal-packages/testcontainers/src/docker.ts new file mode 100644 index 00000000000..f9da9162d2a --- /dev/null +++ b/internal-packages/testcontainers/src/docker.ts @@ -0,0 +1,150 @@ +import { x } from "tinyexec"; + +function stringToLines(str: string): string[] { + return str.split("\n").filter(Boolean); +} + +function lineToWords(line: string): string[] { + return line.trim().split(/\s+/); +} + +async function getDockerNetworks(): Promise { + try { + const result = await x("docker", ["network", "ls" /* , "--no-trunc" */]); + return stringToLines(result.stdout); + } catch (error) { + console.error(error); + return ["error: check additional logs for more details"]; + } +} + +async function getDockerContainers(): Promise { + try { + const result = await x("docker", ["ps", "-a" /* , "--no-trunc" */]); + return stringToLines(result.stdout); + } catch (error) { + console.error(error); + return ["error: check additional logs for more details"]; + } +} + +type DockerResource = { id: string; name: string }; + +type DockerNetworkAttachment = DockerResource & { + containers: string[]; +}; + +export async function getDockerNetworkAttachments(): Promise { + let attachments: DockerNetworkAttachment[] = []; + let networks: DockerResource[] = []; + + try { + const result = await x("docker", [ + "network", + "ls", + "--format", + '{{.ID | printf "%.12s"}} {{.Name}}', + ]); + + const lines = stringToLines(result.stdout); + + networks = lines.map((line) => { + const [id, name] = lineToWords(line); + return { id, name }; + }); + } catch (err) { + console.error("Failed to list docker networks:", err); + } + + for (const { id, name } of networks) { + try { + // Get containers, one per line: id name\n + const containersResult = await x("docker", [ + "network", + "inspect", + "--format", + '{{range $k, $v := .Containers}}{{$k | printf "%.12s"}} {{$v.Name}}\n{{end}}', + id, + ]); + + const containers = stringToLines(containersResult.stdout); + + attachments.push({ id, name, containers }); + } catch (err) { + console.error(`Failed to inspect network ${id}:`, err); + attachments.push({ id, name, containers: [] }); + } + } + + return attachments; +} + +type DockerContainerNetwork = DockerResource & { + networks: string[]; +}; + +export async function getDockerContainerNetworks(): Promise { + let results: DockerContainerNetwork[] = []; + let containers: DockerResource[] = []; + + try { + const result = await x("docker", [ + "ps", + "-a", + "--format", + '{{.ID | printf "%.12s"}} {{.Names}}', + ]); + + const lines = stringToLines(result.stdout); + + containers = lines.map((line) => { + const [id, name] = lineToWords(line); + return { id, name }; + }); + } catch (err) { + console.error("Failed to list docker containers:", err); + } + + for (const { id, name } of containers) { + try { + const inspectResult = await x("docker", [ + "inspect", + "--format", + '{{ range $k, $v := .NetworkSettings.Networks }}{{ $k | printf "%.12s" }} {{ $v.Name }}\n{{ end }}', + id, + ]); + + const networks = stringToLines(inspectResult.stdout); + + results.push({ id, name, networks }); + } catch (err) { + console.error(`Failed to inspect container ${id}:`, err); + results.push({ id, name: String(err), networks: [] }); + } + } + + return results; +} + +export type DockerDiagnostics = { + containers?: string[]; + networks?: string[]; + containerNetworks?: DockerContainerNetwork[]; + networkAttachments?: DockerNetworkAttachment[]; +}; + +export async function getDockerDiagnostics(): Promise { + const [containers, networks, networkAttachments, containerNetworks] = await Promise.all([ + getDockerContainers(), + getDockerNetworks(), + getDockerNetworkAttachments(), + getDockerContainerNetworks(), + ]); + + return { + containers, + networks, + containerNetworks, + networkAttachments, + }; +} diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index 7e8f94a3557..ef36de754c3 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -4,9 +4,14 @@ import { PrismaClient } from "@trigger.dev/database"; import { RedisOptions } from "ioredis"; import { Network, type StartedNetwork } from "testcontainers"; import { TaskContext, test } from "vitest"; -import { createElectricContainer, createPostgresContainer, createRedisContainer } from "./utils"; -import { x } from "tinyexec"; -import { isCI, env } from "std-env"; +import { + createElectricContainer, + createPostgresContainer, + createRedisContainer, + useContainer, + withContainerSetup, +} from "./utils"; +import { getTaskMetadata, logCleanup, logSetup } from "./logs"; export { assertNonNullable } from "./utils"; export { StartedRedisContainer }; @@ -33,218 +38,6 @@ type ContainerWithElectricContext = NetworkContext & PostgresContext & ElectricC type Use = (value: T) => Promise; -let cleanupOrder = 0; -let activeCleanups = 0; - -/** - * Logs the cleanup of a resource. - * @param resource - The resource that is being cleaned up. - * @param promise - The cleanup promise to await.. - */ -async function logCleanup( - resource: string, - promise: Promise, - metadata: Record = {} -) { - const start = new Date(); - const order = cleanupOrder++; - const activeAtStart = ++activeCleanups; - - let error: unknown = null; - - try { - await promise; - } catch (err) { - error = err instanceof Error ? err.message : String(err); - } - - const end = new Date(); - const durationMs = end.getTime() - start.getTime(); - const activeAtEnd = --activeCleanups; - const parallel = activeAtStart > 1 || activeAtEnd > 0; - - if (!isCI) { - return; - } - - let dockerDiagnostics: DockerDiagnostics = {}; - - // Only run docker diagnostics if there was an error or cleanup took longer than 5s - if (error || durationMs > 5000 || env.DOCKER_DIAGNOSTICS) { - try { - dockerDiagnostics = await getDockerDiagnostics(); - } catch (diagnosticErr) { - console.error("Failed to get docker diagnostics:", diagnosticErr); - } - } - - console.log( - JSON.stringify({ - type: "cleanup", - order, - resource, - durationMs, - start: start.toISOString(), - end: end.toISOString(), - parallel, - error, - activeAtStart, - activeAtEnd, - ...metadata, - ...dockerDiagnostics, - }) - ); -} - -function stringToLines(str: string): string[] { - return str.split("\n").filter(Boolean); -} - -function lineToWords(line: string): string[] { - return line.trim().split(/\s+/); -} - -async function getDockerNetworks(): Promise { - try { - const result = await x("docker", ["network", "ls" /* , "--no-trunc" */]); - return stringToLines(result.stdout); - } catch (error) { - console.error(error); - return ["error: check additional logs for more details"]; - } -} - -async function getDockerContainers(): Promise { - try { - const result = await x("docker", ["ps", "-a" /* , "--no-trunc" */]); - return stringToLines(result.stdout); - } catch (error) { - console.error(error); - return ["error: check additional logs for more details"]; - } -} - -type DockerResource = { id: string; name: string }; - -type DockerNetworkAttachment = DockerResource & { - containers: string[]; -}; - -export async function getDockerNetworkAttachments(): Promise { - let attachments: DockerNetworkAttachment[] = []; - let networks: DockerResource[] = []; - - try { - const result = await x("docker", [ - "network", - "ls", - "--format", - '{{.ID | printf "%.12s"}} {{.Name}}', - ]); - - const lines = stringToLines(result.stdout); - - networks = lines.map((line) => { - const [id, name] = lineToWords(line); - return { id, name }; - }); - } catch (err) { - console.error("Failed to list docker networks:", err); - } - - for (const { id, name } of networks) { - try { - // Get containers, one per line: id name\n - const containersResult = await x("docker", [ - "network", - "inspect", - "--format", - '{{range $k, $v := .Containers}}{{$k | printf "%.12s"}} {{$v.Name}}\n{{end}}', - id, - ]); - - const containers = stringToLines(containersResult.stdout); - - attachments.push({ id, name, containers }); - } catch (err) { - console.error(`Failed to inspect network ${id}:`, err); - attachments.push({ id, name, containers: [] }); - } - } - - return attachments; -} - -type DockerContainerNetwork = DockerResource & { - networks: string[]; -}; - -export async function getDockerContainerNetworks(): Promise { - let results: DockerContainerNetwork[] = []; - let containers: DockerResource[] = []; - - try { - const result = await x("docker", [ - "ps", - "-a", - "--format", - '{{.ID | printf "%.12s"}} {{.Names}}', - ]); - - const lines = stringToLines(result.stdout); - - containers = lines.map((line) => { - const [id, name] = lineToWords(line); - return { id, name }; - }); - } catch (err) { - console.error("Failed to list docker containers:", err); - } - - for (const { id, name } of containers) { - try { - const inspectResult = await x("docker", [ - "inspect", - "--format", - '{{ range $k, $v := .NetworkSettings.Networks }}{{ $k | printf "%.12s" }} {{ $v.Name }}\n{{ end }}', - id, - ]); - - const networks = stringToLines(inspectResult.stdout); - - results.push({ id, name, networks }); - } catch (err) { - console.error(`Failed to inspect container ${id}:`, err); - results.push({ id, name: String(err), networks: [] }); - } - } - - return results; -} - -type DockerDiagnostics = { - containers?: string[]; - networks?: string[]; - containerNetworks?: DockerContainerNetwork[]; - networkAttachments?: DockerNetworkAttachment[]; -}; - -async function getDockerDiagnostics(): Promise { - const [containers, networks, networkAttachments, containerNetworks] = await Promise.all([ - getDockerContainers(), - getDockerNetworks(), - getDockerNetworkAttachments(), - getDockerContainerNetworks(), - ]); - - return { - containers, - networks, - containerNetworks, - networkAttachments, - }; -} - const network = async ({ task }: TaskContext, use: Use) => { const testName = task.name; @@ -255,7 +48,7 @@ const network = async ({ task }: TaskContext, use: Use) => { const startDurationMs = Date.now() - start; const metadata = { - testName, + ...getTaskMetadata(task), networkId: network.getId().slice(0, 12), networkName: network.getName(), startDurationMs, @@ -275,31 +68,13 @@ const postgresContainer = async ( { network, task }: { network: StartedNetwork } & TaskContext, use: Use ) => { - const testName = task.name; - - logSetup("postgresContainer: starting", { testName }); - - const start = Date.now(); - const { container } = await createPostgresContainer(network); - const startDurationMs = Date.now() - start; - - const metadata = { - testName, - containerId: container.getId().slice(0, 12), - containerName: container.getName(), - containerNetworkNames: container.getNetworkNames(), - startDurationMs, - }; - - logSetup("postgresContainer: started", metadata); + const { container, metadata } = await withContainerSetup({ + name: "postgresContainer", + task, + setup: createPostgresContainer(network), + }); - try { - await use(container); - } finally { - // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. - // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await logCleanup("postgresContainer", container.stop({ timeout: 30 }), metadata); - } + await useContainer("postgresContainer", { container, task, use: () => use(container) }); }; const prisma = async ( @@ -327,60 +102,20 @@ const prisma = async ( export const postgresTest = test.extend({ network, postgresContainer, prisma }); -let setupOrder = 0; - -function logSetup(resource: string, metadata: Record) { - const order = setupOrder++; - - if (!isCI) { - return; - } - - console.log( - JSON.stringify({ - type: "setup", - order, - resource, - timestamp: new Date().toISOString(), - ...metadata, - }) - ); -} - const redisContainer = async ( { network, task }: { network: StartedNetwork } & TaskContext, use: Use ) => { - const testName = task.name; - - logSetup("redisContainer: starting", { testName }); - - const start = Date.now(); - - const { container } = await createRedisContainer({ - port: 6379, - network, + const { container, metadata } = await withContainerSetup({ + name: "redisContainer", + task, + setup: createRedisContainer({ + port: 6379, + network, + }), }); - const startDurationMs = Date.now() - start; - - const metadata = { - containerName: container.getName(), - containerId: container.getId().slice(0, 12), - containerNetworkNames: container.getNetworkNames(), - startDurationMs, - testName, - }; - - logSetup("redisContainer: started", metadata); - - try { - await use(container); - } finally { - // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. - // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await logCleanup("redisContainer", container.stop({ timeout: 30 }), metadata); - } + await useContainer("redisContainer", { container, task, use: () => use(container) }); }; const redisOptions = async ( @@ -426,30 +161,13 @@ const electricOrigin = async ( }: { postgresContainer: StartedPostgreSqlContainer; network: StartedNetwork } & TaskContext, use: Use ) => { - const testName = task.name; - - logSetup("electricOrigin: starting", { testName }); - - const start = Date.now(); - const { origin, container } = await createElectricContainer(postgresContainer, network); - const startDurationMs = Date.now() - start; - - const metadata = { - testName, - containerId: container.getId().slice(0, 12), - containerName: container.getName(), - startDurationMs, - }; - - logSetup("electricOrigin: started", metadata); + const { origin, container, metadata } = await withContainerSetup({ + name: "electricContainer", + task, + setup: createElectricContainer(postgresContainer, network), + }); - try { - await use(origin); - } finally { - // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. - // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await logCleanup("electricContainer", container.stop({ timeout: 30 }), metadata); - } + await useContainer("electricContainer", { container, task, use: () => use(origin) }); }; export const containerTest = test.extend({ diff --git a/internal-packages/testcontainers/src/logs.ts b/internal-packages/testcontainers/src/logs.ts new file mode 100644 index 00000000000..1a844c3df94 --- /dev/null +++ b/internal-packages/testcontainers/src/logs.ts @@ -0,0 +1,101 @@ +import { env, isCI } from "std-env"; +import { TaskContext } from "vitest"; +import { DockerDiagnostics, getDockerDiagnostics } from "./docker"; +import { StartedTestContainer } from "testcontainers"; + +let setupOrder = 0; + +export function logSetup(resource: string, metadata: Record) { + const order = setupOrder++; + + if (!isCI) { + return; + } + + console.log( + JSON.stringify({ + type: "setup", + order, + resource, + timestamp: new Date().toISOString(), + ...metadata, + }) + ); +} + +export function getContainerMetadata(container: StartedTestContainer) { + return { + containerName: container.getName(), + containerId: container.getId().slice(0, 12), + containerNetworkNames: container.getNetworkNames(), + }; +} + +export function getTaskMetadata(task: TaskContext["task"]) { + return { + testName: task.name, + }; +} + +let cleanupOrder = 0; +let activeCleanups = 0; + +/** + * Logs the cleanup of a resource. + * @param resource - The resource that is being cleaned up. + * @param promise - The cleanup promise to await.. + */ +export async function logCleanup( + resource: string, + promise: Promise, + metadata: Record = {} +) { + const start = new Date(); + const order = cleanupOrder++; + const activeAtStart = ++activeCleanups; + + let error: unknown = null; + + try { + await promise; + } catch (err) { + error = err instanceof Error ? err.message : String(err); + } + + const end = new Date(); + const durationMs = end.getTime() - start.getTime(); + const activeAtEnd = --activeCleanups; + const parallel = activeAtStart > 1 || activeAtEnd > 0; + + if (!isCI) { + return; + } + + let dockerDiagnostics: DockerDiagnostics = {}; + + // Only run docker diagnostics if there was an error or cleanup took longer than 5s + if (error || durationMs > 5000 || env.DOCKER_DIAGNOSTICS) { + try { + dockerDiagnostics = await getDockerDiagnostics(); + } catch (diagnosticErr) { + console.error("Failed to get docker diagnostics:", diagnosticErr); + } + } + + console.log( + JSON.stringify({ + type: "cleanup", + order, + resource, + durationMs, + start: start.toISOString(), + end: end.toISOString(), + parallel, + error, + activeAtStart, + activeAtEnd, + ...metadata, + ...dockerDiagnostics, + }) + ); +} diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts index 5f71fefbae4..7efe4192013 100644 --- a/internal-packages/testcontainers/src/utils.ts +++ b/internal-packages/testcontainers/src/utils.ts @@ -4,9 +4,11 @@ import { tryCatch } from "@trigger.dev/core"; import Redis from "ioredis"; import path from "path"; import { isDebug } from "std-env"; -import { GenericContainer, StartedNetwork, Wait } from "testcontainers"; +import { GenericContainer, StartedNetwork, StartedTestContainer, Wait } from "testcontainers"; import { x } from "tinyexec"; -import { expect } from "vitest"; +import { expect, TaskContext } from "vitest"; +import { getContainerMetadata, getTaskMetadata, logCleanup } from "./logs"; +import { logSetup } from "./logs"; export async function createPostgresContainer(network: StartedNetwork) { const container = await new PostgreSqlContainer("docker.io/postgres:14") @@ -149,3 +151,56 @@ export function assertNonNullable(value: T): asserts value is NonNullable expect(value).toBeDefined(); expect(value).not.toBeNull(); } + +export async function withContainerSetup({ + name, + task, + setup, +}: { + name: string; + task: TaskContext["task"]; + setup: Promise; +}): Promise }> { + const testName = task.name; + logSetup(`${name}: starting`, { testName }); + + const start = Date.now(); + const result = await setup; + const startDurationMs = Date.now() - start; + + const metadata = { + ...getTaskMetadata(task), + ...getContainerMetadata(result.container), + startDurationMs, + }; + + logSetup(`${name}: started`, metadata); + + return { ...result, metadata }; +} + +export async function useContainer( + name: string, + { + container, + task, + use, + }: { container: TContainer; task: TaskContext["task"]; use: () => Promise } +) { + const metadata = { + ...getTaskMetadata(task), + ...getContainerMetadata(container), + useDurationMs: 0, + }; + + try { + const start = Date.now(); + await use(); + const useDurationMs = Date.now() - start; + metadata.useDurationMs = useDurationMs; + } finally { + // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. + // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second + await logCleanup(name, container.stop({ timeout: 30 }), metadata); + } +} From aecde2e2c13ff4d129e9ae3db4632136b369f74a Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 12:23:15 +0100 Subject: [PATCH 16/19] fix unit tests workflow --- .github/workflows/testcontainers.yml | 85 ---------------------------- .github/workflows/unit-tests.yml | 12 +++- 2 files changed, 10 insertions(+), 87 deletions(-) delete mode 100644 .github/workflows/testcontainers.yml diff --git a/.github/workflows/testcontainers.yml b/.github/workflows/testcontainers.yml deleted file mode 100644 index 9fb7df4f4c1..00000000000 --- a/.github/workflows/testcontainers.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: "๐Ÿณ Testcontainers" - -on: - workflow_call: - workflow_dispatch: - push: - -jobs: - unitTests: - name: "๐Ÿงช Unit Tests (run ${{ matrix.run }})" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - steps: - - name: ๐Ÿ”ง Disable IPv6 - run: | - sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 - sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 - sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1 - - - name: ๐Ÿ”ง Configure docker address pool - run: | - CONFIG='{ - "default-address-pools" : [ - { - "base" : "172.17.0.0/12", - "size" : 20 - }, - { - "base" : "192.168.0.0/16", - "size" : 24 - } - ] - }' - mkdir -p /etc/docker - echo "$CONFIG" | sudo tee /etc/docker/daemon.json - - - name: ๐Ÿ”ง Restart docker - run: sudo systemctl restart docker - - - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: โŽ” Setup pnpm - uses: pnpm/action-setup@v4 - with: - version: 8.15.5 - - - name: โŽ” Setup node - uses: buildjet/setup-node@v4 - with: - node-version: 20.11.1 - cache: "pnpm" - - # ..to avoid rate limits when pulling images - - name: ๐Ÿณ Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: ๐Ÿ“ฅ Download deps - run: pnpm install --frozen-lockfile - - - name: ๐Ÿ“€ Generate Prisma Client - run: pnpm run generate - - - name: ๐Ÿงช Run Webapp Unit Tests - run: pnpm run test:webapp - env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres - DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres - SESSION_SECRET: "secret" - MAGIC_LINK_SECRET: "secret" - ENCRYPTION_KEY: "secret" - - - name: ๐Ÿงช Run Package Unit Tests - run: pnpm run test:packages - - - name: ๐Ÿงช Run Internal Unit Tests - run: pnpm run test:internal diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index df4dce01820..382ee5617f6 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -8,7 +8,13 @@ jobs: name: "๐Ÿงช Unit Tests" runs-on: ubuntu-latest steps: - - name: ๐Ÿ”ง Configure docker + - name: ๐Ÿ”ง Disable IPv6 + run: | + sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1 + + - name: ๐Ÿ”ง Configure docker address pool run: | CONFIG='{ "default-address-pools" : [ @@ -24,7 +30,9 @@ jobs: }' mkdir -p /etc/docker echo "$CONFIG" | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker + + - name: ๐Ÿ”ง Restart docker daemon + run: sudo systemctl restart docker - name: โฌ‡๏ธ Checkout repo uses: actions/checkout@v4 From b8b17f10099b3b5f3ababa66125c0db208140feb Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 12:24:38 +0100 Subject: [PATCH 17/19] decrease container cleanup timeout --- internal-packages/testcontainers/src/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts index 7efe4192013..dec2093539b 100644 --- a/internal-packages/testcontainers/src/utils.ts +++ b/internal-packages/testcontainers/src/utils.ts @@ -201,6 +201,6 @@ export async function useContainer( } finally { // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately. // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second - await logCleanup(name, container.stop({ timeout: 30 }), metadata); + await logCleanup(name, container.stop({ timeout: 10 }), metadata); } } From 7439b3237197a8c35883e44d7b6e9db1a9dcce80 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 15:12:44 +0100 Subject: [PATCH 18/19] fix types --- .../testcontainers/src/docker.ts | 22 ++++++++++++++----- .../testcontainers/tsconfig.json | 1 + 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/internal-packages/testcontainers/src/docker.ts b/internal-packages/testcontainers/src/docker.ts index f9da9162d2a..45cacb98aab 100644 --- a/internal-packages/testcontainers/src/docker.ts +++ b/internal-packages/testcontainers/src/docker.ts @@ -48,10 +48,15 @@ export async function getDockerNetworkAttachments(): Promise { + for (const line of lines) { const [id, name] = lineToWords(line); - return { id, name }; - }); + + if (!id || !name) { + continue; + } + + networks.push({ id, name }); + } } catch (err) { console.error("Failed to list docker networks:", err); } @@ -97,10 +102,15 @@ export async function getDockerContainerNetworks(): Promise { + for (const line of lines) { const [id, name] = lineToWords(line); - return { id, name }; - }); + + if (!id || !name) { + continue; + } + + containers.push({ id, name }); + } } catch (err) { console.error("Failed to list docker containers:", err); } diff --git a/internal-packages/testcontainers/tsconfig.json b/internal-packages/testcontainers/tsconfig.json index e5cea6ed2d9..15d4754cb6a 100644 --- a/internal-packages/testcontainers/tsconfig.json +++ b/internal-packages/testcontainers/tsconfig.json @@ -13,6 +13,7 @@ "skipLibCheck": true, "noEmit": true, "strict": true, + "noUncheckedIndexedAccess": true, "paths": { "@trigger.dev/core": ["../../packages/core/src/index"], "@trigger.dev/core/*": ["../../packages/core/src/*"], From dcb62f520bb5b09fae867d04a031064750837660 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 5 May 2025 22:37:55 +0100 Subject: [PATCH 19/19] fix webapp typecheck --- apps/webapp/tsconfig.check.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/webapp/tsconfig.check.json b/apps/webapp/tsconfig.check.json index 8839d20eb45..091b4ddb36a 100644 --- a/apps/webapp/tsconfig.check.json +++ b/apps/webapp/tsconfig.check.json @@ -1,6 +1,8 @@ { "extends": "./tsconfig.json", "compilerOptions": { + "lib": ["DOM", "DOM.Iterable", "DOM.AsyncIterable", "ES2022"], + "target": "ES2022", "noEmit": true, "paths": { "~/*": ["./app/*"],