diff --git a/.gitignore b/.gitignore index 5bd9baac..4be5b644 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ snapshot_*.json *.test.js.map traces/ tests/tracing/test-traces/ +playground # Temporary directories from sync workflows extension-temp/ diff --git a/package-lock.json b/package-lock.json index b74c9ccd..8834d4bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,18 @@ { "name": "sentienceapi", - "version": "0.90.15", + "version": "0.90.17", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sentienceapi", - "version": "0.90.15", + "version": "0.90.17", "license": "(MIT OR Apache-2.0)", "dependencies": { "playwright": "^1.40.0", "turndown": "^7.2.2", "uuid": "^9.0.0", + "zhipuai-sdk-nodejs-v4": "^0.1.12", "zod": "^3.22.0" }, "bin": { @@ -1300,8 +1301,18 @@ "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz", + "integrity": "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==", "license": "MIT", - "optional": true + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.4", + "proxy-from-env": "^1.1.0" + } }, "node_modules/babel-jest": { "version": "29.7.0", @@ -1517,6 +1528,12 @@ "node-int64": "^0.4.0" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -1529,7 +1546,6 @@ "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", "license": "MIT", - "optional": true, "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" @@ -1687,7 +1703,6 @@ "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", "license": "MIT", - "optional": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -1801,7 +1816,6 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", "license": "MIT", - "optional": true, "engines": { "node": ">=0.4.0" } @@ -1841,7 +1855,6 @@ "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", "license": "MIT", - "optional": true, "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", @@ -1851,6 +1864,15 @@ "node": ">= 0.4" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/electron-to-chromium": { "version": "1.5.267", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.267.tgz", @@ -1893,7 +1915,6 @@ "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", "license": "MIT", - "optional": true, "engines": { "node": ">= 0.4" } @@ -1903,7 +1924,6 @@ "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", "license": "MIT", - "optional": true, "engines": { "node": ">= 0.4" } @@ -1913,7 +1933,6 @@ "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", "license": "MIT", - "optional": true, "dependencies": { "es-errors": "^1.3.0" }, @@ -1926,7 +1945,6 @@ "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", "license": "MIT", - "optional": true, "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", @@ -2075,12 +2093,31 @@ "node": ">=8" } }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/form-data": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", "license": "MIT", - "optional": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -2149,7 +2186,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -2180,7 +2216,6 @@ "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "license": "MIT", - "optional": true, "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", @@ -2215,7 +2250,6 @@ "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", "license": "MIT", - "optional": true, "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" @@ -2264,7 +2298,6 @@ "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", "license": "MIT", - "optional": true, "engines": { "node": ">= 0.4" }, @@ -2316,7 +2349,6 @@ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", "license": "MIT", - "optional": true, "engines": { "node": ">= 0.4" }, @@ -2329,7 +2361,6 @@ "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", "license": "MIT", - "optional": true, "dependencies": { "has-symbols": "^1.0.3" }, @@ -2344,7 +2375,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "devOptional": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -3236,6 +3266,61 @@ "node": ">=6" } }, + "node_modules/jsonwebtoken": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", + "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", + "license": "MIT", + "dependencies": { + "jws": "^4.0.1", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/jsonwebtoken/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -3276,6 +3361,42 @@ "node": ">=8" } }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", + "license": "MIT" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", + "license": "MIT" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", + "license": "MIT" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -3283,6 +3404,12 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "license": "MIT" + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -3344,7 +3471,6 @@ "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", "license": "MIT", - "optional": true, "engines": { "node": ">= 0.4" } @@ -3375,7 +3501,6 @@ "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", "license": "MIT", - "optional": true, "engines": { "node": ">= 0.6" } @@ -3385,7 +3510,6 @@ "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "license": "MIT", - "optional": true, "dependencies": { "mime-db": "1.52.0" }, @@ -3430,7 +3554,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "devOptional": true, "license": "MIT" }, "node_modules/natural-compare": { @@ -3840,6 +3963,12 @@ "node": ">= 6" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", @@ -3928,6 +4057,26 @@ "node": ">=10" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -4594,6 +4743,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zhipuai-sdk-nodejs-v4": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/zhipuai-sdk-nodejs-v4/-/zhipuai-sdk-nodejs-v4-0.1.12.tgz", + "integrity": "sha512-UaxTvhIZiJOhwHjCx8WwZjkiQzQvSE/yq7uEEeM8zjZ1D1lX+SIDsTnRhnhVqsvpTnFdD9AcwY15mvjtmRy1ug==", + "license": "MIT", + "dependencies": { + "axios": "^1.6.7", + "jsonwebtoken": "^9.0.2" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/package.json b/package.json index b89c7805..ed90a2a4 100644 --- a/package.json +++ b/package.json @@ -41,8 +41,9 @@ "typescript": "^5.0.0" }, "optionalDependencies": { + "@anthropic-ai/sdk": "^0.20.0", "openai": "^4.0.0", - "@anthropic-ai/sdk": "^0.20.0" + "zhipuai-sdk-nodejs-v4": "^0.1.12" }, "files": [ "dist", diff --git a/src/agent.ts b/src/agent.ts index 471cb995..9c95f59f 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -117,6 +117,7 @@ export class SentienceAgent { totalTokens: 0, byAction: [] }; + } /** @@ -171,6 +172,7 @@ export class SentienceAgent { throw new Error(`Snapshot failed: ${snap.error}`); } + // Apply element filtering based on goal const filteredElements = this.filterElements(snap, goal); @@ -182,15 +184,40 @@ export class SentienceAgent { // Emit snapshot event if (this.tracer) { - this.tracer.emit('snapshot', { + const snapshotData: any = { url: filteredSnap.url, + element_count: filteredSnap.elements.length, + timestamp: filteredSnap.timestamp, elements: filteredSnap.elements.slice(0, 50).map(el => ({ id: el.id, bbox: el.bbox, role: el.role, - text: el.text?.substring(0, 100), + text: el.text?.substring(0, 50), })) - }, stepId); + }; + + // Always include screenshot in trace event for studio viewer compatibility + // CloudTraceSink will extract and upload screenshots separately, then remove + // screenshot_base64 from events before uploading the trace file. + if (snap.screenshot) { + // Extract base64 string from data URL if needed + let screenshotBase64: string; + if (snap.screenshot.startsWith('data:image')) { + // Format: "data:image/jpeg;base64,{base64_string}" + screenshotBase64 = snap.screenshot.includes(',') + ? snap.screenshot.split(',', 2)[1] + : snap.screenshot; + } else { + screenshotBase64 = snap.screenshot; + } + + snapshotData.screenshot_base64 = screenshotBase64; + if (snap.screenshot_format) { + snapshotData.screenshot_format = snap.screenshot_format; + } + } + + this.tracer.emit('snapshot', snapshotData, stepId); } // 2. GROUND: Format elements for LLM context diff --git a/src/index.ts b/src/index.ts index 2ef3319b..85320765 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,7 +23,8 @@ export { LLMProvider, LLMResponse, OpenAIProvider, - AnthropicProvider + AnthropicProvider, + GLMProvider } from './llm-provider'; export { SentienceAgent, diff --git a/src/snapshot.ts b/src/snapshot.ts index 3d8cbe6c..a5608404 100644 --- a/src/snapshot.ts +++ b/src/snapshot.ts @@ -43,8 +43,6 @@ function _saveTraceToFile(rawElements: any[], tracePath?: string): void { // Save the raw elements to JSON fs.writeFileSync(filename, JSON.stringify(rawElements, null, 2)); - - console.log(`[SDK] Trace saved to: ${filename}`); } export async function snapshot( @@ -114,6 +112,20 @@ async function snapshotViaExtension( return (window as any).sentience.snapshot(opts); }, opts); + // Extract screenshot format from data URL if not provided by extension + if (result.screenshot && !result.screenshot_format) { + const screenshotDataUrl = result.screenshot; + if (screenshotDataUrl.startsWith('data:image/')) { + // Extract format from "data:image/jpeg;base64,..." or "data:image/png;base64,..." + const formatMatch = screenshotDataUrl.split(';')[0].split('/')[1]; + if (formatMatch === 'jpeg' || formatMatch === 'jpg') { + result.screenshot_format = 'jpeg'; + } else if (formatMatch === 'png') { + result.screenshot_format = 'png'; + } + } + } + // Save trace if requested if (options.save_trace && result.raw_elements) { _saveTraceToFile(result.raw_elements, options.trace_path); @@ -217,6 +229,21 @@ async function snapshotViaApi( const apiResult = await response.json(); + // Extract screenshot format from data URL if not provided by extension + let screenshotFormat = rawResult.screenshot_format; + if (rawResult.screenshot && !screenshotFormat) { + const screenshotDataUrl = rawResult.screenshot; + if (screenshotDataUrl.startsWith('data:image/')) { + // Extract format from "data:image/jpeg;base64,..." or "data:image/png;base64,..." + const formatMatch = screenshotDataUrl.split(';')[0].split('/')[1]; + if (formatMatch === 'jpeg' || formatMatch === 'jpg') { + screenshotFormat = 'jpeg'; + } else if (formatMatch === 'png') { + screenshotFormat = 'png'; + } + } + } + // Merge API result with local data (screenshot, etc.) const snapshotData: Snapshot = { status: apiResult.status || 'success', @@ -225,7 +252,7 @@ async function snapshotViaApi( viewport: apiResult.viewport || rawResult.viewport, elements: apiResult.elements || [], screenshot: rawResult.screenshot, // Keep local screenshot - screenshot_format: rawResult.screenshot_format, + screenshot_format: screenshotFormat, error: apiResult.error, }; diff --git a/src/tracing/cloud-sink.ts b/src/tracing/cloud-sink.ts index 9f08c0ee..bef4b4c8 100644 --- a/src/tracing/cloud-sink.ts +++ b/src/tracing/cloud-sink.ts @@ -75,9 +75,13 @@ export class CloudTraceSink extends TraceSink { private apiUrl: string; private logger?: SentienceLogger; - // File size tracking (NEW) + // File size tracking private traceFileSizeBytes: number = 0; private screenshotTotalSizeBytes: number = 0; + private screenshotCount: number = 0; // Track number of screenshots extracted + + // Upload success flag + private uploadSuccessful: boolean = false; /** * Create a new CloudTraceSink @@ -221,7 +225,6 @@ export class CloudTraceSink extends TraceSink { console.error(` Local trace preserved at: ${this.tempFilePath}`); }); - console.log('📤 [Sentience] Trace upload started in background'); return; } @@ -250,7 +253,7 @@ export class CloudTraceSink extends TraceSink { // 2. Generate index after closing file this.generateIndex(); - // 2. Read and compress trace data (using async operations) + // 2. Check trace file exists try { await fsPromises.access(this.tempFilePath); } catch { @@ -258,13 +261,27 @@ export class CloudTraceSink extends TraceSink { return; } - const traceData = await fsPromises.readFile(this.tempFilePath); + // 3. Extract screenshots from trace events + const screenshots = await this._extractScreenshotsFromTrace(); + this.screenshotCount = screenshots.size; + + // 4. Upload screenshots separately + if (screenshots.size > 0) { + await this._uploadScreenshots(screenshots); + } + + // 5. Create cleaned trace file (without screenshot_base64) + const cleanedTracePath = this.tempFilePath.replace('.jsonl', '.cleaned.jsonl'); + await this._createCleanedTrace(cleanedTracePath); + + // 6. Read and compress cleaned trace + const traceData = await fsPromises.readFile(cleanedTracePath); const compressedData = zlib.gzipSync(traceData); - // Measure trace file size (NEW) + // Measure trace file size this.traceFileSizeBytes = compressedData.length; - // Log file sizes if logger is provided (NEW) + // Log file sizes if logger is provided if (this.logger) { this.logger.info( `Trace file size: ${(this.traceFileSizeBytes / 1024 / 1024).toFixed(2)} MB` @@ -274,15 +291,18 @@ export class CloudTraceSink extends TraceSink { ); } - // 3. Upload to cloud via pre-signed URL - console.log( - `📤 [Sentience] Uploading trace to cloud (${compressedData.length} bytes)...` - ); + // 7. Upload cleaned trace to cloud + if (this.logger) { + this.logger.info(`Uploading trace to cloud (${compressedData.length} bytes)`); + } const statusCode = await this._uploadToCloud(compressedData); if (statusCode === 200) { - console.log('✅ [Sentience] Trace uploaded successfully'); + this.uploadSuccessful = true; + if (this.logger) { + this.logger.info('Trace uploaded successfully'); + } // Upload trace index file await this._uploadIndex(); @@ -290,9 +310,17 @@ export class CloudTraceSink extends TraceSink { // Call /v1/traces/complete to report file sizes await this._completeTrace(); - // 4. Delete temp file on success - await fsPromises.unlink(this.tempFilePath); + // 8. Delete files on success + await this._cleanupFiles(); + + // Clean up temporary cleaned trace file + try { + await fsPromises.unlink(cleanedTracePath); + } catch { + // Ignore cleanup errors + } } else { + this.uploadSuccessful = false; console.error(`❌ [Sentience] Upload failed: HTTP ${statusCode}`); console.error(` Local trace preserved at: ${this.tempFilePath}`); } @@ -323,6 +351,7 @@ export class CloudTraceSink extends TraceSink { stats: { trace_file_size_bytes: this.traceFileSizeBytes, screenshot_total_size_bytes: this.screenshotTotalSizeBytes, + screenshot_count: this.screenshotCount, }, }); @@ -380,7 +409,7 @@ export class CloudTraceSink extends TraceSink { writeTraceIndex(this.tempFilePath); } catch (error: any) { // Non-fatal: log but don't crash - console.log(`⚠️ Failed to generate trace index: ${error.message}`); + this.logger?.warn(`Failed to generate trace index: ${error.message}`); } } @@ -420,14 +449,17 @@ export class CloudTraceSink extends TraceSink { const indexSize = compressedIndex.length; this.logger?.info(`Index file size: ${(indexSize / 1024).toFixed(2)} KB`); - - console.log(`📤 [Sentience] Uploading trace index (${indexSize} bytes)...`); + if (this.logger) { + this.logger.info(`Uploading trace index (${indexSize} bytes)`); + } // Upload index to cloud storage const statusCode = await this._uploadIndexToCloud(uploadUrlResponse, compressedIndex); if (statusCode === 200) { - console.log('✅ [Sentience] Trace index uploaded successfully'); + if (this.logger) { + this.logger.info('Trace index uploaded successfully'); + } // Delete local index file after successful upload try { @@ -437,12 +469,10 @@ export class CloudTraceSink extends TraceSink { } } else { this.logger?.warn(`Index upload failed: HTTP ${statusCode}`); - console.log(`⚠️ [Sentience] Index upload failed: HTTP ${statusCode}`); } } catch (error: any) { // Non-fatal: log but don't crash this.logger?.warn(`Error uploading trace index: ${error.message}`); - console.log(`⚠️ [Sentience] Error uploading trace index: ${error.message}`); } } @@ -548,6 +578,351 @@ export class CloudTraceSink extends TraceSink { }); } + /** + * Extract screenshots from trace events. + * + * @returns Map of sequence number to screenshot data + */ + private async _extractScreenshotsFromTrace(): Promise> { + const screenshots = new Map(); + let sequence = 0; + + try { + const traceContent = await fsPromises.readFile(this.tempFilePath, 'utf-8'); + const lines = traceContent.split('\n'); + + for (const line of lines) { + if (!line.trim()) { + continue; + } + + try { + const event = JSON.parse(line); + // Check if this is a snapshot event with screenshot + if (event.type === 'snapshot') { + const data = event.data || {}; + const screenshotBase64 = data.screenshot_base64; + + if (screenshotBase64) { + sequence += 1; + screenshots.set(sequence, { + base64: screenshotBase64, + format: data.screenshot_format || 'jpeg', + stepId: event.step_id, + }); + } + } + } catch { + // Skip invalid JSON lines + continue; + } + } + } catch (error: any) { + this.logger?.error(`Error extracting screenshots: ${error.message}`); + } + + return screenshots; + } + + /** + * Create trace file without screenshot_base64 fields. + * + * @param outputPath - Path to write cleaned trace file + */ + private async _createCleanedTrace(outputPath: string): Promise { + try { + const traceContent = await fsPromises.readFile(this.tempFilePath, 'utf-8'); + const lines = traceContent.split('\n'); + const cleanedLines: string[] = []; + + for (const line of lines) { + if (!line.trim()) { + continue; + } + + try { + const event = JSON.parse(line); + // Remove screenshot_base64 from snapshot events + if (event.type === 'snapshot' && event.data) { + const cleanedData: any = {}; + for (const [key, value] of Object.entries(event.data)) { + if (key !== 'screenshot_base64' && key !== 'screenshot_format') { + cleanedData[key] = value; + } + } + event.data = cleanedData; + } + + cleanedLines.push(JSON.stringify(event)); + } catch { + // Skip invalid JSON lines + continue; + } + } + + await fsPromises.writeFile(outputPath, cleanedLines.join('\n') + '\n', 'utf-8'); + } catch (error: any) { + this.logger?.error(`Error creating cleaned trace: ${error.message}`); + throw error; + } + } + + /** + * Request pre-signed upload URLs for screenshots from gateway. + * + * @param sequences - List of screenshot sequence numbers + * @returns Map of sequence number to upload URL + */ + private async _requestScreenshotUrls(sequences: number[]): Promise> { + if (!this.apiKey || sequences.length === 0) { + return new Map(); + } + + return new Promise((resolve) => { + const url = new URL(`${this.apiUrl}/v1/screenshots/init`); + const protocol = url.protocol === 'https:' ? https : http; + + const body = JSON.stringify({ + run_id: this.runId, + sequences, + }); + + const options = { + hostname: url.hostname, + port: url.port || (url.protocol === 'https:' ? 443 : 80), + path: url.pathname + url.search, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(body), + Authorization: `Bearer ${this.apiKey}`, + }, + timeout: 10000, // 10 second timeout + }; + + const req = protocol.request(options, (res) => { + let data = ''; + res.on('data', (chunk) => { + data += chunk; + }); + res.on('end', () => { + if (res.statusCode === 200) { + try { + const response = JSON.parse(data); + const uploadUrls = response.upload_urls || {}; + const urlMap = new Map(); + + // Gateway returns sequences as strings in JSON, convert to int keys + for (const [seqStr, url] of Object.entries(uploadUrls)) { + urlMap.set(parseInt(seqStr, 10), url as string); + } + + resolve(urlMap); + } catch { + this.logger?.warn('Failed to parse screenshot upload URLs response'); + resolve(new Map()); + } + } else { + this.logger?.warn(`Failed to get screenshot URLs: HTTP ${res.statusCode}`); + resolve(new Map()); + } + }); + }); + + req.on('error', (error) => { + this.logger?.warn(`Error requesting screenshot URLs: ${error.message}`); + resolve(new Map()); + }); + + req.on('timeout', () => { + req.destroy(); + this.logger?.warn('Screenshot URLs request timeout'); + resolve(new Map()); + }); + + req.write(body); + req.end(); + }); + } + + /** + * Upload screenshots extracted from trace events. + * + * Steps: + * 1. Request pre-signed URLs from gateway (/v1/screenshots/init) + * 2. Decode base64 to image bytes + * 3. Upload screenshots in parallel (10 concurrent workers) + * 4. Track upload progress + * + * @param screenshots - Map of sequence to screenshot data + */ + private async _uploadScreenshots( + screenshots: Map + ): Promise { + if (screenshots.size === 0) { + return; + } + + // 1. Request pre-signed URLs from gateway + const sequences = Array.from(screenshots.keys()).sort((a, b) => a - b); + const uploadUrls = await this._requestScreenshotUrls(sequences); + + if (uploadUrls.size === 0) { + this.logger?.warn( + 'No screenshot upload URLs received, skipping upload. This may indicate API key permission issue, gateway error, or network problem.' + ); + return; + } + + // 2. Upload screenshots in parallel + const uploadPromises: Promise[] = []; + + for (const [seq, url] of uploadUrls.entries()) { + const screenshotData = screenshots.get(seq); + if (!screenshotData) { + continue; + } + + const uploadPromise = this._uploadSingleScreenshot(seq, url, screenshotData); + uploadPromises.push(uploadPromise); + } + + // Wait for all uploads (max 10 concurrent) + const results = await Promise.allSettled(uploadPromises.slice(0, 10)); + + // Process remaining uploads in batches of 10 + for (let i = 10; i < uploadPromises.length; i += 10) { + const batch = uploadPromises.slice(i, i + 10); + const batchResults = await Promise.allSettled(batch); + results.push(...batchResults); + } + + // Count successes and failures + let uploadedCount = 0; + const failedSequences: number[] = []; + + for (let i = 0; i < results.length; i++) { + const result = results[i]; + if (result.status === 'fulfilled' && result.value) { + uploadedCount++; + } else { + failedSequences.push(sequences[i]); + } + } + + // 3. Report results + const totalCount = uploadUrls.size; + if (uploadedCount === totalCount) { + const totalSizeMB = this.screenshotTotalSizeBytes / 1024 / 1024; + if (this.logger) { + this.logger.info( + `All ${totalCount} screenshots uploaded successfully (total size: ${totalSizeMB.toFixed(2)} MB)` + ); + } + } else { + if (this.logger) { + this.logger.warn( + `Uploaded ${uploadedCount}/${totalCount} screenshots. Failed sequences: ${failedSequences.length > 0 ? failedSequences.join(', ') : 'none'}` + ); + } + } + } + + /** + * Upload a single screenshot to pre-signed URL. + * + * @param sequence - Screenshot sequence number + * @param uploadUrl - Pre-signed upload URL + * @param screenshotData - Screenshot data with base64 and format + * @returns True if upload successful, false otherwise + */ + private async _uploadSingleScreenshot( + sequence: number, + uploadUrl: string, + screenshotData: { base64: string; format: string; stepId?: string } + ): Promise { + try { + // Decode base64 to image bytes + const imageBytes = Buffer.from(screenshotData.base64, 'base64'); + const imageSize = imageBytes.length; + + // Update total size + this.screenshotTotalSizeBytes += imageSize; + + // Upload to pre-signed URL + const statusCode = await this._uploadScreenshotToCloud(uploadUrl, imageBytes, screenshotData.format as 'png' | 'jpeg'); + + if (statusCode === 200) { + return true; + } else { + this.logger?.warn(`Screenshot ${sequence} upload failed: HTTP ${statusCode}`); + return false; + } + } catch (error: any) { + this.logger?.warn(`Screenshot ${sequence} upload error: ${error.message}`); + return false; + } + } + + /** + * Upload screenshot data to cloud using pre-signed URL + */ + private async _uploadScreenshotToCloud( + uploadUrl: string, + data: Buffer, + format: 'png' | 'jpeg' + ): Promise { + return new Promise((resolve, reject) => { + const url = new URL(uploadUrl); + const protocol = url.protocol === 'https:' ? https : http; + + const options = { + hostname: url.hostname, + port: url.port || (url.protocol === 'https:' ? 443 : 80), + path: url.pathname + url.search, + method: 'PUT', + headers: { + 'Content-Type': `image/${format}`, + 'Content-Length': data.length, + }, + timeout: 30000, // 30 second timeout per screenshot + }; + + const req = protocol.request(options, (res) => { + res.on('data', () => {}); + res.on('end', () => { + resolve(res.statusCode || 500); + }); + }); + + req.on('error', (error) => { + reject(error); + }); + + req.on('timeout', () => { + req.destroy(); + reject(new Error('Screenshot upload timeout')); + }); + + req.write(data); + req.end(); + }); + } + + /** + * Delete local files after successful upload. + */ + private async _cleanupFiles(): Promise { + // Delete trace file + try { + if (fs.existsSync(this.tempFilePath)) { + await fsPromises.unlink(this.tempFilePath); + } + } catch { + // Ignore cleanup errors + } + } + /** * Get unique identifier for this sink */ diff --git a/src/types.ts b/src/types.ts index 3313cebf..7e08ecba 100644 --- a/src/types.ts +++ b/src/types.ts @@ -50,6 +50,18 @@ export interface Snapshot { requires_license?: boolean; } +/** + * Metadata for a stored screenshot. + * Used by CloudTraceSink to track screenshots before upload. + */ +export interface ScreenshotMetadata { + sequence: number; + format: "png" | "jpeg"; + sizeBytes: number; + stepId: string | null; + filepath: string; +} + export interface ActionResult { success: boolean; duration_ms: number; diff --git a/tests/tracing/cloud-sink.test.ts b/tests/tracing/cloud-sink.test.ts index b7b59d46..64324b97 100644 --- a/tests/tracing/cloud-sink.test.ts +++ b/tests/tracing/cloud-sink.test.ts @@ -72,7 +72,31 @@ describe('CloudTraceSink', () => { if (file.startsWith('test-run-')) { const filePath = path.join(persistentCacheDir, file); if (fs.existsSync(filePath)) { - fs.unlinkSync(filePath); + const stats = fs.statSync(filePath); + if (stats.isDirectory()) { + // Delete directory and its contents + const dirFiles = fs.readdirSync(filePath); + dirFiles.forEach((dirFile) => { + fs.unlinkSync(path.join(filePath, dirFile)); + }); + fs.rmdirSync(filePath); + } else { + fs.unlinkSync(filePath); + } + } + } + // Also clean up screenshot directories + if (file.endsWith('_screenshots')) { + const dirPath = path.join(persistentCacheDir, file); + if (fs.existsSync(dirPath)) { + const stats = fs.statSync(dirPath); + if (stats.isDirectory()) { + const dirFiles = fs.readdirSync(dirPath); + dirFiles.forEach((dirFile) => { + fs.unlinkSync(path.join(dirPath, dirFile)); + }); + fs.rmdirSync(dirPath); + } } } }); diff --git a/tests/tracing/screenshot-storage.test.ts b/tests/tracing/screenshot-storage.test.ts new file mode 100644 index 00000000..e6fb5a56 --- /dev/null +++ b/tests/tracing/screenshot-storage.test.ts @@ -0,0 +1,290 @@ +/** + * Tests for screenshot extraction and upload in CloudTraceSink + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { CloudTraceSink, SentienceLogger } from '../../src/tracing/cloud-sink'; + +// Mock logger for testing +class MockLogger implements SentienceLogger { + public logs: string[] = []; + + info(message: string): void { + this.logs.push(`[INFO] ${message}`); + } + + warn(message: string): void { + this.logs.push(`[WARN] ${message}`); + } + + error(message: string): void { + this.logs.push(`[ERROR] ${message}`); + } +} + +describe('Screenshot Extraction and Upload', () => { + let uploadUrl: string; + let runId: string; + let cacheDir: string; + + beforeEach(() => { + uploadUrl = 'https://sentience.nyc3.digitaloceanspaces.com/user123/run456/trace.jsonl.gz'; + runId = `test-screenshot-${Date.now()}`; + cacheDir = path.join(os.homedir(), '.sentience', 'traces', 'pending'); + }); + + afterEach(() => { + // Cleanup test files + const tracePath = path.join(cacheDir, `${runId}.jsonl`); + const cleanedTracePath = path.join(cacheDir, `${runId}.cleaned.jsonl`); + + if (fs.existsSync(tracePath)) { + fs.unlinkSync(tracePath); + } + + if (fs.existsSync(cleanedTracePath)) { + fs.unlinkSync(cleanedTracePath); + } + }); + + describe('_extractScreenshotsFromTrace', () => { + it('should extract screenshots from trace events', async () => { + const sink = new CloudTraceSink(uploadUrl, runId); + + // Create a trace file with screenshot events + const testImageBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + + // Emit a snapshot event with screenshot + sink.emit({ + v: 1, + type: 'snapshot', + ts: '2026-01-01T00:00:00.000Z', + run_id: runId, + seq: 1, + step_id: 'step-1', + data: { + url: 'https://example.com', + element_count: 10, + screenshot_base64: testImageBase64, + screenshot_format: 'png', + }, + }); + + // Close to write file + await sink.close(false); + + // Wait a bit for file to be written + await new Promise(resolve => setTimeout(resolve, 100)); + + // Extract screenshots + const screenshots = await (sink as any)._extractScreenshotsFromTrace(); + + expect(screenshots.size).toBe(1); + expect(screenshots.get(1)).toBeDefined(); + expect(screenshots.get(1)?.base64).toBe(testImageBase64); + expect(screenshots.get(1)?.format).toBe('png'); + expect(screenshots.get(1)?.stepId).toBe('step-1'); + }); + + it('should handle multiple screenshots', async () => { + const sink = new CloudTraceSink(uploadUrl, runId); + const testImageBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + + // Emit multiple snapshot events with screenshots + for (let i = 1; i <= 3; i++) { + sink.emit({ + v: 1, + type: 'snapshot', + ts: '2026-01-01T00:00:00.000Z', + run_id: runId, + seq: i, + step_id: `step-${i}`, + data: { + url: 'https://example.com', + element_count: 10, + screenshot_base64: testImageBase64, + screenshot_format: 'png', + }, + }); + } + + await sink.close(false); + await new Promise(resolve => setTimeout(resolve, 100)); + + const screenshots = await (sink as any)._extractScreenshotsFromTrace(); + expect(screenshots.size).toBe(3); + }); + + it('should skip events without screenshots', async () => { + const sink = new CloudTraceSink(uploadUrl, runId); + + // Emit snapshot without screenshot + sink.emit({ + v: 1, + type: 'snapshot', + ts: '2026-01-01T00:00:00.000Z', + run_id: runId, + seq: 1, + data: { + url: 'https://example.com', + element_count: 10, + // No screenshot_base64 + }, + }); + + await sink.close(false); + await new Promise(resolve => setTimeout(resolve, 100)); + + const screenshots = await (sink as any)._extractScreenshotsFromTrace(); + expect(screenshots.size).toBe(0); + }); + }); + + describe('_createCleanedTrace', () => { + it('should remove screenshot_base64 from events', async () => { + const sink = new CloudTraceSink(uploadUrl, runId); + const testImageBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + + // Emit snapshot event with screenshot + sink.emit({ + v: 1, + type: 'snapshot', + ts: '2026-01-01T00:00:00.000Z', + run_id: runId, + seq: 1, + data: { + url: 'https://example.com', + element_count: 10, + screenshot_base64: testImageBase64, + screenshot_format: 'png', + }, + }); + + await sink.close(false); + await new Promise(resolve => setTimeout(resolve, 100)); + + // Create cleaned trace + const cleanedTracePath = path.join(cacheDir, `${runId}.cleaned.jsonl`); + await (sink as any)._createCleanedTrace(cleanedTracePath); + + // Read cleaned trace + const cleanedContent = fs.readFileSync(cleanedTracePath, 'utf-8'); + const cleanedEvent = JSON.parse(cleanedContent.trim()); + + // Verify screenshot fields are removed + expect(cleanedEvent.data.screenshot_base64).toBeUndefined(); + expect(cleanedEvent.data.screenshot_format).toBeUndefined(); + expect(cleanedEvent.data.url).toBe('https://example.com'); + expect(cleanedEvent.data.element_count).toBe(10); + }); + + it('should preserve other event types unchanged', async () => { + const sink = new CloudTraceSink(uploadUrl, runId); + + // Emit non-snapshot event + sink.emit({ + v: 1, + type: 'action', + ts: '2026-01-01T00:00:00.000Z', + run_id: runId, + seq: 1, + data: { + action: 'click', + element_id: 123, + }, + }); + + await sink.close(false); + await new Promise(resolve => setTimeout(resolve, 100)); + + const cleanedTracePath = path.join(cacheDir, `${runId}.cleaned.jsonl`); + await (sink as any)._createCleanedTrace(cleanedTracePath); + + const cleanedContent = fs.readFileSync(cleanedTracePath, 'utf-8'); + const cleanedEvent = JSON.parse(cleanedContent.trim()); + + // Verify action event is unchanged + expect(cleanedEvent.type).toBe('action'); + expect(cleanedEvent.data.action).toBe('click'); + expect(cleanedEvent.data.element_id).toBe(123); + }); + }); + + describe('_requestScreenshotUrls', () => { + it('should request URLs from gateway', async () => { + const apiKey = 'sk_test_123'; + const sink = new CloudTraceSink(uploadUrl, runId, apiKey); + + // Mock HTTP request + const originalRequest = require('https').request; + const mockUrls = { + '1': 'https://sentience.nyc3.digitaloceanspaces.com/user123/run456/screenshots/step_0001.png?signature=...', + '2': 'https://sentience.nyc3.digitaloceanspaces.com/user123/run456/screenshots/step_0002.png?signature=...', + }; + + let requestCalled = false; + require('https').request = jest.fn((options: any, callback: any) => { + requestCalled = true; + const mockRes = { + statusCode: 200, + on: jest.fn((event: string, handler: any) => { + if (event === 'data') { + handler(JSON.stringify({ upload_urls: mockUrls })); + } else if (event === 'end') { + handler(); + } + }), + }; + setTimeout(() => callback(mockRes), 0); + return { + write: jest.fn(), + end: jest.fn(), + on: jest.fn(), + }; + }); + + const result = await (sink as any)._requestScreenshotUrls([1, 2]); + + expect(requestCalled).toBe(true); + expect(result.size).toBe(2); + expect(result.get(1)).toBe(mockUrls['1']); + expect(result.get(2)).toBe(mockUrls['2']); + + require('https').request = originalRequest; + sink.close(false); + }); + + it('should return empty map on failure', async () => { + const apiKey = 'sk_test_123'; + const sink = new CloudTraceSink(uploadUrl, runId, apiKey); + + // Mock HTTP request with failure + const originalRequest = require('https').request; + require('https').request = jest.fn((options: any, callback: any) => { + const mockRes = { + statusCode: 500, + on: jest.fn((event: string, handler: any) => { + if (event === 'end') { + handler(); + } + }), + }; + setTimeout(() => callback(mockRes), 0); + return { + write: jest.fn(), + end: jest.fn(), + on: jest.fn(), + }; + }); + + const result = await (sink as any)._requestScreenshotUrls([1, 2]); + + expect(result.size).toBe(0); + + require('https').request = originalRequest; + sink.close(false); + }); + }); +});