From 7ab8b89506d8a11c6125de49e23258877c5990de Mon Sep 17 00:00:00 2001 From: rcholic Date: Thu, 1 Jan 2026 23:41:52 -0800 Subject: [PATCH 1/4] trace should include snapshot elements --- src/agent.ts | 23 +++++++++++++++++------ src/tracing/indexer.ts | 27 +++++++++++++++++++-------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index 5a8f8eed..8c7b7bfa 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -214,15 +214,26 @@ export class SentienceAgent { // Emit snapshot event if (this.tracer) { + // Include ALL elements with full data for DOM tree display + // Use snap.elements (all elements) not filteredSnap.elements const snapshotData: any = { - url: filteredSnap.url, - element_count: filteredSnap.elements.length, - timestamp: filteredSnap.timestamp, - elements: filteredSnap.elements.slice(0, 50).map(el => ({ + url: snap.url, + element_count: snap.elements.length, + timestamp: snap.timestamp, + elements: snap.elements.map(el => ({ id: el.id, - bbox: el.bbox, role: el.role, - text: el.text?.substring(0, 50), + text: el.text, + importance: el.importance, + bbox: el.bbox, + visual_cues: el.visual_cues, + in_viewport: el.in_viewport, + is_occluded: el.is_occluded, + z_index: el.z_index, + rerank_index: el.rerank_index, + heuristic_index: el.heuristic_index, + ml_probability: el.ml_probability, + ml_score: el.ml_score, })) }; diff --git a/src/tracing/indexer.ts b/src/tracing/indexer.ts index 77c9fef5..a9e0ab04 100644 --- a/src/tracing/indexer.ts +++ b/src/tracing/indexer.ts @@ -57,14 +57,25 @@ function computeSnapshotDigest(snapshotData: any): string { const elements = snapshotData.elements || []; // Canonicalize elements - const canonicalElements = elements.map((elem: any) => ({ - id: elem.id, - role: elem.role || '', - text_norm: normalizeText(elem.text), - bbox: roundBBox(elem.bbox || { x: 0, y: 0, width: 0, height: 0 }), - is_primary: elem.is_primary || false, - is_clickable: elem.is_clickable || false, - })); + const canonicalElements = elements.map((elem: any) => { + // Extract is_primary and is_clickable from visual_cues if present + const visualCues = elem.visual_cues || {}; + const isPrimary = (typeof visualCues === 'object' && visualCues !== null) + ? (visualCues.is_primary || false) + : (elem.is_primary || false); + const isClickable = (typeof visualCues === 'object' && visualCues !== null) + ? (visualCues.is_clickable || false) + : (elem.is_clickable || false); + + return { + id: elem.id, + role: elem.role || '', + text_norm: normalizeText(elem.text), + bbox: roundBBox(elem.bbox || { x: 0, y: 0, width: 0, height: 0 }), + is_primary: isPrimary, + is_clickable: isClickable, + }; + }); // Sort by element id for determinism canonicalElements.sort((a: { id?: number }, b: { id?: number }) => (a.id || 0) - (b.id || 0)); From 348ecd2428d4588c6c0751d749b9e9f198482cdb Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 2 Jan 2026 05:58:01 -0800 Subject: [PATCH 2/4] use trace cloud path in index --- src/tracing/cloud-sink.ts | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/tracing/cloud-sink.ts b/src/tracing/cloud-sink.ts index 7dbbfdde..be7b3cc5 100644 --- a/src/tracing/cloud-sink.ts +++ b/src/tracing/cloud-sink.ts @@ -609,9 +609,30 @@ export class CloudTraceSink extends TraceSink { return; } - // Read and compress index file - const indexData = await fsPromises.readFile(indexPath); - const compressedIndex = zlib.gzipSync(indexData); + // Read index file and update trace_file.path to cloud storage path + const indexContent = await fsPromises.readFile(indexPath, 'utf-8'); + const indexJson = JSON.parse(indexContent); + + // Extract cloud storage path from trace upload URL + // uploadUrl format: https://...digitaloceanspaces.com/traces/{run_id}.jsonl.gz + // Extract path: traces/{run_id}.jsonl.gz + try { + const parsedUrl = new URL(this.uploadUrl); + // Extract path after domain (e.g., /traces/run-123.jsonl.gz -> traces/run-123.jsonl.gz) + const cloudTracePath = parsedUrl.pathname.startsWith('/') + ? parsedUrl.pathname.substring(1) + : parsedUrl.pathname; + // Update trace_file.path in index + if (indexJson.trace_file && typeof indexJson.trace_file === 'object') { + indexJson.trace_file.path = cloudTracePath; + } + } catch (error: any) { + this.logger?.warn(`Failed to extract cloud path from upload URL: ${error.message}`); + } + + // Serialize updated index to JSON + const updatedIndexData = Buffer.from(JSON.stringify(indexJson, null, 2), 'utf-8'); + const compressedIndex = zlib.gzipSync(updatedIndexData); const indexSize = compressedIndex.length; this.indexFileSizeBytes = indexSize; // Track index file size From e02dc6edffff415563328de6172f213d50297a08 Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 2 Jan 2026 06:11:54 -0800 Subject: [PATCH 3/4] fix to seq number --- src/tracing/cloud-sink.ts | 5 ++++- src/tracing/jsonl-sink.ts | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/tracing/cloud-sink.ts b/src/tracing/cloud-sink.ts index be7b3cc5..780fe72e 100644 --- a/src/tracing/cloud-sink.ts +++ b/src/tracing/cloud-sink.ts @@ -572,7 +572,10 @@ export class CloudTraceSink extends TraceSink { private generateIndex(): void { try { const { writeTraceIndex } = require('./indexer'); - writeTraceIndex(this.tempFilePath); + // Use frontend format to ensure 'step' field is present (1-based) + // Frontend derives sequence from step.step - 1, so step must be valid + const indexPath = this.tempFilePath.replace('.jsonl', '.index.json'); + writeTraceIndex(this.tempFilePath, indexPath, true); } catch (error: any) { // Non-fatal: log but don't crash this.logger?.warn(`Failed to generate trace index: ${error.message}`); diff --git a/src/tracing/jsonl-sink.ts b/src/tracing/jsonl-sink.ts index 942b76f9..9a4d0aa0 100644 --- a/src/tracing/jsonl-sink.ts +++ b/src/tracing/jsonl-sink.ts @@ -162,7 +162,10 @@ export class JsonlTraceSink extends TraceSink { private generateIndex(): void { try { const { writeTraceIndex } = require('./indexer'); - writeTraceIndex(this.path); + // Use frontend format to ensure 'step' field is present (1-based) + // Frontend derives sequence from step.step - 1, so step must be valid + const indexPath = this.path.replace(/\.jsonl$/, '.index.json'); + writeTraceIndex(this.path, indexPath, true); } catch (error: any) { // Non-fatal: log but don't crash console.log(`⚠️ Failed to generate trace index: ${error.message}`); From f13a834b5cbc72791aa72ff0c463ff37b77efd80 Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 2 Jan 2026 06:28:04 -0800 Subject: [PATCH 4/4] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5f6726ff..c506ebe1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sentienceapi", - "version": "0.91.0", + "version": "0.91.1", "description": "TypeScript SDK for Sentience AI Agent Browser Automation", "main": "dist/index.js", "types": "dist/index.d.ts",