From 4eeabd4dbff03fdd088fe6026260942d1d462dfa Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 12:46:36 -0800 Subject: [PATCH 1/6] phase 1 and 2 for Tracing --- package-lock.json | 22 ++ package.json | 2 + src/index.ts | 9 + src/tracing/index.ts | 10 + src/tracing/jsonl-sink.ts | 101 ++++++++++ src/tracing/sink.ts | 26 +++ src/tracing/tracer.ts | 155 ++++++++++++++ src/tracing/types.ts | 79 ++++++++ tests/tracing/jsonl-sink.test.ts | 148 ++++++++++++++ tests/tracing/regression.test.ts | 151 ++++++++++++++ tests/tracing/tracer.test.ts | 336 +++++++++++++++++++++++++++++++ 11 files changed, 1039 insertions(+) create mode 100644 src/tracing/index.ts create mode 100644 src/tracing/jsonl-sink.ts create mode 100644 src/tracing/sink.ts create mode 100644 src/tracing/tracer.ts create mode 100644 src/tracing/types.ts create mode 100644 tests/tracing/jsonl-sink.test.ts create mode 100644 tests/tracing/regression.test.ts create mode 100644 tests/tracing/tracer.test.ts diff --git a/package-lock.json b/package-lock.json index a3e52148..48810c1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "playwright": "^1.40.0", "turndown": "^7.2.2", + "uuid": "^9.0.0", "zod": "^3.22.0" }, "bin": { @@ -20,6 +21,7 @@ "@types/jest": "^29.5.14", "@types/node": "^20.0.0", "@types/turndown": "^5.0.3", + "@types/uuid": "^9.0.0", "jest": "^29.0.0", "ts-jest": "^29.0.0", "ts-node": "^10.9.0", @@ -1145,6 +1147,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/uuid": { + "version": "9.0.8", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", + "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/yargs": { "version": "17.0.35", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", @@ -4381,6 +4390,19 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/package.json b/package.json index c87b39c9..3859d91d 100644 --- a/package.json +++ b/package.json @@ -24,12 +24,14 @@ "dependencies": { "playwright": "^1.40.0", "turndown": "^7.2.2", + "uuid": "^9.0.0", "zod": "^3.22.0" }, "devDependencies": { "@types/jest": "^29.5.14", "@types/node": "^20.0.0", "@types/turndown": "^5.0.3", + "@types/uuid": "^9.0.0", "jest": "^29.0.0", "ts-jest": "^29.0.0", "ts-node": "^10.9.0", diff --git a/src/index.ts b/src/index.ts index 7cecab75..cd3d90a0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,3 +40,12 @@ export { ActionParameters } from './conversational-agent'; +// Tracing Layer (v0.3.1+) +export { + Tracer, + TraceSink, + JsonlTraceSink, + TraceEvent, + TraceEventData +} from './tracing'; + diff --git a/src/tracing/index.ts b/src/tracing/index.ts new file mode 100644 index 00000000..e49e5650 --- /dev/null +++ b/src/tracing/index.ts @@ -0,0 +1,10 @@ +/** + * Tracing Module + * + * Exports all tracing functionality + */ + +export * from './types'; +export * from './sink'; +export * from './jsonl-sink'; +export * from './tracer'; diff --git a/src/tracing/jsonl-sink.ts b/src/tracing/jsonl-sink.ts new file mode 100644 index 00000000..5add2b4f --- /dev/null +++ b/src/tracing/jsonl-sink.ts @@ -0,0 +1,101 @@ +/** + * JSONL Trace Sink + * + * Writes trace events to a local JSONL (JSON Lines) file + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { TraceSink } from './sink'; + +/** + * JsonlTraceSink writes trace events to a JSONL file (one JSON object per line) + */ +export class JsonlTraceSink extends TraceSink { + private path: string; + private writeStream: fs.WriteStream; + private closed: boolean = false; + + /** + * Create a new JSONL trace sink + * @param filePath - Path to the JSONL file (will be created if doesn't exist) + */ + constructor(filePath: string) { + super(); + this.path = filePath; + + // Create parent directories if needed + const dir = path.dirname(filePath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + // Open file in append mode with line buffering + this.writeStream = fs.createWriteStream(filePath, { + flags: 'a', + encoding: 'utf-8', + }); + } + + /** + * Emit a trace event (write as JSON line) + * @param event - Event dictionary + */ + emit(event: Record): void { + if (this.closed) { + console.warn('[JsonlTraceSink] Attempted to emit after close()'); + return; + } + + try { + const jsonLine = JSON.stringify(event) + '\n'; + this.writeStream.write(jsonLine); + } catch (error) { + // Log error but don't crash agent execution + console.error('[JsonlTraceSink] Failed to write event:', error); + } + } + + /** + * Close the sink and flush buffered data + */ + async close(): Promise { + if (this.closed) { + return; + } + + this.closed = true; + + return new Promise((resolve, reject) => { + this.writeStream.end((err?: Error | null) => { + if (err) { + console.error('[JsonlTraceSink] Error closing stream:', err); + reject(err); + } else { + resolve(); + } + }); + }); + } + + /** + * Get sink type identifier + */ + getSinkType(): string { + return `JsonlTraceSink(${this.path})`; + } + + /** + * Get file path + */ + getPath(): string { + return this.path; + } + + /** + * Check if sink is closed + */ + isClosed(): boolean { + return this.closed; + } +} diff --git a/src/tracing/sink.ts b/src/tracing/sink.ts new file mode 100644 index 00000000..52f44f37 --- /dev/null +++ b/src/tracing/sink.ts @@ -0,0 +1,26 @@ +/** + * TraceSink Abstract Class + * + * Defines the interface for trace event sinks (local files, cloud storage, etc.) + */ + +/** + * Abstract base class for trace sinks + */ +export abstract class TraceSink { + /** + * Emit a trace event + * @param event - Event dictionary to emit + */ + abstract emit(event: Record): void; + + /** + * Close the sink and flush buffered data + */ + abstract close(): Promise; + + /** + * Get unique identifier for this sink (for debugging) + */ + abstract getSinkType(): string; +} diff --git a/src/tracing/tracer.ts b/src/tracing/tracer.ts new file mode 100644 index 00000000..f1f3166c --- /dev/null +++ b/src/tracing/tracer.ts @@ -0,0 +1,155 @@ +/** + * Tracer Class + * + * High-level API for emitting trace events with automatic sequencing and timestamps + */ + +import { TraceSink } from './sink'; +import { TraceEvent, TraceEventData } from './types'; + +/** + * Tracer provides a high-level API for recording agent execution traces + */ +export class Tracer { + private runId: string; + private sink: TraceSink; + private seq: number; + + /** + * Create a new Tracer + * @param runId - Unique run identifier (UUID) + * @param sink - TraceSink implementation (e.g., JsonlTraceSink) + */ + constructor(runId: string, sink: TraceSink) { + this.runId = runId; + this.sink = sink; + this.seq = 0; + } + + /** + * Emit a trace event + * @param eventType - Type of event (e.g., 'run_start', 'snapshot') + * @param data - Event-specific payload + * @param stepId - Optional step UUID + */ + emit( + eventType: string, + data: TraceEventData, + stepId?: string + ): void { + this.seq += 1; + + // Generate timestamps + const tsMs = Date.now(); + const ts = new Date(tsMs).toISOString(); + + const event: TraceEvent = { + v: 1, + type: eventType, + ts, + ts_ms: tsMs, + run_id: this.runId, + seq: this.seq, + data, + }; + + if (stepId) { + event.step_id = stepId; + } + + this.sink.emit(event); + } + + /** + * Emit run_start event + * @param agent - Agent type (e.g., 'SentienceAgent') + * @param llmModel - Optional LLM model name + * @param config - Optional configuration + */ + emitRunStart( + agent: string, + llmModel?: string, + config?: Record + ): void { + const data: TraceEventData = { agent }; + if (llmModel) data.llm_model = llmModel; + if (config) data.config = config; + + this.emit('run_start', data); + } + + /** + * Emit step_start event + * @param stepId - Step UUID + * @param stepIndex - Step number (1-indexed) + * @param goal - Goal description + * @param attempt - Retry attempt number (0 = first try) + * @param preUrl - Optional URL before step execution + */ + emitStepStart( + stepId: string, + stepIndex: number, + goal: string, + attempt: number = 0, + preUrl?: string + ): void { + const data: TraceEventData = { + step_id: stepId, + step_index: stepIndex, + goal, + attempt, + }; + + if (preUrl) { + data.url = preUrl; + } + + this.emit('step_start', data, stepId); + } + + /** + * Emit run_end event + * @param steps - Total number of steps executed + */ + emitRunEnd(steps: number): void { + this.emit('run_end', { steps }); + } + + /** + * Emit error event + * @param stepId - Step UUID where error occurred + * @param error - Error message + * @param attempt - Retry attempt number + */ + emitError(stepId: string, error: string, attempt: number = 0): void { + this.emit('error', { step_id: stepId, error, attempt }, stepId); + } + + /** + * Close the underlying sink (flush buffered data) + */ + async close(): Promise { + await this.sink.close(); + } + + /** + * Get run ID + */ + getRunId(): string { + return this.runId; + } + + /** + * Get current sequence number + */ + getSeq(): number { + return this.seq; + } + + /** + * Get sink type (for debugging) + */ + getSinkType(): string { + return this.sink.getSinkType(); + } +} diff --git a/src/tracing/types.ts b/src/tracing/types.ts new file mode 100644 index 00000000..03f6b5d0 --- /dev/null +++ b/src/tracing/types.ts @@ -0,0 +1,79 @@ +/** + * Tracing Types + * + * Schema v1 - Compatible with Python SDK + */ + +/** + * TraceEvent represents a single event in an agent execution trace + */ +export interface TraceEvent { + /** Schema version (always 1 for now) */ + v: number; + + /** Event type (e.g., 'run_start', 'snapshot', 'action') */ + type: string; + + /** ISO 8601 timestamp */ + ts: string; + + /** Run UUID */ + run_id: string; + + /** Sequence number (monotonically increasing) */ + seq: number; + + /** Event-specific payload */ + data: Record; + + /** Optional step UUID (for step-scoped events) */ + step_id?: string; + + /** Optional Unix timestamp in milliseconds */ + ts_ms?: number; +} + +/** + * TraceEventData contains common fields for event payloads + */ +export interface TraceEventData { + // Common fields + goal?: string; + step_index?: number; + attempt?: number; + step_id?: string; + + // Snapshot data + url?: string; + elements?: Array<{ + id: number; + bbox: { x: number; y: number; width: number; height: number }; + role: string; + text?: string; + }>; + + // LLM response data + model?: string; + prompt_tokens?: number; + completion_tokens?: number; + response_text?: string; + + // Action data + action_type?: string; + element_id?: number; + text?: string; + key?: string; + success?: boolean; + + // Error data + error?: string; + + // Run metadata + agent?: string; + llm_model?: string; + config?: Record; + steps?: number; + + // Allow additional properties + [key: string]: any; +} diff --git a/tests/tracing/jsonl-sink.test.ts b/tests/tracing/jsonl-sink.test.ts new file mode 100644 index 00000000..f026f9f8 --- /dev/null +++ b/tests/tracing/jsonl-sink.test.ts @@ -0,0 +1,148 @@ +/** + * Tests for JsonlTraceSink + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { JsonlTraceSink } from '../../src/tracing/jsonl-sink'; + +describe('JsonlTraceSink', () => { + const testDir = path.join(__dirname, 'test-traces'); + const testFile = path.join(testDir, 'test.jsonl'); + + beforeEach(() => { + // Clean up test directory + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + // Clean up test directory + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true }); + } + }); + + it('should create parent directories if they do not exist', () => { + const sink = new JsonlTraceSink(testFile); + expect(fs.existsSync(testDir)).toBe(true); + sink.close(); + }); + + it('should emit events as JSON lines', async () => { + const sink = new JsonlTraceSink(testFile); + + sink.emit({ type: 'test1', data: 'hello' }); + sink.emit({ type: 'test2', data: 'world' }); + + await sink.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const lines = content.trim().split('\n'); + + expect(lines.length).toBe(2); + expect(JSON.parse(lines[0])).toEqual({ type: 'test1', data: 'hello' }); + expect(JSON.parse(lines[1])).toEqual({ type: 'test2', data: 'world' }); + }); + + it('should append to existing file', async () => { + // Write first batch + const sink1 = new JsonlTraceSink(testFile); + sink1.emit({ seq: 1 }); + await sink1.close(); + + // Write second batch + const sink2 = new JsonlTraceSink(testFile); + sink2.emit({ seq: 2 }); + await sink2.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const lines = content.trim().split('\n'); + + expect(lines.length).toBe(2); + expect(JSON.parse(lines[0])).toEqual({ seq: 1 }); + expect(JSON.parse(lines[1])).toEqual({ seq: 2 }); + }); + + it('should handle close() multiple times gracefully', async () => { + const sink = new JsonlTraceSink(testFile); + sink.emit({ test: true }); + + await sink.close(); + await sink.close(); // Should not throw + + expect(sink.isClosed()).toBe(true); + }); + + it('should warn when emitting after close', async () => { + const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(); + + const sink = new JsonlTraceSink(testFile); + await sink.close(); + + sink.emit({ test: true }); // Should warn + + expect(consoleWarnSpy).toHaveBeenCalledWith( + expect.stringContaining('Attempted to emit after close()') + ); + + consoleWarnSpy.mockRestore(); + }); + + it('should return correct sink type', () => { + const sink = new JsonlTraceSink(testFile); + expect(sink.getSinkType()).toBe(`JsonlTraceSink(${testFile})`); + sink.close(); + }); + + it('should return file path', () => { + const sink = new JsonlTraceSink(testFile); + expect(sink.getPath()).toBe(testFile); + sink.close(); + }); + + it('should handle write errors gracefully', async () => { + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(); + + const sink = new JsonlTraceSink(testFile); + + // Create a circular reference (will fail JSON.stringify) + const circular: any = { a: 1 }; + circular.self = circular; + + sink.emit(circular); // Should log error but not crash + + expect(consoleErrorSpy).toHaveBeenCalled(); + + consoleErrorSpy.mockRestore(); + await sink.close(); + }); + + it('should write valid JSON for complex objects', async () => { + const sink = new JsonlTraceSink(testFile); + + const complexEvent = { + v: 1, + type: 'snapshot', + ts: '2025-12-26T10:00:00.000Z', + run_id: 'test-run', + seq: 1, + data: { + url: 'https://example.com', + elements: [ + { id: 1, text: 'Hello', bbox: { x: 0, y: 0, width: 100, height: 50 } }, + { id: 2, text: null, bbox: { x: 100, y: 0, width: 100, height: 50 } }, + ], + }, + }; + + sink.emit(complexEvent); + await sink.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const parsed = JSON.parse(content.trim()); + + expect(parsed).toEqual(complexEvent); + }); +}); diff --git a/tests/tracing/regression.test.ts b/tests/tracing/regression.test.ts new file mode 100644 index 00000000..f4a195f4 --- /dev/null +++ b/tests/tracing/regression.test.ts @@ -0,0 +1,151 @@ +/** + * Regression Tests for Tracing + * + * Ensures tracing additions don't break existing SDK functionality + */ + +import { Tracer, JsonlTraceSink, TraceEvent, TraceEventData, TraceSink } from '../../src/tracing'; + +describe('Tracing Module - Regression Tests', () => { + describe('Exports', () => { + it('should export all tracing classes and types', () => { + expect(Tracer).toBeDefined(); + expect(JsonlTraceSink).toBeDefined(); + expect(TraceSink).toBeDefined(); + }); + + it('should export TypeScript types', () => { + // Type-only check - ensures types are exported + const event: TraceEvent = { + v: 1, + type: 'test', + ts: new Date().toISOString(), + run_id: 'test', + seq: 1, + data: {}, + }; + + const data: TraceEventData = { + goal: 'test goal', + }; + + expect(event).toBeDefined(); + expect(data).toBeDefined(); + }); + }); + + describe('Backward Compatibility', () => { + it('should not require uuid package for basic usage', () => { + // Users can provide their own run IDs (no uuid required) + const sink = new JsonlTraceSink('/tmp/test.jsonl'); + const tracer = new Tracer('my-custom-run-id', sink); + + expect(tracer.getRunId()).toBe('my-custom-run-id'); + + tracer.close(); + }); + + it('should work with string run IDs', () => { + const sink = new JsonlTraceSink('/tmp/test.jsonl'); + + // Should accept any string + const tracer1 = new Tracer('simple-id', sink); + expect(tracer1.getRunId()).toBe('simple-id'); + + const tracer2 = new Tracer('123e4567-e89b-12d3-a456-426614174000', sink); + expect(tracer2.getRunId()).toBe('123e4567-e89b-12d3-a456-426614174000'); + + tracer1.close(); + tracer2.close(); + }); + }); + + describe('Module Structure', () => { + it('should have proper TypeScript module structure', () => { + // Ensure classes can be instantiated + const sink = new JsonlTraceSink('/tmp/test.jsonl'); + expect(sink).toBeInstanceOf(JsonlTraceSink); + expect(sink).toBeInstanceOf(TraceSink); + + const tracer = new Tracer('test', sink); + expect(tracer).toBeInstanceOf(Tracer); + + tracer.close(); + }); + + it('should allow extending TraceSink', () => { + class CustomSink extends TraceSink { + emit(event: Record): void { + // Custom implementation + } + + async close(): Promise { + // Custom implementation + } + + getSinkType(): string { + return 'CustomSink'; + } + } + + const customSink = new CustomSink(); + expect(customSink).toBeInstanceOf(TraceSink); + expect(customSink.getSinkType()).toBe('CustomSink'); + }); + }); + + describe('Performance', () => { + it('should have minimal overhead for event emission', () => { + const sink = new JsonlTraceSink('/tmp/perf-test.jsonl'); + const tracer = new Tracer('perf-test', sink); + + const start = Date.now(); + + // Emit 1000 events + for (let i = 0; i < 1000; i++) { + tracer.emit('test', { index: i }); + } + + const duration = Date.now() - start; + + // Should complete in less than 1 second (very generous threshold) + expect(duration).toBeLessThan(1000); + + tracer.close(); + }); + }); + + describe('Memory Management', () => { + it('should not leak memory on close', async () => { + const sink = new JsonlTraceSink('/tmp/memory-test.jsonl'); + const tracer = new Tracer('memory-test', sink); + + tracer.emit('test', { data: 'test' }); + + await tracer.close(); + + // Attempting to emit after close should be safe (no crash) + sink.emit({ test: 'after close' }); + + expect(sink.isClosed()).toBe(true); + }); + }); + + describe('Error Resilience', () => { + it('should handle errors gracefully without crashing', () => { + const sink = new JsonlTraceSink('/tmp/error-test.jsonl'); + const tracer = new Tracer('error-test', sink); + + // Should not throw + expect(() => { + tracer.emit('test', {}); + tracer.emitRunStart('Agent'); + tracer.emitStepStart('step-1', 1, 'goal'); + tracer.emitError('step-1', 'error message'); + tracer.emitRunEnd(1); + }).not.toThrow(); + + tracer.close(); + }); + }); +}); diff --git a/tests/tracing/tracer.test.ts b/tests/tracing/tracer.test.ts new file mode 100644 index 00000000..7c31b12f --- /dev/null +++ b/tests/tracing/tracer.test.ts @@ -0,0 +1,336 @@ +/** + * Tests for Tracer + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { Tracer } from '../../src/tracing/tracer'; +import { JsonlTraceSink } from '../../src/tracing/jsonl-sink'; +import { TraceSink } from '../../src/tracing/sink'; +import { TraceEvent } from '../../src/tracing/types'; + +describe('Tracer', () => { + const testDir = path.join(__dirname, 'test-traces'); + const testFile = path.join(testDir, 'tracer-test.jsonl'); + + beforeEach(() => { + // Clean up test directory + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + // Clean up test directory + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true }); + } + }); + + describe('Basic functionality', () => { + it('should create tracer with run ID and sink', () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run-123', sink); + + expect(tracer.getRunId()).toBe('test-run-123'); + expect(tracer.getSeq()).toBe(0); + + tracer.close(); + }); + + it('should auto-increment sequence numbers', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emit('event1', {}); + expect(tracer.getSeq()).toBe(1); + + tracer.emit('event2', {}); + expect(tracer.getSeq()).toBe(2); + + tracer.emit('event3', {}); + expect(tracer.getSeq()).toBe(3); + + await tracer.close(); + }); + + it('should generate timestamps in ISO 8601 format', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + const before = Date.now(); + tracer.emit('test', { data: 'test' }); + const after = Date.now(); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.ts).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/); + expect(event.ts_ms).toBeGreaterThanOrEqual(before); + expect(event.ts_ms).toBeLessThanOrEqual(after); + }); + + it('should include all required fields in emitted events', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run-456', sink); + + tracer.emit('test_event', { key: 'value' }, 'step-123'); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.v).toBe(1); + expect(event.type).toBe('test_event'); + expect(event.ts).toBeDefined(); + expect(event.ts_ms).toBeDefined(); + expect(event.run_id).toBe('test-run-456'); + expect(event.seq).toBe(1); + expect(event.data).toEqual({ key: 'value' }); + expect(event.step_id).toBe('step-123'); + }); + + it('should omit step_id if not provided', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emit('test_event', { key: 'value' }); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.step_id).toBeUndefined(); + }); + }); + + describe('Convenience methods', () => { + it('should emit run_start event', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emitRunStart('SentienceAgent', 'gpt-4o', { timeout: 30000 }); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.type).toBe('run_start'); + expect(event.data.agent).toBe('SentienceAgent'); + expect(event.data.llm_model).toBe('gpt-4o'); + expect(event.data.config).toEqual({ timeout: 30000 }); + expect(event.step_id).toBeUndefined(); + }); + + it('should emit run_start with optional fields', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emitRunStart('SentienceAgent'); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.type).toBe('run_start'); + expect(event.data.agent).toBe('SentienceAgent'); + expect(event.data.llm_model).toBeUndefined(); + expect(event.data.config).toBeUndefined(); + }); + + it('should emit step_start event', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emitStepStart('step-001', 1, 'Click the button', 0, 'https://example.com'); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.type).toBe('step_start'); + expect(event.step_id).toBe('step-001'); + expect(event.data.step_id).toBe('step-001'); + expect(event.data.step_index).toBe(1); + expect(event.data.goal).toBe('Click the button'); + expect(event.data.attempt).toBe(0); + expect(event.data.url).toBe('https://example.com'); + }); + + it('should emit step_start with default attempt and no URL', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emitStepStart('step-002', 2, 'Type text'); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.type).toBe('step_start'); + expect(event.data.attempt).toBe(0); + expect(event.data.url).toBeUndefined(); + }); + + it('should emit run_end event', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emitRunEnd(5); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.type).toBe('run_end'); + expect(event.data.steps).toBe(5); + expect(event.step_id).toBeUndefined(); + }); + + it('should emit error event', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emitError('step-003', 'Element not found', 2); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.type).toBe('error'); + expect(event.step_id).toBe('step-003'); + expect(event.data.step_id).toBe('step-003'); + expect(event.data.error).toBe('Element not found'); + expect(event.data.attempt).toBe(2); + }); + + it('should emit error with default attempt', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emitError('step-004', 'Timeout'); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.data.attempt).toBe(0); + }); + }); + + describe('Integration', () => { + it('should produce valid event sequence', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + // Simulate agent execution + tracer.emitRunStart('SentienceAgent', 'gpt-4o'); + tracer.emitStepStart('step-1', 1, 'Navigate to page', 0, 'https://start.com'); + tracer.emit('snapshot', { url: 'https://start.com', elements: [] }, 'step-1'); + tracer.emit('action', { action_type: 'click', element_id: 5 }, 'step-1'); + tracer.emitStepStart('step-2', 2, 'Fill form', 0, 'https://form.com'); + tracer.emit('action', { action_type: 'type', text: 'test' }, 'step-2'); + tracer.emitRunEnd(2); + + await tracer.close(); + + const content = fs.readFileSync(testFile, 'utf-8'); + const lines = content.trim().split('\n'); + const events = lines.map(line => JSON.parse(line) as TraceEvent); + + expect(events.length).toBe(7); + + // Check sequence numbers + events.forEach((event, index) => { + expect(event.seq).toBe(index + 1); + }); + + // Check event types + expect(events[0].type).toBe('run_start'); + expect(events[1].type).toBe('step_start'); + expect(events[2].type).toBe('snapshot'); + expect(events[3].type).toBe('action'); + expect(events[4].type).toBe('step_start'); + expect(events[5].type).toBe('action'); + expect(events[6].type).toBe('run_end'); + + // Check all events have same run_id + events.forEach(event => { + expect(event.run_id).toBe('test-run'); + }); + + // Check step IDs + expect(events[1].step_id).toBe('step-1'); + expect(events[2].step_id).toBe('step-1'); + expect(events[3].step_id).toBe('step-1'); + expect(events[4].step_id).toBe('step-2'); + expect(events[5].step_id).toBe('step-2'); + }); + }); + + describe('Error handling', () => { + it('should close sink when tracer is closed', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emit('test', {}); + await tracer.close(); + + expect(sink.isClosed()).toBe(true); + }); + + it('should work with custom sink implementations', async () => { + class MockSink extends TraceSink { + public events: any[] = []; + public closeCount = 0; + + emit(event: Record): void { + this.events.push(event); + } + + async close(): Promise { + this.closeCount++; + } + + getSinkType(): string { + return 'MockSink'; + } + } + + const mockSink = new MockSink(); + const tracer = new Tracer('test-run', mockSink); + + tracer.emit('event1', { data: 1 }); + tracer.emit('event2', { data: 2 }); + + expect(mockSink.events.length).toBe(2); + expect(mockSink.events[0].type).toBe('event1'); + expect(mockSink.events[1].type).toBe('event2'); + + await tracer.close(); + expect(mockSink.closeCount).toBe(1); + }); + }); + + describe('getSinkType', () => { + it('should return sink type from underlying sink', () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + expect(tracer.getSinkType()).toBe(`JsonlTraceSink(${testFile})`); + + tracer.close(); + }); + }); +}); From db851a7b37784a7defed771f7dcd9cf951a8437a Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 13:06:22 -0800 Subject: [PATCH 2/6] phase 1 and 2 for Tracing --- README.md | 130 +++++++++++ examples/agent-with-tracing.ts | 144 ++++++++++++ examples/trace-replay-demo.ts | 161 +++++++++++++ package.json | 2 + src/agent.ts | 76 ++++++- src/tracing/jsonl-sink.ts | 12 +- tests/tracing/agent-integration.test.ts | 291 ++++++++++++++++++++++++ tests/tracing/agent-regression.test.ts | 186 +++++++++++++++ 8 files changed, 997 insertions(+), 5 deletions(-) create mode 100644 examples/agent-with-tracing.ts create mode 100644 examples/trace-replay-demo.ts create mode 100644 tests/tracing/agent-integration.test.ts create mode 100644 tests/tracing/agent-regression.test.ts diff --git a/README.md b/README.md index 19391de5..9c2eb8a1 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,136 @@ await browser.close(); --- +## Agent Execution Tracing (NEW in v0.3.1) + +Record complete agent execution traces for debugging, analysis, and replay. Traces capture every step, snapshot, LLM decision, and action in a structured JSONL format. + +### Quick Start: Agent with Tracing + +```typescript +import { + SentienceBrowser, + SentienceAgent, + OpenAIProvider, + Tracer, + JsonlTraceSink +} from 'sentience-ts'; +import { randomUUID } from 'crypto'; + +const browser = await SentienceBrowser.create({ apiKey: process.env.SENTIENCE_API_KEY }); +const llm = new OpenAIProvider(process.env.OPENAI_API_KEY!, 'gpt-4o'); + +// Create a tracer +const runId = randomUUID(); +const sink = new JsonlTraceSink(`traces/${runId}.jsonl`); +const tracer = new Tracer(runId, sink); + +// Create agent with tracer +const agent = new SentienceAgent(browser, llm, 50, true, tracer); + +// Emit run_start +tracer.emitRunStart('SentienceAgent', 'gpt-4o'); + +try { + await browser.getPage().goto('https://google.com'); + + // Every action is automatically traced! + await agent.act('Click the search box'); + await agent.act("Type 'sentience ai' into the search field"); + await agent.act('Press Enter'); + + tracer.emitRunEnd(3); +} finally { + // Flush trace to disk + await agent.closeTracer(); + await browser.close(); +} + +console.log(`āœ… Trace saved to: traces/${runId}.jsonl`); +``` + +### What Gets Traced + +Each agent action generates multiple events: + +1. **step_start** - Before action execution (goal, URL, attempt) +2. **snapshot** - Page state with all interactive elements +3. **llm_response** - LLM decision (model, tokens, response) +4. **action** - Executed action (type, element ID, success) +5. **error** - Any failures (error message, retry attempt) + +**Example trace output:** +```jsonl +{"v":1,"type":"run_start","ts":"2025-12-26T10:00:00.000Z","run_id":"abc-123","seq":1,"data":{"agent":"SentienceAgent","llm_model":"gpt-4o"}} +{"v":1,"type":"step_start","ts":"2025-12-26T10:00:01.000Z","run_id":"abc-123","seq":2,"step_id":"step-1","data":{"step_index":1,"goal":"Click the search box","attempt":0,"url":"https://google.com"}} +{"v":1,"type":"snapshot","ts":"2025-12-26T10:00:01.500Z","run_id":"abc-123","seq":3,"step_id":"step-1","data":{"url":"https://google.com","elements":[...]}} +{"v":1,"type":"llm_response","ts":"2025-12-26T10:00:02.000Z","run_id":"abc-123","seq":4,"step_id":"step-1","data":{"model":"gpt-4o","prompt_tokens":250,"completion_tokens":10,"response_text":"CLICK(42)"}} +{"v":1,"type":"action","ts":"2025-12-26T10:00:02.500Z","run_id":"abc-123","seq":5,"step_id":"step-1","data":{"action_type":"click","element_id":42,"success":true}} +{"v":1,"type":"run_end","ts":"2025-12-26T10:00:03.000Z","run_id":"abc-123","seq":6,"data":{"steps":1}} +``` + +### Reading and Analyzing Traces + +```typescript +import * as fs from 'fs'; + +// Read trace file +const content = fs.readFileSync(`traces/${runId}.jsonl`, 'utf-8'); +const events = content.trim().split('\n').map(JSON.parse); + +console.log(`Total events: ${events.length}`); + +// Analyze events +events.forEach(event => { + console.log(`[${event.seq}] ${event.type} - ${event.ts}`); +}); + +// Filter by type +const actions = events.filter(e => e.type === 'action'); +console.log(`Actions taken: ${actions.length}`); + +// Get token usage +const llmEvents = events.filter(e => e.type === 'llm_response'); +const totalTokens = llmEvents.reduce((sum, e) => sum + (e.data.prompt_tokens || 0) + (e.data.completion_tokens || 0), 0); +console.log(`Total tokens: ${totalTokens}`); +``` + +### Tracing Without Agent (Manual) + +You can also use the tracer directly for custom workflows: + +```typescript +import { Tracer, JsonlTraceSink } from 'sentience-ts'; +import { randomUUID } from 'crypto'; + +const runId = randomUUID(); +const sink = new JsonlTraceSink(`traces/${runId}.jsonl`); +const tracer = new Tracer(runId, sink); + +// Emit custom events +tracer.emit('custom_event', { + message: 'Something happened', + details: { foo: 'bar' } +}); + +// Use convenience methods +tracer.emitRunStart('MyAgent', 'gpt-4o'); +tracer.emitStepStart('step-1', 1, 'Do something'); +tracer.emitError('step-1', 'Something went wrong'); +tracer.emitRunEnd(1); + +// Flush to disk +await tracer.close(); +``` + +### Schema Compatibility + +Traces are **100% compatible** with Python SDK traces - use the same tools to analyze traces from both TypeScript and Python agents! + +**See full example:** [examples/agent-with-tracing.ts](examples/agent-with-tracing.ts) + +--- + ## Agent Layer Examples ### Google Search (6 lines of code) diff --git a/examples/agent-with-tracing.ts b/examples/agent-with-tracing.ts new file mode 100644 index 00000000..39ae7440 --- /dev/null +++ b/examples/agent-with-tracing.ts @@ -0,0 +1,144 @@ +/** + * Agent with Tracing Example + * + * Demonstrates how to record agent execution traces for debugging and analysis + * + * Usage: + * ts-node examples/agent-with-tracing.ts + * or + * npm run example:tracing + */ + +import { SentienceBrowser } from '../src/browser'; +import { SentienceAgent } from '../src/agent'; +import { OpenAIProvider } from '../src/llm-provider'; +import { Tracer } from '../src/tracing/tracer'; +import { JsonlTraceSink } from '../src/tracing/jsonl-sink'; +import { randomUUID } from 'crypto'; +import * as fs from 'fs'; +import * as path from 'path'; + +async function main() { + // Get API keys from environment + const sentienceKey = process.env.SENTIENCE_API_KEY; + const openaiKey = process.env.OPENAI_API_KEY; + + if (!sentienceKey || !openaiKey) { + console.error('Error: Missing API keys'); + console.error('Please set SENTIENCE_API_KEY and OPENAI_API_KEY environment variables'); + process.exit(1); + } + + console.log('šŸš€ Starting Agent with Tracing Demo\n'); + + // Create browser and LLM + const browser = await SentienceBrowser.create({ apiKey: sentienceKey }); + const llm = new OpenAIProvider(openaiKey, 'gpt-4o-mini'); + + // Create traces directory + const tracesDir = path.join(__dirname, '..', 'traces'); + if (!fs.existsSync(tracesDir)) { + fs.mkdirSync(tracesDir, { recursive: true }); + } + + // Create tracer + const runId = randomUUID(); + const traceFile = path.join(tracesDir, `${runId}.jsonl`); + const sink = new JsonlTraceSink(traceFile); + const tracer = new Tracer(runId, sink); + + console.log(`šŸ“ Trace file: ${traceFile}`); + console.log(`šŸ†” Run ID: ${runId}\n`); + + // Create agent with tracer + const agent = new SentienceAgent(browser, llm, 50, true, tracer); + + // Emit run_start event + tracer.emitRunStart('SentienceAgent', 'gpt-4o-mini', { + example: 'agent-with-tracing', + timestamp: new Date().toISOString(), + }); + + try { + // Navigate to Google + console.log('🌐 Navigating to Google...\n'); + const page = browser.getPage(); + await page.goto('https://www.google.com'); + await page.waitForLoadState('networkidle'); + await new Promise(resolve => setTimeout(resolve, 1000)); + + // Execute agent actions (automatically traced!) + console.log('šŸ¤– Executing agent actions...\n'); + + await agent.act('Click the search box'); + await agent.act("Type 'artificial intelligence' into the search field"); + await agent.act('Press Enter key'); + + // Wait for results + await new Promise(resolve => setTimeout(resolve, 2000)); + + // Emit run_end event + tracer.emitRunEnd(3); + + console.log('\nāœ… Agent execution completed successfully!\n'); + + // Display token usage + const stats = agent.getTokenStats(); + console.log('šŸ“Š Token Usage:'); + console.log(` Total Prompt Tokens: ${stats.totalPromptTokens}`); + console.log(` Total Completion Tokens: ${stats.totalCompletionTokens}`); + console.log(` Total Tokens: ${stats.totalTokens}\n`); + + } catch (error: any) { + console.error('āŒ Error during execution:', error.message); + tracer.emitError('main', error.message, 0); + } finally { + // Flush trace to disk + console.log('šŸ’¾ Flushing trace to disk...'); + await agent.closeTracer(); + await browser.close(); + } + + // Read and analyze the trace + console.log('\nšŸ“– Trace Analysis:\n'); + + const content = fs.readFileSync(traceFile, 'utf-8'); + const events = content.trim().split('\n').map(line => JSON.parse(line)); + + console.log(` Total events: ${events.length}`); + + // Count by type + const eventTypes = events.reduce((acc: Record, e) => { + acc[e.type] = (acc[e.type] || 0) + 1; + return acc; + }, {}); + + console.log(' Event breakdown:'); + Object.entries(eventTypes).forEach(([type, count]) => { + console.log(` - ${type}: ${count}`); + }); + + // Show event sequence + console.log('\n Event sequence:'); + events.forEach((event, i) => { + const stepInfo = event.step_id ? ` [step: ${event.step_id.substring(0, 8)}]` : ''; + console.log(` [${event.seq}] ${event.type}${stepInfo} - ${event.ts}`); + }); + + // Calculate total tokens from trace + const llmEvents = events.filter(e => e.type === 'llm_response'); + const totalTokensFromTrace = llmEvents.reduce( + (sum, e) => sum + (e.data.prompt_tokens || 0) + (e.data.completion_tokens || 0), + 0 + ); + + console.log(`\n Total tokens (from trace): ${totalTokensFromTrace}`); + + console.log(`\n✨ Trace saved to: ${traceFile}`); + console.log(' You can analyze this file with any JSONL parser!\n'); +} + +main().catch(error => { + console.error('Fatal error:', error); + process.exit(1); +}); diff --git a/examples/trace-replay-demo.ts b/examples/trace-replay-demo.ts new file mode 100644 index 00000000..553eca1c --- /dev/null +++ b/examples/trace-replay-demo.ts @@ -0,0 +1,161 @@ +/** + * Trace Replay Demo + * + * Demonstrates how to read and analyze trace files + * + * Usage: + * ts-node examples/trace-replay-demo.ts + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { TraceEvent } from '../src/tracing/types'; + +function analyzeTrace(filePath: string) { + if (!fs.existsSync(filePath)) { + console.error(`āŒ Error: Trace file not found: ${filePath}`); + process.exit(1); + } + + console.log('šŸ“– Reading trace file...\n'); + + const content = fs.readFileSync(filePath, 'utf-8'); + const events: TraceEvent[] = content + .trim() + .split('\n') + .map(line => JSON.parse(line)); + + console.log(`āœ… Loaded ${events.length} events\n`); + + // Extract metadata + const runStart = events.find(e => e.type === 'run_start'); + const runEnd = events.find(e => e.type === 'run_end'); + + if (runStart) { + console.log('šŸ Run Metadata:'); + console.log(` Run ID: ${runStart.run_id}`); + console.log(` Agent: ${runStart.data.agent}`); + console.log(` LLM Model: ${runStart.data.llm_model || 'N/A'}`); + console.log(` Started: ${runStart.ts}`); + if (runEnd) { + console.log(` Ended: ${runEnd.ts}`); + console.log(` Total Steps: ${runEnd.data.steps}`); + } + console.log(); + } + + // Event type breakdown + const eventTypes = events.reduce((acc: Record, e) => { + acc[e.type] = (acc[e.type] || 0) + 1; + return acc; + }, {}); + + console.log('šŸ“Š Event Types:'); + Object.entries(eventTypes) + .sort(([, a], [, b]) => b - a) + .forEach(([type, count]) => { + console.log(` ${type.padEnd(15)} : ${count}`); + }); + console.log(); + + // Step analysis + const stepStarts = events.filter(e => e.type === 'step_start'); + console.log(`šŸ”„ Steps (${stepStarts.length} total):`); + stepStarts.forEach(step => { + console.log(` [Step ${step.data.step_index}] ${step.data.goal}`); + console.log(` URL: ${step.data.url}`); + console.log(` Attempt: ${step.data.attempt}`); + }); + console.log(); + + // Token usage analysis + const llmEvents = events.filter(e => e.type === 'llm_response'); + if (llmEvents.length > 0) { + const totalPromptTokens = llmEvents.reduce( + (sum, e) => sum + (e.data.prompt_tokens || 0), + 0 + ); + const totalCompletionTokens = llmEvents.reduce( + (sum, e) => sum + (e.data.completion_tokens || 0), + 0 + ); + + console.log('šŸ’¬ LLM Usage:'); + console.log(` Total Calls: ${llmEvents.length}`); + console.log(` Prompt Tokens: ${totalPromptTokens}`); + console.log(` Completion Tokens: ${totalCompletionTokens}`); + console.log(` Total Tokens: ${totalPromptTokens + totalCompletionTokens}`); + console.log(); + + console.log(' Decisions:'); + llmEvents.forEach((event, i) => { + const responsePreview = event.data.response_text?.substring(0, 50) || 'N/A'; + console.log(` [${i + 1}] ${responsePreview}...`); + }); + console.log(); + } + + // Action analysis + const actions = events.filter(e => e.type === 'action'); + if (actions.length > 0) { + console.log(`⚔ Actions (${actions.length} total):`); + actions.forEach((action, i) => { + const status = action.data.success ? 'āœ…' : 'āŒ'; + const actionType = action.data.action_type || 'unknown'; + const details = action.data.element_id + ? `element ${action.data.element_id}` + : action.data.text + ? `text: "${action.data.text}"` + : action.data.key + ? `key: ${action.data.key}` + : ''; + + console.log(` [${i + 1}] ${status} ${actionType} ${details}`); + }); + console.log(); + } + + // Error analysis + const errors = events.filter(e => e.type === 'error'); + if (errors.length > 0) { + console.log(`āŒ Errors (${errors.length} total):`); + errors.forEach((error, i) => { + console.log(` [${i + 1}] Attempt ${error.data.attempt}: ${error.data.error}`); + }); + console.log(); + } else { + console.log('āœ… No errors recorded\n'); + } + + // Timeline + console.log('ā±ļø Timeline:'); + events.forEach(event => { + const time = new Date(event.ts).toLocaleTimeString(); + const stepInfo = event.step_id ? ` [${event.step_id.substring(0, 8)}]` : ''; + const icon = { + run_start: 'šŸ', + run_end: 'šŸ', + step_start: 'ā–¶ļø', + snapshot: 'šŸ“ø', + llm_response: '🧠', + action: '⚔', + error: 'āŒ', + }[event.type] || '•'; + + console.log(` [${event.seq.toString().padStart(3)}] ${time} ${icon} ${event.type}${stepInfo}`); + }); + console.log(); +} + +// CLI usage +const args = process.argv.slice(2); + +if (args.length === 0) { + console.log('Usage: ts-node examples/trace-replay-demo.ts '); + console.log('\nExample:'); + console.log(' ts-node examples/trace-replay-demo.ts traces/abc-123.jsonl'); + process.exit(1); +} + +const traceFile = path.resolve(args[0]); +analyzeTrace(traceFile); diff --git a/package.json b/package.json index 3859d91d..44d5ff9b 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,8 @@ "example:agent-claude": "ts-node examples/agent-with-anthropic.ts", "example:conversational-google": "ts-node examples/conversational-google-search.ts", "example:conversational-amazon": "ts-node examples/conversational-amazon-shopping.ts", + "example:tracing": "ts-node examples/agent-with-tracing.ts", + "example:trace-replay": "ts-node examples/trace-replay-demo.ts", "cli": "ts-node src/cli.ts" }, "bin": { diff --git a/src/agent.ts b/src/agent.ts index cdb46166..02287334 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -8,6 +8,8 @@ import { snapshot, SnapshotOptions } from './snapshot'; import { click, typeText, press } from './actions'; import { Snapshot, Element, ActionResult } from './types'; import { LLMProvider, LLMResponse } from './llm-provider'; +import { Tracer } from './tracing/tracer'; +import { randomUUID } from 'crypto'; /** * Execution result from agent.act() @@ -82,6 +84,8 @@ export class SentienceAgent { private llm: LLMProvider; private snapshotLimit: number; private verbose: boolean; + private tracer?: Tracer; + private stepCount: number; private history: HistoryEntry[]; private tokenUsage: TokenStats; @@ -91,17 +95,21 @@ export class SentienceAgent { * @param llm - LLM provider (OpenAIProvider, AnthropicProvider, etc.) * @param snapshotLimit - Maximum elements to include in context (default: 50) * @param verbose - Print execution logs (default: true) + * @param tracer - Optional tracer for recording execution (default: undefined) */ constructor( browser: SentienceBrowser, llm: LLMProvider, snapshotLimit: number = 50, - verbose: boolean = true + verbose: boolean = true, + tracer?: Tracer ) { this.browser = browser; this.llm = llm; this.snapshotLimit = snapshotLimit; this.verbose = verbose; + this.tracer = tracer; + this.stepCount = 0; this.history = []; this.tokenUsage = { totalPromptTokens: 0, @@ -136,6 +144,16 @@ export class SentienceAgent { console.log('='.repeat(70)); } + // Increment step counter and generate step ID + this.stepCount += 1; + const stepId = randomUUID(); + + // Emit step_start event + if (this.tracer) { + const currentUrl = this.browser.getPage().url(); + this.tracer.emitStepStart(stepId, this.stepCount, goal, 0, currentUrl); + } + for (let attempt = 0; attempt <= maxRetries; attempt++) { try { // 1. OBSERVE: Get refined semantic snapshot @@ -161,6 +179,19 @@ export class SentienceAgent { elements: filteredElements }; + // Emit snapshot event + if (this.tracer) { + this.tracer.emit('snapshot', { + url: filteredSnap.url, + elements: filteredSnap.elements.slice(0, 50).map(el => ({ + id: el.id, + bbox: el.bbox, + role: el.role, + text: el.text?.substring(0, 100), + })) + }, stepId); + } + // 2. GROUND: Format elements for LLM context const context = this.buildContext(filteredSnap, goal); @@ -171,6 +202,16 @@ export class SentienceAgent { console.log(`🧠 LLM Decision: ${llmResponse.content}`); } + // Emit LLM response event + if (this.tracer) { + this.tracer.emit('llm_response', { + model: llmResponse.modelName, + prompt_tokens: llmResponse.promptTokens, + completion_tokens: llmResponse.completionTokens, + response_text: llmResponse.content.substring(0, 500), + }, stepId); + } + // Track token usage this.trackTokens(goal, llmResponse); @@ -185,6 +226,17 @@ export class SentienceAgent { result.attempt = attempt; result.goal = goal; + // Emit action event + if (this.tracer) { + this.tracer.emit('action', { + action_type: result.action, + element_id: result.elementId, + text: result.text, + key: result.key, + success: result.success, + }, stepId); + } + // 5. RECORD: Track history this.history.push({ goal, @@ -203,6 +255,11 @@ export class SentienceAgent { return result; } catch (error: any) { + // Emit error event + if (this.tracer) { + this.tracer.emitError(stepId, error.message, attempt); + } + if (attempt < maxRetries) { if (this.verbose) { console.log(`āš ļø Retry ${attempt + 1}/${maxRetries}: ${error.message}`); @@ -474,6 +531,7 @@ Examples: */ clearHistory(): void { this.history = []; + this.stepCount = 0; this.tokenUsage = { totalPromptTokens: 0, totalCompletionTokens: 0, @@ -481,4 +539,20 @@ Examples: byAction: [] }; } + + /** + * Close the tracer and flush events to disk + */ + async closeTracer(): Promise { + if (this.tracer) { + await this.tracer.close(); + } + } + + /** + * Get the tracer instance (if any) + */ + getTracer(): Tracer | undefined { + return this.tracer; + } } diff --git a/src/tracing/jsonl-sink.ts b/src/tracing/jsonl-sink.ts index 5add2b4f..c0755800 100644 --- a/src/tracing/jsonl-sink.ts +++ b/src/tracing/jsonl-sink.ts @@ -66,14 +66,18 @@ export class JsonlTraceSink extends TraceSink { this.closed = true; - return new Promise((resolve, reject) => { + // Check if stream exists and is writable + if (!this.writeStream || this.writeStream.destroyed) { + return; + } + + return new Promise((resolve) => { this.writeStream.end((err?: Error | null) => { if (err) { console.error('[JsonlTraceSink] Error closing stream:', err); - reject(err); - } else { - resolve(); } + // Always resolve, don't reject on close errors + resolve(); }); }); } diff --git a/tests/tracing/agent-integration.test.ts b/tests/tracing/agent-integration.test.ts new file mode 100644 index 00000000..b44387f8 --- /dev/null +++ b/tests/tracing/agent-integration.test.ts @@ -0,0 +1,291 @@ +/** + * Agent Integration Tests with Tracing + * + * Tests that SentienceAgent works correctly with tracer enabled/disabled + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { SentienceAgent } from '../../src/agent'; +import { Tracer } from '../../src/tracing/tracer'; +import { JsonlTraceSink } from '../../src/tracing/jsonl-sink'; +import { TraceEvent } from '../../src/tracing/types'; + +// Mock browser and LLM +const mockBrowser: any = { + getPage: () => ({ + url: () => 'https://example.com', + }), +}; + +const mockLLM: any = { + generate: async () => ({ + content: 'FINISH()', + modelName: 'mock-model', + promptTokens: 100, + completionTokens: 20, + totalTokens: 120, + }), +}; + +describe('Agent Integration with Tracing', () => { + const testDir = path.join(__dirname, 'test-traces'); + const testFile = path.join(testDir, 'agent-test.jsonl'); + + beforeEach(() => { + // Clean up test directory + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + // Clean up test directory + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true }); + } + }); + + describe('Backward Compatibility (No Tracer)', () => { + it('should work without tracer (existing behavior)', async () => { + const agent = new SentienceAgent(mockBrowser, mockLLM); + + // Mock snapshot + const mockSnapshot = jest.spyOn(require('../../src/snapshot'), 'snapshot'); + mockSnapshot.mockResolvedValue({ + status: 'success', + url: 'https://example.com', + elements: [ + { id: 1, role: 'button', text: 'Click me', importance: 0.8, bbox: { x: 0, y: 0, width: 100, height: 50 }, visual_cues: {} }, + ], + }); + + const result = await agent.act('Finish task'); + + expect(result.success).toBe(true); + expect(result.action).toBe('finish'); + + mockSnapshot.mockRestore(); + }); + + it('should not have tracer-related side effects', async () => { + const agent = new SentienceAgent(mockBrowser, mockLLM); + + expect(agent.getTracer()).toBeUndefined(); + + // closeTracer should be safe to call even without tracer + await agent.closeTracer(); + }); + }); + + describe('Agent with Tracer', () => { + it('should accept tracer parameter', () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + const agent = new SentienceAgent(mockBrowser, mockLLM, 50, true, tracer); + + expect(agent.getTracer()).toBe(tracer); + + agent.closeTracer(); + }); + + it('should emit events during act() execution', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + const agent = new SentienceAgent(mockBrowser, mockLLM, 50, false, tracer); + + // Mock snapshot + const mockSnapshot = jest.spyOn(require('../../src/snapshot'), 'snapshot'); + mockSnapshot.mockResolvedValue({ + status: 'success', + url: 'https://example.com', + elements: [ + { + id: 1, + role: 'button', + text: 'Submit', + importance: 0.9, + bbox: { x: 10, y: 20, width: 100, height: 40 }, + visual_cues: { is_clickable: true }, + }, + ], + }); + + await agent.act('Complete the task'); + await agent.closeTracer(); + + mockSnapshot.mockRestore(); + + // Read trace file + const content = fs.readFileSync(testFile, 'utf-8'); + const lines = content.trim().split('\n'); + const events = lines.map(line => JSON.parse(line) as TraceEvent); + + // Should have at least: step_start, snapshot, llm_response, action + expect(events.length).toBeGreaterThanOrEqual(4); + + // Check event types + const eventTypes = events.map(e => e.type); + expect(eventTypes).toContain('step_start'); + expect(eventTypes).toContain('snapshot'); + expect(eventTypes).toContain('llm_response'); + expect(eventTypes).toContain('action'); + + // Verify all events have same run_id + const runIds = new Set(events.map(e => e.run_id)); + expect(runIds.size).toBe(1); + + // Verify step_id is set for step events + const stepStartEvent = events.find(e => e.type === 'step_start'); + expect(stepStartEvent?.step_id).toBeDefined(); + + const snapshotEvent = events.find(e => e.type === 'snapshot'); + expect(snapshotEvent?.step_id).toBe(stepStartEvent?.step_id); + }); + + it('should emit error events on failure', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + const agent = new SentienceAgent(mockBrowser, mockLLM, 50, false, tracer); + + // Mock snapshot to fail + const mockSnapshot = jest.spyOn(require('../../src/snapshot'), 'snapshot'); + mockSnapshot.mockRejectedValue(new Error('Snapshot failed')); + + try { + await agent.act('Do something', 1); // maxRetries = 1 + } catch (error) { + // Expected to fail + } + + await agent.closeTracer(); + mockSnapshot.mockRestore(); + + // Read trace file + const content = fs.readFileSync(testFile, 'utf-8'); + const lines = content.trim().split('\n'); + const events = lines.map(line => JSON.parse(line) as TraceEvent); + + // Should have step_start and error events + const eventTypes = events.map(e => e.type); + expect(eventTypes).toContain('step_start'); + expect(eventTypes).toContain('error'); + + // Check error event + const errorEvent = events.find(e => e.type === 'error'); + expect(errorEvent?.data.error).toContain('Snapshot failed'); + }); + + it('should track step count across multiple actions', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + const agent = new SentienceAgent(mockBrowser, mockLLM, 50, false, tracer); + + // Mock snapshot + const mockSnapshot = jest.spyOn(require('../../src/snapshot'), 'snapshot'); + mockSnapshot.mockResolvedValue({ + status: 'success', + url: 'https://example.com', + elements: [], + }); + + await agent.act('First action'); + await agent.act('Second action'); + await agent.act('Third action'); + + await agent.closeTracer(); + mockSnapshot.mockRestore(); + + // Read trace file + const content = fs.readFileSync(testFile, 'utf-8'); + const lines = content.trim().split('\n'); + const events = lines.map(line => JSON.parse(line) as TraceEvent); + + // Get all step_start events + const stepStarts = events.filter(e => e.type === 'step_start'); + expect(stepStarts.length).toBe(3); + + // Verify step indices + expect(stepStarts[0].data.step_index).toBe(1); + expect(stepStarts[1].data.step_index).toBe(2); + expect(stepStarts[2].data.step_index).toBe(3); + + // Verify goals + expect(stepStarts[0].data.goal).toBe('First action'); + expect(stepStarts[1].data.goal).toBe('Second action'); + expect(stepStarts[2].data.goal).toBe('Third action'); + }); + + it('should preserve agent functionality with tracer', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + const agent = new SentienceAgent(mockBrowser, mockLLM, 50, false, tracer); + + // Mock snapshot + const mockSnapshot = jest.spyOn(require('../../src/snapshot'), 'snapshot'); + mockSnapshot.mockResolvedValue({ + status: 'success', + url: 'https://example.com', + elements: [], + }); + + const result = await agent.act('Test goal'); + + // Verify result structure unchanged + expect(result.success).toBe(true); + expect(result.goal).toBe('Test goal'); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + expect(result.attempt).toBe(0); + + // Verify history tracking still works + const history = agent.getHistory(); + expect(history.length).toBe(1); + expect(history[0].goal).toBe('Test goal'); + + // Verify token tracking still works + const tokenStats = agent.getTokenStats(); + expect(tokenStats.totalPromptTokens).toBeGreaterThan(0); + + await agent.closeTracer(); + mockSnapshot.mockRestore(); + }); + }); + + describe('clearHistory with tracer', () => { + it('should reset step count when clearing history', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + const agent = new SentienceAgent(mockBrowser, mockLLM, 50, false, tracer); + + // Mock snapshot + const mockSnapshot = jest.spyOn(require('../../src/snapshot'), 'snapshot'); + mockSnapshot.mockResolvedValue({ + status: 'success', + url: 'https://example.com', + elements: [], + }); + + await agent.act('First'); + await agent.act('Second'); + + agent.clearHistory(); + + await agent.act('After clear'); + await agent.closeTracer(); + + mockSnapshot.mockRestore(); + + // Read trace file + const content = fs.readFileSync(testFile, 'utf-8'); + const lines = content.trim().split('\n'); + const events = lines.map(line => JSON.parse(line) as TraceEvent); + + const stepStarts = events.filter(e => e.type === 'step_start'); + + // Step indices should be 1, 2, then reset to 1 + expect(stepStarts[0].data.step_index).toBe(1); + expect(stepStarts[1].data.step_index).toBe(2); + expect(stepStarts[2].data.step_index).toBe(1); // Reset! + }); + }); +}); diff --git a/tests/tracing/agent-regression.test.ts b/tests/tracing/agent-regression.test.ts new file mode 100644 index 00000000..637c7890 --- /dev/null +++ b/tests/tracing/agent-regression.test.ts @@ -0,0 +1,186 @@ +/** + * Agent Regression Tests + * + * Ensures agent modifications for tracing don't break existing functionality + */ + +import { SentienceAgent } from '../../src/agent'; + +describe('Agent Regression Tests (Tracing Integration)', () => { + describe('Constructor Backward Compatibility', () => { + it('should accept 4 parameters (without tracer)', () => { + const mockBrowser: any = { + getPage: () => ({ url: () => 'https://test.com' }), + }; + const mockLLM: any = {}; + + // Old signature: (browser, llm, snapshotLimit, verbose) + const agent = new SentienceAgent(mockBrowser, mockLLM, 50, true); + + expect(agent).toBeDefined(); + expect(agent.getTracer()).toBeUndefined(); + }); + + it('should accept 5 parameters (with tracer)', () => { + const mockBrowser: any = { + getPage: () => ({ url: () => 'https://test.com' }), + }; + const mockLLM: any = {}; + const mockTracer: any = { + emit: jest.fn(), + emitStepStart: jest.fn(), + close: jest.fn(), + }; + + // New signature: (browser, llm, snapshotLimit, verbose, tracer) + const agent = new SentienceAgent(mockBrowser, mockLLM, 50, true, mockTracer); + + expect(agent).toBeDefined(); + expect(agent.getTracer()).toBe(mockTracer); + }); + + it('should accept minimal parameters', () => { + const mockBrowser: any = { + getPage: () => ({ url: () => 'https://test.com' }), + }; + const mockLLM: any = {}; + + // Minimal signature: (browser, llm) + const agent = new SentienceAgent(mockBrowser, mockLLM); + + expect(agent).toBeDefined(); + expect(agent.getTracer()).toBeUndefined(); + }); + }); + + describe('Method Signatures', () => { + let agent: SentienceAgent; + + beforeEach(() => { + const mockBrowser: any = { + getPage: () => ({ url: () => 'https://test.com' }), + }; + const mockLLM: any = {}; + agent = new SentienceAgent(mockBrowser, mockLLM); + }); + + it('should have getTokenStats method', () => { + const stats = agent.getTokenStats(); + expect(stats).toBeDefined(); + expect(stats.totalPromptTokens).toBe(0); + expect(stats.totalCompletionTokens).toBe(0); + expect(stats.totalTokens).toBe(0); + expect(stats.byAction).toEqual([]); + }); + + it('should have getHistory method', () => { + const history = agent.getHistory(); + expect(history).toBeDefined(); + expect(Array.isArray(history)).toBe(true); + expect(history.length).toBe(0); + }); + + it('should have clearHistory method', () => { + agent.clearHistory(); + expect(agent.getHistory().length).toBe(0); + }); + + it('should have new closeTracer method', async () => { + // Should not throw even without tracer + await expect(agent.closeTracer()).resolves.not.toThrow(); + }); + + it('should have new getTracer method', () => { + expect(agent.getTracer()).toBeUndefined(); + }); + }); + + describe('Return Types', () => { + it('should maintain AgentActResult interface', () => { + const mockBrowser: any = { + getPage: () => ({ url: () => 'https://test.com' }), + }; + const mockLLM: any = {}; + const agent = new SentienceAgent(mockBrowser, mockLLM); + + // Type check - this will fail at compile time if interface changed + const checkType = (result: any) => { + const typed: { + success: boolean; + action?: string; + elementId?: number; + text?: string; + key?: string; + outcome?: string; + urlChanged?: boolean; + durationMs: number; + attempt: number; + goal: string; + error?: string; + message?: string; + } = result; + return typed; + }; + + expect(checkType).toBeDefined(); + }); + + it('should maintain HistoryEntry interface', () => { + const mockBrowser: any = { + getPage: () => ({ url: () => 'https://test.com' }), + }; + const mockLLM: any = {}; + const agent = new SentienceAgent(mockBrowser, mockLLM); + + const history = agent.getHistory(); + + // Type check + const checkType = (entry: any) => { + const typed: { + goal: string; + action: string; + result: any; + success: boolean; + attempt: number; + durationMs: number; + } = entry; + return typed; + }; + + expect(checkType).toBeDefined(); + }); + + it('should maintain TokenStats interface', () => { + const mockBrowser: any = { + getPage: () => ({ url: () => 'https://test.com' }), + }; + const mockLLM: any = {}; + const agent = new SentienceAgent(mockBrowser, mockLLM); + + const stats = agent.getTokenStats(); + + // Type check + const typed: { + totalPromptTokens: number; + totalCompletionTokens: number; + totalTokens: number; + byAction: Array<{ + goal: string; + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; + model?: string; + }>; + } = stats; + + expect(typed).toBeDefined(); + }); + }); + + describe('Imports', () => { + it('should not break existing imports', () => { + // This test verifies that the import still works + expect(SentienceAgent).toBeDefined(); + }); + }); +}); From 99a489172e1a01fdbe8e34e742ffc47324d00758 Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 15:04:49 -0800 Subject: [PATCH 3/6] fix jsonl sink --- src/tracing/jsonl-sink.ts | 52 ++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/src/tracing/jsonl-sink.ts b/src/tracing/jsonl-sink.ts index c0755800..d0ac8148 100644 --- a/src/tracing/jsonl-sink.ts +++ b/src/tracing/jsonl-sink.ts @@ -13,7 +13,7 @@ import { TraceSink } from './sink'; */ export class JsonlTraceSink extends TraceSink { private path: string; - private writeStream: fs.WriteStream; + private writeStream: fs.WriteStream | null = null; private closed: boolean = false; /** @@ -24,17 +24,33 @@ export class JsonlTraceSink extends TraceSink { super(); this.path = filePath; - // Create parent directories if needed + // Create parent directories if needed (synchronously) const dir = path.dirname(filePath); - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } + try { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } - // Open file in append mode with line buffering - this.writeStream = fs.createWriteStream(filePath, { - flags: 'a', - encoding: 'utf-8', - }); + // Verify directory is writable + fs.accessSync(dir, fs.constants.W_OK); + + // Open file in append mode with line buffering + this.writeStream = fs.createWriteStream(filePath, { + flags: 'a', + encoding: 'utf-8', + autoClose: true, + }); + + // Handle stream errors (suppress logging if stream is closed) + this.writeStream.on('error', (error) => { + if (!this.closed) { + console.error('[JsonlTraceSink] Stream error:', error); + } + }); + } catch (error) { + console.error('[JsonlTraceSink] Failed to initialize sink:', error); + this.writeStream = null; + } } /** @@ -47,6 +63,11 @@ export class JsonlTraceSink extends TraceSink { return; } + if (!this.writeStream) { + console.error('[JsonlTraceSink] Write stream not available'); + return; + } + try { const jsonLine = JSON.stringify(event) + '\n'; this.writeStream.write(jsonLine); @@ -71,10 +92,17 @@ export class JsonlTraceSink extends TraceSink { return; } + // Store reference to satisfy TypeScript null checks + const stream = this.writeStream; + + // Remove error listener to prevent late errors + stream.removeAllListeners('error'); + return new Promise((resolve) => { - this.writeStream.end((err?: Error | null) => { + stream.end((err?: Error | null) => { if (err) { - console.error('[JsonlTraceSink] Error closing stream:', err); + // Silently ignore close errors in production + // (they're logged during stream lifetime if needed) } // Always resolve, don't reject on close errors resolve(); From d677e99345a181f60b86536fdfa8bf7f93059987 Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 15:05:20 -0800 Subject: [PATCH 4/6] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 44d5ff9b..f778c19b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sentience-ts", - "version": "0.3.0", + "version": "0.12.1", "description": "TypeScript SDK for Sentience AI Agent Browser Automation", "main": "dist/index.js", "types": "dist/index.d.ts", From 532e153ac8d24c1247e8ec43133aff074b5610ab Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 17:28:28 -0800 Subject: [PATCH 5/6] Tracer support added & verified with agent --- .gitattributes | 1 + .npmignore | 1 + docs/QUERY_DSL.md | 1 + examples/click-rect-demo.ts | 1 + sentience-chrome/content.js | 4 ++-- sentience-chrome/injected_api.js | 6 +++--- src/expect.ts | 1 + src/inspector.ts | 1 + src/screenshot.ts | 1 + src/types.ts | 1 + tests/query.test.ts | 1 + tests/screenshot.test.ts | 1 + tests/tsconfig.json | 1 + 13 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.gitattributes b/.gitattributes index faa01efd..ba825f17 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,3 +11,4 @@ *.yml text eol=lf *.yaml text eol=lf + diff --git a/.npmignore b/.npmignore index 41705fae..d0fcf259 100644 --- a/.npmignore +++ b/.npmignore @@ -43,3 +43,4 @@ docs/ *.tmp *.temp + diff --git a/docs/QUERY_DSL.md b/docs/QUERY_DSL.md index 1384c9d0..06b8bf0f 100644 --- a/docs/QUERY_DSL.md +++ b/docs/QUERY_DSL.md @@ -506,3 +506,4 @@ const center = query(snap, 'bbox.x>400 bbox.x<600 bbox.y>300 bbox.y<500'); - [Type Definitions](../../spec/sdk-types.md) - [Snapshot Schema](../../spec/snapshot.schema.json) + diff --git a/examples/click-rect-demo.ts b/examples/click-rect-demo.ts index 4526c769..32ae8e80 100644 --- a/examples/click-rect-demo.ts +++ b/examples/click-rect-demo.ts @@ -97,3 +97,4 @@ async function main() { main().catch(console.error); + diff --git a/sentience-chrome/content.js b/sentience-chrome/content.js index e625a772..26c70a24 100644 --- a/sentience-chrome/content.js +++ b/sentience-chrome/content.js @@ -94,7 +94,7 @@ function handleSnapshotRequest(data) { } if (response?.success) { - console.log(`[Sentience Bridge] āœ“ WASM processing complete in ${duration.toFixed(1)}ms`); + // console.log(`[Sentience Bridge] āœ“ WASM processing complete in ${duration.toFixed(1)}ms`); window.postMessage({ type: 'SENTIENCE_SNAPSHOT_RESULT', requestId: data.requestId, @@ -129,4 +129,4 @@ function handleSnapshotRequest(data) { } } -console.log('[Sentience Bridge] Ready - Extension ID:', chrome.runtime.id); +// console.log('[Sentience Bridge] Ready - Extension ID:', chrome.runtime.id); diff --git a/sentience-chrome/injected_api.js b/sentience-chrome/injected_api.js index 8081667e..b8946b73 100644 --- a/sentience-chrome/injected_api.js +++ b/sentience-chrome/injected_api.js @@ -1,7 +1,7 @@ // injected_api.js - MAIN WORLD (NO WASM! CSP-Resistant!) // This script ONLY collects raw DOM data and sends it to background for processing (async () => { - console.log('[SentienceAPI] Initializing (CSP-Resistant Mode)...'); + // console.log('[SentienceAPI] Initializing (CSP-Resistant Mode)...'); // Wait for Extension ID from content.js const getExtensionId = () => document.documentElement.dataset.sentienceExtensionId; @@ -22,7 +22,7 @@ return; } - console.log('[SentienceAPI] Extension ID:', extId); + // console.log('[SentienceAPI] Extension ID:', extId); // Registry for click actions (still needed for click() function) window.sentience_registry = []; @@ -802,5 +802,5 @@ } }; - console.log('[SentienceAPI] āœ“ Ready! (CSP-Resistant - WASM runs in background)'); + // console.log('[SentienceAPI] āœ“ Ready! (CSP-Resistant - WASM runs in background)'); })(); diff --git a/src/expect.ts b/src/expect.ts index 73bc7c1a..41811955 100644 --- a/src/expect.ts +++ b/src/expect.ts @@ -93,3 +93,4 @@ export function expect(browser: SentienceBrowser, selector: QuerySelector): Expe return new Expectation(browser, selector); } + diff --git a/src/inspector.ts b/src/inspector.ts index 2ee09701..ffc8705f 100644 --- a/src/inspector.ts +++ b/src/inspector.ts @@ -165,3 +165,4 @@ export function inspect(browser: SentienceBrowser): Inspector { return new Inspector(browser); } + diff --git a/src/screenshot.ts b/src/screenshot.ts index e7ee1381..8e9b26e9 100644 --- a/src/screenshot.ts +++ b/src/screenshot.ts @@ -48,3 +48,4 @@ export async function screenshot( return `data:${mimeType};base64,${base64Data}`; } + diff --git a/src/types.ts b/src/types.ts index 67009979..a777a3f6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -75,3 +75,4 @@ export interface QuerySelectorObject { export type QuerySelector = string | QuerySelectorObject; + diff --git a/tests/query.test.ts b/tests/query.test.ts index 339ccef5..c064c172 100644 --- a/tests/query.test.ts +++ b/tests/query.test.ts @@ -248,3 +248,4 @@ describe('find', () => { }); }); + diff --git a/tests/screenshot.test.ts b/tests/screenshot.test.ts index 9787260a..0e661b36 100644 --- a/tests/screenshot.test.ts +++ b/tests/screenshot.test.ts @@ -82,3 +82,4 @@ describe('screenshot', () => { }); }); + diff --git a/tests/tsconfig.json b/tests/tsconfig.json index 1a4de77a..e3af6f53 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -8,3 +8,4 @@ "include": ["**/*.test.ts"] } + From 5f49620350c5b50cf6edb3ed2af12d4d6a47fdf2 Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 17:43:33 -0800 Subject: [PATCH 6/6] fix tests --- tests/tracing/agent-integration.test.ts | 3 ++- tests/tracing/tracer.test.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/tracing/agent-integration.test.ts b/tests/tracing/agent-integration.test.ts index b44387f8..1ca2b448 100644 --- a/tests/tracing/agent-integration.test.ts +++ b/tests/tracing/agent-integration.test.ts @@ -33,10 +33,11 @@ describe('Agent Integration with Tracing', () => { const testFile = path.join(testDir, 'agent-test.jsonl'); beforeEach(() => { - // Clean up test directory + // Clean up and recreate test directory if (fs.existsSync(testDir)) { fs.rmSync(testDir, { recursive: true }); } + fs.mkdirSync(testDir, { recursive: true }); }); afterEach(() => { diff --git a/tests/tracing/tracer.test.ts b/tests/tracing/tracer.test.ts index 7c31b12f..3c1dcae5 100644 --- a/tests/tracing/tracer.test.ts +++ b/tests/tracing/tracer.test.ts @@ -14,10 +14,11 @@ describe('Tracer', () => { const testFile = path.join(testDir, 'tracer-test.jsonl'); beforeEach(() => { - // Clean up test directory + // Clean up and recreate test directory if (fs.existsSync(testDir)) { fs.rmSync(testDir, { recursive: true }); } + fs.mkdirSync(testDir, { recursive: true }); }); afterEach(() => {