From 3dff8e0e3bbbe2bff0b4536027972624a8847fc6 Mon Sep 17 00:00:00 2001 From: rcholic Date: Tue, 6 Jan 2026 11:27:45 -0800 Subject: [PATCH 1/4] human like typing --- .prettierignore | 1 + .prettierrc.json | 1 + README.md | 27 ++++++ src/actions.ts | 102 +++++++++++++++++++- src/index.ts | 2 +- src/utils/browser.ts | 1 + src/utils/llm-response-builder.ts | 1 + src/utils/trace-file-manager.ts | 1 + tests/actions.test.ts | 117 ++++++++++++++++++++++- tests/mocks/browser-mock.ts | 1 + tests/utils/action-executor.test.ts | 1 + tests/utils/llm-response-builder.test.ts | 1 + 12 files changed, 251 insertions(+), 5 deletions(-) diff --git a/.prettierignore b/.prettierignore index 18073bdd..0d880a7d 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,3 +7,4 @@ coverage/ *.min.js package-lock.json + diff --git a/.prettierrc.json b/.prettierrc.json index aaccf62e..b519b84b 100644 --- a/.prettierrc.json +++ b/.prettierrc.json @@ -7,3 +7,4 @@ "useTabs": false, "arrowParens": "avoid" } + diff --git a/README.md b/README.md index ec355537..53308ba4 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,33 @@ await browser.close(); --- +## 🆕 What's New (2026-01-06) + +### Human-like Typing +Add realistic delays between keystrokes to mimic human typing: +```typescript +// Type instantly (default) +await typeText(browser, elementId, 'Hello World'); + +// Type with human-like delay (~10ms between keystrokes) +await typeText(browser, elementId, 'Hello World', false, 10); +``` + +### Scroll to Element +Scroll elements into view with smooth animation: +```typescript +const snap = await snapshot(browser); +const button = find(snap, 'role=button text~"Submit"'); + +// Scroll element into view with smooth animation +await scrollTo(browser, button.id); + +// Scroll instantly to top of viewport +await scrollTo(browser, button.id, 'instant', 'start'); +``` + +--- +

📊 Agent Execution Tracing (NEW in v0.3.1)

 diff --git a/src/actions.ts b/src/actions.ts index 9954a337..c07707e6 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -223,6 +223,7 @@ export async function click( * @param elementId - Element ID from snapshot (must be a text input element) * @param text - Text to type * @param takeSnapshot - Take snapshot after action (default: false) + * @param delayMs - Delay between keystrokes in milliseconds for human-like typing (default: 0) * @returns ActionResult with success status, outcome, duration, and optional snapshot * * @example @@ -230,7 +231,11 @@ export async function click( * const snap = await snapshot(browser); * const searchBox = find(snap, 'role=searchbox'); * if (searchBox) { + * // Type instantly (default behavior) * await typeText(browser, searchBox.id, 'Hello World'); + * + * // Type with human-like delay (~10ms between keystrokes) + * await typeText(browser, searchBox.id, 'Hello World', false, 10); * } * ``` */ @@ -238,7 +243,8 @@ export async function typeText( browser: IBrowser, elementId: number, text: string, - takeSnapshot: boolean = false + takeSnapshot: boolean = false, + delayMs: number = 0 ): Promise { const page = browser.getPage(); if (!page) { @@ -270,8 +276,98 @@ export async function typeText( }; } - // Type using Playwright keyboard - await page.keyboard.type(text); + // Type using Playwright keyboard with optional delay between keystrokes + await page.keyboard.type(text, { delay: delayMs }); + + const durationMs = Date.now() - startTime; + const urlAfter = page.url(); + const urlChanged = urlBefore !== urlAfter; + + const outcome = urlChanged ? 'navigated' : 'dom_updated'; + + let snapshotAfter: Snapshot | undefined; + if (takeSnapshot) { + snapshotAfter = await snapshot(browser); + } + + return { + success: true, + duration_ms: durationMs, + outcome, + url_changed: urlChanged, + snapshot_after: snapshotAfter, + }; +} + +/** + * Scroll an element into view + * + * Scrolls the page so that the specified element is visible in the viewport. + * Uses the element registry to find the element and scrollIntoView() to scroll it. + * + * @param browser - SentienceBrowser instance + * @param elementId - Element ID from snapshot to scroll into view + * @param behavior - Scroll behavior: 'smooth' for animated scroll, 'instant' for immediate (default: 'smooth') + * @param block - Vertical alignment: 'start', 'center', 'end', 'nearest' (default: 'center') + * @param takeSnapshot - Take snapshot after action (default: false) + * @returns ActionResult with success status, outcome, duration, and optional snapshot + * + * @example + * ```typescript + * const snap = await snapshot(browser); + * const button = find(snap, 'role=button[name="Submit"]'); + * if (button) { + * // Scroll element into view with smooth animation + * await scrollTo(browser, button.id); + * + * // Scroll instantly to top of viewport + * await scrollTo(browser, button.id, 'instant', 'start'); + * } + * ``` + */ +export async function scrollTo( + browser: IBrowser, + elementId: number, + behavior: 'smooth' | 'instant' | 'auto' = 'smooth', + block: 'start' | 'center' | 'end' | 'nearest' = 'center', + takeSnapshot: boolean = false +): Promise { + const page = browser.getPage(); + if (!page) { + throw new Error('Browser not started. Call start() first.'); + } + const startTime = Date.now(); + const urlBefore = page.url(); + + // Scroll element into view using the element registry + const scrolled = await BrowserEvaluator.evaluate( + page, + (args: { id: number; behavior: string; block: string }) => { + const el = (window as any).sentience_registry[args.id]; + if (el && el.scrollIntoView) { + el.scrollIntoView({ + behavior: args.behavior, + block: args.block, + inline: 'nearest', + }); + return true; + } + return false; + }, + { id: elementId, behavior, block } + ); + + if (!scrolled) { + return { + success: false, + duration_ms: Date.now() - startTime, + outcome: 'error', + error: { code: 'scroll_failed', reason: 'Element not found or not scrollable' }, + }; + } + + // Wait a bit for scroll to complete (especially for smooth scrolling) + await page.waitForTimeout(behavior === 'smooth' ? 500 : 100); const durationMs = Date.now() - startTime; const urlAfter = page.url(); diff --git a/src/index.ts b/src/index.ts index 66394d33..17a41f28 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,7 @@ export { SentienceBrowser } from './browser'; export { snapshot, SnapshotOptions } from './snapshot'; export { query, find, parseSelector } from './query'; -export { click, typeText, press, clickRect, ClickRect } from './actions'; +export { click, typeText, press, scrollTo, clickRect, ClickRect } from './actions'; export { waitFor } from './wait'; export { expect, Expectation } from './expect'; export { Inspector, inspect } from './inspector'; diff --git a/src/utils/browser.ts b/src/utils/browser.ts index 4d1cb96b..77a1d25b 100644 --- a/src/utils/browser.ts +++ b/src/utils/browser.ts @@ -39,3 +39,4 @@ export async function saveStorageState(context: BrowserContext, filePath: string fs.writeFileSync(filePath, JSON.stringify(storageState, null, 2)); console.log(`✅ [Sentience] Saved storage state to ${filePath}`); } + diff --git a/src/utils/llm-response-builder.ts b/src/utils/llm-response-builder.ts index 099e6df0..8c7b924d 100644 --- a/src/utils/llm-response-builder.ts +++ b/src/utils/llm-response-builder.ts @@ -135,3 +135,4 @@ export class LLMResponseBuilder { }; } } + diff --git a/src/utils/trace-file-manager.ts b/src/utils/trace-file-manager.ts index 9265ffbd..846edb21 100644 --- a/src/utils/trace-file-manager.ts +++ b/src/utils/trace-file-manager.ts @@ -173,3 +173,4 @@ export class TraceFileManager { } } } + diff --git a/tests/actions.test.ts b/tests/actions.test.ts index 84c8ca69..22970fa5 100644 --- a/tests/actions.test.ts +++ b/tests/actions.test.ts @@ -2,7 +2,7 @@ * Tests for actions (click, type, press, clickRect) */ -import { SentienceBrowser, click, typeText, press, clickRect, snapshot, find, BBox } from '../src'; +import { SentienceBrowser, click, typeText, press, scrollTo, clickRect, snapshot, find, BBox } from '../src'; import { createTestBrowser, getPageOrThrow } from './test-utils'; describe('Actions', () => { @@ -119,6 +119,121 @@ describe('Actions', () => { }, 60000); }); + describe('scrollTo', () => { + it('should scroll an element into view', async () => { + const browser = await createTestBrowser(); + + try { + const page = getPageOrThrow(browser); + await page.goto('https://example.com'); + await page.waitForLoadState('networkidle', { timeout: 10000 }); + + const snap = await snapshot(browser); + // Find an element to scroll to + const elements = snap.elements.filter(el => el.role === 'link'); + + if (elements.length > 0) { + // Get the last element which might be out of viewport + const element = elements.length > 1 ? elements[elements.length - 1] : elements[0]; + const result = await scrollTo(browser, element.id); + expect(result.success).toBe(true); + expect(result.duration_ms).toBeGreaterThan(0); + expect(['navigated', 'dom_updated']).toContain(result.outcome); + } + } finally { + await browser.close(); + } + }, 60000); + + it('should scroll with instant behavior', async () => { + const browser = await createTestBrowser(); + + try { + const page = getPageOrThrow(browser); + await page.goto('https://example.com'); + await page.waitForLoadState('networkidle', { timeout: 10000 }); + + const snap = await snapshot(browser); + const elements = snap.elements.filter(el => el.role === 'link'); + + if (elements.length > 0) { + const element = elements[0]; + const result = await scrollTo(browser, element.id, 'instant', 'start'); + expect(result.success).toBe(true); + expect(result.duration_ms).toBeGreaterThan(0); + } + } finally { + await browser.close(); + } + }, 60000); + + it('should take snapshot after scroll when requested', async () => { + const browser = await createTestBrowser(); + + try { + const page = getPageOrThrow(browser); + await page.goto('https://example.com'); + await page.waitForLoadState('networkidle', { timeout: 10000 }); + + const snap = await snapshot(browser); + const elements = snap.elements.filter(el => el.role === 'link'); + + if (elements.length > 0) { + const element = elements[0]; + const result = await scrollTo(browser, element.id, 'smooth', 'center', true); + expect(result.success).toBe(true); + expect(result.snapshot_after).toBeDefined(); + expect(result.snapshot_after?.status).toBe('success'); + } + } finally { + await browser.close(); + } + }, 60000); + + it('should fail for invalid element ID', async () => { + const browser = await createTestBrowser(); + + try { + const page = getPageOrThrow(browser); + await page.goto('https://example.com'); + await page.waitForLoadState('networkidle', { timeout: 10000 }); + + // Try to scroll to non-existent element + const result = await scrollTo(browser, 99999); + expect(result.success).toBe(false); + expect(result.error).toBeDefined(); + expect(result.error?.code).toBe('scroll_failed'); + } finally { + await browser.close(); + } + }, 60000); + }); + + describe('typeText with delay', () => { + it('should type text with human-like delay', async () => { + const browser = await createTestBrowser(); + + try { + const page = getPageOrThrow(browser); + await page.goto('https://example.com'); + await page.waitForLoadState('networkidle', { timeout: 10000 }); + + const snap = await snapshot(browser); + const textbox = find(snap, 'role=textbox'); + + if (textbox) { + // Test with 10ms delay between keystrokes + const result = await typeText(browser, textbox.id, 'hello', false, 10); + expect(result.success).toBe(true); + // Duration should be longer due to delays (at least 5 chars * 10ms = 50ms) + expect(result.duration_ms).toBeGreaterThanOrEqual(50); + } + } finally { + await browser.close(); + } + }, 60000); + }); + describe('clickRect', () => { it('should click at rectangle center using rect dict', async () => { const browser = await createTestBrowser(); diff --git a/tests/mocks/browser-mock.ts b/tests/mocks/browser-mock.ts index 9faa2a0e..26224579 100644 --- a/tests/mocks/browser-mock.ts +++ b/tests/mocks/browser-mock.ts @@ -141,3 +141,4 @@ export class MockBrowser implements IBrowser { return this.mockPage; } } + diff --git a/tests/utils/action-executor.test.ts b/tests/utils/action-executor.test.ts index c7f84aba..4c313726 100644 --- a/tests/utils/action-executor.test.ts +++ b/tests/utils/action-executor.test.ts @@ -152,3 +152,4 @@ describe('ActionExecutor', () => { }); }); }); + diff --git a/tests/utils/llm-response-builder.test.ts b/tests/utils/llm-response-builder.test.ts index 976cfcaf..17c6915b 100644 --- a/tests/utils/llm-response-builder.test.ts +++ b/tests/utils/llm-response-builder.test.ts @@ -101,3 +101,4 @@ describe('LLMResponseBuilder', () => { }); }); }); + From 567f5e918244050db0c00971c69fa24f2eaaf93d Mon Sep 17 00:00:00 2001 From: rcholic Date: Tue, 6 Jan 2026 15:54:09 -0800 Subject: [PATCH 2/4] export async api --- package-lock.json | 4 ++-- src/utils/browser.ts | 1 - src/utils/llm-response-builder.ts | 1 - src/utils/trace-file-manager.ts | 1 - 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 84e09742..78694626 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "sentienceapi", - "version": "0.91.1", + "version": "0.92.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sentienceapi", - "version": "0.91.1", + "version": "0.92.2", "license": "(MIT OR Apache-2.0)", "dependencies": { "playwright": "^1.40.0", diff --git a/src/utils/browser.ts b/src/utils/browser.ts index 77a1d25b..4d1cb96b 100644 --- a/src/utils/browser.ts +++ b/src/utils/browser.ts @@ -39,4 +39,3 @@ export async function saveStorageState(context: BrowserContext, filePath: string fs.writeFileSync(filePath, JSON.stringify(storageState, null, 2)); console.log(`✅ [Sentience] Saved storage state to ${filePath}`); } - diff --git a/src/utils/llm-response-builder.ts b/src/utils/llm-response-builder.ts index 8c7b924d..099e6df0 100644 --- a/src/utils/llm-response-builder.ts +++ b/src/utils/llm-response-builder.ts @@ -135,4 +135,3 @@ export class LLMResponseBuilder { }; } } - diff --git a/src/utils/trace-file-manager.ts b/src/utils/trace-file-manager.ts index 846edb21..9265ffbd 100644 --- a/src/utils/trace-file-manager.ts +++ b/src/utils/trace-file-manager.ts @@ -173,4 +173,3 @@ export class TraceFileManager { } } } - From 157de7495b6ed04a57ee23be5689ad65fb5fbcf1 Mon Sep 17 00:00:00 2001 From: rcholic Date: Tue, 6 Jan 2026 15:55:26 -0800 Subject: [PATCH 3/4] formatted --- tests/actions.test.ts | 12 +++++++++++- tests/mocks/browser-mock.ts | 1 - tests/utils/action-executor.test.ts | 1 - tests/utils/llm-response-builder.test.ts | 1 - 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/actions.test.ts b/tests/actions.test.ts index 22970fa5..7bfeb8fe 100644 --- a/tests/actions.test.ts +++ b/tests/actions.test.ts @@ -2,7 +2,17 @@ * Tests for actions (click, type, press, clickRect) */ -import { SentienceBrowser, click, typeText, press, scrollTo, clickRect, snapshot, find, BBox } from '../src'; +import { + SentienceBrowser, + click, + typeText, + press, + scrollTo, + clickRect, + snapshot, + find, + BBox, +} from '../src'; import { createTestBrowser, getPageOrThrow } from './test-utils'; describe('Actions', () => { diff --git a/tests/mocks/browser-mock.ts b/tests/mocks/browser-mock.ts index 26224579..9faa2a0e 100644 --- a/tests/mocks/browser-mock.ts +++ b/tests/mocks/browser-mock.ts @@ -141,4 +141,3 @@ export class MockBrowser implements IBrowser { return this.mockPage; } } - diff --git a/tests/utils/action-executor.test.ts b/tests/utils/action-executor.test.ts index 4c313726..c7f84aba 100644 --- a/tests/utils/action-executor.test.ts +++ b/tests/utils/action-executor.test.ts @@ -152,4 +152,3 @@ describe('ActionExecutor', () => { }); }); }); - diff --git a/tests/utils/llm-response-builder.test.ts b/tests/utils/llm-response-builder.test.ts index 17c6915b..976cfcaf 100644 --- a/tests/utils/llm-response-builder.test.ts +++ b/tests/utils/llm-response-builder.test.ts @@ -101,4 +101,3 @@ describe('LLMResponseBuilder', () => { }); }); }); - From 93f6228d450330614a82ee6446a73a93a54a8c08 Mon Sep 17 00:00:00 2001 From: rcholic Date: Tue, 6 Jan 2026 18:42:39 -0800 Subject: [PATCH 4/4] tests added --- .prettierignore | 1 + .prettierrc.json | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.prettierignore b/.prettierignore index 0d880a7d..d37bf329 100644 --- a/.prettierignore +++ b/.prettierignore @@ -8,3 +8,4 @@ coverage/ package-lock.json + diff --git a/.prettierrc.json b/.prettierrc.json index b519b84b..aaccf62e 100644 --- a/.prettierrc.json +++ b/.prettierrc.json @@ -7,4 +7,3 @@ "useTabs": false, "arrowParens": "avoid" } -