From 5413a6534c000978f96b85c0533f53d3556152c4 Mon Sep 17 00:00:00 2001 From: rcholic Date: Sat, 17 Jan 2026 19:59:34 -0800 Subject: [PATCH 1/3] lang chain example --- examples/lang-chain/README.md | 16 ++++ examples/lang-chain/sentience-tools-demo.ts | 101 ++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 examples/lang-chain/README.md create mode 100644 examples/lang-chain/sentience-tools-demo.ts diff --git a/examples/lang-chain/README.md b/examples/lang-chain/README.md new file mode 100644 index 0000000..98570b2 --- /dev/null +++ b/examples/lang-chain/README.md @@ -0,0 +1,16 @@ +### LangChain examples (TypeScript) + +These examples show how to wrap Sentience TS SDK primitives as LangChain JS tools. + +Install (example): + +```bash +npm install sentienceapi @langchain/core zod +``` + +Run: + +```bash +npx ts-node examples/lang-chain/sentience-tools-demo.ts +``` + diff --git a/examples/lang-chain/sentience-tools-demo.ts b/examples/lang-chain/sentience-tools-demo.ts new file mode 100644 index 0000000..80115fa --- /dev/null +++ b/examples/lang-chain/sentience-tools-demo.ts @@ -0,0 +1,101 @@ +/** + * Example: Wrap Sentience TS SDK primitives as LangChain JS tools. + * + * Install (example): + * npm install sentienceapi @langchain/core zod + * + * Run: + * npx ts-node examples/lang-chain/sentience-tools-demo.ts + */ + +import { DynamicStructuredTool } from '@langchain/core/tools'; +import { z } from 'zod'; + +import { SentienceBrowser, snapshot, click, typeText, press, read } from '../../src/index'; + +async function main() { + const apiKey = process.env.SENTIENCE_API_KEY as string | undefined; + const browser = new SentienceBrowser(apiKey, undefined, false); + + await browser.start(); + await browser.getPage().goto('https://example.com'); + + // Tool: snapshot_state + const snapshotState = new DynamicStructuredTool({ + name: 'sentience_snapshot_state', + description: + 'Observe: take a bounded snapshot (default limit=50) and return elements with ids/roles/bboxes.', + schema: z.object({ + limit: z.number().int().min(1).max(500).default(50), + }), + func: async ({ limit }) => { + const snap = await snapshot(browser, { limit }); + return JSON.stringify( + { + url: snap.url, + elements: snap.elements.map(e => ({ + id: e.id, + role: e.role, + text: e.text, + bbox: e.bbox, + importance: e.importance, + })), + }, + null, + 2 + ); + }, + }); + + // Tool: click(element_id) + const clickTool = new DynamicStructuredTool({ + name: 'sentience_click', + description: 'Act: click an element by elementId from snapshot.', + schema: z.object({ + elementId: z.number().int(), + }), + func: async ({ elementId }) => JSON.stringify(await click(browser, elementId)), + }); + + // Tool: type_text(element_id, text) + const typeTool = new DynamicStructuredTool({ + name: 'sentience_type_text', + description: 'Act: type text into an element by elementId from snapshot.', + schema: z.object({ + elementId: z.number().int(), + text: z.string(), + }), + func: async ({ elementId, text }) => JSON.stringify(await typeText(browser, elementId, text)), + }); + + // Tool: press_key(key) + const pressTool = new DynamicStructuredTool({ + name: 'sentience_press_key', + description: 'Act: press a keyboard key (Enter/Escape/Tab/etc.).', + schema: z.object({ + key: z.string(), + }), + func: async ({ key }) => JSON.stringify(await press(browser, key)), + }); + + // Tool: read_page(format) + const readTool = new DynamicStructuredTool({ + name: 'sentience_read_page', + description: 'Observe: read page content as text/markdown/raw HTML.', + schema: z.object({ + format: z.enum(['raw', 'text', 'markdown']).default('text'), + }), + func: async ({ format }) => JSON.stringify(await read(browser, { format })), + }); + + const tools = [snapshotState, clickTool, typeTool, pressTool, readTool]; + console.log('Created LangChain tools:', tools.map(t => t.name)); + + await browser.close(); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); + From 8f310b8cbe6e8f13fee258ddc2bf5793345b63bf Mon Sep 17 00:00:00 2001 From: rcholic Date: Sat, 17 Jan 2026 23:35:22 -0800 Subject: [PATCH 2/3] nearby_text --- src/types.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/types.ts b/src/types.ts index 8a8c0ff..6cfd10e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -17,6 +17,7 @@ export interface Viewport { export interface VisualCues { is_primary: boolean; background_color_name: string | null; + fallback_background_color_name: string | null; is_clickable: boolean; } @@ -50,6 +51,9 @@ export interface Element { // Hyperlink URL (for link elements) href?: string; + /** Nearby static text (best-effort, usually only for top-ranked elements) */ + nearby_text?: string | null; + // ===== v1 state-aware assertion fields (optional) ===== /** Best-effort accessible name/label for controls (distinct from visible text) */ name?: string | null; From c2e914c9f4a5943043d010c3c37a4f3d06e6e8ad Mon Sep 17 00:00:00 2001 From: rcholic Date: Sat, 17 Jan 2026 23:59:28 -0800 Subject: [PATCH 3/3] fix tests --- src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.ts b/src/types.ts index 6cfd10e..896dc40 100644 --- a/src/types.ts +++ b/src/types.ts @@ -17,7 +17,7 @@ export interface Viewport { export interface VisualCues { is_primary: boolean; background_color_name: string | null; - fallback_background_color_name: string | null; + fallback_background_color_name?: string | null; is_clickable: boolean; }