From 32cff8758e10b2307e1ddd83bea35e5f19e2f84e Mon Sep 17 00:00:00 2001 From: Anas Lecaillon Date: Mon, 8 Dec 2025 10:26:20 +0100 Subject: [PATCH] Added on.('network' listener) --- .../core/lib/v3/tests/page-network.spec.ts | 356 ++++++++++++++++++ packages/core/lib/v3/types/public/page.ts | 3 + .../core/lib/v3/understudy/networkMessage.ts | 181 +++++++++ packages/core/lib/v3/understudy/page.ts | 242 +++++++++++- 4 files changed, 769 insertions(+), 13 deletions(-) create mode 100644 packages/core/lib/v3/tests/page-network.spec.ts create mode 100644 packages/core/lib/v3/understudy/networkMessage.ts diff --git a/packages/core/lib/v3/tests/page-network.spec.ts b/packages/core/lib/v3/tests/page-network.spec.ts new file mode 100644 index 000000000..10105dbc1 --- /dev/null +++ b/packages/core/lib/v3/tests/page-network.spec.ts @@ -0,0 +1,356 @@ +import { test, expect } from "@playwright/test"; +import { V3 } from "../v3"; +import { v3TestConfig } from "./v3.config"; + +test.describe("Page Network Events", () => { + let v3: V3; + + test.beforeEach(async () => { + v3 = new V3(v3TestConfig); + await v3.init(); + }); + + test.afterEach(async () => { + await v3?.close?.().catch(() => {}); + }); + + test("should capture network request events", async () => { + const page = v3.context.pages()[0]; + const requests: any[] = []; + + page.on("network", (message) => { + if (message.type() === "request") { + requests.push(message); + } + }); + + await page.goto("https://example.com"); + + expect(requests.length).toBeGreaterThan(0); + const mainRequest = requests.find((r) => r.url().includes("example.com")); + expect(mainRequest).toBeDefined(); + expect(mainRequest?.method()).toBe("GET"); + }); + + test("should capture network response events", async () => { + const page = v3.context.pages()[0]; + const responses: any[] = []; + + page.on("network", (message) => { + if (message.type() === "response") { + responses.push(message); + } + }); + + await page.goto("https://example.com"); + + expect(responses.length).toBeGreaterThan(0); + const mainResponse = responses.find((r) => r.url().includes("example.com")); + expect(mainResponse).toBeDefined(); + expect(mainResponse?.status()).toBe(200); + expect(mainResponse?.statusText()).toBeDefined(); + }); + + test("should provide resource type information", async () => { + const page = v3.context.pages()[0]; + const messages: any[] = []; + + page.on("network", (message) => { + messages.push(message); + }); + + await page.goto("https://example.com"); + + const documentRequest = messages.find( + (m) => m.resourceType() === "Document", + ); + expect(documentRequest).toBeDefined(); + }); + + test("should support once() for single event", async () => { + const page = v3.context.pages()[0]; + let callCount = 0; + + page.once("network", (message) => { + callCount++; + expect(message).toBeDefined(); + }); + + await page.goto("https://example.com"); + + // Even though multiple network events occur, once() should only fire once + expect(callCount).toBe(1); + }); + + test("should support removing listeners with off()", async () => { + const page = v3.context.pages()[0]; + let callCount = 0; + + const listener = (message: any) => { + callCount++; + }; + + page.on("network", listener); + await page.goto("https://example.com"); + + const firstCallCount = callCount; + expect(firstCallCount).toBeGreaterThan(0); + + page.off("network", listener); + callCount = 0; + + await page.goto("https://example.com"); + expect(callCount).toBe(0); + }); + + test("should provide request headers", async () => { + const page = v3.context.pages()[0]; + let foundHeaders = false; + + page.on("network", (message) => { + if (message.type() === "request") { + const headers = message.requestHeaders(); + if (headers && Object.keys(headers).length > 0) { + foundHeaders = true; + } + } + }); + + await page.goto("https://example.com"); + expect(foundHeaders).toBe(true); + }); + + test("should provide response headers", async () => { + const page = v3.context.pages()[0]; + let foundHeaders = false; + + page.on("network", (message) => { + if (message.type() === "response") { + const headers = message.responseHeaders(); + if (headers && Object.keys(headers).length > 0) { + foundHeaders = true; + } + } + }); + + await page.goto("https://example.com"); + expect(foundHeaders).toBe(true); + }); + + test("should provide MIME type for responses", async () => { + const page = v3.context.pages()[0]; + let foundMimeType = false; + + page.on("network", (message) => { + if (message.type() === "response") { + const mimeType = message.mimeType(); + if (mimeType && mimeType.includes("text/html")) { + foundMimeType = true; + } + } + }); + + await page.goto("https://example.com"); + expect(foundMimeType).toBe(true); + }); + + test("should track frame and loader IDs", async () => { + const page = v3.context.pages()[0]; + let hasFrameId = false; + let hasLoaderId = false; + + page.on("network", (message) => { + if (message.frameId()) hasFrameId = true; + if (message.loaderId()) hasLoaderId = true; + }); + + await page.goto("https://example.com"); + expect(hasFrameId).toBe(true); + expect(hasLoaderId).toBe(true); + }); + + test("should provide unique request IDs", async () => { + const page = v3.context.pages()[0]; + const requestIds = new Set(); + + page.on("network", (message) => { + requestIds.add(message.requestId()); + }); + + await page.goto("https://example.com"); + expect(requestIds.size).toBeGreaterThan(0); + }); + + test("should support toString() method", async () => { + const page = v3.context.pages()[0]; + let foundToString = false; + + page.on("network", (message) => { + const str = message.toString(); + expect(typeof str).toBe("string"); + expect(str.length).toBeGreaterThan(0); + if (str.includes("Request") || str.includes("Response")) { + foundToString = true; + } + }); + + await page.goto("https://example.com"); + expect(foundToString).toBe(true); + }); + + test("should provide page reference", async () => { + const page = v3.context.pages()[0]; + let hasPageRef = false; + + page.on("network", (message) => { + const messagePage = message.page(); + if (messagePage === page) { + hasPageRef = true; + } + }); + + await page.goto("https://example.com"); + expect(hasPageRef).toBe(true); + }); + + test("should capture both requests and responses for same URL", async () => { + const page = v3.context.pages()[0]; + const events: { type: string; url: string }[] = []; + + page.on("network", (message) => { + events.push({ + type: message.type(), + url: message.url(), + }); + }); + + await page.goto("https://example.com"); + + const exampleEvents = events.filter((e) => e.url.includes("example.com")); + const hasRequest = exampleEvents.some((e) => e.type === "request"); + const hasResponse = exampleEvents.some((e) => e.type === "response"); + + expect(hasRequest).toBe(true); + expect(hasResponse).toBe(true); + }); + + test("should work across multiple pages", async () => { + const page1 = v3.context.pages()[0]; + const page2 = await v3.context.newPage(); + + const page1Events: string[] = []; + const page2Events: string[] = []; + + page1.on("network", (message) => { + page1Events.push(message.url()); + }); + + page2.on("network", (message) => { + page2Events.push(message.url()); + }); + + await page1.goto("https://example.com"); + await page2.goto("https://httpbin.org/html"); + + expect(page1Events.some((url) => url.includes("example.com"))).toBe(true); + expect(page1Events.some((url) => url.includes("httpbin.org"))).toBe(false); + + expect(page2Events.some((url) => url.includes("httpbin.org"))).toBe(true); + expect(page2Events.some((url) => url.includes("example.com"))).toBe(false); + + await page2.close(); + }); + + test("should support multiple simultaneous listeners", async () => { + const page = v3.context.pages()[0]; + let listener1Called = false; + let listener2Called = false; + let listener3Called = false; + + page.on("network", () => { + listener1Called = true; + }); + + page.on("network", () => { + listener2Called = true; + }); + + page.on("network", () => { + listener3Called = true; + }); + + await page.goto("https://example.com"); + + expect(listener1Called).toBe(true); + expect(listener2Called).toBe(true); + expect(listener3Called).toBe(true); + }); + + test("should handle errors in listeners gracefully", async () => { + const page = v3.context.pages()[0]; + let goodListenerCalled = false; + + page.on("network", () => { + throw new Error("Listener error"); + }); + + page.on("network", () => { + goodListenerCalled = true; + }); + + await page.goto("https://example.com"); + + // The second listener should still be called even if first throws + expect(goodListenerCalled).toBe(true); + }); + + test("should filter by resource type", async () => { + const page = v3.context.pages()[0]; + const documentRequests: any[] = []; + const imageRequests: any[] = []; + + page.on("network", (message) => { + if (message.resourceType() === "Document") { + documentRequests.push(message); + } else if (message.resourceType() === "Image") { + imageRequests.push(message); + } + }); + + await page.goto("https://example.com"); + + expect(documentRequests.length).toBeGreaterThan(0); + }); + + test("should provide POST data for POST requests", async () => { + const page = v3.context.pages()[0]; + let foundPostData = false; + + page.on("network", (message) => { + if (message.type() === "request" && message.method() === "POST") { + const postData = message.postData(); + if (postData) { + foundPostData = true; + } + } + }); + + await page.goto("https://httpbin.org/forms/post"); + await page.evaluate(() => { + const form = document.querySelector("form"); + if (form) { + const input = form.querySelector( + 'input[name="custname"]', + ) as HTMLInputElement; + if (input) input.value = "test"; + form.submit(); + } + }); + + await page.waitForLoadState("load").catch(() => {}); + + // POST data may or may not be captured depending on timing and form behavior + // Soft expectation - POST data capture is timing-dependent + expect.soft(foundPostData).toBe(true); + }); +}); diff --git a/packages/core/lib/v3/types/public/page.ts b/packages/core/lib/v3/types/public/page.ts index f141f3d67..097cc16ac 100644 --- a/packages/core/lib/v3/types/public/page.ts +++ b/packages/core/lib/v3/types/public/page.ts @@ -9,5 +9,8 @@ export type AnyPage = PlaywrightPage | PuppeteerPage | PatchrightPage | Page; export { ConsoleMessage } from "../../understudy/consoleMessage"; export type { ConsoleListener } from "../../understudy/consoleMessage"; +export { NetworkMessage } from "../../understudy/networkMessage"; +export type { NetworkListener } from "../../understudy/networkMessage"; + export type LoadState = "load" | "domcontentloaded" | "networkidle"; export { Response } from "../../understudy/response"; diff --git a/packages/core/lib/v3/understudy/networkMessage.ts b/packages/core/lib/v3/understudy/networkMessage.ts new file mode 100644 index 000000000..16af6cd49 --- /dev/null +++ b/packages/core/lib/v3/understudy/networkMessage.ts @@ -0,0 +1,181 @@ +import type { Protocol } from "devtools-protocol"; +import type { Page } from "./page"; + +export type NetworkListener = (message: NetworkMessage) => void; + +export type NetworkMessageType = "request" | "response"; + +export interface NetworkMessageData { + type: NetworkMessageType; + requestId: string; + frameId?: string; + loaderId?: string; + url: string; + method?: string; + resourceType?: Protocol.Network.ResourceType; + timestamp: number; + // Request-specific fields + requestHeaders?: Protocol.Network.Headers; + postData?: string; + // Response-specific fields + status?: number; + statusText?: string; + responseHeaders?: Protocol.Network.Headers; + mimeType?: string; + fromCache?: boolean; + fromServiceWorker?: boolean; +} + +/** + * NetworkMessage + * + * Represents a network request or response message captured via CDP. + * Similar to ConsoleMessage, this provides a convenient wrapper around + * the raw CDP events for network activity. + */ +export class NetworkMessage { + private readonly data: NetworkMessageData; + private readonly pageRef?: Page; + + constructor(data: NetworkMessageData, pageRef?: Page) { + this.data = data; + this.pageRef = pageRef; + } + + /** + * Returns the type of network event: "request" or "response" + */ + type(): NetworkMessageType { + return this.data.type; + } + + /** + * Returns the unique request identifier + */ + requestId(): string { + return this.data.requestId; + } + + /** + * Returns the frame ID associated with this network event + */ + frameId(): string | undefined { + return this.data.frameId; + } + + /** + * Returns the loader ID associated with this network event + */ + loaderId(): string | undefined { + return this.data.loaderId; + } + + /** + * Returns the URL of the request + */ + url(): string { + return this.data.url; + } + + /** + * Returns the HTTP method (GET, POST, etc.) + */ + method(): string | undefined { + return this.data.method; + } + + /** + * Returns the resource type (Document, Stylesheet, Image, etc.) + */ + resourceType(): Protocol.Network.ResourceType | undefined { + return this.data.resourceType; + } + + /** + * Returns the timestamp when the event occurred + */ + timestamp(): number { + return this.data.timestamp; + } + + /** + * Returns the request headers (if available) + */ + requestHeaders(): Protocol.Network.Headers | undefined { + return this.data.requestHeaders; + } + + /** + * Returns the POST data (if available for requests) + */ + postData(): string | undefined { + return this.data.postData; + } + + /** + * Returns the HTTP status code (for responses) + */ + status(): number | undefined { + return this.data.status; + } + + /** + * Returns the HTTP status text (for responses) + */ + statusText(): string | undefined { + return this.data.statusText; + } + + /** + * Returns the response headers (if available) + */ + responseHeaders(): Protocol.Network.Headers | undefined { + return this.data.responseHeaders; + } + + /** + * Returns the MIME type (for responses) + */ + mimeType(): string | undefined { + return this.data.mimeType; + } + + /** + * Returns whether the response was served from cache + */ + fromCache(): boolean { + return this.data.fromCache ?? false; + } + + /** + * Returns whether the response was served from a service worker + */ + fromServiceWorker(): boolean { + return this.data.fromServiceWorker ?? false; + } + + /** + * Returns the Page that owns this network message + */ + page(): Page | undefined { + return this.pageRef; + } + + /** + * Returns the raw event data + */ + raw(): NetworkMessageData { + return { ...this.data }; + } + + /** + * Returns a string representation of the network message + */ + toString(): string { + if (this.data.type === "request") { + return `[Request] ${this.data.method ?? "GET"} ${this.data.url}`; + } else { + return `[Response] ${this.data.status ?? "???"} ${this.data.url}`; + } + } +} diff --git a/packages/core/lib/v3/understudy/page.ts b/packages/core/lib/v3/understudy/page.ts index f558659a3..ee981e68e 100644 --- a/packages/core/lib/v3/understudy/page.ts +++ b/packages/core/lib/v3/understudy/page.ts @@ -15,6 +15,7 @@ import { LifecycleWatcher } from "./lifecycleWatcher"; import { NavigationResponseTracker } from "./navigationResponseTracker"; import { Response, isSerializableResponse } from "./response"; import { ConsoleMessage, ConsoleListener } from "./consoleMessage"; +import { NetworkMessage, NetworkListener } from "./networkMessage"; import type { StagehandAPIClient } from "../api"; import type { LocalBrowserLaunchOptions } from "../types/public"; import type { Locator } from "./locator"; @@ -63,6 +64,10 @@ const LIFECYCLE_NAME: Record = { networkidle: "networkIdle", }; +const EVENTS = ["console", "network"]; + +type Event = (typeof EVENTS)[number]; + export class Page { /** Every CDP child session this page owns (top-level + adopted OOPIF sessions). */ private readonly sessions = new Map(); // sessionId -> session @@ -97,6 +102,14 @@ export class Page { string, (evt: Protocol.Runtime.ConsoleAPICalledEvent) => void >(); + private readonly networkListeners = new Set(); + private readonly networkHandlers = new Map< + string, + { + onRequest: (evt: Protocol.Network.RequestWillBeSentEvent) => void; + onResponse: (evt: Protocol.Network.ResponseReceivedEvent) => void; + } + >(); /** Document-start scripts installed across every session this page owns. */ private readonly initScripts: string[] = []; @@ -427,6 +440,10 @@ export class Page { this.installConsoleTap(childSession); } + if (this.networkListeners.size > 0) { + this.installNetworkTap(childSession); + } + // session will start emitting its own page events; mark ownership seed now this.registry.adoptChildSession( childSession.id ?? "child", @@ -527,11 +544,49 @@ export class Page { this.networkManager.untrackSession(sessionId); } - public on(event: "console", listener: ConsoleListener): Page { - if (event !== "console") { - throw new StagehandInvalidArgumentError(`Unsupported event: ${event}`); + public on(event: "console", listener: ConsoleListener): Page; + public on(event: "network", listener: NetworkListener): Page; + public on(event: Event, listener: ConsoleListener | NetworkListener): Page { + if (EVENTS.includes(event)) { + switch (event) { + case "console": + return this.onConsoleEvent(listener as ConsoleListener); + case "network": + return this.onNetworkEvent(listener as NetworkListener); + } + } + throw new StagehandInvalidArgumentError(`Unsupported event: ${event}`); + } + + public once(event: "console", listener: ConsoleListener): Page; + public once(event: "network", listener: NetworkListener): Page; + public once(event: Event, listener: ConsoleListener | NetworkListener): Page { + if (EVENTS.includes(event)) { + switch (event) { + case "console": + return this.onceConsoleEvent(listener as ConsoleListener); + case "network": + return this.onceNetworkEvent(listener as NetworkListener); + } + } + throw new StagehandInvalidArgumentError(`Unsupported event: ${event}`); + } + + public off(event: "console", listener: ConsoleListener): Page; + public off(event: "network", listener: NetworkListener): Page; + public off(event: Event, listener: ConsoleListener | NetworkListener): Page { + if (EVENTS.includes(event)) { + switch (event) { + case "console": + return this.offConsoleEvent(listener as ConsoleListener); + case "network": + return this.offNetworkEvent(listener as NetworkListener); + } } + throw new StagehandInvalidArgumentError(`Unsupported event: ${event}`); + } + public onConsoleEvent(listener: ConsoleListener): Page { const firstListener = this.consoleListeners.size === 0; this.consoleListeners.add(listener); @@ -542,11 +597,7 @@ export class Page { return this; } - public once(event: "console", listener: ConsoleListener): Page { - if (event !== "console") { - throw new StagehandInvalidArgumentError(`Unsupported event: ${event}`); - } - + public onceConsoleEvent(listener: ConsoleListener): Page { const wrapper: ConsoleListener = (message) => { this.off("console", wrapper); listener(message); @@ -555,11 +606,7 @@ export class Page { return this.on("console", wrapper); } - public off(event: "console", listener: ConsoleListener): Page { - if (event !== "console") { - throw new StagehandInvalidArgumentError(`Unsupported event: ${event}`); - } - + public offConsoleEvent(listener: ConsoleListener): Page { this.consoleListeners.delete(listener); if (this.consoleListeners.size === 0) { @@ -569,6 +616,36 @@ export class Page { return this; } + public onNetworkEvent(listener: NetworkListener): Page { + const firstListener = this.networkListeners.size === 0; + this.networkListeners.add(listener); + + if (firstListener) { + this.ensureNetworkTaps(); + } + + return this; + } + + public onceNetworkEvent(listener: NetworkListener): Page { + const wrapper: NetworkListener = (event) => { + this.off("network", wrapper); + listener(event); + }; + + return this.on("network", wrapper); + } + + public offNetworkEvent(listener: NetworkListener): Page { + this.networkListeners.delete(listener); + + if (this.networkListeners.size === 0) { + this.removeAllNetworkTaps(); + } + + return this; + } + // ---------------- MAIN APIs ---------------- public targetId(): string { @@ -646,6 +723,8 @@ export class Page { this.networkManager.dispose(); this.removeAllConsoleTaps(); this.consoleListeners.clear(); + this.removeAllNetworkTaps(); + this.networkListeners.clear(); } public getFullFrameTree(): Protocol.Page.FrameTree { @@ -752,6 +831,143 @@ export class Page { } } + private ensureNetworkTaps(): void { + if (this.networkListeners.size === 0) return; + + this.installNetworkTap(this.mainSession); + for (const session of this.sessions.values()) { + this.installNetworkTap(session); + } + } + + private installNetworkTap(session: CDPSessionLike): void { + const key = this.sessionKey(session); + if (this.networkHandlers.has(key)) return; + + void session.send("Network.enable").catch(() => {}); + + const onRequest = (evt: Protocol.Network.RequestWillBeSentEvent) => { + this.emitNetworkRequest(evt); + }; + + const onResponse = (evt: Protocol.Network.ResponseReceivedEvent) => { + this.emitNetworkResponse(evt); + }; + + session.on( + "Network.requestWillBeSent", + onRequest, + ); + + session.on( + "Network.responseReceived", + onResponse, + ); + + this.networkHandlers.set(key, { onRequest, onResponse }); + } + + private teardownNetworkTap(key: string): void { + const handlers = this.networkHandlers.get(key); + if (!handlers) return; + + const session = this.resolveSessionByKey(key); + if (session) { + session.off("Network.requestWillBeSent", handlers.onRequest); + session.off("Network.responseReceived", handlers.onResponse); + } + this.networkHandlers.delete(key); + } + + private removeAllNetworkTaps(): void { + for (const key of [...this.networkHandlers.keys()]) { + this.teardownNetworkTap(key); + } + } + + private emitNetworkRequest( + evt: Protocol.Network.RequestWillBeSentEvent, + ): void { + if (this.networkListeners.size === 0) return; + + const networkMessage = new NetworkMessage( + { + type: "request", + requestId: evt.requestId, + frameId: evt.frameId, + loaderId: evt.loaderId, + url: evt.request.url, + method: evt.request.method, + resourceType: evt.type, + timestamp: Date.now(), + requestHeaders: evt.request.headers, + postData: evt.request.postData, + }, + this, + ); + + const listeners = [...this.networkListeners]; + + for (const listener of listeners) { + try { + listener(networkMessage); + } catch (error) { + v3Logger({ + category: "page", + message: "Network listener threw on request", + level: 2, + auxiliary: { + error: { value: String(error), type: "string" }, + url: { value: evt.request.url, type: "string" }, + }, + }); + } + } + } + + private emitNetworkResponse( + evt: Protocol.Network.ResponseReceivedEvent, + ): void { + if (this.networkListeners.size === 0) return; + + const networkMessage = new NetworkMessage( + { + type: "response", + requestId: evt.requestId, + frameId: evt.frameId, + loaderId: evt.loaderId, + url: evt.response.url, + resourceType: evt.type, + timestamp: Date.now(), + status: evt.response.status, + statusText: evt.response.statusText, + responseHeaders: evt.response.headers, + mimeType: evt.response.mimeType, + fromCache: evt.response.fromDiskCache || evt.response.fromPrefetchCache, + fromServiceWorker: evt.response.fromServiceWorker, + }, + this, + ); + + const listeners = [...this.networkListeners]; + + for (const listener of listeners) { + try { + listener(networkMessage); + } catch (error) { + v3Logger({ + category: "page", + message: "Network listener threw on response", + level: 2, + auxiliary: { + error: { value: String(error), type: "string" }, + url: { value: evt.response.url, type: "string" }, + }, + }); + } + } + } + // -------- Convenience APIs delegated to the current main frame -------- /**