Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 84 additions & 5 deletions src/agent-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,21 @@ export interface EventuallyOptions {
timeoutMs?: number;
pollMs?: number;
snapshotOptions?: Record<string, any>;
/**
* Optional: increase snapshot `limit` across retries (additive schedule).
*
* Useful on long/virtualized pages where a small element limit can miss targets.
*/
snapshotLimitGrowth?: {
/** Defaults to snapshotOptions.limit if present, else 50. */
startLimit?: number;
/** Defaults to startLimit. */
step?: number;
/** Defaults to 500. */
maxLimit?: number;
/** 'only_on_fail' (default) grows on attempt>1; 'all' always applies schedule. */
applyOn?: 'only_on_fail' | 'all';
};
/** If set, `.eventually()` will treat snapshots below this confidence as failures and resnapshot. */
minConfidence?: number;
/** Max number of snapshot attempts to get above minConfidence before declaring exhaustion. */
Expand Down Expand Up @@ -133,6 +148,7 @@ export class AssertionHandle {
const timeoutMs = options.timeoutMs ?? 10_000;
const pollMs = options.pollMs ?? 250;
const snapshotOptions = options.snapshotOptions;
const snapshotLimitGrowth = options.snapshotLimitGrowth;
const minConfidence = options.minConfidence;
const maxSnapshotAttempts = options.maxSnapshotAttempts ?? 3;
const visionProvider = options.visionProvider;
Expand All @@ -143,10 +159,45 @@ export class AssertionHandle {
let attempt = 0;
let snapshotAttempt = 0;
let lastOutcome: ReturnType<Predicate> | null = null;
let snapshotLimit: number | null = null;

const clampLimit = (n: number): number => {
if (!Number.isFinite(n)) return 50;
if (n < 1) return 1;
if (n > 500) return 500;
return Math.floor(n);
};

const growthApplyOn = snapshotLimitGrowth?.applyOn ?? 'only_on_fail';
const startLimit = clampLimit(
snapshotLimitGrowth?.startLimit ??
(typeof snapshotOptions?.limit === 'number' ? snapshotOptions.limit : 50)
);
const step = clampLimit(snapshotLimitGrowth?.step ?? startLimit);
const maxLimit = clampLimit(snapshotLimitGrowth?.maxLimit ?? 500);

const limitForAttempt = (attempt1: number): number => {
const base = startLimit + step * Math.max(0, attempt1 - 1);
return clampLimit(Math.min(maxLimit, base));
};

while (true) {
attempt += 1;
await this.runtime.snapshot(snapshotOptions);

const perAttemptOptions: Record<string, any> = { ...(snapshotOptions ?? {}) };
snapshotLimit = null;
if (snapshotLimitGrowth) {
const apply =
growthApplyOn === 'all' ||
attempt === 1 ||
(lastOutcome !== null && lastOutcome.passed === false);
snapshotLimit = apply ? limitForAttempt(attempt) : startLimit;
perAttemptOptions.limit = snapshotLimit;
} else if (typeof perAttemptOptions.limit === 'number') {
snapshotLimit = clampLimit(perAttemptOptions.limit);
}

await this.runtime.snapshot(perAttemptOptions);
snapshotAttempt += 1;

const diagnostics = this.runtime.lastSnapshot?.diagnostics;
Expand All @@ -173,7 +224,13 @@ export class AssertionHandle {
lastOutcome,
this.label,
this.required,
{ eventually: true, attempt, snapshot_attempt: snapshotAttempt, final: false },
{
eventually: true,
attempt,
snapshot_attempt: snapshotAttempt,
snapshot_limit: snapshotLimit,
final: false,
},
false
);

Expand Down Expand Up @@ -215,6 +272,7 @@ export class AssertionHandle {
eventually: true,
attempt,
snapshot_attempt: snapshotAttempt,
snapshot_limit: snapshotLimit,
final: true,
vision_fallback: true,
},
Expand Down Expand Up @@ -251,6 +309,7 @@ export class AssertionHandle {
eventually: true,
attempt,
snapshot_attempt: snapshotAttempt,
snapshot_limit: snapshotLimit,
final: true,
exhausted: true,
},
Expand All @@ -271,6 +330,7 @@ export class AssertionHandle {
eventually: true,
attempt,
snapshot_attempt: snapshotAttempt,
snapshot_limit: snapshotLimit,
final: true,
timeout: true,
},
Expand All @@ -295,7 +355,13 @@ export class AssertionHandle {
lastOutcome,
this.label,
this.required,
{ eventually: true, attempt, final: false },
{
eventually: true,
attempt,
snapshot_attempt: snapshotAttempt,
snapshot_limit: snapshotLimit,
final: false,
},
false
);

Expand All @@ -305,7 +371,13 @@ export class AssertionHandle {
lastOutcome,
this.label,
this.required,
{ eventually: true, attempt, final: true },
{
eventually: true,
attempt,
snapshot_attempt: snapshotAttempt,
snapshot_limit: snapshotLimit,
final: true,
},
true
);
return true;
Expand All @@ -317,7 +389,14 @@ export class AssertionHandle {
lastOutcome,
this.label,
this.required,
{ eventually: true, attempt, final: true, timeout: true },
{
eventually: true,
attempt,
snapshot_attempt: snapshotAttempt,
snapshot_limit: snapshotLimit,
final: true,
timeout: true,
},
true
);
if (this.required) {
Expand Down
51 changes: 49 additions & 2 deletions src/asserts/expect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,17 @@ export interface EventuallyConfig {
poll?: number;
/** Max number of retry attempts (default 3) */
maxRetries?: number;
/**
* Optional: increase snapshot `limit` across retries (additive schedule).
*
* This mirrors AgentRuntime's AssertionHandle.eventually() behavior.
*/
snapshotLimitGrowth?: {
startLimit?: number;
step?: number;
maxLimit?: number;
applyOn?: 'only_on_fail' | 'all';
};
}

/**
Expand Down Expand Up @@ -456,14 +467,20 @@ export const expect = Object.assign(
*/
export class EventuallyWrapper {
private _predicate: Predicate;
private _config: Required<EventuallyConfig>;
private _config: {
timeout: number;
poll: number;
maxRetries: number;
snapshotLimitGrowth?: EventuallyConfig['snapshotLimitGrowth'];
};

constructor(predicate: Predicate, config: EventuallyConfig = {}) {
this._predicate = predicate;
this._config = {
timeout: config.timeout ?? DEFAULT_TIMEOUT,
poll: config.poll ?? DEFAULT_POLL,
maxRetries: config.maxRetries ?? DEFAULT_MAX_RETRIES,
snapshotLimitGrowth: config.snapshotLimitGrowth,
};
}

Expand All @@ -482,6 +499,22 @@ export class EventuallyWrapper {
let lastOutcome: AssertOutcome | null = null;
let attempts = 0;

const growth = this._config.snapshotLimitGrowth;
const clampLimit = (n: number): number => {
if (!Number.isFinite(n)) return 50;
if (n < 1) return 1;
if (n > 500) return 500;
return Math.floor(n);
};
const growthApplyOn = growth?.applyOn ?? 'only_on_fail';
const startLimit = clampLimit(growth?.startLimit ?? 50);
const step = clampLimit(growth?.step ?? startLimit);
const maxLimit = clampLimit(growth?.maxLimit ?? 500);
const limitForAttempt = (attempt1: number): number => {
const base = startLimit + step * Math.max(0, attempt1 - 1);
return clampLimit(Math.min(maxLimit, base));
};

while (true) {
// Check timeout (higher precedence than maxRetries)
const elapsed = Date.now() - startTime;
Expand Down Expand Up @@ -513,7 +546,21 @@ export class EventuallyWrapper {
// Take fresh snapshot if not first attempt
if (attempts > 0) {
try {
const freshSnapshot = await snapshotFn();
// If snapshotFn supports kwargs (e.g. runtime.snapshot), pass adaptive limit.
let freshSnapshot: AssertContext['snapshot'];
const attempt1 = attempts + 1;
if (growth) {
const apply =
growthApplyOn === 'all' || (growthApplyOn === 'only_on_fail' && lastOutcome);
const snapLimit = apply ? limitForAttempt(attempt1) : startLimit;
try {
freshSnapshot = await (snapshotFn as any)({ limit: snapLimit });
} catch {
freshSnapshot = await snapshotFn();
}
} else {
freshSnapshot = await snapshotFn();
}
ctx = {
snapshot: freshSnapshot,
url: freshSnapshot?.url ?? ctx.url,
Expand Down
Loading