From 72834980d20e1eb220988e47a96bf504cc10c18a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chaonan=E2=80=9D?= Date: Tue, 18 Aug 2026 15:45:57 +0800 Subject: [PATCH 1/4] fix(record): record switch_tab and keep Trace v3 states per tab Co-authored-by: Cursor --- .../lib/__tests__/recording-runtime.test.ts | 21 +- .../src/lib/__tests__/step-buffer.test.ts | 39 +- .../src/lib/__tests__/tab-coordinator.test.ts | 29 ++ .../lib/__tests__/trace-reducer-v2.test.ts | 11 + .../lib/__tests__/trace-reducer-v3.test.ts | 15 + .../src/lib/recording/observation-session.ts | 8 +- .../src/lib/recording/recording-runtime.ts | 94 +++-- .../src/lib/recording/step-buffer.ts | 29 +- .../src/lib/recording/tab-coordinator.ts | 62 +++ .../src/lib/recording/trace-builder-v3.ts | 6 +- .../src/lib/recording/trace-reducer-v2.ts | 1 + .../src/lib/recording/trace-reducer-v3.ts | 24 +- apps/extension/src/lib/recording/types.ts | 1 + .../src/tools/__tests__/record-steps.test.ts | 384 +++++++++++++++++- apps/extension/src/tools/record.ts | 189 +++++---- apps/extension/src/transport/types.ts | 3 + crates/bsk-cli/src/cli/record.rs | 1 + .../schema/tool_record_await_result.json | 30 ++ .../schema/tool_record_start_params.json | 7 + .../schema/tool_record_stop_result.json | 30 ++ crates/bsk-protocol/schema/trace.json | 30 ++ crates/bsk-protocol/schema/trace_step.json | 30 ++ crates/bsk-protocol/schema/trace_step_v3.json | 30 ++ crates/bsk-protocol/schema/trace_v3.json | 30 ++ crates/bsk-protocol/src/tools/record.rs | 20 + crates/bsk-protocol/src/tools/record_v3.rs | 17 + 26 files changed, 1001 insertions(+), 140 deletions(-) create mode 100644 apps/extension/src/lib/__tests__/tab-coordinator.test.ts create mode 100644 apps/extension/src/lib/recording/tab-coordinator.ts diff --git a/apps/extension/src/lib/__tests__/recording-runtime.test.ts b/apps/extension/src/lib/__tests__/recording-runtime.test.ts index 9e931385..34af5176 100644 --- a/apps/extension/src/lib/__tests__/recording-runtime.test.ts +++ b/apps/extension/src/lib/__tests__/recording-runtime.test.ts @@ -11,11 +11,11 @@ vi.mock("../recording/observation-capture", async (importOriginal) => { import { ObservationNodeIndex } from "../recording/observation-capture"; import { RecordingObservationRuntime } from "../recording/recording-runtime"; -function observation() { +function observation(url = "https://example.com/") { return { rootFrameId: "root", index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }), - url: "https://example.com/", + url, title: "Example", vomText: '@vom 1\nRootWebArea "Example"', truncated: false, @@ -130,4 +130,21 @@ describe("RecordingObservationRuntime", () => { expect(dumped).not.toContain("backendNodeId"); expect(dumped).not.toContain("ObservationNodeIndex"); }); + + it("captures a tab transition without sharing observation cursors", async () => { + captureRecordingObservation + .mockResolvedValueOnce(observation("https://example.com/first")) + .mockResolvedValueOnce(observation("https://example.com/second")); + const recording = runtime(); + + await recording.captureInitial(4); + const transition = await recording.captureTabTransition(4, 5); + + expect(transition).toMatchObject({ + preStateId: "s1", + postStateId: "s2", + targetUrl: "https://example.com/second", + }); + expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]); + }); }); diff --git a/apps/extension/src/lib/__tests__/step-buffer.test.ts b/apps/extension/src/lib/__tests__/step-buffer.test.ts index dcf333cd..32c538e3 100644 --- a/apps/extension/src/lib/__tests__/step-buffer.test.ts +++ b/apps/extension/src/lib/__tests__/step-buffer.test.ts @@ -3,7 +3,7 @@ import { appendRecordedPayload, observeRecordedNavigation } from "../recording/s describe("recording-step-buffer", () => { it("stores semantic click without summary", () => { - const buffer = { steps: [], pendingNavigation: false }; + const buffer = { steps: [], navigation: { pendingNavigation: false } }; appendRecordedPayload(buffer, { op: "click", target: { tag: "button", role: "button", name: "发布" }, @@ -15,11 +15,11 @@ describe("recording-step-buffer", () => { captureTarget: { tag: "button", role: "button", name: "发布" }, }, ]); - expect(buffer.pendingNavigation).toBe(true); + expect(buffer.navigation.pendingNavigation).toBe(true); }); it("keeps the hovered element description as a capture fallback", () => { - const buffer = { steps: [], pendingNavigation: false }; + const buffer = { steps: [], navigation: { pendingNavigation: false } }; appendRecordedPayload( buffer, { @@ -59,9 +59,11 @@ describe("recording-step-buffer", () => { captureTarget: { tag: "button", role: "button", name: "发布" }, }, ], - currentUrl: "https://example.com/a", - pendingNavigation: true, - pendingNavigationDeadline: Date.now() + 5_000, + navigation: { + currentUrl: "https://example.com/a", + pendingNavigation: true, + pendingNavigationDeadline: Date.now() + 5_000, + }, }; observeRecordedNavigation(buffer, "https://example.com/b", true); expect(buffer.steps).toEqual([ @@ -83,9 +85,11 @@ describe("recording-step-buffer", () => { values: ["tech"], }, ], - currentUrl: "https://example.com/list", - pendingNavigation: true, - pendingNavigationDeadline: Date.now() + 5_000, + navigation: { + currentUrl: "https://example.com/list", + pendingNavigation: true, + pendingNavigationDeadline: Date.now() + 5_000, + }, }; observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true); expect(buffer.steps).toEqual([ @@ -101,8 +105,7 @@ describe("recording-step-buffer", () => { it("emits navigate for uncaused URL changes", () => { const buffer = { steps: [], - currentUrl: "https://example.com/a", - pendingNavigation: false, + navigation: { currentUrl: "https://example.com/a", pendingNavigation: false }, }; const result = observeRecordedNavigation(buffer, "https://example.com/b", false); expect(result).toEqual({ kind: "appended", index: 0 }); @@ -115,8 +118,10 @@ describe("recording-step-buffer", () => { it("asks the recorder to coalesce redirect hops instead of emitting each one", () => { const buffer = { steps: [], - currentUrl: "https://passport.example/login", - pendingNavigation: false, + navigation: { + currentUrl: "https://passport.example/login", + pendingNavigation: false, + }, }; const hop1 = observeRecordedNavigation( buffer, @@ -139,14 +144,16 @@ describe("recording-step-buffer", () => { url: "https://app.example/dashboard", }); expect(buffer.steps).toEqual([]); - expect(buffer.currentUrl).toBe("https://app.example/dashboard"); + expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard"); }); it("does not reuse redirect metadata for a later content-observed URL change", () => { const buffer = { steps: [], - currentUrl: "https://example.com/redirect", - pendingNavigation: false, + navigation: { + currentUrl: "https://example.com/redirect", + pendingNavigation: false, + }, }; const result = observeRecordedNavigation(buffer, "https://example.com/spa"); diff --git a/apps/extension/src/lib/__tests__/tab-coordinator.test.ts b/apps/extension/src/lib/__tests__/tab-coordinator.test.ts new file mode 100644 index 00000000..037c850a --- /dev/null +++ b/apps/extension/src/lib/__tests__/tab-coordinator.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { RecordingTabCoordinator } from "../recording/tab-coordinator"; + +describe("RecordingTabCoordinator", () => { + it("keeps navigation state isolated per tab", () => { + const tabs = new RecordingTabCoordinator(4, "https://example.com/first"); + const first = tabs.navigation(4); + first.pendingNavigation = true; + const second = tabs.navigation(5, "https://example.com/second"); + + expect(second).toEqual({ + currentUrl: "https://example.com/second", + pendingNavigation: false, + }); + expect(tabs.navigation(4)).toBe(first); + expect(tabs.navigation(4).pendingNavigation).toBe(true); + }); + + it("invalidates an earlier activation when a newer tab becomes active", () => { + const tabs = new RecordingTabCoordinator(4); + const second = tabs.noteActivation(5); + const third = tabs.noteActivation(6); + + expect(tabs.isLatest(second)).toBe(false); + expect(tabs.isLatest(third)).toBe(true); + expect(tabs.commit(6)).toBe(4); + expect(tabs.currentTabId).toBe(6); + }); +}); diff --git a/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts b/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts index 2b7fb1d7..4a9e375b 100644 --- a/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts +++ b/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts @@ -27,6 +27,17 @@ describe("shouldRecordPress", () => { }); describe("reduceTraceSteps", () => { + it("does not expose internal tab transitions in trace v2", () => { + const trace = reduceTraceSteps([ + { + op: "switch_tab", + preStateId: "s1", + postStateId: "s2", + }, + ]); + expect(trace.steps).toEqual([]); + }); + it("builds steps with pages dictionary and page id references", () => { const drafts: RecordingDraftStep[] = [ { diff --git a/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts b/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts index df60cf7a..54370857 100644 --- a/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts +++ b/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts @@ -33,6 +33,21 @@ describe("trace reducer v3", () => { expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id"); }); + it("emits tab transitions only for callers that advertised support", () => { + const drafts: RecordingDraftStep[] = [ + { + op: "switch_tab", + preStateId: "s1", + postStateId: "s2", + }, + ]; + + expect(reduceTraceStepsV3(drafts).steps).toEqual([]); + expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([ + { op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } }, + ]); + }); + it("collapses redirect hops while retaining draft-to-step identity", () => { const drafts: RecordingDraftStep[] = [ { op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" }, diff --git a/apps/extension/src/lib/recording/observation-session.ts b/apps/extension/src/lib/recording/observation-session.ts index d2f0148e..604391a4 100644 --- a/apps/extension/src/lib/recording/observation-session.ts +++ b/apps/extension/src/lib/recording/observation-session.ts @@ -28,7 +28,13 @@ function abortableDelay(ms: number, signal?: AbortSignal): Promise { } function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft { - return draft.op !== "navigate" && draft.op !== "scroll"; + return ( + draft.op === "click" || + draft.op === "hover" || + draft.op === "fill" || + draft.op === "press" || + draft.op === "select" + ); } export class RecordingObservationSession { diff --git a/apps/extension/src/lib/recording/recording-runtime.ts b/apps/extension/src/lib/recording/recording-runtime.ts index f3bdae7c..42a0261b 100644 --- a/apps/extension/src/lib/recording/recording-runtime.ts +++ b/apps/extension/src/lib/recording/recording-runtime.ts @@ -1,6 +1,7 @@ import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; import type { StopReason, TraceV3 } from "@/transport/types"; import type { DocumentSettleScope } from "./document-settle"; +import type { RegisteredObservation } from "./observation-capture"; import { RecordingObservationSession } from "./observation-session"; import { inferMissingPostStates, SettleController } from "./settle-controller"; import { RecordingStateRegistry } from "./state-registry"; @@ -10,11 +11,11 @@ import type { RecordingDraftStep, StepAnnotation } from "./types"; interface TabRecordingContext { session: RecordingObservationSession; settle: SettleController; - initialCapture: PendingInitialCapture | null; + pendingCapture: PendingCapture | null; } -interface PendingInitialCapture { - promise: Promise; +interface PendingCapture { + promise: Promise; abort: AbortController; } @@ -56,32 +57,71 @@ export class RecordingObservationRuntime { tabsApi: this.#tabsApi, tabId, }), - initialCapture: null, + pendingCapture: null, }; this.#contexts.set(tabId, context); return context; } - async captureInitial(tabId: number): Promise { + async #capture(tabId: number): Promise { const context = this.#context(tabId); - if (context.session.cursor.lastSettled) return; - if (context.initialCapture) return context.initialCapture.promise; + if (context.pendingCapture) return context.pendingCapture.promise; const abort = new AbortController(); - let pending!: PendingInitialCapture; + let pending!: PendingCapture; const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal); pending = { abort, - promise: promise - .then(() => {}) - .finally(() => { - if (context.initialCapture === pending) context.initialCapture = null; - }), + promise: promise.finally(() => { + if (context.pendingCapture === pending) context.pendingCapture = null; + }), }; - context.initialCapture = pending; + context.pendingCapture = pending; return pending.promise; } + async captureInitial(tabId: number): Promise { + const context = this.#context(tabId); + if (context.session.cursor.lastSettled) return; + await this.#capture(tabId); + } + + async captureTabTransition( + fromTabId: number, + toTabId: number, + ): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> { + const from = this.#context(fromTabId); + await this.#flushContext(from); + if (!from.session.cursor.lastSettled) { + try { + await this.captureInitial(fromTabId); + } catch { + // A target-tab observation can still preserve the forward transition. + } + } + + const to = this.#context(toTabId); + let target = to.session.cursor.lastSettled; + try { + target = await this.#capture(toTabId); + } catch { + // Reuse the last settled target state when a refresh races navigation. + } + + const source = from.session.cursor.lastSettled; + return { + ...(source ? { preStateId: source.stateId } : {}), + ...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}), + }; + } + + bindTabTransition( + draft: Extract, + draftId: number, + ): void { + if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId); + } + async processDraft( tabId: number, drafts: RecordingDraftStep[], @@ -119,23 +159,27 @@ export class RecordingObservationRuntime { this.#context(tabId).settle.scheduleRedirect(drafts, url); } - async flushRedirects(): Promise { + async #flushContext(context: TabRecordingContext): Promise { + if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]); + await context.settle.flushRedirects(); + await context.settle.flush(); + } + + async flushRedirects(tabId?: number): Promise { + if (tabId !== undefined) { + await this.#contexts.get(tabId)?.settle.flushRedirects(); + return; + } for (const context of this.#contexts.values()) await context.settle.flushRedirects(); } async flush(): Promise { - await Promise.allSettled( - [...this.#contexts.values()].flatMap((context) => - context.initialCapture ? [context.initialCapture.promise] : [], - ), - ); - await this.flushRedirects(); - for (const context of this.#contexts.values()) await context.settle.flush(); + for (const context of this.#contexts.values()) await this.#flushContext(context); } async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise { const context = this.#context(tabId); - if (context.initialCapture) await Promise.allSettled([context.initialCapture.promise]); + if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]); if (!context.session.cursor.lastSettled) { try { await this.captureInitial(tabId); @@ -149,7 +193,7 @@ export class RecordingObservationRuntime { cancel(): void { for (const context of this.#contexts.values()) { - context.initialCapture?.abort.abort(); + context.pendingCapture?.abort.abort(); context.settle.cancel(); } } @@ -161,6 +205,7 @@ export class RecordingObservationRuntime { startUrl?: string; stoppedBy: StopReason; bskVersion: string; + includeTabSwitches?: boolean; }): TraceV3 { return buildTraceV3({ registry: this.#registry, @@ -172,6 +217,7 @@ export class RecordingObservationRuntime { stoppedBy: input.stoppedBy, bskVersion: input.bskVersion, redactValues: this.#redactValues, + includeTabSwitches: input.includeTabSwitches, }); } } diff --git a/apps/extension/src/lib/recording/step-buffer.ts b/apps/extension/src/lib/recording/step-buffer.ts index cb71e176..c1d9a000 100644 --- a/apps/extension/src/lib/recording/step-buffer.ts +++ b/apps/extension/src/lib/recording/step-buffer.ts @@ -4,6 +4,10 @@ import type { RecordingDraftStep, TargetMatchHint } from "./types"; export interface RecordingStepBuffer { steps: RecordingDraftStep[]; + navigation: RecordingNavigationCursor; +} + +export interface RecordingNavigationCursor { currentUrl?: string; pendingNavigation: boolean; pendingNavigationDeadline?: number; @@ -87,22 +91,23 @@ export function observeRecordedNavigation( transitionType?: string, transitionQualifiers?: string[], ): NavigationObserveResult { - if (!url || url === buffer.currentUrl) return { kind: "noop" }; - buffer.currentUrl = url; + const navigation = buffer.navigation; + if (!url || url === navigation.currentUrl) return { kind: "noop" }; + navigation.currentUrl = url; const pendingIsCurrent = - buffer.pendingNavigation && - (buffer.pendingNavigationDeadline === undefined || - buffer.pendingNavigationDeadline >= Date.now()); + navigation.pendingNavigation && + (navigation.pendingNavigationDeadline === undefined || + navigation.pendingNavigationDeadline >= Date.now()); if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) { - buffer.pendingNavigation = false; - buffer.pendingNavigationDeadline = undefined; + navigation.pendingNavigation = false; + navigation.pendingNavigationDeadline = undefined; const annotatedIndex = annotateLastStepNavigation(buffer, url); if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex }; } else { - buffer.pendingNavigation = false; - buffer.pendingNavigationDeadline = undefined; + navigation.pendingNavigation = false; + navigation.pendingNavigationDeadline = undefined; } if (hasRedirectQualifier(transitionQualifiers)) { @@ -135,14 +140,14 @@ export function appendRecordedPayload( return result.kind === "appended" ? result.index : null; } const step = toDraftStep( - { ...payload, page_url: payload.page_url ?? buffer.currentUrl }, + { ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl }, targetHint, ); if (!step) return null; buffer.steps.push(step); if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") { - buffer.pendingNavigation = payload.expects_navigation === true; - buffer.pendingNavigationDeadline = buffer.pendingNavigation + buffer.navigation.pendingNavigation = payload.expects_navigation === true; + buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation ? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS : undefined; } diff --git a/apps/extension/src/lib/recording/tab-coordinator.ts b/apps/extension/src/lib/recording/tab-coordinator.ts new file mode 100644 index 00000000..0dbe185b --- /dev/null +++ b/apps/extension/src/lib/recording/tab-coordinator.ts @@ -0,0 +1,62 @@ +import type { RecordingNavigationCursor } from "./step-buffer"; + +export interface TabActivation { + tabId: number; + revision: number; +} + +export class RecordingTabCoordinator { + readonly #navigationByTab = new Map(); + #activeTabId: number; + #currentTabId: number; + #activationRevision = 0; + + constructor(initialTabId: number, initialUrl?: string) { + this.#activeTabId = initialTabId; + this.#currentTabId = initialTabId; + this.#navigationByTab.set(initialTabId, { + currentUrl: initialUrl, + pendingNavigation: false, + }); + } + + get activeTabId(): number { + return this.#activeTabId; + } + + get currentTabId(): number { + return this.#currentTabId; + } + + noteActivation(tabId: number): TabActivation { + this.#activeTabId = tabId; + this.#activationRevision += 1; + return { tabId, revision: this.#activationRevision }; + } + + isLatest(activation: TabActivation): boolean { + return ( + activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision + ); + } + + commit(tabId: number): number { + const previousTabId = this.#currentTabId; + this.#currentTabId = tabId; + return previousTabId; + } + + navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor { + const existing = this.#navigationByTab.get(tabId); + if (existing) { + if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl; + return existing; + } + const created: RecordingNavigationCursor = { + currentUrl: fallbackUrl, + pendingNavigation: false, + }; + this.#navigationByTab.set(tabId, created); + return created; + } +} diff --git a/apps/extension/src/lib/recording/trace-builder-v3.ts b/apps/extension/src/lib/recording/trace-builder-v3.ts index 7fc1feec..5da20738 100644 --- a/apps/extension/src/lib/recording/trace-builder-v3.ts +++ b/apps/extension/src/lib/recording/trace-builder-v3.ts @@ -40,8 +40,12 @@ export function buildTraceV3(input: { stoppedBy: StopReason; bskVersion: string; redactValues?: boolean; + includeTabSwitches?: boolean; }): TraceV3 { - const reduced = reduceTraceStepsV3(input.drafts, { redactValues: input.redactValues }); + const reduced = reduceTraceStepsV3(input.drafts, { + includeTabSwitches: input.includeTabSwitches, + redactValues: input.redactValues, + }); const entries = publishedEntries(input.registry, reduced.steps); const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`])); const annotationsByState = new Map(); diff --git a/apps/extension/src/lib/recording/trace-reducer-v2.ts b/apps/extension/src/lib/recording/trace-reducer-v2.ts index 35399389..dacf581f 100644 --- a/apps/extension/src/lib/recording/trace-reducer-v2.ts +++ b/apps/extension/src/lib/recording/trace-reducer-v2.ts @@ -162,6 +162,7 @@ function toV2Step( effectForNavigation(step.navigatedTo, urlToId), ); case "scroll": + case "switch_tab": return null; } } diff --git a/apps/extension/src/lib/recording/trace-reducer-v3.ts b/apps/extension/src/lib/recording/trace-reducer-v3.ts index 7ffe91f1..a28ad808 100644 --- a/apps/extension/src/lib/recording/trace-reducer-v3.ts +++ b/apps/extension/src/lib/recording/trace-reducer-v3.ts @@ -58,7 +58,16 @@ function selection(values: string[], labels?: string[]): Array<{ value: string; })); } -function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolean): StepV3 | null { +interface ReduceDraftOptions { + includeTabSwitches: boolean; + redactValues: boolean; +} + +function reduceDraft( + draft: RecordingDraftStep, + id: number, + options: ReduceDraftOptions, +): StepV3 | null { if (!shouldIncludeDraft(draft)) return null; const state = draft.preStateId ?? draft.postStateId; const resultState = draft.postStateId ?? draft.preStateId; @@ -68,6 +77,8 @@ function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolea switch (draft.op) { case "navigate": return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) }; + case "switch_tab": + return options.includeTabSwitches ? { op: "switch_tab", ...common } : null; case "click": return { op: "click", @@ -81,7 +92,7 @@ function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolea target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), }; case "fill": - const fillIsRedacted = redactValues || draft.redacted === true; + const fillIsRedacted = options.redactValues || draft.redacted === true; return { op: "fill", ...common, @@ -105,7 +116,7 @@ function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolea op: "select", ...common, target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), - ...(!redactValues ? { selection: selection(draft.values, draft.labels) } : {}), + ...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}), }; case "scroll": return { op: "scroll", ...common }; @@ -119,12 +130,15 @@ export interface ReducedTraceV3 { export function reduceTraceStepsV3( steps: RecordingDraftStep[], - options: { redactValues?: boolean } = {}, + options: { includeTabSwitches?: boolean; redactValues?: boolean } = {}, ): ReducedTraceV3 { const output: StepV3[] = []; const stepIdByDraftId = new Map(); for (const { draft, draftIds } of collapseRedirects(steps)) { - const step = reduceDraft(draft, output.length + 1, options.redactValues ?? false); + const step = reduceDraft(draft, output.length + 1, { + includeTabSwitches: options.includeTabSwitches === true, + redactValues: options.redactValues === true, + }); if (!step) continue; output.push(step); for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id); diff --git a/apps/extension/src/lib/recording/types.ts b/apps/extension/src/lib/recording/types.ts index 96df3fee..9767ad62 100644 --- a/apps/extension/src/lib/recording/types.ts +++ b/apps/extension/src/lib/recording/types.ts @@ -69,6 +69,7 @@ export type RecordingDraftStep = DraftTarget & DraftNavigationEffect) | ({ op: "scroll" } & DraftStateLink) + | ({ op: "switch_tab" } & DraftStateLink) | ({ op: "navigate"; url: string; diff --git a/apps/extension/src/tools/__tests__/record-steps.test.ts b/apps/extension/src/tools/__tests__/record-steps.test.ts index 4189d888..4b6a02df 100644 --- a/apps/extension/src/tools/__tests__/record-steps.test.ts +++ b/apps/extension/src/tools/__tests__/record-steps.test.ts @@ -15,7 +15,12 @@ import { const AGENT_WINDOW_ID = 100; const TAB_ID = 4; const START_URL = "https://example.com/"; -const RECORD_START_V3 = { session_id: "abcd", url: START_URL, trace_version: 3 as const }; +const RECORD_START_V3 = { + session_id: "abcd", + url: START_URL, + trace_version: 3 as const, + supports_tab_switch_steps: true, +}; const stepSequenceByProducer = new Map(); function asTraceV3(trace: RecordedTrace): TraceV3 { @@ -48,6 +53,7 @@ function chromeEvent unknown>() { function installChrome() { const runtimeOnMessage = chromeEvent(); + const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>(); const webNavigationOnCompleted = chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>(); const webNavigationOnCommitted = @@ -57,7 +63,7 @@ function installChrome() { vi.stubGlobal("chrome", { runtime: { onMessage: runtimeOnMessage }, tabs: { - onActivated: chromeEvent(), + onActivated: tabsOnActivated, onCreated: chromeEvent(), onUpdated: chromeEvent(), }, @@ -66,7 +72,7 @@ function installChrome() { onCommitted: webNavigationOnCommitted, }, }); - return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted }; + return { runtimeOnMessage, tabsOnActivated, webNavigationOnCompleted, webNavigationOnCommitted }; } function fakeManager() { @@ -118,7 +124,10 @@ type FakeCdp = CdpRunner & { const LONG_IDLE_MS = 10_000; /** AX-only page: DOMSnapshot calls throw so capture falls back to the AX tree. */ -function makeFakeCdp(trees?: unknown[], options?: { failCaptures?: boolean }): FakeCdp { +function makeFakeCdp( + trees?: unknown[], + options?: { failCaptures?: boolean; treesByTab?: Record }, +): FakeCdp { type EventListener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => void; const events: EventListener[] = []; const script = [...(trees ?? [])]; @@ -153,6 +162,8 @@ function makeFakeCdp(trees?: unknown[], options?: { failCaptures?: boolean }): F "Accessibility.enable": () => ({}), "Accessibility.getFullAXTree": (_params, _tabId) => { if (failing) throw new Error("Execution context was destroyed"); + const tabTree = options?.treesByTab?.[_tabId]; + if (tabTree) return tabTree; if (script.length === 0) return axTree("Example Domain", ["Submit"]); return script.length === 1 ? script[0] : script.shift(); }, @@ -202,6 +213,31 @@ function makeTabsApi() { }; } +function makeMultiTabsApi(tabs: chrome.tabs.Tab[]) { + const byId = new Map(tabs.flatMap((tab) => (typeof tab.id === "number" ? [[tab.id, tab]] : []))); + return { + get: async (tabId: number) => { + const tab = byId.get(tabId); + if (!tab) throw new Error(`tab ${tabId} closed`); + return tab; + }, + query: async (query: chrome.tabs.QueryInfo) => + [...byId.values()].filter( + (tab) => + (query.windowId === undefined || tab.windowId === query.windowId) && + (query.active === undefined || tab.active === query.active), + ), + activate(tabId: number) { + for (const tab of byId.values()) tab.active = tab.id === tabId; + }, + goTo(tabId: number, url: string, title: string) { + const tab = byId.get(tabId); + if (!tab) throw new Error(`tab ${tabId} closed`); + Object.assign(tab, { url, title }); + }, + }; +} + /** Outlast a settle on a quiet page plus the per-tab observation cooldown. */ function settleWait(): Promise { return new Promise((resolve) => setTimeout(resolve, 800)); @@ -212,16 +248,19 @@ function runtimeOnMessageEmit( requestId: string, step: unknown, tabId = TAB_ID, -): void { + active = true, +): ReturnType { const producerId = `test-producer-${tabId}`; const producerKey = `${requestId}:${producerId}`; const sequence = (stepSequenceByProducer.get(producerKey) ?? 0) + 1; stepSequenceByProducer.set(producerKey, sequence); + const sendResponse = vi.fn(); chromeApi.runtimeOnMessage.emit( { type: RECORD_STEP, requestId, producerId, sequence, step }, - { tab: { id: tabId } } as chrome.runtime.MessageSender, - () => {}, + { tab: { id: tabId, active } } as chrome.runtime.MessageSender, + sendResponse, ); + return sendResponse; } describe("recorded user steps reach the exported trace", () => { @@ -1173,4 +1212,335 @@ describe("recorded user steps reach the exported trace", () => { ); expect(result).toMatchObject({ code: "invalid_params" }); }); + + it("records tab transitions and binds actions to each tab's observation", async () => { + const chromeApi = installChrome(); + const manager = fakeManager(); + const secondUrl = "https://example.com/second"; + const tabsApi = makeMultiTabsApi([ + { + id: TAB_ID, + windowId: AGENT_WINDOW_ID, + active: true, + status: "complete", + url: START_URL, + title: "First tab", + } as chrome.tabs.Tab, + { + id: 5, + windowId: AGENT_WINDOW_ID, + active: false, + status: "complete", + url: secondUrl, + title: "Second tab", + } as chrome.tabs.Tab, + ]); + const cdp = makeFakeCdp(undefined, { + treesByTab: { + [TAB_ID]: axTree("First tab", ["First action"]), + 5: axTree("Second tab", ["Second action"]), + }, + }); + let requestId = ""; + const sendToTab = vi.fn(async (_tabId: number, message: unknown) => { + const typed = message as { type?: string; requestId?: string }; + if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId; + return { ok: true }; + }); + const deps = { tabsApi, sendToTab, cdp }; + + await handleRecordStart(manager, RECORD_START_V3, deps); + attachRecordStepListener(deps); + tabsApi.activate(5); + chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID }); + await settleWait(); + runtimeOnMessageEmit( + chromeApi, + requestId, + { + op: "click", + page_url: secondUrl, + target: { role: "button", name: "Second action", tag: "button" }, + }, + 5, + ); + await settleWait(); + + tabsApi.activate(TAB_ID); + chromeApi.tabsOnActivated.emit({ tabId: TAB_ID, windowId: AGENT_WINDOW_ID }); + await settleWait(); + runtimeOnMessageEmit(chromeApi, requestId, { + op: "click", + page_url: START_URL, + target: { role: "button", name: "First action", tag: "button" }, + }); + await settleWait(); + + const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps); + const trace = asTraceV3((stopped as RecordStopResult).trace); + const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url; + const switches = trace.steps.filter((step) => step.op === "switch_tab"); + const clicks = trace.steps.filter((step) => step.op === "click"); + + expect(switches).toHaveLength(2); + expect(clicks).toHaveLength(2); + expect(stateUrl(switches[0]!.state)).toBe(START_URL); + expect(stateUrl(switches[0]!.result.state)).toBe(secondUrl); + expect(stateUrl(clicks[0]!.state)).toBe(secondUrl); + expect(stateUrl(switches[1]!.result.state)).toBe(START_URL); + expect(stateUrl(clicks[1]!.state)).toBe(START_URL); + }); + + it("keeps switch_tab internal when the v3 caller did not advertise support", async () => { + const chromeApi = installChrome(); + const manager = fakeManager(); + const tabsApi = makeMultiTabsApi([ + { + id: TAB_ID, + windowId: AGENT_WINDOW_ID, + active: true, + status: "complete", + url: START_URL, + } as chrome.tabs.Tab, + { + id: 5, + windowId: AGENT_WINDOW_ID, + active: false, + status: "complete", + url: "https://example.com/second", + } as chrome.tabs.Tab, + ]); + const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() }; + + await handleRecordStart( + manager, + { session_id: "abcd", url: START_URL, trace_version: 3 }, + deps, + ); + tabsApi.activate(5); + chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID }); + await settleWait(); + + const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps); + const trace = asTraceV3((stopped as RecordStopResult).trace); + expect(trace.steps.some((step) => step.op === "switch_tab")).toBe(false); + }); + + it("keeps the latest rapid tab activation when an earlier rearm acknowledges late", async () => { + const chromeApi = installChrome(); + const manager = fakeManager(); + const tabsApi = makeMultiTabsApi( + [TAB_ID, 5, 6].map( + (id) => + ({ + id, + windowId: AGENT_WINDOW_ID, + active: id === TAB_ID, + status: "complete", + url: id === TAB_ID ? START_URL : `https://example.com/tab-${id}`, + }) as chrome.tabs.Tab, + ), + ); + let releaseSecond!: () => void; + const secondGate = new Promise((resolve) => { + releaseSecond = resolve; + }); + let announceSecond!: () => void; + const secondStarted = new Promise((resolve) => { + announceSecond = resolve; + }); + const sendToTab = vi.fn(async (tabId: number, message: unknown) => { + if ((message as { type?: string }).type === RECORD_START && tabId === 5) { + announceSecond(); + await secondGate; + } + return { ok: true }; + }); + const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() }; + + await handleRecordStart(manager, RECORD_START_V3, deps); + tabsApi.activate(5); + chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID }); + await secondStarted; + tabsApi.activate(6); + chromeApi.tabsOnActivated.emit({ tabId: 6, windowId: AGENT_WINDOW_ID }); + await settleWait(); + releaseSecond(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps); + const trace = asTraceV3((stopped as RecordStopResult).trace); + const switches = trace.steps.filter((step) => step.op === "switch_tab"); + const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url; + expect(switches).toHaveLength(1); + expect(stateUrl(switches[0]!.result.state)).toBe("https://example.com/tab-6"); + }); + + it("keeps pending action settles scoped to their tab during an immediate switch", async () => { + const chromeApi = installChrome(); + const manager = fakeManager(); + const secondUrl = "https://example.com/second"; + const tabsApi = makeMultiTabsApi([ + { + id: TAB_ID, + windowId: AGENT_WINDOW_ID, + active: true, + status: "complete", + url: START_URL, + } as chrome.tabs.Tab, + { + id: 5, + windowId: AGENT_WINDOW_ID, + active: false, + status: "complete", + url: secondUrl, + } as chrome.tabs.Tab, + ]); + let requestId = ""; + const sendToTab = vi.fn(async (_tabId: number, message: unknown) => { + const typed = message as { type?: string; requestId?: string }; + if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId; + return { ok: true }; + }); + const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() }; + + await handleRecordStart(manager, RECORD_START_V3, deps); + attachRecordStepListener(deps); + runtimeOnMessageEmit(chromeApi, requestId, { + op: "click", + page_url: START_URL, + target: { role: "button", name: "First action", tag: "button" }, + }); + tabsApi.activate(5); + chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID }); + runtimeOnMessageEmit( + chromeApi, + requestId, + { + op: "click", + page_url: secondUrl, + target: { role: "button", name: "Second action", tag: "button" }, + }, + 5, + ); + await settleWait(); + + const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps); + const trace = asTraceV3((stopped as RecordStopResult).trace); + const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url; + + expect(trace.steps.map((step) => step.op)).toEqual(["click", "switch_tab", "click"]); + expect(stateUrl(trace.steps[0]!.result.state)).toBe(START_URL); + expect(stateUrl(trace.steps[1]!.result.state)).toBe(secondUrl); + expect(stateUrl(trace.steps[2]!.state)).toBe(secondUrl); + }); + + it("tracks the same navigation URL independently for each tab", async () => { + const chromeApi = installChrome(); + const manager = fakeManager(); + const tabsApi = makeMultiTabsApi([ + { + id: TAB_ID, + windowId: AGENT_WINDOW_ID, + active: true, + status: "complete", + url: START_URL, + } as chrome.tabs.Tab, + { + id: 5, + windowId: AGENT_WINDOW_ID, + active: false, + status: "complete", + url: "https://example.com/second", + } as chrome.tabs.Tab, + ]); + const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() }; + + await handleRecordStart(manager, RECORD_START_V3, deps); + tabsApi.activate(5); + chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID }); + await settleWait(); + tabsApi.goTo(5, START_URL, "Same URL in second tab"); + chromeApi.webNavigationOnCommitted.emit({ + tabId: 5, + frameId: 0, + url: START_URL, + transitionType: "typed", + transitionQualifiers: ["from_address_bar"], + } as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails); + await settleWait(); + + const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps); + const trace = asTraceV3((stopped as RecordStopResult).trace); + expect( + trace.steps.filter((step) => step.op === "navigate" && step.to === START_URL), + ).toHaveLength(1); + }); + + it("drops late input and frame navigation from an inactive tab", async () => { + const chromeApi = installChrome(); + const manager = fakeManager(); + const secondUrl = "https://example.com/second"; + const tabsApi = makeMultiTabsApi([ + { + id: TAB_ID, + windowId: AGENT_WINDOW_ID, + active: true, + status: "complete", + url: START_URL, + } as chrome.tabs.Tab, + { + id: 5, + windowId: AGENT_WINDOW_ID, + active: false, + status: "complete", + url: secondUrl, + } as chrome.tabs.Tab, + ]); + let requestId = ""; + const sendToTab = vi.fn(async (_tabId: number, message: unknown) => { + const typed = message as { type?: string; requestId?: string }; + if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId; + return { ok: true }; + }); + const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() }; + + await handleRecordStart(manager, RECORD_START_V3, deps); + attachRecordStepListener(deps); + tabsApi.activate(5); + chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID }); + await settleWait(); + const lateAck = runtimeOnMessageEmit( + chromeApi, + requestId, + { + op: "fill", + page_url: START_URL, + target: { role: "textbox", name: "Draft", tag: "input" }, + value: "late", + }, + TAB_ID, + false, + ); + expect(lateAck).toHaveBeenCalledWith({ ok: true, sequence: 1 }); + chromeApi.webNavigationOnCommitted.emit({ + tabId: TAB_ID, + frameId: 0, + url: "https://example.com/late", + transitionType: "link", + transitionQualifiers: [], + } as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails); + chromeApi.webNavigationOnCommitted.emit({ + tabId: 5, + frameId: 7, + url: "https://frame.example/late", + transitionType: "link", + transitionQualifiers: [], + } as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails); + await new Promise((resolve) => setTimeout(resolve, 50)); + + const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps); + const trace = asTraceV3((stopped as RecordStopResult).trace); + expect(trace.steps.map((step) => step.op)).toEqual(["switch_tab"]); + }); }); diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index 9aa125d6..8ebceca7 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -19,7 +19,12 @@ import { type RecordStopMessage, } from "@/lib/record-bridge"; import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime"; -import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer"; +import { + appendRecordedPayload, + observeRecordedNavigation, + type RecordingStepBuffer, +} from "@/lib/recording/step-buffer"; +import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator"; import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2"; import type { RecordingDraftStep } from "@/lib/recording/types"; import type { SessionManager } from "@/session-manager/manager"; @@ -48,7 +53,7 @@ import { interface ActiveRecording { requestId: string; - tabId: number; + tabs: RecordingTabCoordinator; agentWindowId: number; startUrl?: string; purpose?: string; @@ -56,14 +61,12 @@ interface ActiveRecording { startedAt: string; startedAtMs: number; traceVersion: 2 | 3; + supportsTabSwitchSteps: boolean; finishPromise: Promise; resolveFinish: (trace: RecordedTrace) => void; rejectFinish: (err: Error) => void; settled: boolean; finishAttempt: Promise | null; - currentUrl?: string; - pendingNavigation: boolean; - pendingNavigationDeadline?: number; observation: RecordingObservationRuntime | null; stoppedBy: StopReason; /** Navigation callbacks tracked from event receipt through action enqueue. */ @@ -79,8 +82,13 @@ interface ActiveRecording { lastStepSequenceByProducer: Map; } -function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise): void { - recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {}); +function enqueueRecordingAction( + recording: ActiveRecording, + task: () => Promise, +): Promise { + const queued = recording.actionQueue.then(task, task); + recording.actionQueue = queued.catch(() => {}); + return queued; } const recordings = new Map(); @@ -198,6 +206,7 @@ function buildTrace(recording: ActiveRecording): RecordedTrace { startUrl: recording.startUrl, stoppedBy: recording.stoppedBy, bskVersion: BSK_TRACE_VERSION, + includeTabSwitches: recording.supportsTabSwitchSteps, }); } if (recording.traceVersion === 3) { @@ -211,6 +220,38 @@ function buildTrace(recording: ActiveRecording): RecordedTrace { }); } +function stepBufferFor( + recording: ActiveRecording, + tabId: number, + fallbackUrl?: string, +): RecordingStepBuffer { + return { + steps: recording.steps, + navigation: recording.tabs.navigation(tabId, fallbackUrl), + }; +} + +async function activateRecordingTab( + recording: ActiveRecording, + targetTabId: number, +): Promise { + const previousTabId = recording.tabs.currentTabId; + if (targetTabId === previousTabId) return; + + const transition = recording.observation + ? await recording.observation.captureTabTransition(previousTabId, targetTabId) + : {}; + const { targetUrl, ...stateLinks } = transition; + const draft: Extract = { + op: "switch_tab", + ...stateLinks, + }; + recording.steps.push(draft); + recording.observation?.bindTabTransition(draft, recording.steps.length); + recording.tabs.navigation(targetTabId, targetUrl); + recording.tabs.commit(targetTabId); +} + async function processRecordedStep( recording: ActiveRecording, draftIndex: number, @@ -307,7 +348,8 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): ( if (!isRecordStepMessage(message)) return false; for (const recording of recordings.values()) { if (recording.requestId !== message.requestId) continue; - const sourceTabId = sender.tab?.id ?? recording.tabId; + const sourceTabId = sender.tab?.id ?? recording.tabs.currentTabId; + const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId; const producerKey = `${sourceTabId}:${message.producerId}`; const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1; if (message.sequence < expectedSequence) { @@ -325,9 +367,19 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): ( recording.lastStepSequenceByProducer.set(producerKey, message.sequence); enqueueRecordingAction(recording, async () => { - await recording.observation?.flushRedirects(); + if (sourceTabId !== recording.tabs.currentTabId) { + if (!sourceWasActive) return; + if (sourceTabId !== recording.tabs.activeTabId) + recording.tabs.noteActivation(sourceTabId); + await activateRecordingTab(recording, sourceTabId); + } + await recording.observation?.flushRedirects(sourceTabId); const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined; - const draftIndex = appendRecordedPayload(recording, message.step, targetHint); + const draftIndex = appendRecordedPayload( + stepBufferFor(recording, sourceTabId, message.step.page_url), + message.step, + targetHint, + ); if (draftIndex !== null) { await processRecordedStep(recording, draftIndex, sourceTabId); } @@ -359,7 +411,7 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()): function findRecordingByTabId(tabId: number): ActiveRecording | null { for (const recording of recordings.values()) { - if (recording.tabId === tabId && !recording.settled) return recording; + if (recording.tabs.currentTabId === tabId && !recording.settled) return recording; } return null; } @@ -396,7 +448,7 @@ async function clearRearmTimersForRecording( recording: ActiveRecording, deps: RecordDeps, ): Promise { - clearRearmTimer(recording.tabId); + clearRearmTimer(recording.tabs.currentTabId); try { const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId }); for (const tab of tabs) { @@ -412,12 +464,12 @@ async function stopRecordingOnAllAgentTabs( deps: RecordDeps, ): Promise { const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId }; - let tabIds = [recording.tabId]; + let tabIds = [recording.tabs.currentTabId]; try { const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId }); tabIds = [ ...new Set([ - recording.tabId, + recording.tabs.currentTabId, ...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])), ]), ]; @@ -428,11 +480,11 @@ async function stopRecordingOnAllAgentTabs( for (const tabId of tabIds) { try { const response = await deps.sendToTab(tabId, stopMsg); - if (tabId === recording.tabId && !isRecordStartAck(response)) { + if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) { throw new Error("content script did not confirm recorded steps"); } } catch { - if (tabId === recording.tabId) { + if (tabId === recording.tabs.currentTabId) { throw new Error("failed to flush recorded steps"); } } @@ -450,6 +502,7 @@ async function rearmRecording( recording: ActiveRecording, targetTabId: number, deps: RecordDeps, + activation?: TabActivation, ): Promise { // Do NOT toggle automation-bypass here: each retry used to increment the // content-script counter, and a single stop decrement left the ControlOverlay @@ -463,7 +516,10 @@ async function rearmRecording( }; try { await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab); - recording.tabId = targetTabId; + if (activation) { + if (!recording.tabs.isLatest(activation)) return true; + await enqueueRecordingAction(recording, () => activateRecordingTab(recording, targetTabId)); + } return true; } catch { if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) { @@ -474,7 +530,7 @@ async function rearmRecording( return false; } -function scheduleRearmForTab(tabId: number, deps: RecordDeps): void { +function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void { const existing = rearmTimers.get(tabId); if (existing) clearTimeout(existing); rearmTimers.set( @@ -483,7 +539,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void { rearmTimers.delete(tabId); void (async () => { const current = await findRecordingForTab(tabId, deps); - if (current) await rearmRecording(current, tabId, deps); + if (current) await rearmRecording(current, tabId, deps, activation); })(); }, RECORD_REARM_DEBOUNCE_MS), ); @@ -502,7 +558,12 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): () }; const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => { - scheduleRearmForTab(activeInfo.tabId, deps); + for (const recording of recordings.values()) { + if (recording.settled || recording.agentWindowId !== activeInfo.windowId) continue; + const activation = recording.tabs.noteActivation(activeInfo.tabId); + scheduleRearmForTab(activeInfo.tabId, deps, activation); + return; + } }; chrome.tabs.onCreated.addListener(onCreated); @@ -522,54 +583,39 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps transitionQualifiers?: string[], ) => { if (!url) return; - const direct = findRecordingByTabId(tabId); - const candidates = direct - ? direct.acceptingNavigation - ? [direct] - : [] - : [...recordings.values()].filter( - (recording) => !recording.settled && recording.acceptingNavigation, + const candidates = [...recordings.values()].filter( + (recording) => + !recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId, + ); + for (const recording of candidates) { + const queued = enqueueRecordingAction(recording, async () => { + if (tabId !== recording.tabs.currentTabId) { + await activateRecordingTab(recording, tabId); + } + const result = observeRecordedNavigation( + stepBufferFor(recording, tabId, url), + url, + causedByAction, + transitionType, + transitionQualifiers, ); - if (candidates.length === 0) return; - - let resolveTracked!: () => void; - const tracked = new Promise((resolve) => { - resolveTracked = resolve; - }); - for (const candidate of candidates) candidate.navigationCallbacks.add(tracked); - - void (async () => { - try { - const recording = await findRecordingForTab(tabId, deps); - if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) { + if (result.kind === "coalesce_redirect") { + recording.observation?.scheduleRedirect(tabId, recording.steps, result.url); return; } - enqueueRecordingAction(recording, async () => { - const result = observeRecordedNavigation( - recording, - url, - causedByAction, - transitionType, - transitionQualifiers, - ); - if (result.kind === "coalesce_redirect") { - recording.observation?.scheduleRedirect(tabId, recording.steps, result.url); - return; - } - - if (result.kind === "noop") return; - recording.observation?.clearRedirect(tabId); - if (result.kind === "appended") { - await processRecordedStep(recording, result.index, tabId); - } else { - recording.observation?.scheduleSettle(tabId, recording.steps, result.index); - } - }); - } finally { - for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked); - resolveTracked(); - } - })(); + + if (result.kind === "noop") return; + recording.observation?.clearRedirect(tabId); + if (result.kind === "appended") { + await processRecordedStep(recording, result.index, tabId); + } else { + recording.observation?.scheduleSettle(tabId, recording.steps, result.index); + } + }); + const tracked = queued.catch(() => {}); + recording.navigationCallbacks.add(tracked); + void tracked.finally(() => recording.navigationCallbacks.delete(tracked)); + } }; const onMainFrameComplete = (tabId: number, url?: string) => { void (async () => { @@ -651,7 +697,8 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): sendResponse({ active: false }); return; } - await rearmRecording(recording, tabId, deps); + const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined; + await rearmRecording(recording, tabId, deps, activation); sendResponse({ active: true, requestId: recording.requestId, @@ -716,7 +763,7 @@ async function finishRecordingAttempt( if (!(await drainRecordingToStability(recording))) { return null; } - await recording.observation?.settleTrailing(recording.tabId, recording.steps); + await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps); recording.settled = true; recordings.delete(sessionId); @@ -768,7 +815,7 @@ export async function handleRecordStart( const redactValues = params.redact_values ?? false; recordings.set(params.session_id, { requestId, - tabId: target.tabId, + tabs: new RecordingTabCoordinator(target.tabId, navigateUrl), agentWindowId: ctx.agentWindowId, startUrl: navigateUrl, ...(params.purpose ? { purpose: params.purpose } : {}), @@ -776,14 +823,12 @@ export async function handleRecordStart( startedAt: new Date(startedAtMs).toISOString(), startedAtMs, traceVersion: traceVersionOrErr, + supportsTabSwitchSteps: params.supports_tab_switch_steps === true, finishPromise, resolveFinish, rejectFinish, settled: false, finishAttempt: null, - currentUrl: navigateUrl, - pendingNavigation: false, - pendingNavigationDeadline: undefined, observation: traceVersionOrErr === 3 && deps.cdp ? new RecordingObservationRuntime({ @@ -886,7 +931,7 @@ export async function handleRecordStart( return cancelledError(); } active.startUrl = startUrl; - active.currentUrl = startUrl; + active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl; if (isContentScriptRestrictedUrl(startUrl)) { await abortPending(false); diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 351b342a..dc60fe2e 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -780,6 +780,7 @@ export interface SelectedOptionV3 { export type StepV3 = | ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause }) + | ({ op: "switch_tab" } & StepCommonV3) | ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 }) | ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 }) | ({ op: "fill" } & StepCommonV3 & { @@ -823,6 +824,8 @@ export interface RecordStartParams { redact_values?: boolean; /** Omitted means v2; `3` requests a state-linked v3 trace. */ trace_version?: number; + /** Client can decode the v3 `switch_tab` step variant. */ + supports_tab_switch_steps?: boolean; } export interface RecordStartResult { diff --git a/crates/bsk-cli/src/cli/record.rs b/crates/bsk-cli/src/cli/record.rs index fbc2dfe3..b5a84603 100644 --- a/crates/bsk-cli/src/cli/record.rs +++ b/crates/bsk-cli/src/cli/record.rs @@ -115,6 +115,7 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError> max_page_tokens: args.max_page_tokens, redact_values: Some(args.redact_values), trace_version: Some(TRACE_VERSION_V3), + supports_tab_switch_steps: Some(true), }; let start_result = business_rpc::call::( info.sock_path.clone(), diff --git a/crates/bsk-protocol/schema/tool_record_await_result.json b/crates/bsk-protocol/schema/tool_record_await_result.json index df2a122a..26b6ea8c 100644 --- a/crates/bsk-protocol/schema/tool_record_await_result.json +++ b/crates/bsk-protocol/schema/tool_record_await_result.json @@ -480,6 +480,36 @@ } } }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "result", + "state" + ], + "properties": { + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "switch_tab" + ] + }, + "result": { + "$ref": "#/definitions/StepResultV3" + }, + "state": { + "description": "Observation id immediately before this action.", + "type": "string" + } + } + }, { "description": "Fields shared by every step variant (flattened in JSON).", "type": "object", diff --git a/crates/bsk-protocol/schema/tool_record_start_params.json b/crates/bsk-protocol/schema/tool_record_start_params.json index 62616b34..4120938b 100644 --- a/crates/bsk-protocol/schema/tool_record_start_params.json +++ b/crates/bsk-protocol/schema/tool_record_start_params.json @@ -29,6 +29,13 @@ "session_id": { "type": "string" }, + "supports_tab_switch_steps": { + "description": "Client can decode the v3 `switch_tab` step variant.", + "type": [ + "boolean", + "null" + ] + }, "tab_id": { "type": [ "integer", diff --git a/crates/bsk-protocol/schema/tool_record_stop_result.json b/crates/bsk-protocol/schema/tool_record_stop_result.json index c1dfef02..3973df41 100644 --- a/crates/bsk-protocol/schema/tool_record_stop_result.json +++ b/crates/bsk-protocol/schema/tool_record_stop_result.json @@ -480,6 +480,36 @@ } } }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "result", + "state" + ], + "properties": { + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "switch_tab" + ] + }, + "result": { + "$ref": "#/definitions/StepResultV3" + }, + "state": { + "description": "Observation id immediately before this action.", + "type": "string" + } + } + }, { "description": "Fields shared by every step variant (flattened in JSON).", "type": "object", diff --git a/crates/bsk-protocol/schema/trace.json b/crates/bsk-protocol/schema/trace.json index 664e035e..95dfb157 100644 --- a/crates/bsk-protocol/schema/trace.json +++ b/crates/bsk-protocol/schema/trace.json @@ -469,6 +469,36 @@ } } }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "result", + "state" + ], + "properties": { + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "switch_tab" + ] + }, + "result": { + "$ref": "#/definitions/StepResultV3" + }, + "state": { + "description": "Observation id immediately before this action.", + "type": "string" + } + } + }, { "description": "Fields shared by every step variant (flattened in JSON).", "type": "object", diff --git a/crates/bsk-protocol/schema/trace_step.json b/crates/bsk-protocol/schema/trace_step.json index ae749428..b13ef893 100644 --- a/crates/bsk-protocol/schema/trace_step.json +++ b/crates/bsk-protocol/schema/trace_step.json @@ -431,6 +431,36 @@ } } }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "result", + "state" + ], + "properties": { + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "switch_tab" + ] + }, + "result": { + "$ref": "#/definitions/StepResultV3" + }, + "state": { + "description": "Observation id immediately before this action.", + "type": "string" + } + } + }, { "description": "Fields shared by every step variant (flattened in JSON).", "type": "object", diff --git a/crates/bsk-protocol/schema/trace_step_v3.json b/crates/bsk-protocol/schema/trace_step_v3.json index 14a33ee1..49f74eec 100644 --- a/crates/bsk-protocol/schema/trace_step_v3.json +++ b/crates/bsk-protocol/schema/trace_step_v3.json @@ -41,6 +41,36 @@ } } }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "result", + "state" + ], + "properties": { + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "switch_tab" + ] + }, + "result": { + "$ref": "#/definitions/StepResultV3" + }, + "state": { + "description": "Observation id immediately before this action.", + "type": "string" + } + } + }, { "description": "Fields shared by every step variant (flattened in JSON).", "type": "object", diff --git a/crates/bsk-protocol/schema/trace_v3.json b/crates/bsk-protocol/schema/trace_v3.json index 750ce6a7..3557e872 100644 --- a/crates/bsk-protocol/schema/trace_v3.json +++ b/crates/bsk-protocol/schema/trace_v3.json @@ -173,6 +173,36 @@ } } }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "result", + "state" + ], + "properties": { + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "switch_tab" + ] + }, + "result": { + "$ref": "#/definitions/StepResultV3" + }, + "state": { + "description": "Observation id immediately before this action.", + "type": "string" + } + } + }, { "description": "Fields shared by every step variant (flattened in JSON).", "type": "object", diff --git a/crates/bsk-protocol/src/tools/record.rs b/crates/bsk-protocol/src/tools/record.rs index f9c9012a..8ffee160 100644 --- a/crates/bsk-protocol/src/tools/record.rs +++ b/crates/bsk-protocol/src/tools/record.rs @@ -39,6 +39,9 @@ pub struct RecordStartParams { /// Desired trace export format. Omitted means v2; `3` requests v3. #[serde(default, skip_serializing_if = "Option::is_none")] pub trace_version: Option, + /// Client can decode the v3 `switch_tab` step variant. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_tab_switch_steps: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] @@ -423,6 +426,19 @@ mod tests { assert_eq!(round, step); } + #[test] + fn step_switch_tab_round_trips() { + let step = StepV3::SwitchTab { + common: sample_common(2, "s2", "s5"), + }; + let v = serde_json::to_value(&step).unwrap(); + assert_eq!(v["op"], "switch_tab"); + assert_eq!(v["state"], "s2"); + assert_eq!(v["result"]["state"], "s5"); + let round: StepV3 = serde_json::from_value(v).unwrap(); + assert_eq!(round, step); + } + #[test] fn trace_v3_round_trips() { let trace = sample_trace(); @@ -463,10 +479,12 @@ mod tests { max_page_tokens: None, redact_values: None, trace_version: None, + supports_tab_switch_steps: None, }; let legacy_value = serde_json::to_value(legacy).unwrap(); assert!(legacy_value.get("trace_version").is_none()); assert!(legacy_value.get("max_page_tokens").is_none()); + assert!(legacy_value.get("supports_tab_switch_steps").is_none()); let v3 = RecordStartParams { session_id: "session".into(), @@ -476,11 +494,13 @@ mod tests { max_page_tokens: Some(4_000), redact_values: Some(true), trace_version: Some(TRACE_VERSION_V3), + supports_tab_switch_steps: Some(true), }; let v3_value = serde_json::to_value(v3).unwrap(); assert_eq!(v3_value["trace_version"], TRACE_VERSION_V3); assert_eq!(v3_value["max_page_tokens"], 4_000); assert_eq!(v3_value["redact_values"], true); + assert_eq!(v3_value["supports_tab_switch_steps"], true); } #[test] diff --git a/crates/bsk-protocol/src/tools/record_v3.rs b/crates/bsk-protocol/src/tools/record_v3.rs index bbdd97fe..9af00d1c 100644 --- a/crates/bsk-protocol/src/tools/record_v3.rs +++ b/crates/bsk-protocol/src/tools/record_v3.rs @@ -126,6 +126,10 @@ pub enum StepV3 { to: String, cause: NavigationCause, }, + SwitchTab { + #[serde(flatten)] + common: StepCommonV3, + }, Click { #[serde(flatten)] common: StepCommonV3, @@ -340,6 +344,19 @@ mod tests { assert_eq!(round, step); } + #[test] + fn step_switch_tab_round_trips() { + let step = StepV3::SwitchTab { + common: sample_common(2, "s2", "s5"), + }; + let v = serde_json::to_value(&step).unwrap(); + assert_eq!(v["op"], "switch_tab"); + assert_eq!(v["state"], "s2"); + assert_eq!(v["result"]["state"], "s5"); + let round: StepV3 = serde_json::from_value(v).unwrap(); + assert_eq!(round, step); + } + #[test] fn trace_v3_round_trips() { let trace = sample_trace(); From 1a3d3b7de82582f484e81c9748d45930ebceda13 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 19 Aug 2026 15:04:55 +0800 Subject: [PATCH 2/4] fix(record): bug fix --- .../src/lib/__tests__/tab-coordinator.test.ts | 2 +- .../lib/__tests__/trace-reducer-v3.test.ts | 9 +++ .../src/lib/recording/recording-runtime.ts | 8 +-- .../src/lib/recording/tab-coordinator.ts | 4 +- .../src/lib/recording/trace-reducer-v3.ts | 11 +++- .../src/tools/__tests__/record-steps.test.ts | 56 +++++++++++++++++++ apps/extension/src/tools/record.ts | 25 +++++++-- 7 files changed, 99 insertions(+), 16 deletions(-) diff --git a/apps/extension/src/lib/__tests__/tab-coordinator.test.ts b/apps/extension/src/lib/__tests__/tab-coordinator.test.ts index 037c850a..42499d35 100644 --- a/apps/extension/src/lib/__tests__/tab-coordinator.test.ts +++ b/apps/extension/src/lib/__tests__/tab-coordinator.test.ts @@ -23,7 +23,7 @@ describe("RecordingTabCoordinator", () => { expect(tabs.isLatest(second)).toBe(false); expect(tabs.isLatest(third)).toBe(true); - expect(tabs.commit(6)).toBe(4); + tabs.commit(6); expect(tabs.currentTabId).toBe(6); }); }); diff --git a/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts b/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts index 54370857..1018aec0 100644 --- a/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts +++ b/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts @@ -48,6 +48,15 @@ describe("trace reducer v3", () => { ]); }); + it("does not export a tab transition without both observation endpoints", () => { + const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" }; + const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" }; + + expect( + reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps, + ).toEqual([]); + }); + it("collapses redirect hops while retaining draft-to-step identity", () => { const drafts: RecordingDraftStep[] = [ { op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" }, diff --git a/apps/extension/src/lib/recording/recording-runtime.ts b/apps/extension/src/lib/recording/recording-runtime.ts index 42a0261b..e27577f6 100644 --- a/apps/extension/src/lib/recording/recording-runtime.ts +++ b/apps/extension/src/lib/recording/recording-runtime.ts @@ -165,12 +165,8 @@ export class RecordingObservationRuntime { await context.settle.flush(); } - async flushRedirects(tabId?: number): Promise { - if (tabId !== undefined) { - await this.#contexts.get(tabId)?.settle.flushRedirects(); - return; - } - for (const context of this.#contexts.values()) await context.settle.flushRedirects(); + async flushRedirects(tabId: number): Promise { + await this.#contexts.get(tabId)?.settle.flushRedirects(); } async flush(): Promise { diff --git a/apps/extension/src/lib/recording/tab-coordinator.ts b/apps/extension/src/lib/recording/tab-coordinator.ts index 0dbe185b..2782b6f4 100644 --- a/apps/extension/src/lib/recording/tab-coordinator.ts +++ b/apps/extension/src/lib/recording/tab-coordinator.ts @@ -40,10 +40,8 @@ export class RecordingTabCoordinator { ); } - commit(tabId: number): number { - const previousTabId = this.#currentTabId; + commit(tabId: number): void { this.#currentTabId = tabId; - return previousTabId; } navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor { diff --git a/apps/extension/src/lib/recording/trace-reducer-v3.ts b/apps/extension/src/lib/recording/trace-reducer-v3.ts index a28ad808..aded63b9 100644 --- a/apps/extension/src/lib/recording/trace-reducer-v3.ts +++ b/apps/extension/src/lib/recording/trace-reducer-v3.ts @@ -69,6 +69,15 @@ function reduceDraft( options: ReduceDraftOptions, ): StepV3 | null { if (!shouldIncludeDraft(draft)) return null; + if (draft.op === "switch_tab") { + if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null; + return { + op: "switch_tab", + id, + state: draft.preStateId, + result: { state: draft.postStateId }, + }; + } const state = draft.preStateId ?? draft.postStateId; const resultState = draft.postStateId ?? draft.preStateId; if (!state || !resultState) return null; @@ -77,8 +86,6 @@ function reduceDraft( switch (draft.op) { case "navigate": return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) }; - case "switch_tab": - return options.includeTabSwitches ? { op: "switch_tab", ...common } : null; case "click": return { op: "click", diff --git a/apps/extension/src/tools/__tests__/record-steps.test.ts b/apps/extension/src/tools/__tests__/record-steps.test.ts index 4b6a02df..83d108b1 100644 --- a/apps/extension/src/tools/__tests__/record-steps.test.ts +++ b/apps/extension/src/tools/__tests__/record-steps.test.ts @@ -1376,6 +1376,62 @@ describe("recorded user steps reach the exported trace", () => { expect(stateUrl(switches[0]!.result.state)).toBe("https://example.com/tab-6"); }); + it("does not retry an in-flight tab rearm after recording stops", async () => { + const chromeApi = installChrome(); + const manager = fakeManager(); + const tabsApi = makeMultiTabsApi([ + { + id: TAB_ID, + windowId: AGENT_WINDOW_ID, + active: true, + status: "complete", + url: START_URL, + } as chrome.tabs.Tab, + { + id: 5, + windowId: AGENT_WINDOW_ID, + active: false, + status: "complete", + url: "https://example.com/second", + } as chrome.tabs.Tab, + ]); + let failRearm!: () => void; + const rearmGate = new Promise((_resolve, reject) => { + failRearm = () => reject(new Error("rearm failed after stop")); + }); + let announceRearm!: () => void; + const rearmStarted = new Promise((resolve) => { + announceRearm = resolve; + }); + const sendToTab = vi.fn(async (tabId: number, message: unknown) => { + if ((message as { type?: string }).type === RECORD_START && tabId === 5) { + announceRearm(); + await rearmGate; + } + return { ok: true }; + }); + const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() }; + + await handleRecordStart(manager, RECORD_START_V3, deps); + tabsApi.activate(5); + chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID }); + await rearmStarted; + + const stoppedPromise = handleRecordStop(manager, { session_id: "abcd" }, deps); + await vi.waitFor(() => { + expect(sendToTab).toHaveBeenCalledWith(5, expect.objectContaining({ type: RECORD_STOP })); + }); + failRearm(); + const stopped = await stoppedPromise; + await new Promise((resolve) => setTimeout(resolve, 600)); + + const startsForSecondTab = sendToTab.mock.calls.filter( + ([tabId, message]) => tabId === 5 && (message as { type?: string }).type === RECORD_START, + ); + expect(startsForSecondTab).toHaveLength(1); + expect(asTraceV3((stopped as RecordStopResult).trace).steps).toHaveLength(0); + }); + it("keeps pending action settles scoped to their tab during an immediate switch", async () => { const chromeApi = installChrome(); const manager = fakeManager(); diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index 8ebceca7..80dde143 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -91,6 +91,10 @@ function enqueueRecordingAction( return queued; } +function isRecordingFinishing(recording: ActiveRecording): boolean { + return recording.settled || recording.finishAttempt !== null; +} + const recordings = new Map(); const RECORD_START_RETRIES = 3; @@ -171,9 +175,11 @@ async function sendRecordStartWithAck( tabId: number, msg: RecordStartMessage, sendToTab: RecordDeps["sendToTab"], + cancelled?: () => boolean, ): Promise { let lastError: unknown; for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) { + if (cancelled?.()) throw new Error("record start cancelled"); try { const response = await sendToTab(tabId, msg); if (isRecordStartAck(response)) return; @@ -181,6 +187,7 @@ async function sendRecordStartWithAck( } catch (err) { lastError = err; } + if (cancelled?.()) throw new Error("record start cancelled"); if (attempt + 1 < RECORD_START_RETRIES) { await sleep(RECORD_START_RETRY_DELAY_MS); } @@ -508,20 +515,29 @@ async function rearmRecording( // content-script counter, and a single stop decrement left the ControlOverlay // stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay // already hides the control chrome while activeRecord is set. + const isFinishing = () => isRecordingFinishing(recording); for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) { + if (isFinishing()) return false; const startMsg: RecordStartMessage = { type: RECORD_START, requestId: recording.requestId, startedAtMs: recording.startedAtMs, }; try { - await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab); + await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing); + if (isFinishing()) return false; if (activation) { if (!recording.tabs.isLatest(activation)) return true; - await enqueueRecordingAction(recording, () => activateRecordingTab(recording, targetTabId)); + await enqueueRecordingAction(recording, async () => { + if (isFinishing() || !recording.tabs.isLatest(activation)) { + return; + } + await activateRecordingTab(recording, targetTabId); + }); } return true; } catch { + if (isFinishing()) return false; if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) { await sleep(RECORD_REARM_RETRY_DELAY_MS); } @@ -551,7 +567,7 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): () const windowId = tab.windowId; if (tabId === undefined || windowId === undefined) return; for (const recording of recordings.values()) { - if (recording.settled || recording.agentWindowId !== windowId) continue; + if (isRecordingFinishing(recording) || recording.agentWindowId !== windowId) continue; scheduleRearmForTab(tabId, deps); return; } @@ -559,7 +575,8 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): () const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => { for (const recording of recordings.values()) { - if (recording.settled || recording.agentWindowId !== activeInfo.windowId) continue; + if (isRecordingFinishing(recording) || recording.agentWindowId !== activeInfo.windowId) + continue; const activation = recording.tabs.noteActivation(activeInfo.tabId); scheduleRearmForTab(activeInfo.tabId, deps, activation); return; From 887d1ed0b9a20d813edc8c6fdda1fbcd34de4dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chaonan=E2=80=9D?= Date: Wed, 19 Aug 2026 17:28:54 +0800 Subject: [PATCH 3/4] fix(record): drop 'hover' in tracev2 for compatible --- .../lib/__tests__/trace-reducer-v2.test.ts | 9 ++------- .../lib/__tests__/trace-reducer-v3.test.ts | 19 +++++++++++++++++++ .../src/lib/recording/trace-reducer-v2.ts | 11 +++-------- .../src/tools/__tests__/record-steps.test.ts | 3 ++- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts b/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts index 4a9e375b..510a075d 100644 --- a/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts +++ b/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts @@ -171,7 +171,7 @@ describe("reduceTraceSteps", () => { }); }); - it("keeps hover steps before menu clicks", () => { + it("drops hover steps, which only exist in trace v3", () => { const { steps } = reduceTraceSteps( [ { @@ -187,12 +187,7 @@ describe("reduceTraceSteps", () => { ], "https://example.com/app", ); - expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); - expect(steps[0]).toMatchObject({ - op: "hover", - page: "p1", - target: { name: "Account" }, - }); + expect(steps.map((s) => s.op)).toEqual(["click"]); }); it("resolveTraceStartUrl prefers explicit start URL", () => { diff --git a/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts b/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts index 1018aec0..8d0e59f1 100644 --- a/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts +++ b/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts @@ -48,6 +48,25 @@ describe("trace reducer v3", () => { ]); }); + it("keeps hover steps before menu clicks", () => { + const drafts: RecordingDraftStep[] = [ + { + op: "hover", + captureTarget: { tag: "span", role: "button", name: "Account" }, + preStateId: "s1", + postStateId: "s1", + }, + { + op: "click", + captureTarget: { tag: "a", role: "link", name: "Profile" }, + preStateId: "s1", + postStateId: "s1", + }, + ]; + + expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]); + }); + it("does not export a tab transition without both observation endpoints", () => { const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" }; const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" }; diff --git a/apps/extension/src/lib/recording/trace-reducer-v2.ts b/apps/extension/src/lib/recording/trace-reducer-v2.ts index dacf581f..f129bc4d 100644 --- a/apps/extension/src/lib/recording/trace-reducer-v2.ts +++ b/apps/extension/src/lib/recording/trace-reducer-v2.ts @@ -119,14 +119,6 @@ function toV2Step( }, effectForNavigation(step.navigatedTo, urlToId), ); - case "hover": - if (!step.captureTarget) return null; - return { - op: "hover", - id, - page, - target: step.captureTarget, - }; case "fill": if (!step.captureTarget) return null; return { @@ -161,6 +153,9 @@ function toV2Step( }, effectForNavigation(step.navigatedTo, urlToId), ); + // `hover` only exists in Trace v3. Peers that negotiate v2 predate the + // step variant and fail to decode the whole result if we emit it. + case "hover": case "scroll": case "switch_tab": return null; diff --git a/apps/extension/src/tools/__tests__/record-steps.test.ts b/apps/extension/src/tools/__tests__/record-steps.test.ts index 83d108b1..f933b499 100644 --- a/apps/extension/src/tools/__tests__/record-steps.test.ts +++ b/apps/extension/src/tools/__tests__/record-steps.test.ts @@ -1200,7 +1200,8 @@ describe("recorded user steps reach the exported trace", () => { const trace = (stopped as RecordStopResult).trace; expect("version" in trace).toBe(false); expect("pages" in trace && trace.pages.length).toBeGreaterThan(0); - expect("steps" in trace && trace.steps.map((step) => step.op)).toEqual(["hover", "click"]); + // `hover` has no v2 counterpart on peers that negotiate v2. + expect("steps" in trace && trace.steps.map((step) => step.op)).toEqual(["click"]); }); it("rejects unsupported trace_version values", async () => { From c08fed5de373656789951d9a121a78ff1c4f70e1 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 12:18:44 +0800 Subject: [PATCH 4/4] fix(record): saftey policy and rebase bug fix --- .../src/lib/__tests__/recording-runtime.test.ts | 4 +++- .../src/lib/__tests__/tab-coordinator.test.ts | 10 ++++++++++ apps/extension/src/lib/recording/tab-coordinator.ts | 3 ++- apps/extension/src/tools/record.ts | 3 +-- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/apps/extension/src/lib/__tests__/recording-runtime.test.ts b/apps/extension/src/lib/__tests__/recording-runtime.test.ts index 34af5176..c541f3a3 100644 --- a/apps/extension/src/lib/__tests__/recording-runtime.test.ts +++ b/apps/extension/src/lib/__tests__/recording-runtime.test.ts @@ -139,8 +139,10 @@ describe("RecordingObservationRuntime", () => { await recording.captureInitial(4); const transition = await recording.captureTabTransition(4, 5); + await recording.captureInitial(4); + await recording.captureInitial(5); - expect(transition).toMatchObject({ + expect(transition).toEqual({ preStateId: "s1", postStateId: "s2", targetUrl: "https://example.com/second", diff --git a/apps/extension/src/lib/__tests__/tab-coordinator.test.ts b/apps/extension/src/lib/__tests__/tab-coordinator.test.ts index 42499d35..06697e07 100644 --- a/apps/extension/src/lib/__tests__/tab-coordinator.test.ts +++ b/apps/extension/src/lib/__tests__/tab-coordinator.test.ts @@ -26,4 +26,14 @@ describe("RecordingTabCoordinator", () => { tabs.commit(6); expect(tabs.currentTabId).toBe(6); }); + + it("commits the freshly observed URL when returning to an existing tab", () => { + const tabs = new RecordingTabCoordinator(4); + tabs.navigation(5, "https://example.com/old"); + + tabs.commit(5, "https://example.com/new"); + + expect(tabs.currentTabId).toBe(5); + expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new"); + }); }); diff --git a/apps/extension/src/lib/recording/tab-coordinator.ts b/apps/extension/src/lib/recording/tab-coordinator.ts index 2782b6f4..ab207f00 100644 --- a/apps/extension/src/lib/recording/tab-coordinator.ts +++ b/apps/extension/src/lib/recording/tab-coordinator.ts @@ -40,7 +40,8 @@ export class RecordingTabCoordinator { ); } - commit(tabId: number): void { + commit(tabId: number, currentUrl?: string): void { + if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl; this.#currentTabId = tabId; } diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index 80dde143..46f984b1 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -255,8 +255,7 @@ async function activateRecordingTab( }; recording.steps.push(draft); recording.observation?.bindTabTransition(draft, recording.steps.length); - recording.tabs.navigation(targetTabId, targetUrl); - recording.tabs.commit(targetTabId); + recording.tabs.commit(targetTabId, targetUrl); } async function processRecordedStep(