From 615e8666087ceb30475994cbf96f3022423ae632 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 19 Aug 2026 16:33:15 +0800 Subject: [PATCH 1/5] fix(recorder): add iframe support --- .../content/__tests__/record-capture.test.ts | 129 +------ .../__tests__/record-frame-agent.test.ts | 102 ++++++ apps/extension/src/content/record-capture.ts | 161 ++------- .../src/content/recording/frame-agent.ts | 209 +++++++++++ apps/extension/src/entrypoints/background.ts | 3 + apps/extension/src/entrypoints/content.ts | 79 ++--- .../src/entrypoints/record-frame.content.ts | 14 + .../record-frame-coordinator.test.ts | 121 +++++++ .../recording-document-marker.test.ts | 58 +++ .../lib/__tests__/recording-runtime.test.ts | 50 +++ .../src/lib/__tests__/target-matcher.test.ts | 32 ++ .../src/lib/recording/document-marker.ts | 37 ++ .../src/lib/recording/frame-bridge.ts | 65 ++++ .../src/lib/recording/frame-coordinator.ts | 330 ++++++++++++++++++ .../src/lib/recording/observation-capture.ts | 25 +- .../src/lib/recording/recording-runtime.ts | 19 +- .../src/lib/recording/target-matcher.ts | 16 +- apps/extension/src/lib/recording/types.ts | 5 +- .../src/shared/recording-document-identity.ts | 9 + apps/extension/src/tools/record.ts | 39 ++- .../__tests__/record-safe-observation.test.ts | 48 +++ .../src/tools/vom/record-safe-observation.ts | 25 +- 22 files changed, 1267 insertions(+), 309 deletions(-) create mode 100644 apps/extension/src/content/__tests__/record-frame-agent.test.ts create mode 100644 apps/extension/src/content/recording/frame-agent.ts create mode 100644 apps/extension/src/entrypoints/record-frame.content.ts create mode 100644 apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts create mode 100644 apps/extension/src/lib/__tests__/recording-document-marker.test.ts create mode 100644 apps/extension/src/lib/recording/document-marker.ts create mode 100644 apps/extension/src/lib/recording/frame-bridge.ts create mode 100644 apps/extension/src/lib/recording/frame-coordinator.ts create mode 100644 apps/extension/src/shared/recording-document-identity.ts create mode 100644 apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 96df004d..30108d8c 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge"; -import { handleRecordContentMessage, startRecordCapture } from "../record-capture"; +import type { RecordStepPayload } from "@/lib/record-bridge"; +import { startRecordCapture } from "../record-capture"; vi.stubGlobal("chrome", { runtime: { @@ -48,118 +48,6 @@ function click(el: Element): void { el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); } -describe("handleRecordContentMessage stop/cancel", () => { - it("ignores STOP when no recording is active", () => { - const dispose = vi.fn(); - const onStop = vi.fn(); - const setActiveRequestId = vi.fn(); - const setCapture = vi.fn(); - const sendResponse = vi.fn(); - - const needsAsync = handleRecordContentMessage( - { type: RECORD_STOP, requestId: "rec-stale" }, - { - activeRequestId: null, - capture: { dispose }, - setActiveRequestId, - setCapture, - onStart: vi.fn(), - onStop, - }, - sendResponse, - ); - - expect(needsAsync).toBe(false); - expect(dispose).not.toHaveBeenCalled(); - expect(onStop).not.toHaveBeenCalled(); - expect(setActiveRequestId).not.toHaveBeenCalled(); - expect(setCapture).not.toHaveBeenCalled(); - expect(sendResponse).not.toHaveBeenCalled(); - }); - - it("ignores STOP for a mismatched requestId", () => { - const dispose = vi.fn(); - const onStop = vi.fn(); - - const needsAsync = handleRecordContentMessage( - { type: RECORD_STOP, requestId: "rec-other" }, - { - activeRequestId: "rec-1", - capture: { dispose }, - setActiveRequestId: vi.fn(), - setCapture: vi.fn(), - onStart: vi.fn(), - onStop, - }, - vi.fn(), - ); - - expect(needsAsync).toBe(false); - expect(dispose).not.toHaveBeenCalled(); - expect(onStop).not.toHaveBeenCalled(); - }); - - it("keeps a failed STOP retryable and redelivers its recorded step", async () => { - document.body.innerHTML = ``; - const sendMessage = vi.mocked(chrome.runtime.sendMessage); - sendMessage.mockReset(); - sendMessage.mockRejectedValueOnce(new Error("service worker unavailable")); - sendMessage.mockRejectedValueOnce(new Error("service worker still unavailable")); - sendMessage.mockResolvedValueOnce({ ok: true, sequence: 1 }); - - let activeRequestId: string | null = null; - let capture: ReturnType | null = null; - const onStop = vi.fn(); - const dispatch = ( - message: - | { type: typeof RECORD_START; requestId: string; startedAtMs?: number } - | { type: typeof RECORD_STOP; requestId: string }, - sendResponse?: (response: unknown) => void, - ) => - handleRecordContentMessage( - message, - { - activeRequestId, - capture, - setActiveRequestId: (id) => { - activeRequestId = id; - }, - setCapture: (next) => { - capture = next; - }, - onStart: vi.fn(), - onStop, - }, - sendResponse, - ); - - dispatch({ type: RECORD_START, requestId: "rec-retry" }); - const input = document.querySelector("input")!; - input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); - input.value = "dirty final value"; - input.dispatchEvent(new Event("input", { bubbles: true })); - - const firstResponse = vi.fn(); - expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true); - await vi.waitFor(() => - expect(firstResponse).toHaveBeenCalledWith({ - ok: false, - error: "failed to deliver one or more recorded steps", - }), - ); - expect(activeRequestId).toBe("rec-retry"); - expect(onStop).not.toHaveBeenCalled(); - - const retryResponse = vi.fn(); - expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true); - await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true })); - - expect(sendMessage).toHaveBeenCalledTimes(3); - expect(activeRequestId).toBeNull(); - expect(onStop).toHaveBeenCalledTimes(1); - }); -}); - describe("record-capture semantic", () => { let steps: RecordStepPayload[]; @@ -236,6 +124,19 @@ describe("record-capture semantic", () => { ]); }); + it("does not emit page navigation from a child Document capture", () => { + const originalUrl = location.href; + const capture = startRecordCapture("rec-child", (step) => steps.push(step), { + captureNavigation: false, + }); + + history.pushState({}, "", "#inside-frame"); + expect(steps).toEqual([]); + + capture.dispose(); + history.replaceState({}, "", originalUrl); + }); + it("does not record clicks on anonymous layout divs", () => { document.body.innerHTML = `
page chrome
diff --git a/apps/extension/src/content/__tests__/record-frame-agent.test.ts b/apps/extension/src/content/__tests__/record-frame-agent.test.ts new file mode 100644 index 00000000..d81479fc --- /dev/null +++ b/apps/extension/src/content/__tests__/record-frame-agent.test.ts @@ -0,0 +1,102 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { RECORD_DOCUMENT_ATTRIBUTE } from "@/lib/recording/document-marker"; +import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge"; +import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge"; +import { RecordFrameAgent } from "../recording/frame-agent"; + +class PortListeners unknown> { + readonly values = new Set(); + addListener = (listener: T) => this.values.add(listener); + removeListener = (listener: T) => this.values.delete(listener); +} + +function portHarness() { + const onMessage = new PortListeners<(message: unknown) => void>(); + const onDisconnect = new PortListeners<() => void>(); + const outbound: RecordFramePortMessage[] = []; + const port = { + name: "bsk-record-frame", + onMessage, + onDisconnect, + postMessage(message: RecordFramePortMessage) { + outbound.push(message); + if (message.type === "ready") { + queueMicrotask(() => { + for (const listener of onMessage.values) { + listener({ + type: "ready_ack", + requestId: message.requestId, + producerId: message.producerId, + }); + } + }); + } + }, + disconnect: vi.fn(), + } as unknown as chrome.runtime.Port; + return { + port, + outbound, + receive(message: RecordFramePortMessage) { + for (const listener of onMessage.values) listener(message); + }, + }; +} + +describe("RecordFrameAgent", () => { + afterEach(() => { + document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE); + document.body.replaceChildren(); + }); + + it("keeps a failed stop retryable and flushes the final dirty fill", async () => { + const harness = portHarness(); + const sendMessage = vi + .fn<(message: { sequence: number }) => Promise>() + .mockRejectedValueOnce(new Error("offline")) + .mockRejectedValueOnce(new Error("still offline")) + .mockImplementation(async (message) => ({ ok: true, sequence: message.sequence })); + vi.stubGlobal("chrome", { + runtime: { + connect: vi.fn(() => harness.port), + sendMessage, + }, + }); + document.body.innerHTML = ``; + const agent = new RecordFrameAgent(); + + await expect( + agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }), + ).resolves.toEqual({ ok: true }); + expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true); + + const input = document.querySelector("input")!; + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.value = "final value"; + input.dispatchEvent(new Event("input", { bubbles: true })); + + harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" }); + await vi.waitFor(() => + expect(harness.outbound).toContainEqual({ + type: "stopped", + requestId: "rec-1", + commandId: "stop-1", + ok: false, + error: "failed to deliver one or more recorded steps", + }), + ); + expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true); + + harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" }); + await vi.waitFor(() => + expect(harness.outbound).toContainEqual({ + type: "stopped", + requestId: "rec-1", + commandId: "stop-2", + ok: true, + }), + ); + expect(sendMessage).toHaveBeenCalledTimes(3); + expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false); + }); +}); diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 29e596a8..22f7001b 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -12,21 +12,7 @@ import { hasDirectHoverInteractiveSignal, hasStrongHoverExpansionSignal, } from "@/lib/hover-trigger-policy"; -import { - isRecordCancelMessage, - isRecordStartMessage, - isRecordStopMessage, - RECORD_CANCEL, - RECORD_START, - RECORD_STEP, - RECORD_STOP, - type RecordCancelMessage, - type RecordStartAck, - type RecordStartMessage, - type RecordStepPayload, - type RecordStopAck, - type RecordStopMessage, -} from "@/lib/record-bridge"; +import type { RecordStepPayload } from "@/lib/record-bridge"; import { shouldRecordPress } from "@/lib/recording/draft-policy"; import { closestHoverSurfaceCandidate, @@ -36,18 +22,6 @@ import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner, } from "./record-hover-surface"; -import { RecordStepDelivery } from "./record-step-delivery"; - -const stepDeliveries = new Map(); -const pendingStopFlushes = new Map>(); - -function deliveryFor(requestId: string): RecordStepDelivery { - const existing = stepDeliveries.get(requestId); - if (existing) return existing; - const created = new RecordStepDelivery(requestId); - stepDeliveries.set(requestId, created); - return created; -} export interface RecordCaptureController { dispose(): void; @@ -385,6 +359,7 @@ function scheduleInputCompletionCommit( export function startRecordCapture( _requestId: string, sendStep: (step: RecordStepPayload) => void, + options: { captureNavigation?: boolean } = {}, ): RecordCaptureController { const emitStep = (step: RecordStepPayload) => { sendStep({ page_url: location.href, ...step }); @@ -921,26 +896,33 @@ export function startRecordCapture( document.addEventListener("change", onChange, true); document.addEventListener("keydown", onKeyDown, true); - const urlObserver = new MutationObserver(() => emitNavigateIfChanged()); - urlObserver.observe(document, { subtree: true, childList: true }); + const captureNavigation = options.captureNavigation ?? true; + const urlObserver = captureNavigation + ? new MutationObserver(() => emitNavigateIfChanged()) + : null; + urlObserver?.observe(document, { subtree: true, childList: true }); const onUrlEvent = () => emitNavigateIfChanged(); // Wrap: passing commitFillSession directly would forward the DOM event as // the `commit` argument and stamp it onto the recorded fill step. const onPageHide = () => commitFillSession(); - window.addEventListener("hashchange", onUrlEvent); - window.addEventListener("popstate", onUrlEvent); + if (captureNavigation) { + window.addEventListener("hashchange", onUrlEvent); + window.addEventListener("popstate", onUrlEvent); + } window.addEventListener("pagehide", onPageHide); const originalPushState = history.pushState; const originalReplaceState = history.replaceState; - history.pushState = function (...args: Parameters) { - originalPushState.apply(this, args); - emitNavigateIfChanged(navigationActionPending ? true : undefined); - }; - history.replaceState = function (...args: Parameters) { - originalReplaceState.apply(this, args); - emitNavigateIfChanged(navigationActionPending ? true : undefined); - }; + if (captureNavigation) { + history.pushState = function (...args: Parameters) { + originalPushState.apply(this, args); + emitNavigateIfChanged(navigationActionPending ? true : undefined); + }; + history.replaceState = function (...args: Parameters) { + originalReplaceState.apply(this, args); + emitNavigateIfChanged(navigationActionPending ? true : undefined); + }; + } return { dispose() { @@ -954,99 +936,16 @@ export function startRecordCapture( document.removeEventListener("compositionend", onCompositionEnd, true); document.removeEventListener("change", onChange, true); document.removeEventListener("keydown", onKeyDown, true); - urlObserver.disconnect(); - window.removeEventListener("hashchange", onUrlEvent); - window.removeEventListener("popstate", onUrlEvent); + urlObserver?.disconnect(); + if (captureNavigation) { + window.removeEventListener("hashchange", onUrlEvent); + window.removeEventListener("popstate", onUrlEvent); + } window.removeEventListener("pagehide", onPageHide); - history.pushState = originalPushState; - history.replaceState = originalReplaceState; + if (captureNavigation) { + history.pushState = originalPushState; + history.replaceState = originalReplaceState; + } }, }; } - -export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage; - -export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage { - return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg); -} - -export function handleRecordContentMessage( - message: RecordContentMessage, - state: { - activeRequestId: string | null; - capture: RecordCaptureController | null; - setActiveRequestId(id: string | null): void; - setCapture(capture: RecordCaptureController | null): void; - onStart(requestId: string, startedAtMs?: number): void; - onStop(): void; - }, - sendResponse?: (response: RecordStartAck | RecordStopAck) => void, -): boolean { - if (isRecordStartMessage(message)) { - const delivery = deliveryFor(message.requestId); - state.capture?.dispose(); - state.setCapture( - startRecordCapture(message.requestId, (step) => { - delivery.enqueue(step); - }), - ); - state.setActiveRequestId(message.requestId); - state.onStart(message.requestId, message.startedAtMs); - sendResponse?.({ ok: true }); - return sendResponse !== undefined; - } - - if (isRecordStopMessage(message) || isRecordCancelMessage(message)) { - // Require an active recording that matches this requestId — otherwise a - // stray STOP/CANCEL (e.g. after teardown) would still run finishStop and - // clear overlay state even though nothing was capturing. - if (!state.activeRequestId || state.activeRequestId !== message.requestId) { - return false; - } - state.capture?.dispose(); - state.setCapture(null); - const finishStop = () => { - state.onStop(); - state.setActiveRequestId(null); - }; - if (isRecordStopMessage(message) && sendResponse) { - const existingFlush = pendingStopFlushes.get(message.requestId); - if (existingFlush) { - void existingFlush.then(sendResponse); - return true; - } - - const flush = deliveryFor(message.requestId) - .flush() - .then((succeeded): RecordStopAck => { - if (succeeded) { - stepDeliveries.delete(message.requestId); - finishStop(); - } - return succeeded - ? { ok: true } - : { - ok: false, - error: "failed to deliver one or more recorded steps", - }; - }); - pendingStopFlushes.set(message.requestId, flush); - void flush.then(sendResponse).finally(() => { - if (pendingStopFlushes.get(message.requestId) === flush) { - pendingStopFlushes.delete(message.requestId); - } - }); - return true; - } - if (isRecordCancelMessage(message)) { - stepDeliveries.delete(message.requestId); - pendingStopFlushes.delete(message.requestId); - } - finishStop(); - return false; - } - - return false; -} - -export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP }; diff --git a/apps/extension/src/content/recording/frame-agent.ts b/apps/extension/src/content/recording/frame-agent.ts new file mode 100644 index 00000000..68e7b412 --- /dev/null +++ b/apps/extension/src/content/recording/frame-agent.ts @@ -0,0 +1,209 @@ +import { + markRecordingDocument, + type RecordingDocumentMarker, + waitForDocumentElement, +} from "@/lib/recording/document-marker"; +import { + isRecordFrameStartMessage, + RECORD_FRAME_PORT, + RECORD_FRAME_QUERY, + RECORD_FRAME_START, + type RecordFramePortMessage, + type RecordFrameQueryResponse, + type RecordFrameStartMessage, +} from "@/lib/recording/frame-bridge"; +import { type RecordCaptureController, startRecordCapture } from "../record-capture"; +import { RecordStepDelivery } from "../record-step-delivery"; + +interface ActiveFrameRecording { + requestId: string; + producerId: string; + port: chrome.runtime.Port; + delivery: RecordStepDelivery; + capture: RecordCaptureController; + marker: RecordingDocumentMarker; +} + +const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000; + +export class RecordFrameAgent { + #active: ActiveFrameRecording | null = null; + #startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null; + #disposed = false; + + async start( + message: RecordFrameStartMessage, + ): Promise<{ ok: true } | { ok: false; error: string }> { + if (this.#active?.requestId === message.requestId) return { ok: true }; + if (this.#disposed) return { ok: false, error: "recording document is disposed" }; + if (this.#startPromise) { + await this.#startPromise; + return this.start(message); + } + if (this.#active) this.cancel(this.#active.requestId); + const starting = this.#start(message); + this.#startPromise = starting; + try { + return await starting; + } finally { + if (this.#startPromise === starting) this.#startPromise = null; + } + } + + cancel(requestId: string): void { + const active = this.#active; + if (!active || active.requestId !== requestId) return; + active.capture.dispose(); + active.marker.restore(); + active.port.disconnect(); + this.#active = null; + } + + dispose(): void { + this.#disposed = true; + if (this.#active) this.cancel(this.#active.requestId); + } + + async #start( + message: RecordFrameStartMessage, + ): Promise<{ ok: true } | { ok: false; error: string }> { + const producerId = crypto.randomUUID(); + const root = await waitForDocumentElement(); + if (this.#disposed) return { ok: false, error: "recording document is disposed" }; + const marker = markRecordingDocument(producerId, root); + const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT }); + try { + await registerPort(port, message.requestId, producerId); + } catch { + marker.restore(); + port.disconnect(); + return { ok: false, error: "failed to register recording document" }; + } + if (this.#disposed) { + marker.restore(); + port.disconnect(); + return { ok: false, error: "recording document is disposed" }; + } + + const delivery = new RecordStepDelivery(message.requestId, undefined, producerId); + const capture = startRecordCapture( + message.requestId, + (step) => { + marker.ensure(); + delivery.enqueue(step); + }, + { captureNavigation: window.top === window }, + ); + const active: ActiveFrameRecording = { + requestId: message.requestId, + producerId, + port, + delivery, + capture, + marker, + }; + this.#active = active; + port.onMessage.addListener((raw: unknown) => { + const command = raw as Partial; + if (command.type === "stop" && command.requestId === active.requestId) { + void this.#stop(active, command.commandId); + } else if (command.type === "cancel" && command.requestId === active.requestId) { + this.cancel(active.requestId); + } + }); + port.onDisconnect.addListener(() => { + if (this.#active !== active) return; + active.capture.dispose(); + active.marker.restore(); + this.#active = null; + }); + return { ok: true }; + } + + async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise { + if (!commandId || this.#active !== active) return; + active.capture.dispose(); + const succeeded = await active.delivery.flush(); + try { + active.port.postMessage({ + type: "stopped", + requestId: active.requestId, + commandId, + ok: succeeded, + ...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }), + } satisfies RecordFramePortMessage); + } catch { + return; + } + if (!succeeded || this.#active !== active) return; + active.marker.restore(); + this.#active = null; + active.port.disconnect(); + } +} + +function registerPort( + port: chrome.runtime.Port, + requestId: string, + producerId: string, +): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(new Error("recording document registration timed out")); + }, RECORD_FRAME_REGISTER_TIMEOUT_MS); + const cleanup = () => { + clearTimeout(timer); + port.onMessage.removeListener(onMessage); + port.onDisconnect.removeListener(onDisconnect); + }; + const onMessage = (raw: unknown) => { + const message = raw as Partial; + if ( + message.type !== "ready_ack" || + message.requestId !== requestId || + message.producerId !== producerId + ) { + return; + } + cleanup(); + resolve(); + }; + const onDisconnect = () => { + cleanup(); + reject(new Error("recording document port disconnected")); + }; + port.onMessage.addListener(onMessage); + port.onDisconnect.addListener(onDisconnect); + port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage); + }); +} + +export function attachRecordFrameAgent(): () => void { + const agent = new RecordFrameAgent(); + const onMessage = ( + message: unknown, + _sender: chrome.runtime.MessageSender, + sendResponse: (response: { ok: true } | { ok: false; error: string }) => void, + ) => { + if (!isRecordFrameStartMessage(message)) return false; + void agent.start(message).then(sendResponse); + return true; + }; + chrome.runtime.onMessage.addListener(onMessage); + void chrome.runtime + .sendMessage({ type: RECORD_FRAME_QUERY }) + .then((response: RecordFrameQueryResponse | undefined) => { + if (!response?.active || !response.requestId || response.startedAtMs === undefined) return; + return agent.start({ + type: RECORD_FRAME_START, + requestId: response.requestId, + startedAtMs: response.startedAtMs, + }); + }) + .catch(() => {}); + return () => { + chrome.runtime.onMessage.removeListener(onMessage); + agent.dispose(); + }; +} diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts index 4154b4b7..8e819d79 100644 --- a/apps/extension/src/entrypoints/background.ts +++ b/apps/extension/src/entrypoints/background.ts @@ -21,6 +21,7 @@ import { type OverlayMode, } from "@/lib/overlay-bridge"; import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge"; +import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator"; import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag"; import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup"; import { attachSessionEventHandler } from "@/session-manager/event-handler"; @@ -173,6 +174,7 @@ export default defineBackground(() => { const recordDeps = { tabsApi: chrome.tabs, cdp, + frameCoordinator: recordFrameCoordinator, sendToTab: (tabId: number, msg: Parameters[1]) => chrome.tabs.sendMessage(tabId, msg), bypassOverlay: async (tabId: number, enabled: boolean) => { @@ -192,6 +194,7 @@ export default defineBackground(() => { attachRecordStepListener(recordDeps); attachRecordFinishListener(recordDeps); attachRecordQueryListener(recordDeps); + recordFrameCoordinator.attach(); if (typeof chrome.notifications?.onClicked?.addListener === "function") { attachBorrowNotificationClickHandler({ onClicked: chrome.notifications.onClicked, diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index 7552a098..827eca6c 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -11,11 +11,6 @@ import { createHelpRequestData } from "@/content/help-request"; import overlayCss from "@/content/overlay.css?inline"; import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; import { RecordOverlay } from "@/content/RecordOverlay"; -import { - handleRecordContentMessage, - isRecordContentMessage, - type RecordCaptureController, -} from "@/content/record-capture"; import { type CaptureSuppressAck, type CaptureSuppressMessage, @@ -45,11 +40,17 @@ import { } from "@/lib/overlay-bridge"; import { sendInterrupt } from "@/lib/overlay-interrupt-client"; import { + isRecordCancelMessage, + isRecordStartMessage, + isRecordStopMessage, RECORD_FINISH, RECORD_QUERY, + type RecordCancelMessage, type RecordQueryResponse, type RecordStartAck, + type RecordStartMessage, type RecordStopAck, + type RecordStopMessage, } from "@/lib/record-bridge"; import type { BorrowCancelMessage, @@ -69,7 +70,6 @@ export default defineContentScript({ if (window.top !== window) return; const overlays = new OverlayController(); - let recordCapture: RecordCaptureController | null = null; let activeRecordRequestId: string | null = null; let reactRoot: ReactDOM.Root | null = null; let overlayHost: HTMLElement | null = null; @@ -228,8 +228,6 @@ export default defineContentScript({ if (previousHelp) { void sendHelpFinish(previousHelp.id, "cancelled"); } - recordCapture?.dispose(); - recordCapture = null; activeRecordRequestId = null; renderAll(); } @@ -259,6 +257,9 @@ export default defineContentScript({ | HelpRequestMessage | HelpCancelMessage | CaptureSuppressMessage + | RecordStartMessage + | RecordStopMessage + | RecordCancelMessage | OverlayAgentOverlayResetMessage | OverlayAgentStateMessage | OverlayAutomationBypassMessage, @@ -269,41 +270,32 @@ export default defineContentScript({ return captureSuppress.handleMessage(message, sendResponse); } - if (isRecordContentMessage(message)) { - const needsAsync = handleRecordContentMessage( - message, - { - activeRequestId: activeRecordRequestId, - capture: recordCapture, - setActiveRequestId: (id) => { - activeRecordRequestId = id; - }, - setCapture: (capture) => { - recordCapture = capture; - }, - onStart: (requestId, startedAtMs) => { - overlays.setAgentRecordRequest({ - id: requestId, - ...(typeof startedAtMs === "number" ? { startedAtMs } : {}), - onFinish: () => { - void chrome.runtime.sendMessage({ - type: RECORD_FINISH, - requestId, - }); - }, - }); - renderAll(); - }, - onStop: () => { - overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined); - renderAll(); - }, + if (isRecordStartMessage(message)) { + activeRecordRequestId = message.requestId; + overlays.setAgentRecordRequest({ + id: message.requestId, + ...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}), + onFinish: () => { + void chrome.runtime.sendMessage({ + type: RECORD_FINISH, + requestId: message.requestId, + }); }, - sendResponse as unknown as - | ((response: RecordStartAck | RecordStopAck) => void) - | undefined, - ); - return needsAsync; + }); + renderAll(); + (sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true }); + return false; + } + + if (isRecordStopMessage(message) || isRecordCancelMessage(message)) { + if (activeRecordRequestId !== message.requestId) return false; + overlays.clearAgentRecordRequest(message.requestId); + activeRecordRequestId = null; + renderAll(); + if (isRecordStopMessage(message)) { + (sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true }); + } + return false; } if ( @@ -526,9 +518,6 @@ export default defineContentScript({ chrome.runtime.onMessage.removeListener(onMessage); chrome.storage.onChanged.removeListener(onStorageChange); window.removeEventListener("pageshow", onPageShow); - // Restore history hooks / remove capture listeners before the CS unloads. - recordCapture?.dispose(); - recordCapture = null; activeRecordRequestId = null; }); }, diff --git a/apps/extension/src/entrypoints/record-frame.content.ts b/apps/extension/src/entrypoints/record-frame.content.ts new file mode 100644 index 00000000..6b5bdd0e --- /dev/null +++ b/apps/extension/src/entrypoints/record-frame.content.ts @@ -0,0 +1,14 @@ +import { attachRecordFrameAgent } from "@/content/recording/frame-agent"; + +export default defineContentScript({ + matches: [""], + runAt: "document_start", + allFrames: true, + matchAboutBlank: true, + matchOriginAsFallback: true, + + main(ctx) { + const dispose = attachRecordFrameAgent(); + ctx.onInvalidated(dispose); + }, +}); diff --git a/apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts b/apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts new file mode 100644 index 00000000..e5e74499 --- /dev/null +++ b/apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts @@ -0,0 +1,121 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { RECORD_FRAME_PORT } from "../recording/frame-bridge"; +import { RecordFrameCoordinator } from "../recording/frame-coordinator"; + +class ListenerSet unknown> { + readonly listeners = new Set(); + addListener = (listener: T) => this.listeners.add(listener); + removeListener = (listener: T) => this.listeners.delete(listener); +} + +function fakePort(sender: chrome.runtime.MessageSender) { + const messages: unknown[] = []; + const onMessage = new ListenerSet<(message: unknown) => void>(); + const onDisconnect = new ListenerSet<() => void>(); + return { + port: { + name: RECORD_FRAME_PORT, + sender, + onMessage, + onDisconnect, + postMessage: vi.fn((message: unknown) => messages.push(message)), + disconnect: vi.fn(), + } as unknown as chrome.runtime.Port, + messages, + receive(message: unknown) { + for (const listener of onMessage.listeners) listener(message); + }, + }; +} + +describe("RecordFrameCoordinator", () => { + const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>(); + const onMessage = new ListenerSet< + ( + message: unknown, + sender: chrome.runtime.MessageSender, + sendResponse: (response: unknown) => void, + ) => boolean + >(); + + beforeEach(() => { + onConnect.listeners.clear(); + onMessage.listeners.clear(); + vi.stubGlobal("chrome", { + runtime: { onConnect, onMessage }, + }); + }); + + it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => { + const sendToDocument = vi.fn(async () => ({ ok: true })); + const coordinator = new RecordFrameCoordinator({ + getAllFrames: async () => [ + { frameId: 0, documentId: "top-document" }, + { frameId: 7, documentId: "child-document" }, + ], + sendToDocument, + }); + coordinator.attach(); + coordinator.begin("rec-1", 10); + await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true); + + const sender = { + tab: { id: 3 }, + frameId: 7, + documentId: "child-document", + } as chrome.runtime.MessageSender; + const frame = fakePort(sender); + for (const listener of onConnect.listeners) listener(frame.port); + frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" }); + + expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({ + tabId: 3, + documentId: "child-document", + browserFrameId: 7, + producerId: "producer-1", + }); + + const stopping = coordinator.stop("rec-1"); + expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull(); + const stop = frame.messages.find( + (message): message is { type: "stop"; commandId: string } => + typeof message === "object" && + message !== null && + (message as { type?: string }).type === "stop", + ); + expect(stop).toBeDefined(); + frame.receive({ + type: "stopped", + requestId: "rec-1", + commandId: stop!.commandId, + ok: true, + }); + await expect(stopping).resolves.toBe(true); + expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull(); + }); + + it("rejects a producer used from another Document", async () => { + const coordinator = new RecordFrameCoordinator({ + getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }], + sendToDocument: async () => ({ ok: true }), + }); + coordinator.attach(); + coordinator.begin("rec-1", 10); + await coordinator.armTab("rec-1", 3); + const sender = { + tab: { id: 3 }, + frameId: 0, + documentId: "top-document", + } as chrome.runtime.MessageSender; + const frame = fakePort(sender); + for (const listener of onConnect.listeners) listener(frame.port); + frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" }); + + expect( + coordinator.sourceFor("rec-1", "producer-1", { + ...sender, + documentId: "other-document", + }), + ).toBeNull(); + }); +}); diff --git a/apps/extension/src/lib/__tests__/recording-document-marker.test.ts b/apps/extension/src/lib/__tests__/recording-document-marker.test.ts new file mode 100644 index 00000000..cc91fb94 --- /dev/null +++ b/apps/extension/src/lib/__tests__/recording-document-marker.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; +import { markRecordingDocument } from "../recording/document-marker"; +import { ObservationNodeIndex } from "../recording/observation-capture"; + +describe("recording document marker", () => { + it("restores the Document attribute exactly", () => { + document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value"); + const marker = markRecordingDocument("producer-1"); + + expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("producer-1"); + marker.restore(); + expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value"); + document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE); + }); + + it("indexes the CDP scope from the safe frame identity", () => { + const index = new ObservationNodeIndex({ + rootFrameId: "root", + frames: [ + { + frameId: "child", + target: { tabId: 3, sessionId: "oopif-session" }, + recordingDocumentId: "producer-1", + }, + ], + matchNodes: [], + refs: [], + }); + + expect(index.documentScope("producer-1")).toEqual({ + frameId: "child", + target: { tabId: 3, sessionId: "oopif-session" }, + }); + }); + + it("fails closed when a recording identity maps to multiple Documents", () => { + const index = new ObservationNodeIndex({ + rootFrameId: "root", + frames: [ + { + frameId: "left", + target: { tabId: 3, sessionId: "left-session" }, + recordingDocumentId: "duplicate", + }, + { + frameId: "right", + target: { tabId: 3, sessionId: "right-session" }, + recordingDocumentId: "duplicate", + }, + ], + matchNodes: [], + refs: [], + }); + + expect(index.documentScope("duplicate")).toBeUndefined(); + }); +}); diff --git a/apps/extension/src/lib/__tests__/recording-runtime.test.ts b/apps/extension/src/lib/__tests__/recording-runtime.test.ts index c541f3a3..6dfe5244 100644 --- a/apps/extension/src/lib/__tests__/recording-runtime.test.ts +++ b/apps/extension/src/lib/__tests__/recording-runtime.test.ts @@ -10,6 +10,7 @@ vi.mock("../recording/observation-capture", async (importOriginal) => { import { ObservationNodeIndex } from "../recording/observation-capture"; import { RecordingObservationRuntime } from "../recording/recording-runtime"; +import type { RecordingDraftStep } from "../recording/types"; function observation(url = "https://example.com/") { return { @@ -149,4 +150,53 @@ describe("RecordingObservationRuntime", () => { }); expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]); }); + + it("binds an iframe draft to its marked CDP Document scope", async () => { + captureRecordingObservation.mockResolvedValueOnce({ + ...observation(), + index: new ObservationNodeIndex({ + rootFrameId: "root", + frames: [ + { + frameId: "child", + target: { tabId: 7, sessionId: "oopif-session" }, + recordingDocumentId: "producer-1", + }, + ], + matchNodes: [ + { + backendNodeId: 42, + frameId: "child", + tag: "button", + rect: { x: 410, y: 20, w: 100, h: 30 }, + localRect: { x: 10, y: 20, w: 100, h: 30 }, + }, + ], + refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }], + }), + }); + const recording = runtime(); + const drafts: RecordingDraftStep[] = [ + { + op: "click" as const, + captureTarget: { tag: "button", role: "button", name: "Save" }, + targetHint: { + geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" }, + }, + }, + ]; + + await recording.captureInitial(7); + await recording.processDraft(7, drafts, 0, "producer-1"); + recording.cancel(); + + const draft = drafts[0]; + expect(draft?.op).toBe("click"); + if (!draft || draft.op !== "click") throw new Error("expected click draft"); + expect(draft.targetHint).toMatchObject({ + frameId: "child", + geometrySpace: "local", + }); + expect(draft.matchedTarget?.ref).toBe("e1"); + }); }); diff --git a/apps/extension/src/lib/__tests__/target-matcher.test.ts b/apps/extension/src/lib/__tests__/target-matcher.test.ts index d3efae2a..22253515 100644 --- a/apps/extension/src/lib/__tests__/target-matcher.test.ts +++ b/apps/extension/src/lib/__tests__/target-matcher.test.ts @@ -69,6 +69,23 @@ describe("matchObservationTarget", () => { expect(target.ref).toBe("e2"); }); + it("matches iframe capture geometry in the child viewport coordinate space", () => { + const child = node(42, "child", 410); + child.localRect = { x: 10, y: 20, w: 100, h: 30 }; + const target = matchObservationTarget({ + observation: observation( + [child], + [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }], + ), + hint: { + frameId: "child", + geometrySpace: "local", + geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" }, + }, + }); + expect(target.ref).toBe("e1"); + }); + it("restricts a missing frame hint to the root frame", () => { const target = matchObservationTarget({ observation: observation( @@ -81,6 +98,21 @@ describe("matchObservationTarget", () => { expect(target).toEqual({ name: "发布", unmatched: true }); }); + it("fails closed when the source Document has no VOM frame mapping", () => { + const target = matchObservationTarget({ + observation: observation( + [node(42, "root")], + [{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }], + ), + hint: { + frameId: null, + geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" }, + }, + fallback: { tag: "button", role: "button", name: "发布" }, + }); + expect(target).toEqual({ role: "button", name: "发布", unmatched: true }); + }); + it("returns unmatched for ambiguous geometry", () => { const target = matchObservationTarget({ observation: observation( diff --git a/apps/extension/src/lib/recording/document-marker.ts b/apps/extension/src/lib/recording/document-marker.ts new file mode 100644 index 00000000..b1b32261 --- /dev/null +++ b/apps/extension/src/lib/recording/document-marker.ts @@ -0,0 +1,37 @@ +import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; + +export { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; + +export interface RecordingDocumentMarker { + restore(): void; + ensure(): void; +} + +export function waitForDocumentElement(): Promise { + if (document.documentElement) return Promise.resolve(document.documentElement); + return new Promise((resolve) => { + const observer = new MutationObserver(() => { + if (!document.documentElement) return; + observer.disconnect(); + resolve(document.documentElement); + }); + observer.observe(document, { childList: true }); + }); +} + +export function markRecordingDocument( + producerId: string, + root: HTMLElement = document.documentElement, +): RecordingDocumentMarker { + const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE); + const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, producerId); + ensure(); + return { + ensure, + restore() { + if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== producerId) return; + if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE); + else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous); + }, + }; +} diff --git a/apps/extension/src/lib/recording/frame-bridge.ts b/apps/extension/src/lib/recording/frame-bridge.ts new file mode 100644 index 00000000..48af039d --- /dev/null +++ b/apps/extension/src/lib/recording/frame-bridge.ts @@ -0,0 +1,65 @@ +export const RECORD_FRAME_PORT = "bsk-record-frame"; +export const RECORD_FRAME_QUERY = "bsk-record-frame-query"; +export const RECORD_FRAME_START = "bsk-record-frame-start"; + +export interface RecordFrameQueryMessage { + type: typeof RECORD_FRAME_QUERY; +} + +export interface RecordFrameQueryResponse { + active: boolean; + requestId?: string; + startedAtMs?: number; +} + +export interface RecordFrameStartMessage { + type: typeof RECORD_FRAME_START; + requestId: string; + startedAtMs: number; +} + +export type RecordFramePortMessage = + | { + type: "ready"; + requestId: string; + producerId: string; + } + | { + type: "ready_ack"; + requestId: string; + producerId: string; + } + | { + type: "stop"; + requestId: string; + commandId: string; + } + | { + type: "cancel"; + requestId: string; + } + | { + type: "stopped"; + requestId: string; + commandId: string; + ok: boolean; + error?: string; + }; + +export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage { + return ( + typeof value === "object" && + value !== null && + (value as { type?: unknown }).type === RECORD_FRAME_QUERY + ); +} + +export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage { + if (typeof value !== "object" || value === null) return false; + const message = value as Partial; + return ( + message.type === RECORD_FRAME_START && + typeof message.requestId === "string" && + typeof message.startedAtMs === "number" + ); +} diff --git a/apps/extension/src/lib/recording/frame-coordinator.ts b/apps/extension/src/lib/recording/frame-coordinator.ts new file mode 100644 index 00000000..8bdf1886 --- /dev/null +++ b/apps/extension/src/lib/recording/frame-coordinator.ts @@ -0,0 +1,330 @@ +import { + isRecordFrameQueryMessage, + RECORD_FRAME_PORT, + RECORD_FRAME_START, + type RecordFramePortMessage, + type RecordFrameQueryResponse, + type RecordFrameStartMessage, +} from "./frame-bridge"; + +export interface RecordingCaptureScope { + tabId: number; + documentId: string; + browserFrameId: number; + producerId: string; +} + +interface ArmedRecording { + requestId: string; + startedAtMs: number; + tabIds: Set; + agents: Map; + finishing: boolean; +} + +interface FrameAgent extends RecordingCaptureScope { + requestId: string; + port: chrome.runtime.Port; + stopWaiters: Map void>; +} + +interface BrowserFrame { + frameId: number; + documentId?: string; +} + +const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000; + +export interface RecordFrameCoordinatorDeps { + getAllFrames(tabId: number): Promise; + sendToDocument( + tabId: number, + message: RecordFrameStartMessage, + target: { documentId?: string; frameId?: number }, + ): Promise; +} + +function documentKey(tabId: number, documentId: string): string { + return `${tabId}:${documentId}`; +} + +function senderAddress( + sender: chrome.runtime.MessageSender, +): { tabId: number; documentId: string; browserFrameId: number } | null { + const tabId = sender.tab?.id; + const documentId = sender.documentId; + const browserFrameId = sender.frameId; + if ( + typeof tabId !== "number" || + typeof documentId !== "string" || + typeof browserFrameId !== "number" + ) { + return null; + } + return { tabId, documentId, browserFrameId }; +} + +function defaultDeps(): RecordFrameCoordinatorDeps { + return { + async getAllFrames(tabId) { + return (await chrome.webNavigation.getAllFrames({ tabId })) ?? []; + }, + sendToDocument(tabId, message, target) { + return chrome.tabs.sendMessage(tabId, message, target); + }, + }; +} + +export class RecordFrameCoordinator { + readonly #deps: RecordFrameCoordinatorDeps; + readonly #recordings = new Map(); + #attached = false; + + constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) { + this.#deps = deps; + } + + attach(): () => void { + if (this.#attached) return () => {}; + this.#attached = true; + chrome.runtime.onConnect.addListener(this.#onConnect); + chrome.runtime.onMessage.addListener(this.#onMessage); + return () => { + if (!this.#attached) return; + this.#attached = false; + chrome.runtime.onConnect.removeListener(this.#onConnect); + chrome.runtime.onMessage.removeListener(this.#onMessage); + for (const recording of this.#recordings.values()) this.#cancelAgents(recording); + this.#recordings.clear(); + }; + } + + begin(requestId: string, startedAtMs: number): void { + const previous = this.#recordings.get(requestId); + if (previous) this.#cancelAgents(previous); + this.#recordings.set(requestId, { + requestId, + startedAtMs, + tabIds: new Set(), + agents: new Map(), + finishing: false, + }); + } + + async armTab(requestId: string, tabId: number): Promise { + const recording = this.#recordings.get(requestId); + if (!recording || recording.finishing) return false; + recording.tabIds.add(tabId); + + let frames: BrowserFrame[]; + try { + frames = await this.#deps.getAllFrames(tabId); + } catch { + frames = [{ frameId: 0 }]; + } + if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 }); + + const message: RecordFrameStartMessage = { + type: RECORD_FRAME_START, + requestId, + startedAtMs: recording.startedAtMs, + }; + const results = await Promise.all( + frames.map(async (frame) => { + try { + const response = await this.#deps.sendToDocument(tabId, message, { + ...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }), + }); + return { frameId: frame.frameId, started: isStarted(response) }; + } catch { + return { frameId: frame.frameId, started: false }; + } + }), + ); + return results.some((result) => result.frameId === 0 && result.started); + } + + sourceFor( + requestId: string, + producerId: string, + sender: chrome.runtime.MessageSender, + ): RecordingCaptureScope | null { + const address = senderAddress(sender); + const recording = this.#recordings.get(requestId); + if (!address || !recording) return null; + const agent = recording.agents.get(documentKey(address.tabId, address.documentId)); + if (!agent || agent.producerId !== producerId) return null; + return { + tabId: agent.tabId, + documentId: agent.documentId, + browserFrameId: agent.browserFrameId, + producerId: agent.producerId, + }; + } + + async stop(requestId: string): Promise { + const recording = this.#recordings.get(requestId); + if (!recording) return true; + recording.finishing = true; + const commandId = crypto.randomUUID(); + const results = await Promise.all( + [...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)), + ); + const succeeded = results.every(Boolean); + if (succeeded) this.#recordings.delete(requestId); + else recording.finishing = false; + return succeeded; + } + + cancel(requestId: string): void { + const recording = this.#recordings.get(requestId); + if (!recording) return; + this.#cancelAgents(recording); + this.#recordings.delete(requestId); + } + + readonly #onMessage = ( + message: unknown, + sender: chrome.runtime.MessageSender, + sendResponse: (response: RecordFrameQueryResponse) => void, + ): boolean => { + if (!isRecordFrameQueryMessage(message)) return false; + const tabId = sender.tab?.id; + const recording = + typeof tabId === "number" + ? [...this.#recordings.values()].find( + (candidate) => !candidate.finishing && candidate.tabIds.has(tabId), + ) + : undefined; + sendResponse( + recording + ? { + active: true, + requestId: recording.requestId, + startedAtMs: recording.startedAtMs, + } + : { active: false }, + ); + return false; + }; + + readonly #onConnect = (port: chrome.runtime.Port): void => { + if (port.name !== RECORD_FRAME_PORT) return; + const address = senderAddress(port.sender ?? {}); + if (!address) { + port.disconnect(); + return; + } + let registered: FrameAgent | null = null; + port.onMessage.addListener((raw: unknown) => { + const message = raw as Partial; + if (message.type === "ready") { + if (typeof message.requestId !== "string" || typeof message.producerId !== "string") { + port.disconnect(); + return; + } + const requestId = message.requestId; + const producerId = message.producerId; + if (registered) { + if (registered.requestId === requestId && registered.producerId === producerId) { + port.postMessage({ + type: "ready_ack", + requestId, + producerId, + } satisfies RecordFramePortMessage); + } else { + port.disconnect(); + } + return; + } + const recording = this.#recordings.get(requestId); + if ( + !recording || + recording.finishing || + !recording.tabIds.has(address.tabId) || + producerId.length === 0 + ) { + port.disconnect(); + return; + } + const agent: FrameAgent = { + ...address, + requestId, + producerId, + port, + stopWaiters: new Map(), + }; + registered = agent; + const key = documentKey(address.tabId, address.documentId); + const previous = recording.agents.get(key); + if (previous && previous !== agent) previous.port.disconnect(); + recording.agents.set(key, agent); + port.postMessage({ + type: "ready_ack", + requestId, + producerId, + } satisfies RecordFramePortMessage); + return; + } + if ( + message.type === "stopped" && + registered && + message.requestId === registered.requestId && + typeof message.commandId === "string" + ) { + registered.stopWaiters.get(message.commandId)?.(message.ok === true); + registered.stopWaiters.delete(message.commandId); + } + }); + port.onDisconnect.addListener(() => { + if (!registered) return; + for (const resolve of registered.stopWaiters.values()) resolve(true); + const recording = this.#recordings.get(registered.requestId); + const key = documentKey(registered.tabId, registered.documentId); + if (recording?.agents.get(key) === registered) recording.agents.delete(key); + }); + }; + + #stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => { + agent.stopWaiters.delete(commandId); + resolve(false); + }, RECORD_FRAME_STOP_TIMEOUT_MS); + agent.stopWaiters.set(commandId, (ok) => { + clearTimeout(timer); + resolve(ok); + }); + try { + agent.port.postMessage({ + type: "stop", + requestId, + commandId, + } satisfies RecordFramePortMessage); + } catch { + clearTimeout(timer); + agent.stopWaiters.delete(commandId); + resolve(true); + } + }); + } + + #cancelAgents(recording: ArmedRecording): void { + for (const agent of recording.agents.values()) { + try { + agent.port.postMessage({ + type: "cancel", + requestId: recording.requestId, + } satisfies RecordFramePortMessage); + } catch { + // A destroyed Document has no remaining capture state to cancel. + } + } + } +} + +function isStarted(value: unknown): boolean { + return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true; +} + +export const recordFrameCoordinator = new RecordFrameCoordinator(); diff --git a/apps/extension/src/lib/recording/observation-capture.ts b/apps/extension/src/lib/recording/observation-capture.ts index 9c1f4591..efafad4e 100644 --- a/apps/extension/src/lib/recording/observation-capture.ts +++ b/apps/extension/src/lib/recording/observation-capture.ts @@ -28,6 +28,11 @@ export interface RegisteredObservation { url: string; } +export interface RecordingDocumentScope { + frameId: string; + target: CdpTarget; +} + function nodeKey(frameId: string, backendNodeId: number): string { return `${frameId}:${backendNodeId}`; } @@ -40,8 +45,12 @@ export class ObservationNodeIndex { readonly #nodesByFrameTag = new Map(); readonly #refById = new Map(); readonly #refsByFrame = new Map(); + readonly #scopeByProducer = new Map(); - constructor(input: Pick) { + constructor( + input: Pick & + Partial>, + ) { const refByNode = new Map(); for (const ref of input.refs) { const frameId = ref.frameId ?? input.rootFrameId; @@ -51,6 +60,16 @@ export class ObservationNodeIndex { frameRefs.push(ref); this.#refsByFrame.set(frameId, frameRefs); } + for (const frame of input.frames ?? []) { + const documentId = frame.recordingDocumentId; + if (!documentId) continue; + this.#scopeByProducer.set( + documentId, + this.#scopeByProducer.has(documentId) + ? null + : { frameId: frame.frameId, target: frame.target }, + ); + } for (const geometry of input.matchNodes) { const { frameId } = geometry; const entry = { @@ -76,6 +95,10 @@ export class ObservationNodeIndex { refs(frameId: string): readonly RenderedRef[] { return this.#refsByFrame.get(frameId) ?? []; } + + documentScope(producerId: string): RecordingDocumentScope | undefined { + return this.#scopeByProducer.get(producerId) ?? undefined; + } } async function readTabMeta( diff --git a/apps/extension/src/lib/recording/recording-runtime.ts b/apps/extension/src/lib/recording/recording-runtime.ts index e27577f6..d9a396e5 100644 --- a/apps/extension/src/lib/recording/recording-runtime.ts +++ b/apps/extension/src/lib/recording/recording-runtime.ts @@ -126,7 +126,7 @@ export class RecordingObservationRuntime { tabId: number, drafts: RecordingDraftStep[], draftIndex: number, - scope?: DocumentSettleScope, + producerId?: string, ): Promise { const draft = drafts[draftIndex]; if (!draft) return; @@ -138,6 +138,23 @@ export class RecordingObservationRuntime { // Post-action settle can still provide a usable state. } } + const scope = producerId + ? context.session.cursor.lastSettled?.index.documentScope(producerId) + : undefined; + if ( + producerId && + (draft.op === "click" || + draft.op === "hover" || + draft.op === "fill" || + draft.op === "press" || + draft.op === "select") + ) { + draft.targetHint = { + ...(draft.targetHint ?? {}), + frameId: scope?.frameId ?? null, + geometrySpace: "local", + }; + } context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending); context.settle.schedule(drafts, draftIndex, scope); } diff --git a/apps/extension/src/lib/recording/target-matcher.ts b/apps/extension/src/lib/recording/target-matcher.ts index 5cf21aac..9fca298c 100644 --- a/apps/extension/src/lib/recording/target-matcher.ts +++ b/apps/extension/src/lib/recording/target-matcher.ts @@ -14,8 +14,13 @@ function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): bool return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h); } -function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean { - return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect); +function candidateMatches( + candidate: IndexedObservationNode, + geometry: TargetGeometry, + geometrySpace: "top" | "local", +): boolean { + const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect; + return rect != null && rectMatches(geometry.rect, rect); } export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 { @@ -53,12 +58,17 @@ export function matchObservationTarget(input: { hint?: TargetMatchHint; fallback?: CaptureTargetDescriptor; }): TargetDescriptorV3 { + if (input.hint?.frameId === null) return unmatchedTarget(input.fallback); const frameId = input.hint?.frameId ?? input.observation.rootFrameId; const geometry = input.hint?.geometry; if (geometry) { const matches = input.observation.index .candidates(frameId, geometry.tag) - .filter((candidate) => candidate.ref && candidateMatches(candidate, geometry)); + .filter( + (candidate) => + candidate.ref && + candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"), + ); if (matches.length === 1) return descriptor(matches[0]!.ref!); const semanticMatches = matches.filter( (candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback), diff --git a/apps/extension/src/lib/recording/types.ts b/apps/extension/src/lib/recording/types.ts index 9767ad62..4ccffe26 100644 --- a/apps/extension/src/lib/recording/types.ts +++ b/apps/extension/src/lib/recording/types.ts @@ -15,8 +15,9 @@ export interface TargetGeometry { export interface TargetMatchHint { geometry?: TargetGeometry; - /** Missing means the current top frame, never an unrestricted frame search. */ - frameId?: string; + /** Missing means top frame; null means the source Document could not be resolved. */ + frameId?: string | null; + geometrySpace?: "top" | "local"; } export interface StepAnnotation { diff --git a/apps/extension/src/shared/recording-document-identity.ts b/apps/extension/src/shared/recording-document-identity.ts new file mode 100644 index 00000000..a044a5d7 --- /dev/null +++ b/apps/extension/src/shared/recording-document-identity.ts @@ -0,0 +1,9 @@ +export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document"; + +/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */ +export function readRecordingDocumentIdentity( + attrs: Readonly>, +): string | undefined { + const identity = attrs[RECORD_DOCUMENT_ATTRIBUTE]; + return identity ? identity : undefined; +} diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index 46f984b1..c99f4b79 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -18,6 +18,11 @@ import { type RecordStepAck, type RecordStopMessage, } from "@/lib/record-bridge"; +import { + type RecordFrameCoordinator, + type RecordingCaptureScope, + recordFrameCoordinator, +} from "@/lib/recording/frame-coordinator"; import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime"; import { appendRecordedPayload, @@ -262,10 +267,11 @@ async function processRecordedStep( recording: ActiveRecording, draftIndex: number, tabId: number, + producerId?: string, ): Promise { if (!recording.observation) return; try { - await recording.observation.processDraft(tabId, recording.steps, draftIndex); + await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId); } catch (err) { console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err); } @@ -278,6 +284,10 @@ export interface RecordDeps { msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage, ): Promise; bypassOverlay?: (tabId: number, enabled: boolean) => Promise; + frameCoordinator?: Pick< + RecordFrameCoordinator, + "begin" | "armTab" | "sourceFor" | "stop" | "cancel" + >; cdp?: CdpRunner; signal?: AbortSignal; } @@ -288,6 +298,7 @@ function getDefaultDeps(): RecordDeps { defaultDeps = { tabsApi: chromeTabsApi, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), + frameCoordinator: recordFrameCoordinator, }; } return defaultDeps; @@ -354,9 +365,15 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): ( if (!isRecordStepMessage(message)) return false; for (const recording of recordings.values()) { if (recording.requestId !== message.requestId) continue; - const sourceTabId = sender.tab?.id ?? recording.tabs.currentTabId; + const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator + ? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender) + : undefined; + if (deps.frameCoordinator && !source) return false; + const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId; const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId; - const producerKey = `${sourceTabId}:${message.producerId}`; + const producerKey = source + ? `${source.tabId}:${source.documentId}:${source.producerId}` + : `${sourceTabId}:${message.producerId}`; const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1; if (message.sequence < expectedSequence) { sendResponse({ ok: true, sequence: message.sequence }); @@ -387,7 +404,7 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): ( targetHint, ); if (draftIndex !== null) { - await processRecordedStep(recording, draftIndex, sourceTabId); + await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId); } }); sendResponse({ ok: true, sequence: message.sequence }); @@ -469,6 +486,9 @@ async function stopRecordingOnAllAgentTabs( recording: ActiveRecording, deps: RecordDeps, ): Promise { + if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) { + throw new Error("failed to flush one or more recording documents"); + } const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId }; let tabIds = [recording.tabs.currentTabId]; try { @@ -523,6 +543,10 @@ async function rearmRecording( startedAtMs: recording.startedAtMs, }; try { + if (deps.frameCoordinator) { + const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId); + if (!frameStarted) throw new Error("recording document did not start"); + } await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing); if (isFinishing()) return false; if (activation) { @@ -860,12 +884,14 @@ export async function handleRecordStart( actionQueue: Promise.resolve(), lastStepSequenceByProducer: new Map(), }); + deps.frameCoordinator?.begin(requestId, startedAtMs); // Observe navigations for the whole recording lifetime; attach before // optional navigate so the destination load can rearm capture. ensureBrowserObservationListeners(deps); const abortPending = async (notifyContent: boolean) => { recordings.get(params.session_id)?.observation?.cancel(); + deps.frameCoordinator?.cancel(requestId); recordings.delete(params.session_id); releaseBrowserObservationListenersIfIdle(); if (notifyContent) { @@ -982,6 +1008,10 @@ export async function handleRecordStart( const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs }; try { + if (deps.frameCoordinator) { + const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId); + if (!frameStarted) throw new Error("top recording document did not start"); + } await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab); } catch { await abortPending(true); @@ -1101,6 +1131,7 @@ export function clearRecordingForSession(sessionId: string): void { } void clearRearmTimersForRecording(recording, getDefaultDeps()); if (!recording.settled) { + getDefaultDeps().frameCoordinator?.cancel(recording.requestId); recording.observation?.cancel(); recording.settled = true; recording.rejectFinish(new Error("recording cleared")); diff --git a/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts b/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts new file mode 100644 index 00000000..2889d9e0 --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; +import { projectRecordSafeObservation } from "../record-safe-observation"; + +describe("projectRecordSafeObservation", () => { + it("projects the opaque recording identity without exposing captured attributes", () => { + const result = projectRecordSafeObservation({ + rootFrameId: "child", + frameDocuments: [ + { + frameId: "child", + target: { tabId: 4, sessionId: "oopif-session" }, + contextScopeId: "scope-1", + axNodes: [], + domNodes: [ + { + backendNodeId: 1, + parentBackendNodeId: null, + tag: "html", + attrs: { + [RECORD_DOCUMENT_ATTRIBUTE]: "producer-1", + value: "user@example.com", + }, + rect: null, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + ], + }, + ], + rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false }, + }); + + expect(result.frames).toEqual([ + { + frameId: "child", + target: { tabId: 4, sessionId: "oopif-session" }, + recordingDocumentId: "producer-1", + }, + ]); + const dumped = JSON.stringify(result); + expect(dumped).not.toContain("user@example.com"); + expect(dumped).not.toContain("attrs"); + expect(dumped).not.toContain("domNodes"); + expect(dumped).not.toContain("axNodes"); + }); +}); diff --git a/apps/extension/src/tools/vom/record-safe-observation.ts b/apps/extension/src/tools/vom/record-safe-observation.ts index 3b5befd5..b4fd093e 100644 --- a/apps/extension/src/tools/vom/record-safe-observation.ts +++ b/apps/extension/src/tools/vom/record-safe-observation.ts @@ -1,5 +1,6 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom"; import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity"; import type { CapturedSurfaceProbe } from "./capture"; import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture"; @@ -9,6 +10,8 @@ export interface CaptureVomFrame { target: CdpTarget; parentFrameId?: string; ownerBackendNodeId?: number; + /** BrowserSkill-generated identity for the recording agent in this Document. */ + recordingDocumentId?: string; } /** Allowlisted geometry used to match a recorded action to a rendered ref. */ @@ -37,14 +40,20 @@ export interface CaptureVomObservationResult { } function projectFrames(documents: CapturedFrameDocument[]): CaptureVomFrame[] { - return documents.map((document) => ({ - frameId: document.frameId, - target: document.target, - ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}), - ...(document.ownerBackendNodeId !== undefined - ? { ownerBackendNodeId: document.ownerBackendNodeId } - : {}), - })); + return documents.map((document) => { + const recordingDocumentId = document.domNodes + .map((node) => readRecordingDocumentIdentity(node.attrs)) + .find((identity) => identity !== undefined); + return { + frameId: document.frameId, + target: document.target, + ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}), + ...(document.ownerBackendNodeId !== undefined + ? { ownerBackendNodeId: document.ownerBackendNodeId } + : {}), + ...(recordingDocumentId ? { recordingDocumentId } : {}), + }; + }); } function projectMatchNodes(documents: CapturedFrameDocument[]): CaptureVomMatchNode[] { From 9742d83213ace111deb0f14bdd86061bc1b1aea1 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 19 Aug 2026 17:13:16 +0800 Subject: [PATCH 2/5] fix(recorder): bug fix --- apps/extension/src/entrypoints/background.ts | 48 ++++++------- .../src/tools/__tests__/dispatcher.test.ts | 67 +++++++++++++++++ apps/extension/src/tools/dispatcher.ts | 71 +++++++++---------- apps/extension/src/tools/record.ts | 4 ++ 4 files changed, 129 insertions(+), 61 deletions(-) diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts index 8e819d79..4edd0a24 100644 --- a/apps/extension/src/entrypoints/background.ts +++ b/apps/extension/src/entrypoints/background.ts @@ -39,6 +39,7 @@ import { attachRecordFinishListener, attachRecordQueryListener, attachRecordStepListener, + type RecordRuntimeDeps, } from "@/tools/record"; import { detectBrowserMeta } from "@/transport/handshake"; import type { Transport } from "@/transport/transport"; @@ -146,10 +147,33 @@ export default defineBackground(() => { void sessionsLive.syncFromManager(); }, }); + const recordDeps = { + tabsApi: chrome.tabs, + cdp, + frameCoordinator: recordFrameCoordinator, + sendToTab: (tabId: number, msg: Parameters[1]) => + chrome.tabs.sendMessage(tabId, msg), + bypassOverlay: async (tabId: number, enabled: boolean) => { + try { + await chrome.tabs.sendMessage(tabId, { + type: OVERLAY_AUTOMATION_BYPASS, + enabled, + }); + } catch { + // Content script may be unavailable on restricted pages. + } + }, + } satisfies RecordRuntimeDeps; + recordFrameCoordinator.attach(); + attachRecordStepListener(recordDeps); + attachRecordFinishListener(recordDeps); + attachRecordQueryListener(recordDeps); + const dispatcher = new ToolDispatcher({ transport, sessions, cdp, + recording: recordDeps, onSessionsChanged: onOverlaySessionStateChanged, onBrowserControlResumed, approveBorrow: (ctx) => @@ -171,30 +195,6 @@ export default defineBackground(() => { }), }); dispatcher.start(); - const recordDeps = { - tabsApi: chrome.tabs, - cdp, - frameCoordinator: recordFrameCoordinator, - sendToTab: (tabId: number, msg: Parameters[1]) => - chrome.tabs.sendMessage(tabId, msg), - bypassOverlay: async (tabId: number, enabled: boolean) => { - try { - await chrome.tabs.sendMessage(tabId, { - type: OVERLAY_AUTOMATION_BYPASS, - enabled, - }); - } catch { - // Content script may be unavailable on restricted pages. - } - }, - }; - // Message listeners stay up (cheap; fire only for record message types). - // Tab / webNavigation observation attaches lazily while a recording is - // active — see ensureBrowserObservationListeners in tools/record.ts. - attachRecordStepListener(recordDeps); - attachRecordFinishListener(recordDeps); - attachRecordQueryListener(recordDeps); - recordFrameCoordinator.attach(); if (typeof chrome.notifications?.onClicked?.addListener === "function") { attachBorrowNotificationClickHandler({ onClicked: chrome.notifications.onClicked, diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 2e1b9211..4ed32e06 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -8,6 +8,10 @@ import type { RequestFrame, } from "@/transport/types"; import { ToolDispatcher } from "../dispatcher"; +import { + resetBrowserObservationForTests, + setBrowserObservationAttachForTests, +} from "../record"; type TestDispatcherCdp = NonNullable[0]["cdp"]>; @@ -44,9 +48,72 @@ function makeRequest(method: string, params: unknown): RequestFrame { describe("ToolDispatcher", () => { afterEach(() => { + resetBrowserObservationForTests(); vi.unstubAllGlobals(); }); + it("uses the configured recording runtime for start and stop", async () => { + const { transport, sent, deliver } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 4242), + remove: vi.fn(), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + await sessions.start("aa11"); + const tab = { + id: 7, + windowId: 4242, + active: true, + status: "complete", + url: "https://example.com/start", + } as chrome.tabs.Tab; + const frameCoordinator = { + begin: vi.fn(), + armTab: vi.fn(async () => true), + sourceFor: vi.fn(() => null), + stop: vi.fn(async () => true), + cancel: vi.fn(), + }; + const sendToTab = vi.fn(async () => ({ ok: true })); + setBrowserObservationAttachForTests( + () => () => {}, + () => () => {}, + ); + const dispatcher = new ToolDispatcher({ + transport, + sessions, + recording: { + tabsApi: { + get: vi.fn(async () => tab), + query: vi.fn(async () => [tab]), + }, + frameCoordinator, + sendToTab, + }, + }); + dispatcher.start(); + + deliver( + makeRequest("tool.record_start", { + session_id: "aa11", + url: "https://example.com/start", + }), + ); + await vi.waitFor(() => expect(sent).toHaveLength(1)); + + expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } }); + expect(frameCoordinator.begin).toHaveBeenCalledOnce(); + expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7); + + deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } }); + await vi.waitFor(() => expect(sent).toHaveLength(2)); + + expect(frameCoordinator.stop).toHaveBeenCalledOnce(); + expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } }); + }); + it("routes tool.session_start to the SessionManager and replies with the window id", async () => { const { transport, sent, deliver } = fakeTransport(); const sessions = new SessionManager({ diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 21be8956..e0a5ac4d 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -51,7 +51,12 @@ import { handleScreenshot, handleSnapshot, } from "./observation"; -import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record"; +import { + handleRecordAwait, + handleRecordStart, + handleRecordStop, + type RecordRuntimeDeps, +} from "./record"; import { handleSessionStart, handleSessionStop, @@ -99,6 +104,7 @@ export interface DispatcherDeps { transport: Transport; sessions: SessionManager; cdp?: DispatcherCdpRunner; + recording?: RecordRuntimeDeps; /** * Invoked whenever a dispatched RPC may have changed the live * session set (currently `tool.session_start` and @@ -134,6 +140,7 @@ export class ToolDispatcher { private readonly transport: Transport; private readonly sessions: SessionManager; private readonly cdp?: DispatcherCdpRunner; + private readonly recording?: RecordRuntimeDeps; private readonly onSessionsChanged?: () => void; private readonly onBrowserControlResumed?: (sessionId: string) => void; private readonly approveBorrow?: BorrowConfirmationApprover; @@ -153,6 +160,7 @@ export class ToolDispatcher { this.transport = deps.transport; this.sessions = deps.sessions; this.cdp = deps.cdp; + this.recording = deps.recording; this.onSessionsChanged = deps.onSessionsChanged; this.onBrowserControlResumed = deps.onBrowserControlResumed; this.approveBorrow = deps.approveBorrow; @@ -531,44 +539,26 @@ export class ToolDispatcher { signal, }); case "tool.record_start": - return handleRecordStart(this.sessions, req.params as RecordStartParams, { - tabsApi: chromeTabsApi, - sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), - bypassOverlay: async (tabId, enabled) => { - try { - await chrome.tabs.sendMessage(tabId, { - type: OVERLAY_AUTOMATION_BYPASS, - enabled, - }); - } catch { - // Content script may be unavailable on restricted pages. - } - }, - ...(this.cdp ? { cdp: this.cdp } : {}), - signal, - }); + return this.recording + ? handleRecordStart(this.sessions, req.params as RecordStartParams, { + ...this.recording, + signal, + }) + : recordingRuntimeUnavailable(); case "tool.record_stop": - return handleRecordStop(this.sessions, req.params as RecordStopParams, { - tabsApi: chromeTabsApi, - sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), - bypassOverlay: async (tabId, enabled) => { - try { - await chrome.tabs.sendMessage(tabId, { - type: OVERLAY_AUTOMATION_BYPASS, - enabled, - }); - } catch { - // Content script may be unavailable on restricted pages. - } - }, - signal, - }); + return this.recording + ? handleRecordStop(this.sessions, req.params as RecordStopParams, { + ...this.recording, + signal, + }) + : recordingRuntimeUnavailable(); case "tool.record_await": - return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, { - tabsApi: chromeTabsApi, - sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), - signal, - }); + return this.recording + ? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, { + ...this.recording, + signal, + }) + : recordingRuntimeUnavailable(); default: return { code: "unknown_method", @@ -704,6 +694,13 @@ function isRpcError(v: unknown): v is RpcError { ); } +function recordingRuntimeUnavailable(): RpcError { + return { + code: "protocol_error", + message: "recording runtime is unavailable", + }; +} + function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { switch (req.method) { case "tool.tab_create": diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index c99f4b79..31430464 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -292,6 +292,10 @@ export interface RecordDeps { signal?: AbortSignal; } +export type RecordRuntimeDeps = Omit & { + frameCoordinator: NonNullable; +}; + let defaultDeps: RecordDeps | null = null; function getDefaultDeps(): RecordDeps { if (!defaultDeps) { From 3656f0dd3c2ea742431c6e041996ec0cbb3c5559 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 19 Aug 2026 17:35:36 +0800 Subject: [PATCH 3/5] fix(recorder): ci fix --- apps/extension/src/tools/__tests__/dispatcher.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 4ed32e06..c05cdb2a 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -8,10 +8,7 @@ import type { RequestFrame, } from "@/transport/types"; import { ToolDispatcher } from "../dispatcher"; -import { - resetBrowserObservationForTests, - setBrowserObservationAttachForTests, -} from "../record"; +import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record"; type TestDispatcherCdp = NonNullable[0]["cdp"]>; From 616af1099f4a3623a019021360505771a11009f6 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 12:29:32 +0800 Subject: [PATCH 4/5] fix(record): saftey policy and rebase bug fix --- .../__tests__/record-frame-agent.test.ts | 2 +- .../recording-document-marker.test.ts | 18 +++++++-- .../src/lib/recording/document-marker.ts | 12 +++--- .../src/lib/recording/observation-capture.ts | 1 + .../src/shared/recording-document-identity.ts | 17 +++++++- .../__tests__/record-safe-observation.test.ts | 40 +++++++++++++++++-- .../src/tools/vom/record-safe-observation.ts | 8 ++-- 7 files changed, 81 insertions(+), 17 deletions(-) diff --git a/apps/extension/src/content/__tests__/record-frame-agent.test.ts b/apps/extension/src/content/__tests__/record-frame-agent.test.ts index d81479fc..0c0a6517 100644 --- a/apps/extension/src/content/__tests__/record-frame-agent.test.ts +++ b/apps/extension/src/content/__tests__/record-frame-agent.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { RECORD_DOCUMENT_ATTRIBUTE } from "@/lib/recording/document-marker"; import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge"; import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge"; +import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; import { RecordFrameAgent } from "../recording/frame-agent"; class PortListeners unknown> { diff --git a/apps/extension/src/lib/__tests__/recording-document-marker.test.ts b/apps/extension/src/lib/__tests__/recording-document-marker.test.ts index cc91fb94..53fb2afd 100644 --- a/apps/extension/src/lib/__tests__/recording-document-marker.test.ts +++ b/apps/extension/src/lib/__tests__/recording-document-marker.test.ts @@ -1,14 +1,26 @@ import { describe, expect, it } from "vitest"; -import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; +import { + RECORD_DOCUMENT_ATTRIBUTE, + recordingDocumentMarkerValue, +} from "@/shared/recording-document-identity"; import { markRecordingDocument } from "../recording/document-marker"; import { ObservationNodeIndex } from "../recording/observation-capture"; describe("recording document marker", () => { + it("rejects a non-random Document identity", () => { + expect(() => markRecordingDocument("user@example.com")).toThrow( + "recording Document identity must be a random UUID", + ); + }); + it("restores the Document attribute exactly", () => { document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value"); - const marker = markRecordingDocument("producer-1"); + const producerId = "123e4567-e89b-42d3-a456-426614174000"; + const marker = markRecordingDocument(producerId); - expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("producer-1"); + expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe( + recordingDocumentMarkerValue(producerId), + ); marker.restore(); expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value"); document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE); diff --git a/apps/extension/src/lib/recording/document-marker.ts b/apps/extension/src/lib/recording/document-marker.ts index b1b32261..f8f383f0 100644 --- a/apps/extension/src/lib/recording/document-marker.ts +++ b/apps/extension/src/lib/recording/document-marker.ts @@ -1,6 +1,7 @@ -import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; - -export { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; +import { + RECORD_DOCUMENT_ATTRIBUTE, + recordingDocumentMarkerValue, +} from "@/shared/recording-document-identity"; export interface RecordingDocumentMarker { restore(): void; @@ -24,12 +25,13 @@ export function markRecordingDocument( root: HTMLElement = document.documentElement, ): RecordingDocumentMarker { const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE); - const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, producerId); + const markerValue = recordingDocumentMarkerValue(producerId); + const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue); ensure(); return { ensure, restore() { - if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== producerId) return; + if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return; if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE); else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous); }, diff --git a/apps/extension/src/lib/recording/observation-capture.ts b/apps/extension/src/lib/recording/observation-capture.ts index efafad4e..8423f437 100644 --- a/apps/extension/src/lib/recording/observation-capture.ts +++ b/apps/extension/src/lib/recording/observation-capture.ts @@ -1,4 +1,5 @@ import type { RenderedRef } from "@browser-skill/vom"; +import type { CdpTarget } from "@/browser-driver/frame-graph"; import { type CaptureVomMatchNode, type CaptureVomObservationResult, diff --git a/apps/extension/src/shared/recording-document-identity.ts b/apps/extension/src/shared/recording-document-identity.ts index a044a5d7..6cd3bc28 100644 --- a/apps/extension/src/shared/recording-document-identity.ts +++ b/apps/extension/src/shared/recording-document-identity.ts @@ -1,9 +1,22 @@ export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document"; +const RECORDING_DOCUMENT_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:"; + +export function recordingDocumentMarkerValue(identity: string): string { + if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) { + throw new TypeError("recording Document identity must be a random UUID"); + } + return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`; +} + /** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */ export function readRecordingDocumentIdentity( attrs: Readonly>, ): string | undefined { - const identity = attrs[RECORD_DOCUMENT_ATTRIBUTE]; - return identity ? identity : undefined; + const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE]; + if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined; + const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length); + return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined; } diff --git a/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts b/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts index 2889d9e0..4401792a 100644 --- a/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts +++ b/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from "vitest"; -import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; +import { + RECORD_DOCUMENT_ATTRIBUTE, + recordingDocumentMarkerValue, +} from "@/shared/recording-document-identity"; import { projectRecordSafeObservation } from "../record-safe-observation"; describe("projectRecordSafeObservation", () => { it("projects the opaque recording identity without exposing captured attributes", () => { + const recordingDocumentId = "123e4567-e89b-42d3-a456-426614174000"; const result = projectRecordSafeObservation({ rootFrameId: "child", frameDocuments: [ @@ -18,7 +22,7 @@ describe("projectRecordSafeObservation", () => { parentBackendNodeId: null, tag: "html", attrs: { - [RECORD_DOCUMENT_ATTRIBUTE]: "producer-1", + [RECORD_DOCUMENT_ATTRIBUTE]: recordingDocumentMarkerValue(recordingDocumentId), value: "user@example.com", }, rect: null, @@ -36,7 +40,7 @@ describe("projectRecordSafeObservation", () => { { frameId: "child", target: { tabId: 4, sessionId: "oopif-session" }, - recordingDocumentId: "producer-1", + recordingDocumentId, }, ]); const dumped = JSON.stringify(result); @@ -45,4 +49,34 @@ describe("projectRecordSafeObservation", () => { expect(dumped).not.toContain("domNodes"); expect(dumped).not.toContain("axNodes"); }); + + it("does not project a page-controlled marker value as recording identity", () => { + const result = projectRecordSafeObservation({ + rootFrameId: "root", + frameDocuments: [ + { + frameId: "root", + target: { tabId: 4 }, + contextScopeId: "scope-1", + axNodes: [], + domNodes: [ + { + backendNodeId: 1, + parentBackendNodeId: null, + tag: "html", + attrs: { [RECORD_DOCUMENT_ATTRIBUTE]: "bsk:user@example.com" }, + rect: null, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + ], + }, + ], + rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false }, + }); + + expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]); + expect(JSON.stringify(result)).not.toContain("user@example.com"); + }); }); diff --git a/apps/extension/src/tools/vom/record-safe-observation.ts b/apps/extension/src/tools/vom/record-safe-observation.ts index b4fd093e..d58e7577 100644 --- a/apps/extension/src/tools/vom/record-safe-observation.ts +++ b/apps/extension/src/tools/vom/record-safe-observation.ts @@ -41,9 +41,11 @@ export interface CaptureVomObservationResult { function projectFrames(documents: CapturedFrameDocument[]): CaptureVomFrame[] { return documents.map((document) => { - const recordingDocumentId = document.domNodes - .map((node) => readRecordingDocumentIdentity(node.attrs)) - .find((identity) => identity !== undefined); + let recordingDocumentId: string | undefined; + for (const node of document.domNodes) { + recordingDocumentId = readRecordingDocumentIdentity(node.attrs); + if (recordingDocumentId) break; + } return { frameId: document.frameId, target: document.target, From 4aade4b94fd8fb59abfea781aac7c60165acd47f Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 13:16:15 +0800 Subject: [PATCH 5/5] fix(recorder): bug fix, oopif load too late --- .../record-frame-coordinator.test.ts | 98 +++++++++++++++++- .../lib/__tests__/recording-runtime.test.ts | 89 ++++++++++++++++- .../src/lib/recording/frame-coordinator.ts | 99 ++++++++++++++++--- .../src/lib/recording/recording-runtime.ts | 43 +++++++- apps/extension/src/tools/record.ts | 11 ++- 5 files changed, 319 insertions(+), 21 deletions(-) diff --git a/apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts b/apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts index e5e74499..abefbaa1 100644 --- a/apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts +++ b/apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { RECORD_FRAME_PORT } from "../recording/frame-bridge"; +import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge"; import { RecordFrameCoordinator } from "../recording/frame-coordinator"; class ListenerSet unknown> { @@ -46,6 +46,98 @@ describe("RecordFrameCoordinator", () => { }); }); + it("answers child-document queries for the initial tab before it is armed", () => { + const coordinator = new RecordFrameCoordinator({ + getAllFrames: async () => [], + sendToDocument: async () => ({ ok: true }), + }); + coordinator.attach(); + coordinator.begin("rec-1", 10, 3); + + const query = (tabId: number) => { + let response: unknown; + for (const listener of onMessage.listeners) { + listener( + { type: RECORD_FRAME_QUERY }, + { + tab: { id: tabId }, + frameId: 7, + documentId: "child-document", + } as chrome.runtime.MessageSender, + (value) => { + response = value; + }, + ); + } + return response; + }; + + expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 }); + expect(query(4)).toEqual({ active: false }); + }); + + it("starts a child Document that appears after the initial frame snapshot", async () => { + let notifyFrameNavigation: + | ((frame: { + tabId: number; + frameId: number; + documentId?: string; + lifecycle: "committed" | "completed"; + }) => void) + | undefined; + const sendToDocument = vi.fn(async () => ({ ok: true })); + const coordinator = new RecordFrameCoordinator({ + getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }], + sendToDocument, + subscribeFrameNavigation(listener) { + notifyFrameNavigation = listener; + return () => {}; + }, + }); + coordinator.attach(); + const onDocumentReady = vi.fn(); + coordinator.begin("rec-1", 10, 3, onDocumentReady); + await coordinator.armTab("rec-1", 3); + sendToDocument.mockClear(); + + notifyFrameNavigation?.({ + tabId: 3, + frameId: 7, + documentId: "late-child", + lifecycle: "completed", + }); + await vi.waitFor(() => { + expect(sendToDocument).toHaveBeenCalledWith( + 3, + { type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 }, + { documentId: "late-child" }, + ); + }); + expect(onDocumentReady).not.toHaveBeenCalled(); + + const child = fakePort({ + tab: { id: 3 }, + frameId: 7, + documentId: "late-child", + } as chrome.runtime.MessageSender); + for (const listener of onConnect.listeners) listener(child.port); + child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" }); + notifyFrameNavigation?.({ + tabId: 3, + frameId: 7, + documentId: "late-child", + lifecycle: "completed", + }); + await vi.waitFor(() => { + expect(onDocumentReady).toHaveBeenCalledWith({ + tabId: 3, + documentId: "late-child", + browserFrameId: 7, + producerId: "producer-1", + }); + }); + }); + it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => { const sendToDocument = vi.fn(async () => ({ ok: true })); const coordinator = new RecordFrameCoordinator({ @@ -56,7 +148,7 @@ describe("RecordFrameCoordinator", () => { sendToDocument, }); coordinator.attach(); - coordinator.begin("rec-1", 10); + coordinator.begin("rec-1", 10, 3); await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true); const sender = { @@ -100,7 +192,7 @@ describe("RecordFrameCoordinator", () => { sendToDocument: async () => ({ ok: true }), }); coordinator.attach(); - coordinator.begin("rec-1", 10); + coordinator.begin("rec-1", 10, 3); await coordinator.armTab("rec-1", 3); const sender = { tab: { id: 3 }, diff --git a/apps/extension/src/lib/__tests__/recording-runtime.test.ts b/apps/extension/src/lib/__tests__/recording-runtime.test.ts index 6dfe5244..5889e65e 100644 --- a/apps/extension/src/lib/__tests__/recording-runtime.test.ts +++ b/apps/extension/src/lib/__tests__/recording-runtime.test.ts @@ -2,12 +2,18 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; const captureRecordingObservation = vi.hoisted(() => vi.fn()); +const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const)); vi.mock("../recording/observation-capture", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, captureRecordingObservation }; }); +vi.mock("../recording/document-settle", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, waitForDocumentSettled }; +}); + import { ObservationNodeIndex } from "../recording/observation-capture"; import { RecordingObservationRuntime } from "../recording/recording-runtime"; import type { RecordingDraftStep } from "../recording/types"; @@ -31,7 +37,10 @@ function runtime(): RecordingObservationRuntime { } describe("RecordingObservationRuntime", () => { - afterEach(() => captureRecordingObservation.mockReset()); + afterEach(() => { + captureRecordingObservation.mockReset(); + waitForDocumentSettled.mockClear(); + }); it("shares one initial capture and includes it in flush", async () => { let release!: (value: ReturnType) => void; @@ -199,4 +208,82 @@ describe("RecordingObservationRuntime", () => { }); expect(draft.matchedTarget?.ref).toBe("e1"); }); + + it("refreshes a late iframe before matching its first action", async () => { + const iframeObservation = { + ...observation(), + index: new ObservationNodeIndex({ + rootFrameId: "root", + frames: [ + { + frameId: "child", + target: { tabId: 7, sessionId: "oopif-session" }, + recordingDocumentId: "producer-1", + }, + ], + matchNodes: [ + { + backendNodeId: 42, + frameId: "child", + tag: "button", + rect: { x: 410, y: 20, w: 100, h: 30 }, + localRect: { x: 10, y: 20, w: 100, h: 30 }, + }, + ], + refs: [ + { + ref: "e1", + backendNodeId: 42, + frameId: "child", + role: "button", + name: "表格视图", + line: 1, + }, + ], + }), + }; + captureRecordingObservation + .mockResolvedValueOnce(observation()) + .mockResolvedValueOnce(iframeObservation) + .mockResolvedValueOnce(iframeObservation); + const recording = runtime(); + + await recording.captureInitial(7); + await recording.refreshDocument(7, "producer-1"); + const drafts: RecordingDraftStep[] = [ + { + op: "click", + captureTarget: { tag: "button", role: "button", name: "表格视图" }, + targetHint: { + geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" }, + }, + }, + ]; + await recording.processDraft(7, drafts, 0, "producer-1"); + recording.cancel(); + + expect(captureRecordingObservation).toHaveBeenCalledTimes(3); + expect(waitForDocumentSettled).toHaveBeenCalledWith( + expect.anything(), + { frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } }, + { signal: expect.any(AbortSignal) }, + ); + const draft = drafts[0]; + expect(draft?.op).toBe("click"); + if (!draft || draft.op !== "click") throw new Error("expected click draft"); + expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" }); + }); + + it("does not poison final flush when a frame readiness refresh fails", async () => { + captureRecordingObservation + .mockResolvedValueOnce(observation()) + .mockRejectedValueOnce(new Error("child Document replaced")); + const recording = runtime(); + + await recording.captureInitial(7); + await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow( + "child Document replaced", + ); + await expect(recording.flush()).resolves.toBeUndefined(); + }); }); diff --git a/apps/extension/src/lib/recording/frame-coordinator.ts b/apps/extension/src/lib/recording/frame-coordinator.ts index 8bdf1886..52ca0ad6 100644 --- a/apps/extension/src/lib/recording/frame-coordinator.ts +++ b/apps/extension/src/lib/recording/frame-coordinator.ts @@ -19,6 +19,7 @@ interface ArmedRecording { startedAtMs: number; tabIds: Set; agents: Map; + onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise; finishing: boolean; } @@ -33,6 +34,11 @@ interface BrowserFrame { documentId?: string; } +interface BrowserFrameNavigation extends BrowserFrame { + tabId: number; + lifecycle: "committed" | "completed"; +} + const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000; export interface RecordFrameCoordinatorDeps { @@ -42,6 +48,7 @@ export interface RecordFrameCoordinatorDeps { message: RecordFrameStartMessage, target: { documentId?: string; frameId?: number }, ): Promise; + subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void; } function documentKey(tabId: number, documentId: string): string { @@ -72,6 +79,18 @@ function defaultDeps(): RecordFrameCoordinatorDeps { sendToDocument(tabId, message, target) { return chrome.tabs.sendMessage(tabId, message, target); }, + subscribeFrameNavigation(listener) { + const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) => + listener({ ...details, lifecycle: "committed" }); + const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => + listener({ ...details, lifecycle: "completed" }); + chrome.webNavigation.onCommitted.addListener(onCommitted); + chrome.webNavigation.onCompleted.addListener(onCompleted); + return () => { + chrome.webNavigation.onCommitted.removeListener(onCommitted); + chrome.webNavigation.onCompleted.removeListener(onCompleted); + }; + }, }; } @@ -79,6 +98,7 @@ export class RecordFrameCoordinator { readonly #deps: RecordFrameCoordinatorDeps; readonly #recordings = new Map(); #attached = false; + #detachFrameNavigation: (() => void) | null = null; constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) { this.#deps = deps; @@ -89,24 +109,37 @@ export class RecordFrameCoordinator { this.#attached = true; chrome.runtime.onConnect.addListener(this.#onConnect); chrome.runtime.onMessage.addListener(this.#onMessage); + this.#detachFrameNavigation = + this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null; return () => { if (!this.#attached) return; this.#attached = false; chrome.runtime.onConnect.removeListener(this.#onConnect); chrome.runtime.onMessage.removeListener(this.#onMessage); + this.#detachFrameNavigation?.(); + this.#detachFrameNavigation = null; for (const recording of this.#recordings.values()) this.#cancelAgents(recording); this.#recordings.clear(); }; } - begin(requestId: string, startedAtMs: number): void { + begin( + requestId: string, + startedAtMs: number, + initialTabId: number, + onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise, + ): void { const previous = this.#recordings.get(requestId); if (previous) this.#cancelAgents(previous); this.#recordings.set(requestId, { requestId, startedAtMs, - tabIds: new Set(), + // Register the initial tab before navigation begins. A newly-created + // child document can query at document_start before armTab's frame + // snapshot sees it, especially when Chromium promotes it to an OOPIF. + tabIds: new Set([initialTabId]), agents: new Map(), + ...(onDocumentReady ? { onDocumentReady } : {}), finishing: false, }); } @@ -124,21 +157,10 @@ export class RecordFrameCoordinator { } if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 }); - const message: RecordFrameStartMessage = { - type: RECORD_FRAME_START, - requestId, - startedAtMs: recording.startedAtMs, - }; const results = await Promise.all( frames.map(async (frame) => { - try { - const response = await this.#deps.sendToDocument(tabId, message, { - ...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }), - }); - return { frameId: frame.frameId, started: isStarted(response) }; - } catch { - return { frameId: frame.frameId, started: false }; - } + const started = await this.#startFrame(recording, tabId, frame); + return { frameId: frame.frameId, started }; }), ); return results.some((result) => result.frameId === 0 && result.started); @@ -208,6 +230,33 @@ export class RecordFrameCoordinator { return false; }; + readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => { + if (frame.frameId === 0) return; + for (const recording of this.#recordings.values()) { + if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue; + // onCommitted can race document_start, while onCompleted runs after the + // content script is available. Starting twice is safe and closes both + // the early-query and late-SPA-frame gaps. + void (async () => { + const started = await this.#startFrame(recording, frame.tabId, frame); + if (!started || frame.lifecycle !== "completed" || recording.finishing) return; + const agent = frame.documentId + ? recording.agents.get(documentKey(frame.tabId, frame.documentId)) + : [...recording.agents.values()].find( + (candidate) => + candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId, + ); + if (!agent) return; + await recording.onDocumentReady?.({ + tabId: agent.tabId, + documentId: agent.documentId, + browserFrameId: agent.browserFrameId, + producerId: agent.producerId, + }); + })(); + } + }; + readonly #onConnect = (port: chrome.runtime.Port): void => { if (port.name !== RECORD_FRAME_PORT) return; const address = senderAddress(port.sender ?? {}); @@ -309,6 +358,26 @@ export class RecordFrameCoordinator { }); } + async #startFrame( + recording: ArmedRecording, + tabId: number, + frame: BrowserFrame, + ): Promise { + const message: RecordFrameStartMessage = { + type: RECORD_FRAME_START, + requestId: recording.requestId, + startedAtMs: recording.startedAtMs, + }; + try { + const response = await this.#deps.sendToDocument(tabId, message, { + ...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }), + }); + return isStarted(response); + } catch { + return false; + } + } + #cancelAgents(recording: ArmedRecording): void { for (const agent of recording.agents.values()) { try { diff --git a/apps/extension/src/lib/recording/recording-runtime.ts b/apps/extension/src/lib/recording/recording-runtime.ts index d9a396e5..55423738 100644 --- a/apps/extension/src/lib/recording/recording-runtime.ts +++ b/apps/extension/src/lib/recording/recording-runtime.ts @@ -1,6 +1,6 @@ import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; import type { StopReason, TraceV3 } from "@/transport/types"; -import type { DocumentSettleScope } from "./document-settle"; +import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle"; import type { RegisteredObservation } from "./observation-capture"; import { RecordingObservationSession } from "./observation-session"; import { inferMissingPostStates, SettleController } from "./settle-controller"; @@ -12,6 +12,8 @@ interface TabRecordingContext { session: RecordingObservationSession; settle: SettleController; pendingCapture: PendingCapture | null; + frameRefresh: Promise; + frameRefreshAbort: AbortController | null; } interface PendingCapture { @@ -58,6 +60,8 @@ export class RecordingObservationRuntime { tabId, }), pendingCapture: null, + frameRefresh: Promise.resolve(), + frameRefreshAbort: null, }; this.#contexts.set(tabId, context); return context; @@ -86,6 +90,41 @@ export class RecordingObservationRuntime { await this.#capture(tabId); } + /** Refresh the safe observation after a late child Document becomes usable. */ + async refreshDocument(tabId: number, producerId: string): Promise { + const context = this.#context(tabId); + context.frameRefreshAbort?.abort(); + const abort = new AbortController(); + context.frameRefreshAbort = abort; + const previous = context.frameRefresh; + const refresh = (async () => { + await previous; + if (abort.signal.aborted) return; + if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]); + if (abort.signal.aborted) return; + + // The first capture discovers the safe producer -> CDP Document scope. + await this.#capture(tabId); + if (abort.signal.aborted) return; + const scope = context.session.cursor.lastSettled?.index.documentScope(producerId); + if (scope) { + await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal }); + } + if (abort.signal.aborted) return; + + // Capture again after the child SPA is quiet so the next action can be + // matched against refs that actually exist in its pre-action state. + await this.#capture(tabId); + })(); + const tracked = refresh.catch(() => {}); + context.frameRefresh = tracked; + try { + await refresh; + } finally { + if (context.frameRefresh === tracked) context.frameRefreshAbort = null; + } + } + async captureTabTransition( fromTabId: number, toTabId: number, @@ -177,6 +216,7 @@ export class RecordingObservationRuntime { } async #flushContext(context: TabRecordingContext): Promise { + await context.frameRefresh; if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]); await context.settle.flushRedirects(); await context.settle.flush(); @@ -206,6 +246,7 @@ export class RecordingObservationRuntime { cancel(): void { for (const context of this.#contexts.values()) { + context.frameRefreshAbort?.abort(); context.pendingCapture?.abort.abort(); context.settle.cancel(); } diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index 31430464..294788d8 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -888,7 +888,16 @@ export async function handleRecordStart( actionQueue: Promise.resolve(), lastStepSequenceByProducer: new Map(), }); - deps.frameCoordinator?.begin(requestId, startedAtMs); + deps.frameCoordinator?.begin(requestId, startedAtMs, target.tabId, async (scope) => { + const recording = recordings.get(params.session_id); + if (!recording || recording.requestId !== requestId || recording.settled) return; + try { + await recording.observation?.refreshDocument(scope.tabId, scope.producerId); + } catch { + // The normal action-time capture remains available if a child Document + // is replaced while its readiness refresh is in flight. + } + }); // Observe navigations for the whole recording lifetime; attach before // optional navigate so the destination load can rearm capture. ensureBrowserObservationListeners(deps);