diff --git a/browser-extension/common/src/constants.ts b/browser-extension/common/src/constants.ts index b2aa4a6576..fa1528b34a 100644 --- a/browser-extension/common/src/constants.ts +++ b/browser-extension/common/src/constants.ts @@ -45,6 +45,9 @@ export const EXTENSION_MESSAGES = { TRIGGER_OPEN_CURL_MODAL: "triggerOpenCurlModal", STOP_NETWORK_RECORDING: "stopNetworkRecording", GET_NETWORK_RECORDING_STATE: "getNetworkRecordingState", + // v2 body capture: SW → client content script → page script (networkBodyRecorder) start/stop. + START_NETWORK_BODY_CAPTURE: "startNetworkBodyCapture", + STOP_NETWORK_BODY_CAPTURE: "stopNetworkBodyCapture", }; export const EXTENSION_EXTERNAL_MESSAGES = { @@ -79,6 +82,7 @@ export const CLIENT_MESSAGES = { OPEN_CURL_IMPORT_MODAL: "openCurlImportModal", NETWORK_EVENT_CAPTURED: "networkEventCaptured", NETWORK_RECORDING_ENDED: "networkRecordingEnded", + NETWORK_BODY_CAPTURED: "networkBodyCaptured", }; export const STORAGE_TYPE = "local"; diff --git a/browser-extension/common/src/sidepanel/network-recording/NetworkRecordingPanel.tsx b/browser-extension/common/src/sidepanel/network-recording/NetworkRecordingPanel.tsx index e288ef5e92..b290704320 100644 --- a/browser-extension/common/src/sidepanel/network-recording/NetworkRecordingPanel.tsx +++ b/browser-extension/common/src/sidepanel/network-recording/NetworkRecordingPanel.tsx @@ -31,7 +31,7 @@ const formatSize = (bytes: number | undefined): string => { }; // Why a recording ended — mirrors the StopReason union in the service worker. -type StopReason = "user" | "max-duration" | "connection-lost" | "tab-closed"; +type StopReason = "user" | "max-duration" | "connection-lost" | "tab-closed" | "extension-disabled"; // Banner shown for SW-initiated stops. `user` and `tab-closed` show no banner (the user knows / // the panel is gone), so they're absent from this map. @@ -46,6 +46,11 @@ const STOP_BANNERS: Partial { diff --git a/browser-extension/mv3/rollup.config.js b/browser-extension/mv3/rollup.config.js index 3dcd28aa0f..1aa435de84 100644 --- a/browser-extension/mv3/rollup.config.js +++ b/browser-extension/mv3/rollup.config.js @@ -145,4 +145,15 @@ export default [ }, plugins: commonPlugins, }, + { + ...commonConfig, + // Network Interceptor v2 body capture. Uses the global Requestly.Network (web-sdk UMD injected + // separately), so no npm deps to resolve — commonPlugins (no nodeResolve) is sufficient. + input: "src/page-scripts/networkBodyRecorder.js", + output: { + file: `${OUTPUT_DIR}/page-scripts/networkBodyRecorder.ps.js`, + format: "iife", + }, + plugins: commonPlugins, + }, ]; diff --git a/browser-extension/mv3/src/content-scripts/client/pageScriptMessageListener.ts b/browser-extension/mv3/src/content-scripts/client/pageScriptMessageListener.ts index 6f8e5ddc08..5b311410df 100644 --- a/browser-extension/mv3/src/content-scripts/client/pageScriptMessageListener.ts +++ b/browser-extension/mv3/src/content-scripts/client/pageScriptMessageListener.ts @@ -1,6 +1,20 @@ import { CLIENT_MESSAGES, EXTENSION_MESSAGES } from "common/constants"; export const initPageScriptMessageListener = () => { + // SW → page relay for Network Interceptor v2 body capture start/stop control signals. + // The page script (networkBodyRecorder, MAIN world) listens for source "requestly:extension". + chrome.runtime.onMessage.addListener((message) => { + if ( + message?.action === EXTENSION_MESSAGES.START_NETWORK_BODY_CAPTURE || + message?.action === EXTENSION_MESSAGES.STOP_NETWORK_BODY_CAPTURE + ) { + window.postMessage( + { source: "requestly:extension", action: message.action, payload: message.payload }, + window.location.href + ); + } + }); + window.addEventListener("message", function (event) { if (event.source !== window || event.data.source !== "requestly:client") { return; @@ -41,6 +55,14 @@ export const initPageScriptMessageListener = () => { case EXTENSION_MESSAGES.CACHE_SHARED_STATE: chrome.runtime.sendMessage(event.data); break; + case CLIENT_MESSAGES.NETWORK_BODY_CAPTURED: + // Network Interceptor v2: forward a captured XHR/Fetch body+headers to the SW. + // Fire-and-forget; tabId is added in the SW from sender.tab.id. + chrome.runtime.sendMessage({ + action: CLIENT_MESSAGES.NETWORK_BODY_CAPTURED, + payload: event.data.payload, + }); + break; } }); }; diff --git a/browser-extension/mv3/src/page-scripts/networkBodyRecorder.js b/browser-extension/mv3/src/page-scripts/networkBodyRecorder.js new file mode 100644 index 0000000000..b281d87597 --- /dev/null +++ b/browser-extension/mv3/src/page-scripts/networkBodyRecorder.js @@ -0,0 +1,102 @@ +import { CLIENT_MESSAGES, EXTENSION_MESSAGES } from "common/constants"; + +/** + * MAIN-world page script for Network Interceptor v2 — body + header capture for XHR/Fetch. + * + * v1 captures all resource types via chrome.webRequest in the service worker, but webRequest + * cannot read bodies. For XHR/Fetch we instead use the web-sdk Network interceptor (the same + * module session recording uses), which sees request + response headers AND bodies. The service + * worker hard-suppresses webRequest for xhr/fetch, so this is their sole source — no correlation. + * + * The web-sdk UMD (`libs/requestly-web-sdk.js`) is injected before this script and declares a + * top-level `var Requestly` (global binding), so we call `Requestly.Network.intercept(...)` + * directly — no import/bundle needed. (Same global-reference style as sessionRecorderHelper.js.) + * + * Caps: Network.intercept has no size options — those live only on SessionRecorder — so we port + * its `#filterOutLargeNetworkValues` here (media-skip + per-body maxPayloadSize, with error flags). + */ + +// Mirrors web-sdk RQNetworkEventErrorCodes. +const REQUEST_TOO_LARGE = 101; +const RESPONSE_TOO_LARGE = 102; + +const isMediaContentType = (contentType) => /^(image|audio|video)\/.+$/gi.test(contentType || ""); + +const sizeInBytes = (value) => { + if (!value) return NaN; + let str = value; + if (typeof value !== "string") { + try { + str = JSON.stringify(value); + } catch { + return NaN; + } + } + return str.length; +}; + +// Clear over-cap / media bodies in place and collect error codes — a port of the web-sdk's +// SessionRecorder.#filterOutLargeNetworkValues so behaviour matches session recording. +const applyCaps = (data, cfg) => { + const errors = []; + const payload = { ...data }; + + if (cfg.ignoreMediaResponse && isMediaContentType(payload.contentType)) { + payload.response = ""; + } else if (sizeInBytes(payload.response) > cfg.maxPayloadSize) { + payload.response = ""; + errors.push(RESPONSE_TOO_LARGE); + } + + if (sizeInBytes(payload.requestData) > cfg.maxPayloadSize) { + payload.requestData = ""; + errors.push(REQUEST_TOO_LARGE); + } + + payload.errors = errors; + return payload; +}; + +(() => { + let enabled = false; + let registered = false; + let cfg = { maxPayloadSize: 100 * 1024, ignoreMediaResponse: true }; + + const postToExtension = (action, payload) => { + window.postMessage({ source: "requestly:client", action, payload }, window.location.href); + }; + + const registerInterceptorOnce = () => { + if (registered) return; + // The web-sdk UMD declares a top-level `var Requestly`; reference it bare (same as + // sessionRecorderHelper.js does with `Requestly.SessionRecorder`) rather than via window, + // so it resolves the global binding regardless of how the file scope reflects onto window. + if (typeof Requestly === "undefined" || !Requestly?.Network?.intercept) return; // UMD not present yet + registered = true; + // overrideResponse=false → observe only, never block/alter the real response. + Requestly.Network.intercept( + /.*/, + (data) => { + if (!enabled) return; + postToExtension(CLIENT_MESSAGES.NETWORK_BODY_CAPTURED, applyCaps(data, cfg)); + }, + false + ); + }; + + window.addEventListener("message", (event) => { + if (event.source !== window || event.data?.source !== "requestly:extension") return; + + if (event.data.action === EXTENSION_MESSAGES.START_NETWORK_BODY_CAPTURE) { + const incoming = event.data.payload || {}; + if (typeof incoming.maxPayloadSize === "number") cfg.maxPayloadSize = incoming.maxPayloadSize; + if (typeof incoming.ignoreMediaResponse === "boolean") cfg.ignoreMediaResponse = incoming.ignoreMediaResponse; + enabled = true; + registerInterceptorOnce(); + } else if (event.data.action === EXTENSION_MESSAGES.STOP_NETWORK_BODY_CAPTURE) { + // Gate the callback off — do NOT call Network.clearInterceptors() (it would nuke every + // SDK consumer on the page, e.g. session recording). + enabled = false; + } + }); +})(); diff --git a/browser-extension/mv3/src/service-worker/services/messageHandler/listener.ts b/browser-extension/mv3/src/service-worker/services/messageHandler/listener.ts index 997f102ce0..8b4bd23925 100644 --- a/browser-extension/mv3/src/service-worker/services/messageHandler/listener.ts +++ b/browser-extension/mv3/src/service-worker/services/messageHandler/listener.ts @@ -39,6 +39,7 @@ import { getNetworkRecordingState, getNetworkRecordingSummary, handleNetworkRecordingOnClientPageLoad, + onNetworkBodyCaptured, } from "../networkRecording"; export const initExternalMessageListener = () => { @@ -103,6 +104,11 @@ export const initMessageHandler = () => { onSessionRecordingStoppedNotification(sender.tab.id); break; + case CLIENT_MESSAGES.NETWORK_BODY_CAPTURED: + // Network Interceptor v2: an XHR/Fetch body+headers captured by the SDK page script. + onNetworkBodyCaptured(sender.tab?.id, message.payload); + break; + case EXTENSION_MESSAGES.START_RECORDING_EXPLICITLY: startRecordingExplicitly(message.tab ?? sender.tab, message.showWidget); break; diff --git a/browser-extension/mv3/src/service-worker/services/networkRecording/harBuilder.ts b/browser-extension/mv3/src/service-worker/services/networkRecording/harBuilder.ts index 8ec9381bdd..22eaaeea5a 100644 --- a/browser-extension/mv3/src/service-worker/services/networkRecording/harBuilder.ts +++ b/browser-extension/mv3/src/service-worker/services/networkRecording/harBuilder.ts @@ -1,7 +1,33 @@ import { Entry, Header, QueryString } from "har-format"; -/** HAR Entry plus our `_error` extension (set on failed/aborted requests). */ -export type NetworkHarEntry = Entry & { _error?: string }; +/** + * HAR Entry plus our extensions: + * - `_error`: set on failed/aborted requests (webRequest path). + * - `_truncated`: per-body cap codes when the SDK page script dropped an over-size / media body + * (101 = request too large, 102 = response too large). Lets LTS tell "dropped (too large)" + * from a genuinely empty body. Mirrors the web-sdk RQNetworkEventErrorCodes. + */ +export type NetworkHarEntry = Entry & { _error?: string; _truncated?: number[] }; + +/** + * Shape posted by the networkBodyRecorder page script (derived from the web-sdk + * Network interceptor callback). Headers are a plain name→value record. + */ +export interface SdkNetworkPayload { + api?: string; // "xmlhttprequest" | "fetch" + method: string; + url: string; + status: number; + statusText?: string; + requestHeaders?: Record; + responseHeaders?: Record; + requestData?: unknown; + response?: unknown; + contentType?: string; + responseTime?: number; + responseURL?: string; + errors?: number[]; +} /** * The HAR _resourceType enum (Chrome DevTools convention) differs from @@ -173,3 +199,85 @@ export const buildErrorEntry = ( _error: error, }; }; + +/** Record headers → HAR Header[]. */ +const recordToHarHeaders = (headers: Record | undefined): Header[] => + Object.entries(headers || {}).map(([name, value]) => ({ name, value: value ?? "" })); + +/** Coerce an SDK body (string | object | undefined) to a HAR body string. */ +const bodyToText = (body: unknown): string | undefined => { + if (body === undefined || body === null || body === "") return undefined; + if (typeof body === "string") return body; + try { + return JSON.stringify(body); + } catch { + return undefined; + } +}; + +const byteLength = (text: string | undefined): number => (text === undefined ? -1 : text.length); + +/** + * Build a HAR 1.2 Entry from an SDK (web-sdk Network interceptor) payload — the v2 source for + * XHR/Fetch. Unlike the webRequest path this carries request + response BODIES and headers, with + * no correlation needed (the payload is self-complete). `requestId` is extension-assigned. + */ +export const buildSdkEntry = (payload: SdkNetworkPayload, requestId: string): NetworkHarEntry => { + const responseTime = Math.max(0, Math.round(payload.responseTime ?? 0)); + // The SDK doesn't give a start timestamp; derive it so startedDateTime + time are consistent. + const startTime = Date.now() - responseTime; + + const requestText = bodyToText(payload.requestData); + const responseText = bodyToText(payload.response); + const requestContentType = payload.requestHeaders + ? payload.requestHeaders["content-type"] || payload.requestHeaders["Content-Type"] + : undefined; + + const entry: NetworkHarEntry = { + startedDateTime: new Date(startTime).toISOString(), + time: responseTime, + request: { + method: payload.method, + url: payload.url, + httpVersion: "", + cookies: [], + headers: recordToHarHeaders(payload.requestHeaders), + queryString: parseQueryString(payload.url), + headersSize: -1, + bodySize: requestText !== undefined ? requestText.length : -1, + }, + response: { + status: payload.status, + statusText: payload.statusText || "", + httpVersion: "", + cookies: [], + headers: recordToHarHeaders(payload.responseHeaders), + content: { + size: byteLength(responseText), + mimeType: payload.contentType || "", + }, + redirectURL: payload.responseURL && payload.responseURL !== payload.url ? payload.responseURL : "", + headersSize: -1, + bodySize: byteLength(responseText), + }, + cache: {}, + timings: { send: 0, wait: responseTime, receive: 0 }, + _resourceType: "xhr", // SDK only sees xhr/fetch; single-bucket to match v1 (api field has the split) + _request_id: requestId, + _fromCache: null, + }; + + // Only set postData when there's an actual request body (strict HAR: omit otherwise). + if (requestText !== undefined) { + entry.request.postData = { mimeType: requestContentType || "", text: requestText }; + } + // content.text only when a body survived the cap. + if (responseText !== undefined) { + entry.response.content.text = responseText; + } + if (payload.errors && payload.errors.length) { + entry._truncated = payload.errors; + } + + return entry; +}; diff --git a/browser-extension/mv3/src/service-worker/services/networkRecording/index.ts b/browser-extension/mv3/src/service-worker/services/networkRecording/index.ts index c16950ea50..dfc32f696e 100644 --- a/browser-extension/mv3/src/service-worker/services/networkRecording/index.ts +++ b/browser-extension/mv3/src/service-worker/services/networkRecording/index.ts @@ -1,15 +1,34 @@ import { tabService, TAB_SERVICE_DATA } from "../tabService"; -import { CLIENT_MESSAGES } from "common/constants"; +import { CLIENT_MESSAGES, EXTENSION_MESSAGES } from "common/constants"; import { ChangeType } from "common/storage"; +import { injectWebAccessibleScript } from "../utils"; import { isExtensionEnabled } from "../../../utils"; import { onVariableChange, Variable } from "../../variable"; -import { buildCompletedEntry, buildErrorEntry, CorrelationData, NetworkHarEntry } from "./harBuilder"; +import { + buildCompletedEntry, + buildErrorEntry, + buildSdkEntry, + CorrelationData, + NetworkHarEntry, + SdkNetworkPayload, +} from "./harBuilder"; + +// Recording config from the LTS start call. All optional. +// - maxDuration: time cap (no cap when omitted; see isOverMaxDuration). +// - maxPayloadSize: per-body cap (bytes) applied to SDK-captured request/response bodies (v2); +// defaults to DEFAULT_MAX_PAYLOAD_SIZE when omitted. +export interface NetworkRecordingConfig { + maxDuration?: number; + maxPayloadSize?: number; +} + +const DEFAULT_MAX_PAYLOAD_SIZE = 100 * 1024; // 100 KB, matches the web-sdk SessionRecorder default interface NetworkRecordingState { targetTabId: number; url: string; startTime: number; - config: { maxDuration?: number }; + config: NetworkRecordingConfig; // The LTS tab/window that started the recording. On stop we return focus here. // Both may be gone by stop time (user closed the tab/window mid-recording). senderTabId?: number; @@ -116,8 +135,15 @@ const stopKeepaliveIfIdle = () => { // to details.timeStamp + empty request headers. Expected, not an error. // 4. Orphans (started, never completed/errored — cancelled, navigated away) are swept by the // CORRELATION_TTL_MS pass in the keepalive ping. +// +// v2: XHR/Fetch are captured solely by the web-sdk Network interceptor (page script) — it carries +// headers AND bodies. We hard-suppress the webRequest path for "xmlhttprequest" (the resource type +// for both XHR and fetch) so there's exactly one source and no correlation needed for them. +const isSdkOwnedRequest = (type: chrome.webRequest.ResourceType): boolean => type === "xmlhttprequest"; + const onBeforeSendHeaders = (details: chrome.webRequest.WebRequestHeadersDetails) => { if (!activeRecordings.has(details.tabId)) return; + if (isSdkOwnedRequest(details.type)) return; // SDK owns xhr/fetch; don't populate correlationMap for them correlationMap.set(details.requestId, { startTime: details.timeStamp, requestHeaders: details.requestHeaders, @@ -139,6 +165,8 @@ const onRequestCompleted = (details: chrome.webRequest.WebResponseCacheDetails) return; } + if (isSdkOwnedRequest(details.type)) return; // xhr/fetch come from the SDK page script, not webRequest + const correlation = correlationMap.get(details.requestId); correlationMap.delete(details.requestId); @@ -153,6 +181,8 @@ const onRequestError = (details: chrome.webRequest.WebResponseErrorDetails) => { const recording = activeRecordings.get(details.tabId); if (!recording) return; + if (isSdkOwnedRequest(details.type)) return; // xhr/fetch come from the SDK page script, not webRequest + const correlation = correlationMap.get(details.requestId); correlationMap.delete(details.requestId); @@ -185,6 +215,21 @@ const deliverEntry = (tabId: number, entry: NetworkHarEntry) => { }); }; +/** + * v2: an XHR/Fetch body+headers captured by the SDK page script (networkBodyRecorder) arrives + * here via the content-script relay. These are the SOLE source for xhr/fetch (webRequest is + * hard-suppressed for them), so we just build the HAR entry and feed the same buffer + stream + * path as v1 — no correlation. `tabId` comes from the message sender. + */ +export const onNetworkBodyCaptured = (tabId: number | undefined, payload: SdkNetworkPayload | undefined) => { + if (tabId === undefined || !payload) return; + if (!activeRecordings.has(tabId)) return; // not recording this tab (stale page script / race) + + const entry = buildSdkEntry(payload, nextRequestId()); + recordingEntries.get(tabId)?.push(entry); + deliverEntry(tabId, entry); +}; + // Why a recording ended — drives the message the side panel shows. // user – the user clicked Stop in the panel (no banner; just "Stopped") // max-duration – config.maxDuration elapsed (amber banner) @@ -322,6 +367,44 @@ export const initNetworkRecordingPort = () => { }); }; +// --- v2 body capture: inject the web-sdk Network interceptor into the recorded tab ---------- +// The web-sdk UMD exposes the global `Requestly` (incl. Network); networkBodyRecorder.ps.js uses +// it. Both are MAIN-world. executeScript is one-shot, so we re-inject on each navigation of the +// recorded tab (handled by chrome.webNavigation.onCommitted below). The content-script relay +// forwards the start/stop control signals to the page script. + +const injectBodyRecorder = async (tabId: number, frameId = 0) => { + try { + // 1) web-sdk UMD lib (exposes global Requestly.Network) + await injectWebAccessibleScript("libs/requestly-web-sdk.js", { tabId, frameIds: [frameId] }); + // 2) our page script that registers the interceptor + await injectWebAccessibleScript("page-scripts/networkBodyRecorder.ps.js", { tabId, frameIds: [frameId] }); + // 3) start signal with the resolved caps (relayed by the content script to the page) + sendBodyCaptureSignal(tabId, EXTENSION_MESSAGES.START_NETWORK_BODY_CAPTURE); + } catch { + // Injection can fail on restricted pages (e.g. chrome://, strict CSP) — body capture is + // best-effort; webRequest still covers non-xhr/fetch. Don't break the recording. + } +}; + +const sendBodyCaptureSignal = (tabId: number, action: string) => { + const recording = activeRecordings.get(tabId); + const payload = + action === EXTENSION_MESSAGES.START_NETWORK_BODY_CAPTURE + ? { maxPayloadSize: recording?.config.maxPayloadSize, ignoreMediaResponse: true } + : undefined; + // Relayed by the client content script → page (source "requestly:extension"). + chrome.tabs.sendMessage(tabId, { action, payload }).catch(() => {}); +}; + +// Re-inject on navigation of a recorded tab (executeScript is one-shot). Single-tab scoped, +// matching v1's model. Gated to active recordings; main frame only. +chrome.webNavigation.onCommitted.addListener((details) => { + if (details.frameId !== 0) return; + if (!activeRecordings.has(details.tabId)) return; + injectBodyRecorder(details.tabId, 0); +}); + // Synchronously-readable copy of IS_EXTENSION_ENABLED, so startNetworkRecording can reject a start // while the extension is off WITHOUT an async storage read — an await there would push // sidePanel.open() past its user-gesture window and the panel would never open. Seeded at init and @@ -372,12 +455,13 @@ const openPanel = (tabId: number) => { export const startNetworkRecording = ( url: string, - config: { maxDuration?: number } = {}, + config: NetworkRecordingConfig = {}, sender?: { tabId?: number; windowId?: number } ): Promise<{ success: boolean; targetTabId?: number; error?: string }> => { + // NOTE: kept synchronous up to chrome.tabs.create (no await) so the LTS sendMessage user gesture + // survives to the openPanel() call — chrome.sidePanel.open() requires an in-gesture call stack. // Reject a start while the extension is off, so the UI never says "disabled" with a live - // recording. Read from the in-memory cache (NOT an await) to keep the path to openPanel - // synchronous — see isExtensionEnabledCache. + // recording. Read from the in-memory cache (NOT an await) to keep that path synchronous. if (!isExtensionEnabledCache) { return Promise.resolve({ success: false, @@ -400,7 +484,9 @@ export const startNetworkRecording = ( targetTabId: tab.id, url, startTime: Date.now(), - config, + // Resolve maxPayloadSize to its default now so the body page script (v2) can read a + // concrete cap off state without re-defaulting. maxDuration stays undefined = no cap. + config: { ...config, maxPayloadSize: config.maxPayloadSize ?? DEFAULT_MAX_PAYLOAD_SIZE }, senderTabId: sender?.tabId, senderWindowId: sender?.windowId, }; @@ -418,7 +504,15 @@ export const startNetworkRecording = ( addWebRequestListeners(); startKeepalive(); + // Open the panel here, synchronously on the external-message path. chrome.sidePanel.open() + // requires a user gesture and must run within its call stack — the LTS sendMessage provides + // that gesture, but only as long as nothing awaits before this point (hence no async + // isExtensionEnabled check above). handleNetworkRecordingOnClientPageLoad re-opens it on + // later navigations of the recorded tab as a backstop. openPanel(tab.id); + // v2: the body recorder is injected via webNavigation.onCommitted, which fires for this new + // tab's initial navigation (and every later one). No explicit inject here — it would be too + // early (the document isn't committed yet). resolve({ success: true, targetTabId: tab.id }); }); @@ -506,6 +600,8 @@ export const stopNetworkRecording = ( streamCompleteToPorts(targetTabId); // Tell the side panel why it ended so it can show the right stopped state / banner. notifyPanelEnded(targetTabId, reason); + // v2: tell the page script to stop capturing bodies (gates its callback off; no clearInterceptors). + sendBodyCaptureSignal(targetTabId, EXTENSION_MESSAGES.STOP_NETWORK_BODY_CAPTURE); activeRecordings.delete(targetTabId); recordingEntries.delete(targetTabId);