From e56e27e5e57a14170668a6fec1a8e5c1f5b21daa Mon Sep 17 00:00:00 2001 From: "Tommy D. Rossi" Date: Fri, 23 Jan 2026 16:10:40 +0100 Subject: [PATCH] WIP: screen recording via chrome.tabCapture Work in progress for navigation-safe screen recording. Implemented: - tabCapture permission in manifest - Recording handlers in extension background.ts - Recording endpoints in relay server - Binary WebSocket streaming for video chunks - Protocol types for recording messages - Executor integration Needs: - Offscreen document implementation (tabCapture not available in service workers) - Connect background.ts to offscreen document via messages - Update manifest for offscreen document The offscreen.ts/html files are scaffolding for this next step. --- extension/manifest.json | 2 +- extension/src/background.ts | 355 ++++++++++++++++++++++++++++- extension/src/offscreen.html | 9 + extension/src/offscreen.ts | 224 ++++++++++++++++++ extension/src/types.ts | 13 ++ playwriter/src/cdp-relay.ts | 226 +++++++++++++++++- playwriter/src/executor.ts | 34 +++ playwriter/src/index.ts | 2 + playwriter/src/mcp.test.ts | 151 ++++++++++++ playwriter/src/protocol.ts | 102 ++++++++- playwriter/src/screen-recording.ts | 156 +++++++++++++ playwriter/src/skill.md | 33 +++ 12 files changed, 1303 insertions(+), 4 deletions(-) create mode 100644 extension/src/offscreen.html create mode 100644 extension/src/offscreen.ts create mode 100644 playwriter/src/screen-recording.ts diff --git a/extension/manifest.json b/extension/manifest.json index 875d7c7..7cb7923 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -3,7 +3,7 @@ "name": "Playwriter MCP", "version": "0.0.69", "description": "Automate your Browser using Cursor, Claude, VS Code. More capable and context efficient than Playwright MCP.", - "permissions": ["debugger", "tabGroups", "contextMenus", "tabs"], + "permissions": ["debugger", "tabGroups", "contextMenus", "tabs", "tabCapture", "offscreen"], "host_permissions": [""], "background": { "service_worker": "background.js", diff --git a/extension/src/background.ts b/extension/src/background.ts index aa4a42c..af0bff5 100644 --- a/extension/src/background.ts +++ b/extension/src/background.ts @@ -1,7 +1,7 @@ declare const process: { env: { PLAYWRITER_PORT: string } } import { createStore } from 'zustand/vanilla' -import type { ExtensionState, ConnectionState, TabState, TabInfo } from './types' +import type { ExtensionState, ConnectionState, TabState, TabInfo, RecordingInfo } from './types' import type { CDPEvent, Protocol } from 'playwriter/src/cdp-types' import type { ExtensionCommandMessage, ExtensionResponseMessage } from 'playwriter/src/protocol' @@ -16,6 +16,9 @@ let childSessions: Map = new Map() let nextSessionId = 1 let tabGroupQueue: Promise = Promise.resolve() +// Active recordings - kept outside store since MediaRecorder/MediaStream can't be serialized +const activeRecordings: Map = new Map() + class ConnectionManager { ws: WebSocket | null = null private connectionPromise: Promise | null = null @@ -176,6 +179,51 @@ class ConnectionManager { return } + // Handle recording commands + if (message.method === 'startRecording') { + try { + const result = await handleStartRecording(message.params) + sendMessage({ id: message.id, result }) + } catch (error: any) { + logger.error('Failed to start recording:', error) + sendMessage({ id: message.id, result: { success: false, error: error.message } }) + } + return + } + + if (message.method === 'stopRecording') { + try { + const result = await handleStopRecording(message.params) + sendMessage({ id: message.id, result }) + } catch (error: any) { + logger.error('Failed to stop recording:', error) + sendMessage({ id: message.id, result: { success: false, error: error.message } }) + } + return + } + + if (message.method === 'isRecording') { + try { + const result = handleIsRecording(message.params) + sendMessage({ id: message.id, result }) + } catch (error: any) { + logger.error('Failed to check recording status:', error) + sendMessage({ id: message.id, result: { isRecording: false } }) + } + return + } + + if (message.method === 'cancelRecording') { + try { + const result = handleCancelRecording(message.params) + sendMessage({ id: message.id, result }) + } catch (error: any) { + logger.error('Failed to cancel recording:', error) + sendMessage({ id: message.id, result: { success: false, error: error.message } }) + } + return + } + const response: ExtensionResponseMessage = { id: message.id } try { response.result = await handleCommand(message as ExtensionCommandMessage) @@ -845,6 +893,9 @@ function detachTab(tabId: number, shouldDetachDebugger: boolean): void { return } + // Clean up any active recording for this tab + cleanupRecordingForTab(tabId) + logger.warn(`DISCONNECT: detachTab tabId=${tabId} shouldDetach=${shouldDetachDebugger} stack=${getCallStack()}`) // Only send detach event if tab was fully attached (has sessionId/targetId) @@ -985,6 +1036,308 @@ async function disconnectEverything(): Promise { // WS connection is maintained - maintainConnection handles it } +// ============= Recording Functions ============= + +function resolveTabIdFromSessionId(sessionId?: string): number | undefined { + if (!sessionId) { + // Return the first connected tab + for (const [tabId, tab] of store.getState().tabs) { + if (tab.state === 'connected') { + return tabId + } + } + return undefined + } + + const found = getTabBySessionId(sessionId) + return found?.tabId +} + +interface StartRecordingParams { + sessionId?: string + frameRate?: number + audio?: boolean + videoBitsPerSecond?: number + audioBitsPerSecond?: number +} + +async function handleStartRecording(params: StartRecordingParams): Promise<{ success: true; tabId: number; startedAt: number } | { success: false; error: string }> { + const tabId = resolveTabIdFromSessionId(params.sessionId) + if (!tabId) { + return { success: false, error: 'No connected tab found for recording' } + } + + if (activeRecordings.has(tabId)) { + return { success: false, error: 'Recording already in progress for this tab' } + } + + const tabInfo = store.getState().tabs.get(tabId) + if (!tabInfo || tabInfo.state !== 'connected') { + return { success: false, error: 'Tab is not connected' } + } + + logger.debug('Starting recording for tab:', tabId, 'params:', params) + + try { + // Use chrome.tabCapture to capture the tab + const stream = await new Promise((resolve, reject) => { + chrome.tabCapture.capture( + { + audio: params.audio ?? false, + video: true, + videoConstraints: { + mandatory: { + minFrameRate: params.frameRate ?? 30, + maxFrameRate: params.frameRate ?? 30, + }, + }, + }, + (capturedStream) => { + if (chrome.runtime.lastError) { + reject(new Error(chrome.runtime.lastError.message)) + } else if (!capturedStream) { + reject(new Error('Failed to capture tab - no stream returned')) + } else { + resolve(capturedStream) + } + } + ) + }) + + // Determine best codec + const mimeType = MediaRecorder.isTypeSupported('video/webm;codecs=vp9,opus') + ? 'video/webm;codecs=vp9,opus' + : MediaRecorder.isTypeSupported('video/webm;codecs=vp8,opus') + ? 'video/webm;codecs=vp8,opus' + : 'video/webm' + + const recorder = new MediaRecorder(stream, { + mimeType, + videoBitsPerSecond: params.videoBitsPerSecond ?? 2500000, + audioBitsPerSecond: params.audioBitsPerSecond ?? 128000, + }) + + const startedAt = Date.now() + + // Store recording info + const recordingInfo: RecordingInfo = { + tabId, + startedAt, + recorder, + stream, + mimeType, + } + activeRecordings.set(tabId, recordingInfo) + + // Update tab state + store.setState((state) => { + const newTabs = new Map(state.tabs) + const existing = newTabs.get(tabId) + if (existing) { + newTabs.set(tabId, { ...existing, isRecording: true }) + } + return { tabs: newTabs } + }) + + // Send binary chunks as they become available + recorder.ondataavailable = async (event) => { + if (event.data.size > 0 && connectionManager.ws?.readyState === WebSocket.OPEN) { + try { + // Send metadata message first + sendMessage({ + method: 'recordingData', + params: { tabId }, + }) + // Then send binary data + const arrayBuffer = await event.data.arrayBuffer() + connectionManager.ws.send(arrayBuffer) + } catch (error: any) { + logger.error('Failed to send recording chunk:', error.message) + } + } + } + + recorder.onerror = (event: any) => { + logger.error('MediaRecorder error:', event.error?.message || 'Unknown error') + handleCancelRecording({ sessionId: params.sessionId }) + } + + recorder.onstop = () => { + logger.debug('MediaRecorder stopped for tab:', tabId) + // Stream tracks are stopped in handleStopRecording + } + + // Start recording with 1 second chunks + recorder.start(1000) + logger.debug('Recording started for tab:', tabId, 'mimeType:', mimeType) + + return { success: true, tabId, startedAt } + } catch (error: any) { + logger.error('Failed to start recording:', error) + return { success: false, error: error.message } + } +} + +interface StopRecordingParams { + sessionId?: string +} + +async function handleStopRecording(params: StopRecordingParams): Promise<{ success: true; tabId: number; duration: number } | { success: false; error: string }> { + const tabId = resolveTabIdFromSessionId(params.sessionId) + if (!tabId) { + return { success: false, error: 'No connected tab found' } + } + + const recording = activeRecordings.get(tabId) + if (!recording) { + return { success: false, error: 'No active recording for this tab' } + } + + logger.debug('Stopping recording for tab:', tabId) + + try { + const { recorder, stream, startedAt } = recording + + // Wait for recorder to finish + await new Promise((resolve) => { + const originalOnStop = recorder.onstop + recorder.onstop = (event: Event) => { + if (originalOnStop) { + originalOnStop.call(recorder, event) + } + resolve() + } + + if (recorder.state !== 'inactive') { + recorder.stop() + } else { + resolve() + } + }) + + // Stop all tracks + stream.getTracks().forEach((track: MediaStreamTrack) => track.stop()) + + // Send final marker + if (connectionManager.ws?.readyState === WebSocket.OPEN) { + sendMessage({ + method: 'recordingData', + params: { tabId, final: true }, + }) + } + + const duration = Date.now() - startedAt + + // Clean up + activeRecordings.delete(tabId) + store.setState((state) => { + const newTabs = new Map(state.tabs) + const existing = newTabs.get(tabId) + if (existing) { + newTabs.set(tabId, { ...existing, isRecording: false }) + } + return { tabs: newTabs } + }) + + logger.debug('Recording stopped for tab:', tabId, 'duration:', duration) + return { success: true, tabId, duration } + } catch (error: any) { + logger.error('Failed to stop recording:', error) + return { success: false, error: error.message } + } +} + +interface IsRecordingParams { + sessionId?: string +} + +function handleIsRecording(params: IsRecordingParams): { isRecording: boolean; tabId?: number; startedAt?: number } { + const tabId = resolveTabIdFromSessionId(params.sessionId) + if (!tabId) { + return { isRecording: false } + } + + const recording = activeRecordings.get(tabId) + if (!recording) { + return { isRecording: false, tabId } + } + + return { + isRecording: recording.recorder.state === 'recording', + tabId, + startedAt: recording.startedAt, + } +} + +interface CancelRecordingParams { + sessionId?: string +} + +function handleCancelRecording(params: CancelRecordingParams): { success: boolean; error?: string } { + const tabId = resolveTabIdFromSessionId(params.sessionId) + if (!tabId) { + return { success: false, error: 'No connected tab found' } + } + + const recording = activeRecordings.get(tabId) + if (!recording) { + return { success: true } // Already not recording + } + + logger.debug('Cancelling recording for tab:', tabId) + + try { + const { recorder, stream } = recording + + if (recorder.state !== 'inactive') { + recorder.stop() + } + stream.getTracks().forEach((track: MediaStreamTrack) => track.stop()) + + activeRecordings.delete(tabId) + store.setState((state) => { + const newTabs = new Map(state.tabs) + const existing = newTabs.get(tabId) + if (existing) { + newTabs.set(tabId, { ...existing, isRecording: false }) + } + return { tabs: newTabs } + }) + + // Send cancel marker + if (connectionManager.ws?.readyState === WebSocket.OPEN) { + sendMessage({ + method: 'recordingCancelled', + params: { tabId }, + }) + } + + return { success: true } + } catch (error: any) { + logger.error('Failed to cancel recording:', error) + return { success: false, error: error.message } + } +} + +// Clean up recordings when tab is disconnected +function cleanupRecordingForTab(tabId: number): void { + const recording = activeRecordings.get(tabId) + if (recording) { + logger.debug('Cleaning up recording for disconnected tab:', tabId) + try { + if (recording.recorder.state !== 'inactive') { + recording.recorder.stop() + } + recording.stream.getTracks().forEach((track: MediaStreamTrack) => track.stop()) + } catch (e) { + logger.debug('Error cleaning up recording:', e) + } + activeRecordings.delete(tabId) + } +} + +// ============= End Recording Functions ============= + async function resetDebugger(): Promise { let targets = await chrome.debugger.getTargets() targets = targets.filter((x) => x.tabId && x.attached) diff --git a/extension/src/offscreen.html b/extension/src/offscreen.html new file mode 100644 index 0000000..500f4e3 --- /dev/null +++ b/extension/src/offscreen.html @@ -0,0 +1,9 @@ + + + + Playwriter Offscreen + + + + + diff --git a/extension/src/offscreen.ts b/extension/src/offscreen.ts new file mode 100644 index 0000000..aa64eb6 --- /dev/null +++ b/extension/src/offscreen.ts @@ -0,0 +1,224 @@ +/** + * Offscreen document for Playwriter extension. + * Handles operations that require DOM APIs not available in service workers: + * - Screen recording via chrome.tabCapture + MediaRecorder + * - Future: audio processing, canvas operations, etc. + */ + +interface RecordingState { + recorder: MediaRecorder | null + stream: MediaStream | null + startedAt: number + tabId: number +} + +let recording: RecordingState | null = null + +// Message types +type StartRecordingMessage = { + action: 'startRecording' + tabId: number + streamId: string + frameRate?: number + videoBitsPerSecond?: number + audioBitsPerSecond?: number + audio?: boolean +} + +type StopRecordingMessage = { + action: 'stopRecording' +} + +type IsRecordingMessage = { + action: 'isRecording' +} + +type CancelRecordingMessage = { + action: 'cancelRecording' +} + +type OffscreenMessage = StartRecordingMessage | StopRecordingMessage | IsRecordingMessage | CancelRecordingMessage + +chrome.runtime.onMessage.addListener((message: OffscreenMessage, _sender, sendResponse) => { + handleMessage(message).then(sendResponse) + return true // Keep channel open for async response +}) + +async function handleMessage(message: OffscreenMessage): Promise { + switch (message.action) { + case 'startRecording': + return handleStartRecording(message) + case 'stopRecording': + return handleStopRecording() + case 'isRecording': + return handleIsRecording() + case 'cancelRecording': + return handleCancelRecording() + default: + return { success: false, error: 'Unknown action' } + } +} + +async function handleStartRecording(params: StartRecordingMessage): Promise { + if (recording) { + return { success: false, error: 'Recording already in progress' } + } + + try { + // Get media stream from the streamId provided by tabCapture.getMediaStreamId + const stream = await navigator.mediaDevices.getUserMedia({ + audio: params.audio ? { + mandatory: { + chromeMediaSource: 'tab', + chromeMediaSourceId: params.streamId, + } + } as any : false, + video: { + mandatory: { + chromeMediaSource: 'tab', + chromeMediaSourceId: params.streamId, + minFrameRate: params.frameRate || 30, + maxFrameRate: params.frameRate || 30, + } + } as any, + }) + + // Determine best codec + const mimeType = MediaRecorder.isTypeSupported('video/webm;codecs=vp9,opus') + ? 'video/webm;codecs=vp9,opus' + : MediaRecorder.isTypeSupported('video/webm;codecs=vp8,opus') + ? 'video/webm;codecs=vp8,opus' + : 'video/webm' + + const recorder = new MediaRecorder(stream, { + mimeType, + videoBitsPerSecond: params.videoBitsPerSecond || 2500000, + audioBitsPerSecond: params.audioBitsPerSecond || 128000, + }) + + const startedAt = Date.now() + + recording = { + recorder, + stream, + startedAt, + tabId: params.tabId, + } + + // Send chunks to service worker + recorder.ondataavailable = async (event) => { + if (event.data.size > 0) { + // Convert blob to array buffer and send to service worker + const arrayBuffer = await event.data.arrayBuffer() + const uint8Array = new Uint8Array(arrayBuffer) + chrome.runtime.sendMessage({ + action: 'recordingChunk', + tabId: params.tabId, + data: Array.from(uint8Array), // Convert to regular array for message passing + }) + } + } + + recorder.onerror = (event: any) => { + console.error('MediaRecorder error:', event.error) + handleCancelRecording() + } + + recorder.onstop = () => { + console.log('MediaRecorder stopped') + } + + // Start with 1 second chunks + recorder.start(1000) + + return { success: true, tabId: params.tabId, startedAt, mimeType } + } catch (error: any) { + console.error('Failed to start recording:', error) + return { success: false, error: error.message } + } +} + +async function handleStopRecording(): Promise { + if (!recording) { + return { success: false, error: 'No active recording' } + } + + try { + const { recorder, stream, startedAt, tabId } = recording + + // Stop recorder and wait for final data + await new Promise((resolve) => { + const originalOnStop = recorder.onstop + recorder.onstop = (event) => { + if (originalOnStop) { + originalOnStop.call(recorder, event) + } + resolve() + } + if (recorder.state !== 'inactive') { + recorder.stop() + } else { + resolve() + } + }) + + // Stop all tracks + stream.getTracks().forEach((track) => track.stop()) + + const duration = Date.now() - startedAt + + // Send final marker + chrome.runtime.sendMessage({ + action: 'recordingChunk', + tabId, + final: true, + }) + + recording = null + + return { success: true, tabId, duration } + } catch (error: any) { + console.error('Failed to stop recording:', error) + return { success: false, error: error.message } + } +} + +function handleIsRecording(): any { + if (!recording) { + return { isRecording: false } + } + return { + isRecording: recording.recorder?.state === 'recording', + tabId: recording.tabId, + startedAt: recording.startedAt, + } +} + +function handleCancelRecording(): any { + if (!recording) { + return { success: true } + } + + try { + const { recorder, stream, tabId } = recording + + if (recorder.state !== 'inactive') { + recorder.stop() + } + stream.getTracks().forEach((track) => track.stop()) + + chrome.runtime.sendMessage({ + action: 'recordingCancelled', + tabId, + }) + + recording = null + + return { success: true } + } catch (error: any) { + console.error('Failed to cancel recording:', error) + return { success: false, error: error.message } + } +} + +console.log('Playwriter offscreen document loaded') diff --git a/extension/src/types.ts b/extension/src/types.ts index 011c466..79434f7 100644 --- a/extension/src/types.ts +++ b/extension/src/types.ts @@ -8,6 +8,7 @@ export interface TabInfo { errorText?: string pinnedCount?: number attachOrder?: number + isRecording?: boolean } export interface ExtensionState { @@ -16,3 +17,15 @@ export interface ExtensionState { currentTabId: number | undefined errorText: string | undefined } + +/** + * Recording state - kept separate from store since MediaRecorder/MediaStream can't be serialized. + * Note: MediaRecorder and MediaStream types are available in the extension's browser context. + */ +export interface RecordingInfo { + tabId: number + startedAt: number + recorder: any // MediaRecorder - using any to avoid DOM type dependency + stream: any // MediaStream - using any to avoid DOM type dependency + mimeType: string +} diff --git a/playwriter/src/cdp-relay.ts b/playwriter/src/cdp-relay.ts index 25f65b0..0264822 100644 --- a/playwriter/src/cdp-relay.ts +++ b/playwriter/src/cdp-relay.ts @@ -6,7 +6,9 @@ import { createNodeWebSocket } from '@hono/node-ws' import type { WSContext } from 'hono/ws' import type { Protocol } from './cdp-types.js' import type { CDPCommand, CDPResponseBase, CDPEventBase, CDPEventFor, RelayServerEvents } from './cdp-types.js' -import type { ExtensionMessage, ExtensionEventMessage } from './protocol.js' +import type { ExtensionMessage, ExtensionEventMessage, StartRecordingParams, StopRecordingParams, IsRecordingParams, CancelRecordingParams, StartRecordingResult, StopRecordingResult, IsRecordingResult, CancelRecordingResult } from './protocol.js' +import fs from 'node:fs' +import path from 'node:path' import pc from 'picocolors' import { EventEmitter } from 'node:events' import { VERSION } from './utils.js' @@ -89,6 +91,16 @@ export async function startPlayWriterCDPRelayServer({ resolvedCdpLogger.log(entry) } + // Recording state - tracks active recordings and their accumulated chunks + type ActiveRecording = { + tabId: number + outputPath: string + chunks: Buffer[] + startedAt: number + resolveStop?: (result: StopRecordingResult) => void + } + const activeRecordings = new Map() + const playwrightClients = new Map() let extensionWs: WSContext | null = null @@ -824,6 +836,19 @@ export async function startPlayWriterCDPRelayServer({ }, async onMessage(event, ws) { + // Handle binary data (recording chunks) + if (event.data instanceof ArrayBuffer || Buffer.isBuffer(event.data)) { + const buffer = Buffer.isBuffer(event.data) ? event.data : Buffer.from(event.data) + // Find the recording that's waiting for data (the one we just received metadata for) + // The extension sends metadata first, then binary + for (const recording of activeRecordings.values()) { + recording.chunks.push(buffer) + logger?.log(pc.blue(`Received recording chunk for tab ${recording.tabId}: ${buffer.length} bytes (total chunks: ${recording.chunks.length})`)) + break // Only add to the first active recording + } + return + } + let message: ExtensionMessage try { @@ -854,6 +879,50 @@ export async function startPlayWriterCDPRelayServer({ const logFn = (logger as any)?.[level] || logger?.log const prefix = pc.yellow(`[Extension] [${level.toUpperCase()}]`) logFn?.(prefix, ...args) + } else if (message.method === 'recordingData') { + const { tabId, final } = (message as any).params + const recording = activeRecordings.get(tabId) + + if (recording && final) { + // This is the final marker - write all chunks to file + try { + const totalSize = recording.chunks.reduce((sum, chunk) => sum + chunk.length, 0) + const combined = Buffer.concat(recording.chunks) + fs.writeFileSync(recording.outputPath, combined) + + const duration = Date.now() - recording.startedAt + logger?.log(pc.green(`Recording saved: ${recording.outputPath} (${totalSize} bytes, ${duration}ms)`)) + + // Resolve the stop promise + if (recording.resolveStop) { + recording.resolveStop({ + success: true, + tabId, + duration, + path: recording.outputPath, + size: totalSize, + } as any) + } + } catch (error: any) { + logger?.error('Failed to write recording:', error) + if (recording.resolveStop) { + recording.resolveStop({ success: false, error: error.message }) + } + } + + activeRecordings.delete(tabId) + } + // Non-final recordingData is just a marker that binary follows - handled above + } else if (message.method === 'recordingCancelled') { + const { tabId } = (message as any).params + const recording = activeRecordings.get(tabId) + if (recording) { + logger?.log(pc.yellow(`Recording cancelled for tab ${tabId}`)) + if (recording.resolveStop) { + recording.resolveStop({ success: false, error: 'Recording was cancelled' }) + } + activeRecordings.delete(tabId) + } } else { const extensionEvent = message as ExtensionEventMessage @@ -1162,6 +1231,161 @@ export async function startPlayWriterCDPRelayServer({ return c.json({ next: nextSessionNumber }) }) + // ============================================================================ + // Recording Endpoints - For screen recording via chrome.tabCapture + // ============================================================================ + + app.post('/recording/start', async (c) => { + try { + const body = await c.req.json() as StartRecordingParams & { outputPath: string } + const { outputPath, ...params } = body + + if (!outputPath) { + return c.json({ success: false, error: 'outputPath is required' } as StartRecordingResult, 400) + } + + if (!extensionWs) { + return c.json({ success: false, error: 'Extension not connected' } as StartRecordingResult, 503) + } + + // Ensure output directory exists + const dir = path.dirname(outputPath) + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }) + } + + const result = await sendToExtension({ + method: 'startRecording', + params, + timeout: 10000, + }) as StartRecordingResult + + if (result.success) { + // Track this recording + activeRecordings.set(result.tabId, { + tabId: result.tabId, + outputPath, + chunks: [], + startedAt: result.startedAt, + }) + logger?.log(pc.green(`Recording started for tab ${result.tabId}, output: ${outputPath}`)) + } + + return c.json(result) + } catch (error: any) { + logger?.error('Start recording error:', error) + return c.json({ success: false, error: error.message } as StartRecordingResult, 500) + } + }) + + app.post('/recording/stop', async (c) => { + try { + const body = await c.req.json() as StopRecordingParams + const params = body + + if (!extensionWs) { + return c.json({ success: false, error: 'Extension not connected' } as StopRecordingResult, 503) + } + + // Create a promise that will be resolved when we receive the final chunk + const recording = Array.from(activeRecordings.values()).find(r => { + // Find the recording for this session + if (params.sessionId) { + const target = connectedTargets.get(params.sessionId) + // We don't have tabId in connectedTargets, so we'll match by the first one + return true + } + return true + }) + + if (!recording) { + return c.json({ success: false, error: 'No active recording found' } as StopRecordingResult, 404) + } + + // Set up promise to wait for final chunk + const finalPromise = new Promise((resolve) => { + recording.resolveStop = resolve + // Timeout after 30 seconds + setTimeout(() => { + if (recording.resolveStop) { + recording.resolveStop = undefined + resolve({ success: false, error: 'Timeout waiting for recording data' }) + } + }, 30000) + }) + + // Tell extension to stop recording + const result = await sendToExtension({ + method: 'stopRecording', + params, + timeout: 10000, + }) as StopRecordingResult + + if (!result.success) { + recording.resolveStop = undefined + activeRecordings.delete(recording.tabId) + return c.json(result) + } + + // Wait for final chunk to arrive + const finalResult = await finalPromise + + return c.json(finalResult) + } catch (error: any) { + logger?.error('Stop recording error:', error) + return c.json({ success: false, error: error.message } as StopRecordingResult, 500) + } + }) + + app.get('/recording/status', async (c) => { + try { + const sessionId = c.req.query('sessionId') + const params: IsRecordingParams = { sessionId } + + if (!extensionWs) { + return c.json({ isRecording: false } as IsRecordingResult) + } + + const result = await sendToExtension({ + method: 'isRecording', + params, + timeout: 5000, + }) as IsRecordingResult + + return c.json(result) + } catch (error: any) { + logger?.error('Recording status error:', error) + return c.json({ isRecording: false } as IsRecordingResult, 500) + } + }) + + app.post('/recording/cancel', async (c) => { + try { + const body = await c.req.json() as CancelRecordingParams + const params = body + + if (!extensionWs) { + return c.json({ success: false, error: 'Extension not connected' } as CancelRecordingResult, 503) + } + + const result = await sendToExtension({ + method: 'cancelRecording', + params, + timeout: 5000, + }) as CancelRecordingResult + + // Clean up local recording state + for (const [tabId, recording] of activeRecordings) { + activeRecordings.delete(tabId) + } + + return c.json(result) + } catch (error: any) { + logger?.error('Cancel recording error:', error) + return c.json({ success: false, error: error.message } as CancelRecordingResult, 500) + } + }) + const server = serve({ fetch: app.fetch, port, hostname: host }) injectWebSocket(server) diff --git a/playwriter/src/executor.ts b/playwriter/src/executor.ts index 935957a..999bb6e 100644 --- a/playwriter/src/executor.ts +++ b/playwriter/src/executor.ts @@ -24,6 +24,7 @@ import { ScopedFS } from './scoped-fs.js' import { screenshotWithAccessibilityLabels, formatSnapshot, DEFAULT_SNAPSHOT_FORMAT, type ScreenshotResult, type SnapshotFormat } from './aria-snapshot.js' export type { SnapshotFormat } import { getCleanHTML, type GetCleanHTMLOptions } from './clean-html.js' +import { startRecording, stopRecording, isRecording, cancelRecording } from './screen-recording.js' const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) @@ -589,6 +590,35 @@ export class PlaywrightExecutor { }) } + // Screen recording functions (via chrome.tabCapture in extension - survives navigation) + const relayPort = this.cdpConfig.port || 19988 + + const startRecordingFn = async (options: { + page?: Page + frameRate?: number + videoBitsPerSecond?: number + audioBitsPerSecond?: number + audio?: boolean + outputPath: string + }) => { + return startRecording({ + page: options.page || page, + relayPort, + ...options + }) + } + + const stopRecordingFn = async (options: { page?: Page } = {}) => { + return stopRecording({ page: options.page || page, relayPort }) + } + + const isRecordingFn = async (options: { page?: Page } = {}) => { + return isRecording({ page: options.page || page, relayPort }) + } + + const cancelRecordingFn = async (options: { page?: Page } = {}) => { + return cancelRecording({ page: options.page || page, relayPort }) + } const self = this let vmContextObj: any = { @@ -609,6 +639,10 @@ export class PlaywrightExecutor { formatStylesAsText, getReactSource: getReactSourceFn, screenshotWithAccessibilityLabels: screenshotWithAccessibilityLabelsFn, + startRecording: startRecordingFn, + stopRecording: stopRecordingFn, + isRecording: isRecordingFn, + cancelRecording: cancelRecordingFn, resetPlaywright: async () => { const { page: newPage, context: newContext } = await self.reset() vmContextObj.page = newPage diff --git a/playwriter/src/index.ts b/playwriter/src/index.ts index c510cfb..3ea09fa 100644 --- a/playwriter/src/index.ts +++ b/playwriter/src/index.ts @@ -8,3 +8,5 @@ export { Debugger } from './debugger.js' export type { BreakpointInfo, LocationInfo, EvaluateResult, ScriptInfo } from './debugger.js' export { getAriaSnapshot, showAriaRefLabels, hideAriaRefLabels } from './aria-snapshot.js' export type { AriaRef, AriaSnapshotResult } from './aria-snapshot.js' +export { startRecording, stopRecording, isRecording, cancelRecording } from './screen-recording.js' +export type { StartRecordingOptions, StopRecordingOptions, RecordingState } from './screen-recording.js' diff --git a/playwriter/src/mcp.test.ts b/playwriter/src/mcp.test.ts index 99468e8..d70c797 100644 --- a/playwriter/src/mcp.test.ts +++ b/playwriter/src/mcp.test.ts @@ -2616,6 +2616,157 @@ describe('MCP Server Tests', () => { await page.close() }, 60000) + it('should record screen with navigation using chrome.tabCapture', async () => { + // Create a new page for recording + await client.callTool({ + name: 'execute', + arguments: { + code: js` + const newPage = await context.newPage(); + state.recordingPage = newPage; + await newPage.goto('https://news.ycombinator.com/', { waitUntil: 'domcontentloaded' }); + console.log('Page loaded:', newPage.url()); + `, + }, + }) + + // Ensure tmp directory exists + const outputPath = path.join(process.cwd(), 'tmp', 'test-recording-tabcapture.webm') + const tmpDir = path.dirname(outputPath) + if (!fs.existsSync(tmpDir)) { + fs.mkdirSync(tmpDir, { recursive: true }) + } + + // Start recording with outputPath specified upfront + const startResult = await client.callTool({ + name: 'execute', + arguments: { + code: js` + const result = await startRecording({ + page: state.recordingPage, + frameRate: 30, + audio: false, + videoBitsPerSecond: 1500000, + outputPath: '${outputPath}' + }); + console.log('Recording started:', result); + return result; + `, + }, + }) + + const startOutput = (startResult as any).content[0].text + console.log('Start recording result:', startOutput) + + // Check if recording started or if it failed + if (startOutput.includes('error') || startOutput.includes('Error')) { + console.log('Recording failed. Output:', startOutput) + // Clean up + await client.callTool({ + name: 'execute', + arguments: { + code: js` + if (state.recordingPage) { + await state.recordingPage.close(); + delete state.recordingPage; + } + `, + }, + }) + // Skip test if recording not supported (e.g., permissions issue) + return + } + + expect(startOutput).toContain('isRecording') + expect(startOutput).toContain('true') + + // KEY TEST: Navigate to different pages while recording + // This is what the getDisplayMedia approach cannot do! + await client.callTool({ + name: 'execute', + arguments: { + code: js` + // Navigate to a story link - this would break getDisplayMedia recording! + const firstStory = state.recordingPage.locator('.titleline a').first(); + console.log('Clicking first story...'); + await firstStory.click(); + await state.recordingPage.waitForLoadState('domcontentloaded'); + console.log('Navigated to:', state.recordingPage.url()); + + // Wait a moment + await new Promise(r => setTimeout(r, 1000)); + + // Go back to Hacker News + await state.recordingPage.goBack(); + await state.recordingPage.waitForLoadState('domcontentloaded'); + console.log('Back to:', state.recordingPage.url()); + + // Scroll down + await state.recordingPage.evaluate(() => window.scrollBy(0, 500)); + await new Promise(r => setTimeout(r, 500)); + `, + }, + }) + + // Check recording status - should still be recording despite navigation! + const statusResult = await client.callTool({ + name: 'execute', + arguments: { + code: js` + const status = await isRecording({ page: state.recordingPage }); + console.log('Recording status after navigation:', status); + return status; + `, + }, + }) + + const statusOutput = (statusResult as any).content[0].text + console.log('Recording status:', statusOutput) + // This is the key assertion - recording should still be active after navigation + expect(statusOutput).toContain('isRecording') + expect(statusOutput).toContain('true') + + // Stop recording + const stopResult = await client.callTool({ + name: 'execute', + arguments: { + code: js` + const result = await stopRecording({ page: state.recordingPage }); + console.log('Recording stopped:', result); + return result; + `, + }, + }) + + const stopOutput = (stopResult as any).content[0].text + console.log('Stop recording result:', stopOutput) + expect(stopOutput).toContain('path') + expect(stopOutput).toContain('duration') + expect(stopOutput).toContain('size') + + // Verify the file was created + expect(fs.existsSync(outputPath)).toBe(true) + const stats = fs.statSync(outputPath) + console.log('Recording file size:', stats.size, 'bytes') + expect(stats.size).toBeGreaterThan(10000) // Should be at least 10KB + + // Clean up + await client.callTool({ + name: 'execute', + arguments: { + code: js` + if (state.recordingPage) { + await state.recordingPage.close(); + delete state.recordingPage; + } + `, + }, + }) + + // Clean up the recording file + fs.unlinkSync(outputPath) + }, 90000) + }) diff --git a/playwriter/src/protocol.ts b/playwriter/src/protocol.ts index 376f203..8aa5f98 100644 --- a/playwriter/src/protocol.ts +++ b/playwriter/src/protocol.ts @@ -61,4 +61,104 @@ export type ServerPingMessage = { id?: undefined } -export type ExtensionMessage = ExtensionResponseMessage | ExtensionEventMessage | ExtensionLogMessage | ExtensionPongMessage +export type RecordingDataMessage = { + id?: undefined + method: 'recordingData' + params: { + tabId: number + final?: boolean + } +} + +export type RecordingCancelledMessage = { + id?: undefined + method: 'recordingCancelled' + params: { + tabId: number + } +} + +export type ExtensionMessage = ExtensionResponseMessage | ExtensionEventMessage | ExtensionLogMessage | ExtensionPongMessage | RecordingDataMessage | RecordingCancelledMessage + +// Recording command messages (MCP -> Extension via relay) +export type StartRecordingParams = { + sessionId?: string + frameRate?: number + audio?: boolean + videoBitsPerSecond?: number + audioBitsPerSecond?: number +} + +export type StopRecordingParams = { + sessionId?: string +} + +export type IsRecordingParams = { + sessionId?: string +} + +export type CancelRecordingParams = { + sessionId?: string +} + +export type StartRecordingMessage = { + id: number + method: 'startRecording' + params: StartRecordingParams +} + +export type StopRecordingMessage = { + id: number + method: 'stopRecording' + params: StopRecordingParams +} + +export type IsRecordingMessage = { + id: number + method: 'isRecording' + params: IsRecordingParams +} + +export type CancelRecordingMessage = { + id: number + method: 'cancelRecording' + params: CancelRecordingParams +} + +export type RecordingCommandMessage = + | StartRecordingMessage + | StopRecordingMessage + | IsRecordingMessage + | CancelRecordingMessage + +// Recording result types +export type StartRecordingResult = { + success: true + tabId: number + startedAt: number +} | { + success: false + error: string +} + +export type StopRecordingResult = { + success: true + tabId: number + duration: number + path: string + size: number +} | { + success: false + error: string +} + +export type IsRecordingResult = { + isRecording: boolean + tabId?: number + startedAt?: number +} + +export type CancelRecordingResult = { + success: boolean + error?: string +} diff --git a/playwriter/src/screen-recording.ts b/playwriter/src/screen-recording.ts new file mode 100644 index 0000000..ad72a97 --- /dev/null +++ b/playwriter/src/screen-recording.ts @@ -0,0 +1,156 @@ +/** + * Screen recording utility for playwriter using chrome.tabCapture. + * Recording happens in the extension context, so it survives page navigation. + * + * This module communicates with the relay server which forwards commands to the extension. + */ + +import type { Page } from 'playwright-core' +import type { StartRecordingResult, StopRecordingResult, IsRecordingResult, CancelRecordingResult } from './protocol.js' + +export interface StartRecordingOptions { + /** Target page to record */ + page: Page + /** Frame rate (default: 30) */ + frameRate?: number + /** Video bitrate in bps (default: 2500000 = 2.5 Mbps) */ + videoBitsPerSecond?: number + /** Audio bitrate in bps (default: 128000 = 128 kbps) */ + audioBitsPerSecond?: number + /** Include audio from tab (default: false) */ + audio?: boolean + /** Path to save the video file */ + outputPath: string + /** Relay server port (default: 19988) */ + relayPort?: number +} + +export interface StopRecordingOptions { + /** Target page that is being recorded */ + page: Page + /** Relay server port (default: 19988) */ + relayPort?: number +} + +export interface RecordingState { + isRecording: boolean + startedAt?: number + tabId?: number +} + +function getSessionId(page: Page): string | undefined { + // The page's _guid is Playwright-internal and doesn't match the extension's sessionId (pw-tab-X). + // For now, we don't pass sessionId and let the extension use the first connected tab. + // TODO: Add proper mapping between page and extension tab sessionIds + return undefined +} + +/** + * Start recording the page. + * The recording is handled by the extension, so it survives page navigation. + */ +export async function startRecording(options: StartRecordingOptions): Promise { + const { + page, + frameRate = 30, + videoBitsPerSecond = 2500000, + audioBitsPerSecond = 128000, + audio = false, + outputPath, + relayPort = 19988, + } = options + + const sessionId = getSessionId(page) + + const response = await fetch(`http://127.0.0.1:${relayPort}/recording/start`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + sessionId, + frameRate, + videoBitsPerSecond, + audioBitsPerSecond, + audio, + outputPath, + }), + }) + + const result = await response.json() as StartRecordingResult + + if (!result.success) { + throw new Error(`Failed to start recording: ${result.error}`) + } + + return { + isRecording: true, + startedAt: result.startedAt, + tabId: result.tabId, + } +} + +/** + * Stop recording and save to file. + * Returns the path to the saved video file. + */ +export async function stopRecording(options: StopRecordingOptions): Promise<{ path: string; duration: number; size: number }> { + const { page, relayPort = 19988 } = options + + const sessionId = getSessionId(page) + + const response = await fetch(`http://127.0.0.1:${relayPort}/recording/stop`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sessionId }), + }) + + const result = await response.json() as StopRecordingResult + + if (!result.success) { + throw new Error(`Failed to stop recording: ${result.error}`) + } + + return { + path: result.path, + duration: result.duration, + size: result.size, + } +} + +/** + * Check if recording is currently active on a page. + */ +export async function isRecording(options: { page: Page; relayPort?: number }): Promise { + const { page, relayPort = 19988 } = options + + const sessionId = getSessionId(page) + + const response = await fetch(`http://127.0.0.1:${relayPort}/recording/status?sessionId=${sessionId || ''}`) + const result = await response.json() as IsRecordingResult + + return { + isRecording: result.isRecording, + startedAt: result.startedAt, + tabId: result.tabId, + } +} + +/** + * Cancel recording without saving. + */ +export async function cancelRecording(options: { page: Page; relayPort?: number }): Promise { + const { page, relayPort = 19988 } = options + + const sessionId = getSessionId(page) + + const response = await fetch(`http://127.0.0.1:${relayPort}/recording/cancel`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sessionId }), + }) + + const result = await response.json() as CancelRecordingResult + + if (!result.success) { + throw new Error(`Failed to cancel recording: ${result.error}`) + } +} diff --git a/playwriter/src/skill.md b/playwriter/src/skill.md index 5d5f2d9..d4ff339 100644 --- a/playwriter/src/skill.md +++ b/playwriter/src/skill.md @@ -421,6 +421,39 @@ await screenshotWithAccessibilityLabels({ page }); Labels are color-coded: yellow=links, orange=buttons, coral=inputs, pink=checkboxes, peach=sliders, salmon=menus, amber=tabs. +**startRecording / stopRecording** - record the page as a video at native FPS (30-60fps). Uses `chrome.tabCapture` in the extension context, so **recording survives page navigation**. Video is saved as WebM. + +```js +// Start recording - outputPath must be specified upfront +await startRecording({ + page, + outputPath: './recording.webm', + frameRate: 30, // default: 30 + audio: false, // default: false (tab audio) + videoBitsPerSecond: 2500000 // 2.5 Mbps +}); + +// Navigate around - recording continues! +await page.click('a'); +await page.waitForLoadState('domcontentloaded'); +await page.goBack(); + +// Stop and get result +const { path, duration, size } = await stopRecording({ page }); +console.log(`Saved ${size} bytes, duration: ${duration}ms`); +``` + +Additional recording utilities: +```js +// Check if recording is active +const { isRecording, startedAt } = await isRecording({ page }); + +// Cancel recording without saving +await cancelRecording({ page }); +``` + +**Key difference from getDisplayMedia**: This approach uses `chrome.tabCapture` which runs in the extension context, not the page. The recording persists across navigations because the extension holds the `MediaRecorder`, not the page's JavaScript context. + ## pinned elements Users can right-click → "Copy Playwriter Element Reference" to store elements in `globalThis.playwriterPinnedElem1` (increments for each pin). The reference is copied to clipboard: