WIP: screen recording via chrome.tabCapture

Work in progress for navigation-safe screen recording.

Implemented:
- tabCapture permission in manifest
- Recording handlers in extension background.ts
- Recording endpoints in relay server
- Binary WebSocket streaming for video chunks
- Protocol types for recording messages
- Executor integration

Needs:
- Offscreen document implementation (tabCapture not available in service workers)
- Connect background.ts to offscreen document via messages
- Update manifest for offscreen document

The offscreen.ts/html files are scaffolding for this next step.
This commit is contained in:
Tommy D. Rossi
2026-01-23 16:10:40 +01:00
parent 81988c3305
commit e56e27e5e5
12 changed files with 1303 additions and 4 deletions
+225 -1
View File
@@ -6,7 +6,9 @@ import { createNodeWebSocket } from '@hono/node-ws'
import type { WSContext } from 'hono/ws'
import type { Protocol } from './cdp-types.js'
import type { CDPCommand, CDPResponseBase, CDPEventBase, CDPEventFor, RelayServerEvents } from './cdp-types.js'
import type { ExtensionMessage, ExtensionEventMessage } from './protocol.js'
import type { ExtensionMessage, ExtensionEventMessage, StartRecordingParams, StopRecordingParams, IsRecordingParams, CancelRecordingParams, StartRecordingResult, StopRecordingResult, IsRecordingResult, CancelRecordingResult } from './protocol.js'
import fs from 'node:fs'
import path from 'node:path'
import pc from 'picocolors'
import { EventEmitter } from 'node:events'
import { VERSION } from './utils.js'
@@ -89,6 +91,16 @@ export async function startPlayWriterCDPRelayServer({
resolvedCdpLogger.log(entry)
}
// Recording state - tracks active recordings and their accumulated chunks
type ActiveRecording = {
tabId: number
outputPath: string
chunks: Buffer[]
startedAt: number
resolveStop?: (result: StopRecordingResult) => void
}
const activeRecordings = new Map<number, ActiveRecording>()
const playwrightClients = new Map<string, PlaywrightClient>()
let extensionWs: WSContext | null = null
@@ -824,6 +836,19 @@ export async function startPlayWriterCDPRelayServer({
},
async onMessage(event, ws) {
// Handle binary data (recording chunks)
if (event.data instanceof ArrayBuffer || Buffer.isBuffer(event.data)) {
const buffer = Buffer.isBuffer(event.data) ? event.data : Buffer.from(event.data)
// Find the recording that's waiting for data (the one we just received metadata for)
// The extension sends metadata first, then binary
for (const recording of activeRecordings.values()) {
recording.chunks.push(buffer)
logger?.log(pc.blue(`Received recording chunk for tab ${recording.tabId}: ${buffer.length} bytes (total chunks: ${recording.chunks.length})`))
break // Only add to the first active recording
}
return
}
let message: ExtensionMessage
try {
@@ -854,6 +879,50 @@ export async function startPlayWriterCDPRelayServer({
const logFn = (logger as any)?.[level] || logger?.log
const prefix = pc.yellow(`[Extension] [${level.toUpperCase()}]`)
logFn?.(prefix, ...args)
} else if (message.method === 'recordingData') {
const { tabId, final } = (message as any).params
const recording = activeRecordings.get(tabId)
if (recording && final) {
// This is the final marker - write all chunks to file
try {
const totalSize = recording.chunks.reduce((sum, chunk) => sum + chunk.length, 0)
const combined = Buffer.concat(recording.chunks)
fs.writeFileSync(recording.outputPath, combined)
const duration = Date.now() - recording.startedAt
logger?.log(pc.green(`Recording saved: ${recording.outputPath} (${totalSize} bytes, ${duration}ms)`))
// Resolve the stop promise
if (recording.resolveStop) {
recording.resolveStop({
success: true,
tabId,
duration,
path: recording.outputPath,
size: totalSize,
} as any)
}
} catch (error: any) {
logger?.error('Failed to write recording:', error)
if (recording.resolveStop) {
recording.resolveStop({ success: false, error: error.message })
}
}
activeRecordings.delete(tabId)
}
// Non-final recordingData is just a marker that binary follows - handled above
} else if (message.method === 'recordingCancelled') {
const { tabId } = (message as any).params
const recording = activeRecordings.get(tabId)
if (recording) {
logger?.log(pc.yellow(`Recording cancelled for tab ${tabId}`))
if (recording.resolveStop) {
recording.resolveStop({ success: false, error: 'Recording was cancelled' })
}
activeRecordings.delete(tabId)
}
} else {
const extensionEvent = message as ExtensionEventMessage
@@ -1162,6 +1231,161 @@ export async function startPlayWriterCDPRelayServer({
return c.json({ next: nextSessionNumber })
})
// ============================================================================
// Recording Endpoints - For screen recording via chrome.tabCapture
// ============================================================================
app.post('/recording/start', async (c) => {
try {
const body = await c.req.json() as StartRecordingParams & { outputPath: string }
const { outputPath, ...params } = body
if (!outputPath) {
return c.json({ success: false, error: 'outputPath is required' } as StartRecordingResult, 400)
}
if (!extensionWs) {
return c.json({ success: false, error: 'Extension not connected' } as StartRecordingResult, 503)
}
// Ensure output directory exists
const dir = path.dirname(outputPath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
const result = await sendToExtension({
method: 'startRecording',
params,
timeout: 10000,
}) as StartRecordingResult
if (result.success) {
// Track this recording
activeRecordings.set(result.tabId, {
tabId: result.tabId,
outputPath,
chunks: [],
startedAt: result.startedAt,
})
logger?.log(pc.green(`Recording started for tab ${result.tabId}, output: ${outputPath}`))
}
return c.json(result)
} catch (error: any) {
logger?.error('Start recording error:', error)
return c.json({ success: false, error: error.message } as StartRecordingResult, 500)
}
})
app.post('/recording/stop', async (c) => {
try {
const body = await c.req.json() as StopRecordingParams
const params = body
if (!extensionWs) {
return c.json({ success: false, error: 'Extension not connected' } as StopRecordingResult, 503)
}
// Create a promise that will be resolved when we receive the final chunk
const recording = Array.from(activeRecordings.values()).find(r => {
// Find the recording for this session
if (params.sessionId) {
const target = connectedTargets.get(params.sessionId)
// We don't have tabId in connectedTargets, so we'll match by the first one
return true
}
return true
})
if (!recording) {
return c.json({ success: false, error: 'No active recording found' } as StopRecordingResult, 404)
}
// Set up promise to wait for final chunk
const finalPromise = new Promise<StopRecordingResult>((resolve) => {
recording.resolveStop = resolve
// Timeout after 30 seconds
setTimeout(() => {
if (recording.resolveStop) {
recording.resolveStop = undefined
resolve({ success: false, error: 'Timeout waiting for recording data' })
}
}, 30000)
})
// Tell extension to stop recording
const result = await sendToExtension({
method: 'stopRecording',
params,
timeout: 10000,
}) as StopRecordingResult
if (!result.success) {
recording.resolveStop = undefined
activeRecordings.delete(recording.tabId)
return c.json(result)
}
// Wait for final chunk to arrive
const finalResult = await finalPromise
return c.json(finalResult)
} catch (error: any) {
logger?.error('Stop recording error:', error)
return c.json({ success: false, error: error.message } as StopRecordingResult, 500)
}
})
app.get('/recording/status', async (c) => {
try {
const sessionId = c.req.query('sessionId')
const params: IsRecordingParams = { sessionId }
if (!extensionWs) {
return c.json({ isRecording: false } as IsRecordingResult)
}
const result = await sendToExtension({
method: 'isRecording',
params,
timeout: 5000,
}) as IsRecordingResult
return c.json(result)
} catch (error: any) {
logger?.error('Recording status error:', error)
return c.json({ isRecording: false } as IsRecordingResult, 500)
}
})
app.post('/recording/cancel', async (c) => {
try {
const body = await c.req.json() as CancelRecordingParams
const params = body
if (!extensionWs) {
return c.json({ success: false, error: 'Extension not connected' } as CancelRecordingResult, 503)
}
const result = await sendToExtension({
method: 'cancelRecording',
params,
timeout: 5000,
}) as CancelRecordingResult
// Clean up local recording state
for (const [tabId, recording] of activeRecordings) {
activeRecordings.delete(tabId)
}
return c.json(result)
} catch (error: any) {
logger?.error('Cancel recording error:', error)
return c.json({ success: false, error: error.message } as CancelRecordingResult, 500)
}
})
const server = serve({ fetch: app.fetch, port, hostname: host })
injectWebSocket(server)
+34
View File
@@ -24,6 +24,7 @@ import { ScopedFS } from './scoped-fs.js'
import { screenshotWithAccessibilityLabels, formatSnapshot, DEFAULT_SNAPSHOT_FORMAT, type ScreenshotResult, type SnapshotFormat } from './aria-snapshot.js'
export type { SnapshotFormat }
import { getCleanHTML, type GetCleanHTMLOptions } from './clean-html.js'
import { startRecording, stopRecording, isRecording, cancelRecording } from './screen-recording.js'
const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
@@ -589,6 +590,35 @@ export class PlaywrightExecutor {
})
}
// Screen recording functions (via chrome.tabCapture in extension - survives navigation)
const relayPort = this.cdpConfig.port || 19988
const startRecordingFn = async (options: {
page?: Page
frameRate?: number
videoBitsPerSecond?: number
audioBitsPerSecond?: number
audio?: boolean
outputPath: string
}) => {
return startRecording({
page: options.page || page,
relayPort,
...options
})
}
const stopRecordingFn = async (options: { page?: Page } = {}) => {
return stopRecording({ page: options.page || page, relayPort })
}
const isRecordingFn = async (options: { page?: Page } = {}) => {
return isRecording({ page: options.page || page, relayPort })
}
const cancelRecordingFn = async (options: { page?: Page } = {}) => {
return cancelRecording({ page: options.page || page, relayPort })
}
const self = this
let vmContextObj: any = {
@@ -609,6 +639,10 @@ export class PlaywrightExecutor {
formatStylesAsText,
getReactSource: getReactSourceFn,
screenshotWithAccessibilityLabels: screenshotWithAccessibilityLabelsFn,
startRecording: startRecordingFn,
stopRecording: stopRecordingFn,
isRecording: isRecordingFn,
cancelRecording: cancelRecordingFn,
resetPlaywright: async () => {
const { page: newPage, context: newContext } = await self.reset()
vmContextObj.page = newPage
+2
View File
@@ -8,3 +8,5 @@ export { Debugger } from './debugger.js'
export type { BreakpointInfo, LocationInfo, EvaluateResult, ScriptInfo } from './debugger.js'
export { getAriaSnapshot, showAriaRefLabels, hideAriaRefLabels } from './aria-snapshot.js'
export type { AriaRef, AriaSnapshotResult } from './aria-snapshot.js'
export { startRecording, stopRecording, isRecording, cancelRecording } from './screen-recording.js'
export type { StartRecordingOptions, StopRecordingOptions, RecordingState } from './screen-recording.js'
+151
View File
@@ -2616,6 +2616,157 @@ describe('MCP Server Tests', () => {
await page.close()
}, 60000)
it('should record screen with navigation using chrome.tabCapture', async () => {
// Create a new page for recording
await client.callTool({
name: 'execute',
arguments: {
code: js`
const newPage = await context.newPage();
state.recordingPage = newPage;
await newPage.goto('https://news.ycombinator.com/', { waitUntil: 'domcontentloaded' });
console.log('Page loaded:', newPage.url());
`,
},
})
// Ensure tmp directory exists
const outputPath = path.join(process.cwd(), 'tmp', 'test-recording-tabcapture.webm')
const tmpDir = path.dirname(outputPath)
if (!fs.existsSync(tmpDir)) {
fs.mkdirSync(tmpDir, { recursive: true })
}
// Start recording with outputPath specified upfront
const startResult = await client.callTool({
name: 'execute',
arguments: {
code: js`
const result = await startRecording({
page: state.recordingPage,
frameRate: 30,
audio: false,
videoBitsPerSecond: 1500000,
outputPath: '${outputPath}'
});
console.log('Recording started:', result);
return result;
`,
},
})
const startOutput = (startResult as any).content[0].text
console.log('Start recording result:', startOutput)
// Check if recording started or if it failed
if (startOutput.includes('error') || startOutput.includes('Error')) {
console.log('Recording failed. Output:', startOutput)
// Clean up
await client.callTool({
name: 'execute',
arguments: {
code: js`
if (state.recordingPage) {
await state.recordingPage.close();
delete state.recordingPage;
}
`,
},
})
// Skip test if recording not supported (e.g., permissions issue)
return
}
expect(startOutput).toContain('isRecording')
expect(startOutput).toContain('true')
// KEY TEST: Navigate to different pages while recording
// This is what the getDisplayMedia approach cannot do!
await client.callTool({
name: 'execute',
arguments: {
code: js`
// Navigate to a story link - this would break getDisplayMedia recording!
const firstStory = state.recordingPage.locator('.titleline a').first();
console.log('Clicking first story...');
await firstStory.click();
await state.recordingPage.waitForLoadState('domcontentloaded');
console.log('Navigated to:', state.recordingPage.url());
// Wait a moment
await new Promise(r => setTimeout(r, 1000));
// Go back to Hacker News
await state.recordingPage.goBack();
await state.recordingPage.waitForLoadState('domcontentloaded');
console.log('Back to:', state.recordingPage.url());
// Scroll down
await state.recordingPage.evaluate(() => window.scrollBy(0, 500));
await new Promise(r => setTimeout(r, 500));
`,
},
})
// Check recording status - should still be recording despite navigation!
const statusResult = await client.callTool({
name: 'execute',
arguments: {
code: js`
const status = await isRecording({ page: state.recordingPage });
console.log('Recording status after navigation:', status);
return status;
`,
},
})
const statusOutput = (statusResult as any).content[0].text
console.log('Recording status:', statusOutput)
// This is the key assertion - recording should still be active after navigation
expect(statusOutput).toContain('isRecording')
expect(statusOutput).toContain('true')
// Stop recording
const stopResult = await client.callTool({
name: 'execute',
arguments: {
code: js`
const result = await stopRecording({ page: state.recordingPage });
console.log('Recording stopped:', result);
return result;
`,
},
})
const stopOutput = (stopResult as any).content[0].text
console.log('Stop recording result:', stopOutput)
expect(stopOutput).toContain('path')
expect(stopOutput).toContain('duration')
expect(stopOutput).toContain('size')
// Verify the file was created
expect(fs.existsSync(outputPath)).toBe(true)
const stats = fs.statSync(outputPath)
console.log('Recording file size:', stats.size, 'bytes')
expect(stats.size).toBeGreaterThan(10000) // Should be at least 10KB
// Clean up
await client.callTool({
name: 'execute',
arguments: {
code: js`
if (state.recordingPage) {
await state.recordingPage.close();
delete state.recordingPage;
}
`,
},
})
// Clean up the recording file
fs.unlinkSync(outputPath)
}, 90000)
})
+101 -1
View File
@@ -61,4 +61,104 @@ export type ServerPingMessage = {
id?: undefined
}
export type ExtensionMessage = ExtensionResponseMessage | ExtensionEventMessage | ExtensionLogMessage | ExtensionPongMessage
export type RecordingDataMessage = {
id?: undefined
method: 'recordingData'
params: {
tabId: number
final?: boolean
}
}
export type RecordingCancelledMessage = {
id?: undefined
method: 'recordingCancelled'
params: {
tabId: number
}
}
export type ExtensionMessage = ExtensionResponseMessage | ExtensionEventMessage | ExtensionLogMessage | ExtensionPongMessage | RecordingDataMessage | RecordingCancelledMessage
// Recording command messages (MCP -> Extension via relay)
export type StartRecordingParams = {
sessionId?: string
frameRate?: number
audio?: boolean
videoBitsPerSecond?: number
audioBitsPerSecond?: number
}
export type StopRecordingParams = {
sessionId?: string
}
export type IsRecordingParams = {
sessionId?: string
}
export type CancelRecordingParams = {
sessionId?: string
}
export type StartRecordingMessage = {
id: number
method: 'startRecording'
params: StartRecordingParams
}
export type StopRecordingMessage = {
id: number
method: 'stopRecording'
params: StopRecordingParams
}
export type IsRecordingMessage = {
id: number
method: 'isRecording'
params: IsRecordingParams
}
export type CancelRecordingMessage = {
id: number
method: 'cancelRecording'
params: CancelRecordingParams
}
export type RecordingCommandMessage =
| StartRecordingMessage
| StopRecordingMessage
| IsRecordingMessage
| CancelRecordingMessage
// Recording result types
export type StartRecordingResult = {
success: true
tabId: number
startedAt: number
} | {
success: false
error: string
}
export type StopRecordingResult = {
success: true
tabId: number
duration: number
path: string
size: number
} | {
success: false
error: string
}
export type IsRecordingResult = {
isRecording: boolean
tabId?: number
startedAt?: number
}
export type CancelRecordingResult = {
success: boolean
error?: string
}
+156
View File
@@ -0,0 +1,156 @@
/**
* Screen recording utility for playwriter using chrome.tabCapture.
* Recording happens in the extension context, so it survives page navigation.
*
* This module communicates with the relay server which forwards commands to the extension.
*/
import type { Page } from 'playwright-core'
import type { StartRecordingResult, StopRecordingResult, IsRecordingResult, CancelRecordingResult } from './protocol.js'
export interface StartRecordingOptions {
/** Target page to record */
page: Page
/** Frame rate (default: 30) */
frameRate?: number
/** Video bitrate in bps (default: 2500000 = 2.5 Mbps) */
videoBitsPerSecond?: number
/** Audio bitrate in bps (default: 128000 = 128 kbps) */
audioBitsPerSecond?: number
/** Include audio from tab (default: false) */
audio?: boolean
/** Path to save the video file */
outputPath: string
/** Relay server port (default: 19988) */
relayPort?: number
}
export interface StopRecordingOptions {
/** Target page that is being recorded */
page: Page
/** Relay server port (default: 19988) */
relayPort?: number
}
export interface RecordingState {
isRecording: boolean
startedAt?: number
tabId?: number
}
function getSessionId(page: Page): string | undefined {
// The page's _guid is Playwright-internal and doesn't match the extension's sessionId (pw-tab-X).
// For now, we don't pass sessionId and let the extension use the first connected tab.
// TODO: Add proper mapping between page and extension tab sessionIds
return undefined
}
/**
* Start recording the page.
* The recording is handled by the extension, so it survives page navigation.
*/
export async function startRecording(options: StartRecordingOptions): Promise<RecordingState> {
const {
page,
frameRate = 30,
videoBitsPerSecond = 2500000,
audioBitsPerSecond = 128000,
audio = false,
outputPath,
relayPort = 19988,
} = options
const sessionId = getSessionId(page)
const response = await fetch(`http://127.0.0.1:${relayPort}/recording/start`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sessionId,
frameRate,
videoBitsPerSecond,
audioBitsPerSecond,
audio,
outputPath,
}),
})
const result = await response.json() as StartRecordingResult
if (!result.success) {
throw new Error(`Failed to start recording: ${result.error}`)
}
return {
isRecording: true,
startedAt: result.startedAt,
tabId: result.tabId,
}
}
/**
* Stop recording and save to file.
* Returns the path to the saved video file.
*/
export async function stopRecording(options: StopRecordingOptions): Promise<{ path: string; duration: number; size: number }> {
const { page, relayPort = 19988 } = options
const sessionId = getSessionId(page)
const response = await fetch(`http://127.0.0.1:${relayPort}/recording/stop`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ sessionId }),
})
const result = await response.json() as StopRecordingResult
if (!result.success) {
throw new Error(`Failed to stop recording: ${result.error}`)
}
return {
path: result.path,
duration: result.duration,
size: result.size,
}
}
/**
* Check if recording is currently active on a page.
*/
export async function isRecording(options: { page: Page; relayPort?: number }): Promise<RecordingState> {
const { page, relayPort = 19988 } = options
const sessionId = getSessionId(page)
const response = await fetch(`http://127.0.0.1:${relayPort}/recording/status?sessionId=${sessionId || ''}`)
const result = await response.json() as IsRecordingResult
return {
isRecording: result.isRecording,
startedAt: result.startedAt,
tabId: result.tabId,
}
}
/**
* Cancel recording without saving.
*/
export async function cancelRecording(options: { page: Page; relayPort?: number }): Promise<void> {
const { page, relayPort = 19988 } = options
const sessionId = getSessionId(page)
const response = await fetch(`http://127.0.0.1:${relayPort}/recording/cancel`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ sessionId }),
})
const result = await response.json() as CancelRecordingResult
if (!result.success) {
throw new Error(`Failed to cancel recording: ${result.error}`)
}
}
+33
View File
@@ -421,6 +421,39 @@ await screenshotWithAccessibilityLabels({ page });
Labels are color-coded: yellow=links, orange=buttons, coral=inputs, pink=checkboxes, peach=sliders, salmon=menus, amber=tabs.
**startRecording / stopRecording** - record the page as a video at native FPS (30-60fps). Uses `chrome.tabCapture` in the extension context, so **recording survives page navigation**. Video is saved as WebM.
```js
// Start recording - outputPath must be specified upfront
await startRecording({
page,
outputPath: './recording.webm',
frameRate: 30, // default: 30
audio: false, // default: false (tab audio)
videoBitsPerSecond: 2500000 // 2.5 Mbps
});
// Navigate around - recording continues!
await page.click('a');
await page.waitForLoadState('domcontentloaded');
await page.goBack();
// Stop and get result
const { path, duration, size } = await stopRecording({ page });
console.log(`Saved ${size} bytes, duration: ${duration}ms`);
```
Additional recording utilities:
```js
// Check if recording is active
const { isRecording, startedAt } = await isRecording({ page });
// Cancel recording without saving
await cancelRecording({ page });
```
**Key difference from getDisplayMedia**: This approach uses `chrome.tabCapture` which runs in the extension context, not the page. The recording persists across navigations because the extension holds the `MediaRecorder`, not the page's JavaScript context.
## pinned elements
Users can right-click → "Copy Playwriter Element Reference" to store elements in `globalThis.playwriterPinnedElem1` (increments for each pin). The reference is copied to clipboard: