feat: add Chrome flags for automated recording without user interaction
- Add getChromeRestartCommand() with --allowlisted-extension-id and --auto-accept-this-tab-capture flags - Move EXTENSION_IDS to utils.ts for cleaner imports - Add sessionId support for multi-tab recording - Add helpful error message when activeTab permission is missing - Add null check for empty extension result - Document offscreen recording flow with ASCII diagram
This commit is contained in:
Tommy D. Rossi
@@ -1116,8 +1116,9 @@ async function handleStartRecording(params: StartRecordingParams): Promise<Start
|
||||
if (chrome.runtime.lastError) {
|
||||
const errorMsg = chrome.runtime.lastError.message || 'Unknown error'
|
||||
// Chrome returns this error when activeTab permission hasn't been granted
|
||||
// User must click the extension icon at least once per session - this is a Chrome security requirement
|
||||
if (errorMsg.includes('Extension has not been invoked') || errorMsg.includes('activeTab')) {
|
||||
reject(new Error(`${errorMsg}. Click the Playwriter extension icon on this tab to enable recording, then try again.`))
|
||||
reject(new Error(`${errorMsg}. Click the Playwriter extension icon on this tab to enable recording.`))
|
||||
} else {
|
||||
reject(new Error(errorMsg))
|
||||
}
|
||||
|
||||
@@ -1,8 +1,40 @@
|
||||
/**
|
||||
* Offscreen document for Playwriter extension.
|
||||
* Handles operations that require DOM APIs not available in service workers:
|
||||
* - Screen recording via chrome.tabCapture + MediaRecorder
|
||||
* - Future: audio processing, canvas operations, etc.
|
||||
* Offscreen document for Playwriter screen recording.
|
||||
*
|
||||
* WHY OFFSCREEN DOCUMENT?
|
||||
* Manifest V3 service workers cannot use MediaRecorder or getUserMedia directly.
|
||||
* This hidden document provides access to Web APIs while the service worker orchestrates.
|
||||
*
|
||||
* RECORDING FLOW:
|
||||
*
|
||||
* ┌─────────────────┐ HTTP ┌─────────────────┐ WebSocket ┌─────────────────┐
|
||||
* │ User Code │ ────────────► │ Relay Server │ ───────────────►│ Extension │
|
||||
* │ startRecording │ │ /recording/* │ │ background.ts │
|
||||
* └─────────────────┘ └─────────────────┘ └────────┬────────┘
|
||||
* │
|
||||
* ┌─────────────────────────────────────┘
|
||||
* ▼
|
||||
* ┌─────────────────┐
|
||||
* │ Offscreen Doc │ ◄── MediaRecorder
|
||||
* │ (this file) │
|
||||
* └─────────────────┘
|
||||
*
|
||||
* STEP BY STEP:
|
||||
* 1. User calls startRecording() → HTTP POST to relay server
|
||||
* 2. Relay server forwards to extension via WebSocket
|
||||
* 3. Extension calls chrome.tabCapture.getMediaStreamId() to get capture permission
|
||||
* - Requires --allowlisted-extension-id flag OR user clicking extension icon
|
||||
* 4. Extension creates this offscreen document via chrome.offscreen.createDocument()
|
||||
* 5. Extension sends streamId to offscreen document
|
||||
* 6. Offscreen calls navigator.mediaDevices.getUserMedia() with streamId
|
||||
* 7. Offscreen creates MediaRecorder and starts encoding to webm
|
||||
* 8. Chunks are sent back to extension → relay server → written to output file
|
||||
*
|
||||
* KEY APIS:
|
||||
* - chrome.tabCapture.getMediaStreamId() - Extension API, gets capture permission
|
||||
* - chrome.offscreen.createDocument() - Extension API, creates this document
|
||||
* - navigator.mediaDevices.getUserMedia() - Web API, gets MediaStream from streamId
|
||||
* - MediaRecorder - Web API, encodes video to webm
|
||||
*/
|
||||
|
||||
interface OffscreenRecordingState {
|
||||
|
||||
+10
-10
@@ -11,7 +11,7 @@ import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import pc from 'picocolors'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { VERSION } from './utils.js'
|
||||
import { VERSION, EXTENSION_IDS } from './utils.js'
|
||||
import { createCdpLogger, type CdpLogEntry, type CdpLogger } from './cdp-log.js'
|
||||
|
||||
type ConnectedTarget = {
|
||||
@@ -21,11 +21,7 @@ type ConnectedTarget = {
|
||||
}
|
||||
|
||||
// Our extension IDs - allow attaching to our own extension pages for debugging
|
||||
const OUR_EXTENSION_IDS = [
|
||||
'jfeammnjpkecdekppnclgkkffahnhfhe', // Production extension (Chrome Web Store)
|
||||
'pebbngnfojnignonigcnkdilknapkgid', // Dev extension (stable ID from manifest key)
|
||||
]
|
||||
|
||||
const OUR_EXTENSION_IDS = EXTENSION_IDS
|
||||
/**
|
||||
* Checks if a target should be filtered out (not exposed to Playwright).
|
||||
* Filters extension pages, service workers, and other restricted targets,
|
||||
@@ -47,7 +43,7 @@ function isRestrictedTarget(targetInfo: Protocol.Target.TargetInfo): boolean {
|
||||
// Allow our own extension pages
|
||||
if (url.startsWith('chrome-extension://')) {
|
||||
const extensionId = url.replace('chrome-extension://', '').split('/')[0]
|
||||
if (OUR_EXTENSION_IDS.includes(extensionId)) {
|
||||
if (EXTENSION_IDS.includes(extensionId)) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
@@ -461,7 +457,7 @@ export async function startPlayWriterCDPRelayServer({
|
||||
return null
|
||||
}
|
||||
const extensionId = origin.replace('chrome-extension://', '')
|
||||
if (!OUR_EXTENSION_IDS.includes(extensionId)) {
|
||||
if (!EXTENSION_IDS.includes(extensionId)) {
|
||||
return null
|
||||
}
|
||||
return origin
|
||||
@@ -589,7 +585,7 @@ export async function startPlayWriterCDPRelayServer({
|
||||
if (origin) {
|
||||
if (origin.startsWith('chrome-extension://')) {
|
||||
const extensionId = origin.replace('chrome-extension://', '')
|
||||
if (!OUR_EXTENSION_IDS.includes(extensionId)) {
|
||||
if (!EXTENSION_IDS.includes(extensionId)) {
|
||||
logger?.log(pc.red(`Rejecting /cdp WebSocket from unknown extension: ${extensionId}`))
|
||||
return c.text('Forbidden', 403)
|
||||
}
|
||||
@@ -798,7 +794,7 @@ export async function startPlayWriterCDPRelayServer({
|
||||
}
|
||||
|
||||
const extensionId = origin.replace('chrome-extension://', '')
|
||||
if (!OUR_EXTENSION_IDS.includes(extensionId)) {
|
||||
if (!EXTENSION_IDS.includes(extensionId)) {
|
||||
logger?.log(pc.red(`Rejecting /extension WebSocket from unknown extension: ${extensionId}`))
|
||||
return c.text('Forbidden', 403)
|
||||
}
|
||||
@@ -1275,6 +1271,10 @@ export async function startPlayWriterCDPRelayServer({
|
||||
timeout: 10000,
|
||||
}) as StartRecordingResult
|
||||
|
||||
if (!result) {
|
||||
return c.json({ success: false, error: 'Extension returned empty result' } as StartRecordingResult, 500)
|
||||
}
|
||||
|
||||
if (result.success) {
|
||||
// Track this recording
|
||||
activeRecordings.set(result.tabId, {
|
||||
|
||||
@@ -16,6 +16,7 @@ export interface ICDPSession {
|
||||
on(event: string, callback: (params: any) => void): unknown
|
||||
off(event: string, callback: (params: any) => void): unknown
|
||||
detach(): Promise<void>
|
||||
getSessionId?(): string | null
|
||||
}
|
||||
|
||||
interface PendingRequest {
|
||||
@@ -68,6 +69,10 @@ export class CDPSession implements ICDPSession {
|
||||
this.sessionId = sessionId
|
||||
}
|
||||
|
||||
getSessionId(): string | null {
|
||||
return this.sessionId
|
||||
}
|
||||
|
||||
send<K extends keyof ProtocolMapping.Commands>(
|
||||
method: K,
|
||||
params?: ProtocolMapping.Commands[K]['paramsType'][0],
|
||||
|
||||
@@ -595,10 +595,16 @@ export class PlaywrightExecutor {
|
||||
// This permission is granted when the user clicks the Playwriter extension icon on a tab.
|
||||
const relayPort = this.cdpConfig.port || 19988
|
||||
// Recording will work on any tab where the user has clicked the icon.
|
||||
const withRecordingDefaults = <T extends { page?: Page }, R>(
|
||||
fn: (opts: T & { relayPort: number }) => Promise<R>
|
||||
const withRecordingDefaults = <T extends { page?: Page; sessionId?: string }, R>(
|
||||
fn: (opts: T & { relayPort: number; sessionId?: string }) => Promise<R>
|
||||
) => {
|
||||
return (options: T = {} as T) => fn({ page: options.page || page, relayPort, ...options })
|
||||
return async (options: T = {} as T) => {
|
||||
const targetPage = options.page || page
|
||||
// Get sessionId from cached CDP session to identify which tab to record
|
||||
const cdp = await getCDPSession({ page: targetPage })
|
||||
const sessionId = options.sessionId || cdp.getSessionId() || undefined
|
||||
return fn({ page: targetPage, sessionId, relayPort, ...options })
|
||||
}
|
||||
}
|
||||
const self = this
|
||||
|
||||
|
||||
@@ -3,11 +3,10 @@
|
||||
* Recording happens in the extension context, so it survives page navigation.
|
||||
*
|
||||
* This module communicates with the relay server which forwards commands to the extension.
|
||||
*
|
||||
* Note: Recording uses the first connected tab. Multi-tab recording support would require
|
||||
* proper sessionId mapping between Playwright pages and extension tabs.
|
||||
* SessionId (pw-tab-X format) is used to identify which tab to record.
|
||||
*/
|
||||
|
||||
import os from 'node:os'
|
||||
import type { Page } from 'playwright-core'
|
||||
import type {
|
||||
StartRecordingResult,
|
||||
@@ -15,10 +14,44 @@ import type {
|
||||
IsRecordingResult,
|
||||
CancelRecordingResult,
|
||||
} from './protocol.js'
|
||||
import { EXTENSION_IDS } from './utils.js'
|
||||
|
||||
/**
|
||||
* Generate a shell command to quit and restart Chrome with flags that allow automatic tab capture.
|
||||
* This enables screen recording without user interaction (clicking extension icon).
|
||||
*
|
||||
* Required flags:
|
||||
* - --allowlisted-extension-id=<id> - grants the extension special privileges (one per extension)
|
||||
* - --auto-accept-this-tab-capture - auto-accepts tab capture permission requests
|
||||
*/
|
||||
export function getChromeRestartCommand(): string {
|
||||
const platform = os.platform()
|
||||
const flags = EXTENSION_IDS.map(id => `--allowlisted-extension-id=${id}`).join(' ') + ' --auto-accept-this-tab-capture'
|
||||
|
||||
if (platform === 'darwin') {
|
||||
return `osascript -e 'quit app "Google Chrome"' && sleep 1 && open -a "Google Chrome" --args ${flags}`
|
||||
}
|
||||
if (platform === 'win32') {
|
||||
return `taskkill /IM chrome.exe /F & timeout /t 1 & start chrome.exe ${flags}`
|
||||
}
|
||||
// Linux
|
||||
return `pkill chrome; sleep 1; google-chrome ${flags}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an error is related to missing activeTab permission for recording.
|
||||
*/
|
||||
function isActiveTabPermissionError(error: string): boolean {
|
||||
return error.includes('Extension has not been invoked') ||
|
||||
error.includes('activeTab') ||
|
||||
error.includes('enable recording')
|
||||
}
|
||||
|
||||
export interface StartRecordingOptions {
|
||||
/** Target page to record (currently unused - records first connected tab) */
|
||||
/** Target page to record */
|
||||
page: Page
|
||||
/** Session ID (pw-tab-X format) to identify which tab to record */
|
||||
sessionId?: string
|
||||
/** Frame rate (default: 30) */
|
||||
frameRate?: number
|
||||
/** Video bitrate in bps (default: 2500000 = 2.5 Mbps) */
|
||||
@@ -34,8 +67,10 @@ export interface StartRecordingOptions {
|
||||
}
|
||||
|
||||
export interface StopRecordingOptions {
|
||||
/** Target page that is being recorded (currently unused) */
|
||||
/** Target page that is being recorded */
|
||||
page: Page
|
||||
/** Session ID (pw-tab-X format) to identify which tab to stop recording */
|
||||
sessionId?: string
|
||||
/** Relay server port (default: 19988) */
|
||||
relayPort?: number
|
||||
}
|
||||
@@ -52,6 +87,7 @@ export interface RecordingState {
|
||||
*/
|
||||
export async function startRecording(options: StartRecordingOptions): Promise<RecordingState> {
|
||||
const {
|
||||
sessionId,
|
||||
frameRate = 30,
|
||||
videoBitsPerSecond = 2500000,
|
||||
audioBitsPerSecond = 128000,
|
||||
@@ -63,13 +99,27 @@ export async function startRecording(options: StartRecordingOptions): Promise<Re
|
||||
const response = await fetch(`http://127.0.0.1:${relayPort}/recording/start`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ frameRate, videoBitsPerSecond, audioBitsPerSecond, audio, outputPath }),
|
||||
body: JSON.stringify({ sessionId, frameRate, videoBitsPerSecond, audioBitsPerSecond, audio, outputPath }),
|
||||
})
|
||||
|
||||
const result = await response.json() as StartRecordingResult
|
||||
|
||||
if (!result.success) {
|
||||
throw new Error(`Failed to start recording: ${result.error}`)
|
||||
const errorMsg = result.error || 'Unknown error'
|
||||
|
||||
// If the error is about missing activeTab permission, provide helpful guidance
|
||||
if (isActiveTabPermissionError(errorMsg)) {
|
||||
const restartCmd = getChromeRestartCommand()
|
||||
throw new Error(
|
||||
`Failed to start recording: ${errorMsg}\n\n` +
|
||||
`For automated recording, Chrome must be restarted with special flags.\n` +
|
||||
`WARNING: This will close all Chrome windows. Save your work first!\n\n` +
|
||||
` ${restartCmd}\n\n` +
|
||||
`Or click the Playwriter extension icon on the tab once to grant permission.`
|
||||
)
|
||||
}
|
||||
|
||||
throw new Error(`Failed to start recording: ${errorMsg}`)
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -84,12 +134,12 @@ export async function startRecording(options: StartRecordingOptions): Promise<Re
|
||||
* Returns the path to the saved video file.
|
||||
*/
|
||||
export async function stopRecording(options: StopRecordingOptions): Promise<{ path: string; duration: number; size: number }> {
|
||||
const { relayPort = 19988 } = options
|
||||
const { sessionId, relayPort = 19988 } = options
|
||||
|
||||
const response = await fetch(`http://127.0.0.1:${relayPort}/recording/stop`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({}),
|
||||
body: JSON.stringify({ sessionId }),
|
||||
})
|
||||
|
||||
const result = await response.json() as StopRecordingResult
|
||||
@@ -104,10 +154,13 @@ export async function stopRecording(options: StopRecordingOptions): Promise<{ pa
|
||||
/**
|
||||
* Check if recording is currently active.
|
||||
*/
|
||||
export async function isRecording(options: { page: Page; relayPort?: number }): Promise<RecordingState> {
|
||||
const { relayPort = 19988 } = options
|
||||
export async function isRecording(options: { page: Page; sessionId?: string; relayPort?: number }): Promise<RecordingState> {
|
||||
const { sessionId, relayPort = 19988 } = options
|
||||
|
||||
const response = await fetch(`http://127.0.0.1:${relayPort}/recording/status`)
|
||||
const url = sessionId
|
||||
? `http://127.0.0.1:${relayPort}/recording/status?sessionId=${encodeURIComponent(sessionId)}`
|
||||
: `http://127.0.0.1:${relayPort}/recording/status`
|
||||
const response = await fetch(url)
|
||||
const result = await response.json() as IsRecordingResult
|
||||
|
||||
return { isRecording: result.isRecording, startedAt: result.startedAt, tabId: result.tabId }
|
||||
@@ -116,13 +169,13 @@ export async function isRecording(options: { page: Page; relayPort?: number }):
|
||||
/**
|
||||
* Cancel recording without saving.
|
||||
*/
|
||||
export async function cancelRecording(options: { page: Page; relayPort?: number }): Promise<void> {
|
||||
const { relayPort = 19988 } = options
|
||||
export async function cancelRecording(options: { page: Page; sessionId?: string; relayPort?: number }): Promise<void> {
|
||||
const { sessionId, relayPort = 19988 } = options
|
||||
|
||||
const response = await fetch(`http://127.0.0.1:${relayPort}/recording/cancel`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({}),
|
||||
body: JSON.stringify({ sessionId }),
|
||||
})
|
||||
|
||||
const result = await response.json() as CancelRecordingResult
|
||||
|
||||
@@ -3,6 +3,12 @@ import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
// Playwriter extension IDs - used for validation and Chrome flag commands
|
||||
export const EXTENSION_IDS = [
|
||||
'jfeammnjpkecdekppnclgkkffahnhfhe', // Production (Chrome Web Store)
|
||||
'elnnakgjclnapgflmidlpobefkdmapdm', // Dev (loaded unpacked)
|
||||
]
|
||||
|
||||
export function getCdpUrl({ port = 19988, host = '127.0.0.1', token }: { port?: number; host?: string; token?: string } = {}) {
|
||||
const id = `${Math.random().toString(36).substring(2, 15)}_${Date.now()}`
|
||||
const queryString = token ? `?token=${token}` : ''
|
||||
|
||||
Reference in New Issue
Block a user