From df63f418e59dd9e24073d33740dab6b9ea038773 Mon Sep 17 00:00:00 2001
From: "Tommy D. Rossi" <beats.by.morse@gmail.com>
Date: Sat, 24 Jan 2026 17:05:24 +0100
Subject: [PATCH] feat: add Chrome flags for automated recording without user
 interaction

- Add getChromeRestartCommand() with --allowlisted-extension-id and --auto-accept-this-tab-capture flags
- Move EXTENSION_IDS to utils.ts for cleaner imports
- Add sessionId support for multi-tab recording
- Add helpful error message when activeTab permission is missing
- Add null check for empty extension result
- Document offscreen recording flow with ASCII diagram
---
 extension/src/background.ts        |  3 +-
 extension/src/offscreen.ts         | 40 ++++++++++++--
 playwriter/src/cdp-relay.ts        | 20 +++----
 playwriter/src/cdp-session.ts      |  5 ++
 playwriter/src/executor.ts         | 12 +++--
 playwriter/src/screen-recording.ts | 83 ++++++++++++++++++++++++------
 playwriter/src/utils.ts            |  6 +++
 7 files changed, 136 insertions(+), 33 deletions(-)

diff --git a/extension/src/background.ts b/extension/src/background.ts
index 6e8a41a..e002ef3 100644
--- a/extension/src/background.ts
+++ b/extension/src/background.ts
@@ -1116,8 +1116,9 @@ async function handleStartRecording(params: StartRecordingParams): Promise<Start
         if (chrome.runtime.lastError) {
           const errorMsg = chrome.runtime.lastError.message || 'Unknown error'
           // Chrome returns this error when activeTab permission hasn't been granted
+          // User must click the extension icon at least once per session - this is a Chrome security requirement
           if (errorMsg.includes('Extension has not been invoked') || errorMsg.includes('activeTab')) {
-            reject(new Error(`${errorMsg}. Click the Playwriter extension icon on this tab to enable recording, then try again.`))
+            reject(new Error(`${errorMsg}. Click the Playwriter extension icon on this tab to enable recording.`))
           } else {
             reject(new Error(errorMsg))
           }
diff --git a/extension/src/offscreen.ts b/extension/src/offscreen.ts
index 8fbe6cb..b5bb46b 100644
--- a/extension/src/offscreen.ts
+++ b/extension/src/offscreen.ts
@@ -1,8 +1,40 @@
 /**
- * Offscreen document for Playwriter extension.
- * Handles operations that require DOM APIs not available in service workers:
- * - Screen recording via chrome.tabCapture + MediaRecorder
- * - Future: audio processing, canvas operations, etc.
+ * Offscreen document for Playwriter screen recording.
+ * 
+ * WHY OFFSCREEN DOCUMENT?
+ * Manifest V3 service workers cannot use MediaRecorder or getUserMedia directly.
+ * This hidden document provides access to Web APIs while the service worker orchestrates.
+ * 
+ * RECORDING FLOW:
+ * 
+ * ┌─────────────────┐     HTTP      ┌─────────────────┐    WebSocket    ┌─────────────────┐
+ * │  User Code      │ ────────────► │  Relay Server   │ ───────────────►│  Extension      │
+ * │  startRecording │               │  /recording/*   │                 │  background.ts  │
+ * └─────────────────┘               └─────────────────┘                 └────────┬────────┘
+ *                                                                                │
+ *                                          ┌─────────────────────────────────────┘
+ *                                          ▼
+ *                                   ┌─────────────────┐
+ *                                   │  Offscreen Doc  │  ◄── MediaRecorder
+ *                                   │  (this file)    │
+ *                                   └─────────────────┘
+ * 
+ * STEP BY STEP:
+ * 1. User calls startRecording() → HTTP POST to relay server
+ * 2. Relay server forwards to extension via WebSocket
+ * 3. Extension calls chrome.tabCapture.getMediaStreamId() to get capture permission
+ *    - Requires --allowlisted-extension-id flag OR user clicking extension icon
+ * 4. Extension creates this offscreen document via chrome.offscreen.createDocument()
+ * 5. Extension sends streamId to offscreen document
+ * 6. Offscreen calls navigator.mediaDevices.getUserMedia() with streamId
+ * 7. Offscreen creates MediaRecorder and starts encoding to webm
+ * 8. Chunks are sent back to extension → relay server → written to output file
+ * 
+ * KEY APIS:
+ * - chrome.tabCapture.getMediaStreamId() - Extension API, gets capture permission
+ * - chrome.offscreen.createDocument()    - Extension API, creates this document
+ * - navigator.mediaDevices.getUserMedia() - Web API, gets MediaStream from streamId
+ * - MediaRecorder                         - Web API, encodes video to webm
  */
 
 interface OffscreenRecordingState {
diff --git a/playwriter/src/cdp-relay.ts b/playwriter/src/cdp-relay.ts
index 392363e..7366c16 100644
--- a/playwriter/src/cdp-relay.ts
+++ b/playwriter/src/cdp-relay.ts
@@ -11,7 +11,7 @@ import fs from 'node:fs'
 import path from 'node:path'
 import pc from 'picocolors'
 import { EventEmitter } from 'node:events'
-import { VERSION } from './utils.js'
+import { VERSION, EXTENSION_IDS } from './utils.js'
 import { createCdpLogger, type CdpLogEntry, type CdpLogger } from './cdp-log.js'
 
 type ConnectedTarget = {
@@ -21,11 +21,7 @@ type ConnectedTarget = {
 }
 
 // Our extension IDs - allow attaching to our own extension pages for debugging
-const OUR_EXTENSION_IDS = [
-  'jfeammnjpkecdekppnclgkkffahnhfhe', // Production extension (Chrome Web Store)
-  'pebbngnfojnignonigcnkdilknapkgid', // Dev extension (stable ID from manifest key)
-]
-
+const OUR_EXTENSION_IDS = EXTENSION_IDS
 /**
  * Checks if a target should be filtered out (not exposed to Playwright).
  * Filters extension pages, service workers, and other restricted targets,
@@ -47,7 +43,7 @@ function isRestrictedTarget(targetInfo: Protocol.Target.TargetInfo): boolean {
   // Allow our own extension pages
   if (url.startsWith('chrome-extension://')) {
     const extensionId = url.replace('chrome-extension://', '').split('/')[0]
-    if (OUR_EXTENSION_IDS.includes(extensionId)) {
+    if (EXTENSION_IDS.includes(extensionId)) {
       return false
     }
     return true
@@ -461,7 +457,7 @@ export async function startPlayWriterCDPRelayServer({
         return null
       }
       const extensionId = origin.replace('chrome-extension://', '')
-      if (!OUR_EXTENSION_IDS.includes(extensionId)) {
+      if (!EXTENSION_IDS.includes(extensionId)) {
         return null
       }
       return origin
@@ -589,7 +585,7 @@ export async function startPlayWriterCDPRelayServer({
     if (origin) {
       if (origin.startsWith('chrome-extension://')) {
         const extensionId = origin.replace('chrome-extension://', '')
-        if (!OUR_EXTENSION_IDS.includes(extensionId)) {
+        if (!EXTENSION_IDS.includes(extensionId)) {
           logger?.log(pc.red(`Rejecting /cdp WebSocket from unknown extension: ${extensionId}`))
           return c.text('Forbidden', 403)
         }
@@ -798,7 +794,7 @@ export async function startPlayWriterCDPRelayServer({
     }
 
     const extensionId = origin.replace('chrome-extension://', '')
-    if (!OUR_EXTENSION_IDS.includes(extensionId)) {
+    if (!EXTENSION_IDS.includes(extensionId)) {
       logger?.log(pc.red(`Rejecting /extension WebSocket from unknown extension: ${extensionId}`))
       return c.text('Forbidden', 403)
     }
@@ -1275,6 +1271,10 @@ export async function startPlayWriterCDPRelayServer({
         timeout: 10000,
       }) as StartRecordingResult
 
+      if (!result) {
+        return c.json({ success: false, error: 'Extension returned empty result' } as StartRecordingResult, 500)
+      }
+
       if (result.success) {
         // Track this recording
         activeRecordings.set(result.tabId, {
diff --git a/playwriter/src/cdp-session.ts b/playwriter/src/cdp-session.ts
index a171221..ac23aba 100644
--- a/playwriter/src/cdp-session.ts
+++ b/playwriter/src/cdp-session.ts
@@ -16,6 +16,7 @@ export interface ICDPSession {
   on(event: string, callback: (params: any) => void): unknown
   off(event: string, callback: (params: any) => void): unknown
   detach(): Promise<void>
+  getSessionId?(): string | null
 }
 
 interface PendingRequest {
@@ -68,6 +69,10 @@ export class CDPSession implements ICDPSession {
     this.sessionId = sessionId
   }
 
+  getSessionId(): string | null {
+    return this.sessionId
+  }
+
   send<K extends keyof ProtocolMapping.Commands>(
     method: K,
     params?: ProtocolMapping.Commands[K]['paramsType'][0],
diff --git a/playwriter/src/executor.ts b/playwriter/src/executor.ts
index 469119b..7658990 100644
--- a/playwriter/src/executor.ts
+++ b/playwriter/src/executor.ts
@@ -595,10 +595,16 @@ export class PlaywrightExecutor {
       // This permission is granted when the user clicks the Playwriter extension icon on a tab.
       const relayPort = this.cdpConfig.port || 19988
       // Recording will work on any tab where the user has clicked the icon.
-      const withRecordingDefaults = <T extends { page?: Page }, R>(
-        fn: (opts: T & { relayPort: number }) => Promise<R>
+      const withRecordingDefaults = <T extends { page?: Page; sessionId?: string }, R>(
+        fn: (opts: T & { relayPort: number; sessionId?: string }) => Promise<R>
       ) => {
-        return (options: T = {} as T) => fn({ page: options.page || page, relayPort, ...options })
+        return async (options: T = {} as T) => {
+          const targetPage = options.page || page
+          // Get sessionId from cached CDP session to identify which tab to record
+          const cdp = await getCDPSession({ page: targetPage })
+          const sessionId = options.sessionId || cdp.getSessionId() || undefined
+          return fn({ page: targetPage, sessionId, relayPort, ...options })
+        }
       }
       const self = this
 
diff --git a/playwriter/src/screen-recording.ts b/playwriter/src/screen-recording.ts
index e5f5dc1..3d172f2 100644
--- a/playwriter/src/screen-recording.ts
+++ b/playwriter/src/screen-recording.ts
@@ -3,11 +3,10 @@
  * Recording happens in the extension context, so it survives page navigation.
  * 
  * This module communicates with the relay server which forwards commands to the extension.
- * 
- * Note: Recording uses the first connected tab. Multi-tab recording support would require
- * proper sessionId mapping between Playwright pages and extension tabs.
+ * SessionId (pw-tab-X format) is used to identify which tab to record.
  */
 
+import os from 'node:os'
 import type { Page } from 'playwright-core'
 import type {
   StartRecordingResult,
@@ -15,10 +14,44 @@ import type {
   IsRecordingResult,
   CancelRecordingResult,
 } from './protocol.js'
+import { EXTENSION_IDS } from './utils.js'
+
+/**
+ * Generate a shell command to quit and restart Chrome with flags that allow automatic tab capture.
+ * This enables screen recording without user interaction (clicking extension icon).
+ * 
+ * Required flags:
+ * - --allowlisted-extension-id=<id> - grants the extension special privileges (one per extension)
+ * - --auto-accept-this-tab-capture - auto-accepts tab capture permission requests
+ */
+export function getChromeRestartCommand(): string {
+  const platform = os.platform()
+  const flags = EXTENSION_IDS.map(id => `--allowlisted-extension-id=${id}`).join(' ') + ' --auto-accept-this-tab-capture'
+  
+  if (platform === 'darwin') {
+    return `osascript -e 'quit app "Google Chrome"' && sleep 1 && open -a "Google Chrome" --args ${flags}`
+  }
+  if (platform === 'win32') {
+    return `taskkill /IM chrome.exe /F & timeout /t 1 & start chrome.exe ${flags}`
+  }
+  // Linux
+  return `pkill chrome; sleep 1; google-chrome ${flags}`
+}
+
+/**
+ * Check if an error is related to missing activeTab permission for recording.
+ */
+function isActiveTabPermissionError(error: string): boolean {
+  return error.includes('Extension has not been invoked') || 
+         error.includes('activeTab') ||
+         error.includes('enable recording')
+}
 
 export interface StartRecordingOptions {
-  /** Target page to record (currently unused - records first connected tab) */
+  /** Target page to record */
   page: Page
+  /** Session ID (pw-tab-X format) to identify which tab to record */
+  sessionId?: string
   /** Frame rate (default: 30) */
   frameRate?: number
   /** Video bitrate in bps (default: 2500000 = 2.5 Mbps) */
@@ -34,8 +67,10 @@ export interface StartRecordingOptions {
 }
 
 export interface StopRecordingOptions {
-  /** Target page that is being recorded (currently unused) */
+  /** Target page that is being recorded */
   page: Page
+  /** Session ID (pw-tab-X format) to identify which tab to stop recording */
+  sessionId?: string
   /** Relay server port (default: 19988) */
   relayPort?: number
 }
@@ -52,6 +87,7 @@ export interface RecordingState {
  */
 export async function startRecording(options: StartRecordingOptions): Promise<RecordingState> {
   const {
+    sessionId,
     frameRate = 30,
     videoBitsPerSecond = 2500000,
     audioBitsPerSecond = 128000,
@@ -63,13 +99,27 @@ export async function startRecording(options: StartRecordingOptions): Promise<Re
   const response = await fetch(`http://127.0.0.1:${relayPort}/recording/start`, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({ frameRate, videoBitsPerSecond, audioBitsPerSecond, audio, outputPath }),
+    body: JSON.stringify({ sessionId, frameRate, videoBitsPerSecond, audioBitsPerSecond, audio, outputPath }),
   })
 
   const result = await response.json() as StartRecordingResult
 
   if (!result.success) {
-    throw new Error(`Failed to start recording: ${result.error}`)
+    const errorMsg = result.error || 'Unknown error'
+    
+    // If the error is about missing activeTab permission, provide helpful guidance
+    if (isActiveTabPermissionError(errorMsg)) {
+      const restartCmd = getChromeRestartCommand()
+      throw new Error(
+        `Failed to start recording: ${errorMsg}\n\n` +
+        `For automated recording, Chrome must be restarted with special flags.\n` +
+        `WARNING: This will close all Chrome windows. Save your work first!\n\n` +
+        `  ${restartCmd}\n\n` +
+        `Or click the Playwriter extension icon on the tab once to grant permission.`
+      )
+    }
+    
+    throw new Error(`Failed to start recording: ${errorMsg}`)
   }
 
   return {
@@ -84,12 +134,12 @@ export async function startRecording(options: StartRecordingOptions): Promise<Re
  * Returns the path to the saved video file.
  */
 export async function stopRecording(options: StopRecordingOptions): Promise<{ path: string; duration: number; size: number }> {
-  const { relayPort = 19988 } = options
+  const { sessionId, relayPort = 19988 } = options
 
   const response = await fetch(`http://127.0.0.1:${relayPort}/recording/stop`, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({}),
+    body: JSON.stringify({ sessionId }),
   })
 
   const result = await response.json() as StopRecordingResult
@@ -104,10 +154,13 @@ export async function stopRecording(options: StopRecordingOptions): Promise<{ pa
 /**
  * Check if recording is currently active.
  */
-export async function isRecording(options: { page: Page; relayPort?: number }): Promise<RecordingState> {
-  const { relayPort = 19988 } = options
+export async function isRecording(options: { page: Page; sessionId?: string; relayPort?: number }): Promise<RecordingState> {
+  const { sessionId, relayPort = 19988 } = options
 
-  const response = await fetch(`http://127.0.0.1:${relayPort}/recording/status`)
+  const url = sessionId 
+    ? `http://127.0.0.1:${relayPort}/recording/status?sessionId=${encodeURIComponent(sessionId)}`
+    : `http://127.0.0.1:${relayPort}/recording/status`
+  const response = await fetch(url)
   const result = await response.json() as IsRecordingResult
 
   return { isRecording: result.isRecording, startedAt: result.startedAt, tabId: result.tabId }
@@ -116,13 +169,13 @@ export async function isRecording(options: { page: Page; relayPort?: number }):
 /**
  * Cancel recording without saving.
  */
-export async function cancelRecording(options: { page: Page; relayPort?: number }): Promise<void> {
-  const { relayPort = 19988 } = options
+export async function cancelRecording(options: { page: Page; sessionId?: string; relayPort?: number }): Promise<void> {
+  const { sessionId, relayPort = 19988 } = options
 
   const response = await fetch(`http://127.0.0.1:${relayPort}/recording/cancel`, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({}),
+    body: JSON.stringify({ sessionId }),
   })
 
   const result = await response.json() as CancelRecordingResult
diff --git a/playwriter/src/utils.ts b/playwriter/src/utils.ts
index b3b5783..559d8aa 100644
--- a/playwriter/src/utils.ts
+++ b/playwriter/src/utils.ts
@@ -3,6 +3,12 @@ import os from 'node:os'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 
+// Playwriter extension IDs - used for validation and Chrome flag commands
+export const EXTENSION_IDS = [
+  'jfeammnjpkecdekppnclgkkffahnhfhe', // Production (Chrome Web Store)
+  'elnnakgjclnapgflmidlpobefkdmapdm', // Dev (loaded unpacked)
+]
+
 export function getCdpUrl({ port = 19988, host = '127.0.0.1', token }: { port?: number; host?: string; token?: string } = {}) {
   const id = `${Math.random().toString(36).substring(2, 15)}_${Date.now()}`
   const queryString = token ? `?token=${token}` : ''