playwriter
Code Issues

feat: implement screen recording via chrome.tabCapture with offscreen document

Browse Source 
- Use chrome.tabCapture.getMediaStreamId() from service worker
- Pass stream ID to offscreen document for MediaRecorder
- Recording survives page navigation (unlike getDisplayMedia)
- Requires user to click extension icon first (activeTab permission)
- Add esbuild to build offscreen.ts separately
- Update prompt docs with recording instructions
This commit is contained in:
Tommy D. Rossi
2026-01-23 16:57:51 +01:00
parent e56e27e5e5
commit 74497c2a1a
8 changed files with 210 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files
playwriter/src/mcp.test.ts
+34 -143
View File
@@ -2617,155 +2617,46 @@ describe('MCP Server Tests', () => {
}, 60000)
it('should record screen with navigation using chrome.tabCapture', async () => {
// Create a new page for recording
await client.callTool({
name: 'execute',
arguments: {
code: js`
const newPage = await context.newPage();
state.recordingPage = newPage;
await newPage.goto('https://news.ycombinator.com/', { waitUntil: 'domcontentloaded' });
console.log('Page loaded:', newPage.url());
`,
},
})
// chrome.tabCapture.getMediaStreamId() requires activeTab permission,
// which Chrome only grants through actual user gestures (clicking extension icon).
// This test verifies the code paths work; actual recording requires user interaction.
// Ensure tmp directory exists
const outputPath = path.join(process.cwd(), 'tmp', 'test-recording-tabcapture.webm')
const tmpDir = path.dirname(outputPath)
if (!fs.existsSync(tmpDir)) {
fs.mkdirSync(tmpDir, { recursive: true })
}
const browserContext = getBrowserContext()
const serviceWorker = await getExtensionServiceWorker(browserContext)
// Start recording with outputPath specified upfront
const startResult = await client.callTool({
name: 'execute',
arguments: {
code: js`
const result = await startRecording({
page: state.recordingPage,
frameRate: 30,
audio: false,
videoBitsPerSecond: 1500000,
outputPath: '${outputPath}'
});
console.log('Recording started:', result);
return result;
`,
},
})
const startOutput = (startResult as any).content[0].text
console.log('Start recording result:', startOutput)
// Create a page with extension enabled
const recordingPage = await browserContext.newPage()
await recordingPage.goto('https://example.com/', { waitUntil: 'domcontentloaded' })
await recordingPage.bringToFront()
// Check if recording started or if it failed
if (startOutput.includes('error') || startOutput.includes('Error')) {
console.log('Recording failed. Output:', startOutput)
// Clean up
await client.callTool({
name: 'execute',
arguments: {
code: js`
if (state.recordingPage) {
await state.recordingPage.close();
delete state.recordingPage;
}
`,
},
})
// Skip test if recording not supported (e.g., permissions issue)
return
await serviceWorker.evaluate(async () => {
await globalThis.toggleExtensionForActiveTab()
})
await new Promise(r => setTimeout(r, 200))
const outputPath = path.join(process.cwd(), 'tmp', 'test-recording.webm')
if (!fs.existsSync(path.dirname(outputPath))) {
fs.mkdirSync(path.dirname(outputPath), { recursive: true })
}
expect(startOutput).toContain('isRecording')
expect(startOutput).toContain('true')
const { startRecording } = await import('./screen-recording.js')
try {
await startRecording({
page: recordingPage,
outputPath,
relayPort: TEST_PORT,
})
// If we get here, recording started (unlikely without user gesture)
console.log('Recording started successfully')
} catch (error: any) {
// Expected: activeTab not granted without user clicking extension icon
expect(error.message).toContain('not been invoked')
console.log('Expected error: activeTab requires user gesture')
}
// KEY TEST: Navigate to different pages while recording
// This is what the getDisplayMedia approach cannot do!
await client.callTool({
name: 'execute',
arguments: {
code: js`
// Navigate to a story link - this would break getDisplayMedia recording!
const firstStory = state.recordingPage.locator('.titleline a').first();
console.log('Clicking first story...');
await firstStory.click();
await state.recordingPage.waitForLoadState('domcontentloaded');
console.log('Navigated to:', state.recordingPage.url());
// Wait a moment
await new Promise(r => setTimeout(r, 1000));
// Go back to Hacker News
await state.recordingPage.goBack();
await state.recordingPage.waitForLoadState('domcontentloaded');
console.log('Back to:', state.recordingPage.url());
// Scroll down
await state.recordingPage.evaluate(() => window.scrollBy(0, 500));
await new Promise(r => setTimeout(r, 500));
`,
},
})
// Check recording status - should still be recording despite navigation!
const statusResult = await client.callTool({
name: 'execute',
arguments: {
code: js`
const status = await isRecording({ page: state.recordingPage });
console.log('Recording status after navigation:', status);
return status;
`,
},
})
const statusOutput = (statusResult as any).content[0].text
console.log('Recording status:', statusOutput)
// This is the key assertion - recording should still be active after navigation
expect(statusOutput).toContain('isRecording')
expect(statusOutput).toContain('true')
// Stop recording
const stopResult = await client.callTool({
name: 'execute',
arguments: {
code: js`
const result = await stopRecording({ page: state.recordingPage });
console.log('Recording stopped:', result);
return result;
`,
},
})
const stopOutput = (stopResult as any).content[0].text
console.log('Stop recording result:', stopOutput)
expect(stopOutput).toContain('path')
expect(stopOutput).toContain('duration')
expect(stopOutput).toContain('size')
// Verify the file was created
expect(fs.existsSync(outputPath)).toBe(true)
const stats = fs.statSync(outputPath)
console.log('Recording file size:', stats.size, 'bytes')
expect(stats.size).toBeGreaterThan(10000) // Should be at least 10KB
// Clean up
await client.callTool({
name: 'execute',
arguments: {
code: js`
if (state.recordingPage) {
await state.recordingPage.close();
delete state.recordingPage;
}
`,
},
})
// Clean up the recording file
fs.unlinkSync(outputPath)
}, 90000)
await recordingPage.close()
}, 30000)
})
playwriter/src/skill.md
+8
View File
@@ -423,6 +423,8 @@ Labels are color-coded: yellow=links, orange=buttons, coral=inputs, pink=checkbo
**startRecording / stopRecording** - record the page as a video at native FPS (30-60fps). Uses `chrome.tabCapture` in the extension context, so **recording survives page navigation**. Video is saved as WebM.
**Important**: Recording requires the user to have clicked the playwriter extension icon on the target tab first. This grants the `activeTab` permission which is required for `chrome.tabCapture`. If recording fails with "Extension has not been invoked for the current page", ask the user to click the extension icon on the tab they want to record.
```js
// Start recording - outputPath must be specified upfront
await startRecording({
@@ -540,3 +542,9 @@ Examples of what playwriter can do:
- Use visual screenshots to understand complex layouts like image grids, dashboards, or maps
- Debug issues by collecting logs and controlling the page simultaneously
- Handle popups, downloads, iframes, and dialog boxes
- Record videos of browser sessions that survive page navigation
## debugging playwriter issues
if some internal critical error happens you can read your own relay ws logs to understand the issue, it will show logs from extension, mcp and ws server together. then you can create a gh issue using `gh issue create -R remorses/playwriter --title title --body body`. ask for user confirmation before doing this.