From 95bda7acc75f032ff774898b9650f284f0677a62 Mon Sep 17 00:00:00 2001 From: "Tommy D. Rossi" Date: Wed, 25 Feb 2026 12:45:04 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20demo=20video=20creation=20=E2=80=94=20a?= =?UTF-8?q?uto=20speed=20up=20idle=20sections=20during=20recordings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ffmpeg.ts with utilities for video manipulation: - concatenateVideos: merge multiple video files with trim/scale/fps normalization - speedUpSections: speed up arbitrary timestamp ranges in a single ffmpeg pass - computeIdleSections: compute idle gaps from execution timestamps - createDemoVideo: high-level wrapper that takes stopRecording() result and produces a sped-up video where idle time (between execute() calls) is fast-forwarded Executor changes: - Track execution timestamps (start/end in seconds) while recording is active - startRecording wrapper resets tracking state - stopRecording wrapper returns executionTimestamps alongside path/duration/size - cancelRecording wrapper clears tracking state - Expose createDemoVideo in the sandbox VM context The recording timeline looks like: 0s 5s 8s 12s 15s 20s 25s 30s ├───────┼────┼────┼────┼────┼────┼─────┤ ▓▓▓▓▓ ▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓ exec1 exec2 exec3 ░░░░░░░ ░░░░░ ░░░░░ IDLE(4x) IDLE(4x) IDLE(4x) A 1-second buffer (INTERACTION_BUFFER_SECONDS) is kept around each execution at normal speed so viewers see context before/after actions. Requires ffmpeg + ffprobe on the system. --- playwriter/src/executor.ts | 41 +- playwriter/src/ffmpeg.ts | 531 ++++++++++++++++++++++++ playwriter/src/relay-navigation.test.ts | 23 + playwriter/src/skill.md | 33 ++ 4 files changed, 625 insertions(+), 3 deletions(-) create mode 100644 playwriter/src/ffmpeg.ts diff --git a/playwriter/src/executor.ts b/playwriter/src/executor.ts index f61b08b..346c31a 100644 --- a/playwriter/src/executor.ts +++ b/playwriter/src/executor.ts @@ -34,6 +34,7 @@ export type { SnapshotFormat } import { getCleanHTML, type GetCleanHTMLOptions } from './clean-html.js' import { getPageMarkdown, type GetPageMarkdownOptions } from './page-markdown.js' import { startRecording, stopRecording, isRecording, cancelRecording } from './screen-recording.js' +import { createDemoVideo } from './ffmpeg.js' const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) @@ -231,6 +232,12 @@ export class PlaywrightExecutor { private warningEvents: WarningEvent[] = [] private nextWarningEventId = 0 private lastDeliveredWarningEventId = 0 + + // Recording timestamp tracking: when recording is active, each execute() + // call pushes {start, end} (seconds relative to recordingStartedAt). + // Returned by stopRecording() so the model can speed up idle sections. + private recordingStartedAt: number | null = null + private executionTimestamps: Array<{ start: number; end: number }> = [] private activeWarningScopes = new Set() private pagesWithListeners = new WeakSet() private suppressPageCloseWarnings = false @@ -987,10 +994,27 @@ export class PlaywrightExecutor { formatStylesAsText, getReactSource: getReactSourceFn, screenshotWithAccessibilityLabels: screenshotWithAccessibilityLabelsFn, - startRecording: withRecordingDefaults(startRecording), - stopRecording: withRecordingDefaults(stopRecording), + startRecording: async (opts?: Parameters[0]) => { + const result = await withRecordingDefaults(startRecording)(opts) + self.recordingStartedAt = Date.now() + self.executionTimestamps = [] + return result + }, + stopRecording: async (opts?: Parameters[0]) => { + const result = await withRecordingDefaults(stopRecording)(opts) + const executionTimestamps = [...self.executionTimestamps] + self.recordingStartedAt = null + self.executionTimestamps = [] + return { ...result, executionTimestamps } + }, isRecording: withRecordingDefaults(isRecording), - cancelRecording: withRecordingDefaults(cancelRecording), + cancelRecording: async (opts?: Parameters[0]) => { + const result = await withRecordingDefaults(cancelRecording)(opts) + self.recordingStartedAt = null + self.executionTimestamps = [] + return result + }, + createDemoVideo, resetPlaywright: async () => { const { page: newPage, context: newContext } = await self.reset() vmContextObj.page = newPage @@ -1009,11 +1033,22 @@ export class PlaywrightExecutor { const wrappedCode = autoReturn ? `(async () => { return await (${code}) })()` : `(async () => { ${code} })()` const hasExplicitReturn = autoReturn || /\breturn\b/.test(code) + // Track execution timestamps relative to recording start (seconds). + // Used to identify idle gaps that can be sped up in demo videos. + const execStartSec = this.recordingStartedAt !== null + ? (Date.now() - this.recordingStartedAt) / 1000 + : -1 + const result = await Promise.race([ vm.runInContext(wrappedCode, vmContext, { timeout, displayErrors: true }), new Promise((_, reject) => setTimeout(() => reject(new CodeExecutionTimeoutError(timeout)), timeout)), ]) + if (this.recordingStartedAt !== null && execStartSec >= 0) { + const execEndSec = (Date.now() - this.recordingStartedAt) / 1000 + this.executionTimestamps.push({ start: execStartSec, end: execEndSec }) + } + let responseText = formatConsoleLogs(consoleLogs) // Only show return value if user explicitly used return diff --git a/playwriter/src/ffmpeg.ts b/playwriter/src/ffmpeg.ts new file mode 100644 index 0000000..34621ee --- /dev/null +++ b/playwriter/src/ffmpeg.ts @@ -0,0 +1,531 @@ +/** + * FFmpeg utilities for video concatenation and section-based speed manipulation. + * + * Both functions use a single ffmpeg filter_complex pass: trim segments from + * the input, apply setpts for speed, normalize fps/scale, then concat. + * No intermediate files, no multi-pass. + */ + +import { exec } from 'node:child_process' +import path from 'node:path' + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/** Seconds of normal-speed buffer kept before and after each execution */ +export const INTERACTION_BUFFER_SECONDS = 1 + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface InputFile { + path: string + start?: number + end?: number +} + +export interface ConcatenateOptions { + inputFiles: InputFile[] + outputFile: string + outputDimensions: { width: number; height: number } + frameRate: number + signal?: AbortSignal +} + +export interface SpeedSection { + /** Start time in seconds */ + start: number + /** End time in seconds */ + end: number + /** Speed multiplier, e.g. 2 = 2x faster, 0.5 = 2x slower */ + speed: number +} + +export interface SpeedUpSectionsOptions { + inputFile: string + /** Defaults to inputFile with `-fast` suffix before extension */ + outputFile?: string + sections: SpeedSection[] + /** Defaults to input video dimensions (probed via ffprobe) */ + outputDimensions?: { width: number; height: number } + /** Defaults to input video frame rate (probed via ffprobe) */ + frameRate?: number + signal?: AbortSignal +} + +export interface VideoInfo { + width: number + height: number + frameRate: number +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Probe input video for dimensions and frame rate via ffprobe. */ +export async function probeVideo(filePath: string): Promise { + const command = [ + 'ffprobe', + '-v error', + '-select_streams v:0', + '-show_entries stream=width,height,r_frame_rate', + '-of json', + `"${filePath}"`, + ].join(' ') + + const stdout = await runCommand({ command }) + const parsed = JSON.parse(stdout) + const stream = parsed.streams?.[0] + if (!stream) { + throw new Error(`No video stream found in ${filePath}`) + } + + // r_frame_rate is a fraction like "30/1" or "30000/1001" + const [num, den] = (stream.r_frame_rate as string).split('/').map(Number) + + return { + width: stream.width as number, + height: stream.height as number, + frameRate: Math.round(num / den), + } +} + +/** Run a shell command, optionally abortable. Returns stdout. */ +function runCommand({ + command, + signal, +}: { + command: string + signal?: AbortSignal +}): Promise { + return new Promise((resolve, reject) => { + const childProcess = exec(command, (error, stdout, stderr) => { + if (error) { + reject(new Error(`FFmpeg error: ${stderr}`, { cause: error })) + } else { + resolve(stdout) + } + }) + + if (signal) { + signal.addEventListener( + 'abort', + () => { + childProcess.kill() + reject( + signal.reason instanceof Error + ? signal.reason + : new Error('Operation aborted'), + ) + }, + { once: true }, + ) + } + }) +} + +/** Build default output path: `/dir/name-fast.ext` */ +function defaultOutputPath(inputFile: string): string { + const ext = path.extname(inputFile) + const base = path.basename(inputFile, ext) + const dir = path.dirname(inputFile) + return path.join(dir, `${base}-fast${ext}`) +} + +// --------------------------------------------------------------------------- +// Internal segment types +// --------------------------------------------------------------------------- + +interface Segment { + start: number + /** undefined = until end of video */ + end: number | undefined + /** 1 = normal speed */ + speed: number +} + +/** + * Given sorted, non-overlapping SpeedSections, fill gaps with normal-speed + * segments so the entire video is covered. + */ +function buildSegments(sections: SpeedSection[]): Segment[] { + const sorted = [...sections].sort((a, b) => { + return a.start - b.start + }) + + // Validate: no overlaps + for (let i = 1; i < sorted.length; i++) { + if (sorted[i].start < sorted[i - 1].end) { + throw new Error( + `Sections overlap: [${sorted[i - 1].start}-${sorted[i - 1].end}] and [${sorted[i].start}-${sorted[i].end}]`, + ) + } + } + + const segments: Segment[] = [] + let cursor = 0 + + for (const section of sorted) { + // Gap before this section → normal speed + if (section.start > cursor) { + segments.push({ start: cursor, end: section.start, speed: 1 }) + } + // The speed section itself + segments.push({ + start: section.start, + end: section.end, + speed: section.speed, + }) + cursor = section.end + } + + // Trailing normal-speed segment (no end bound → until EOF) + segments.push({ start: cursor, end: undefined, speed: 1 }) + + return segments +} + +/** + * Build the filter string for a single segment. + * Returns `[0:v]trim=...,setpts=...,fps=...,scale=...[vN]` + */ +function buildSegmentFilter({ + segment, + index, + frameRate, + width, + height, +}: { + segment: Segment + index: number + frameRate: number + width: number + height: number +}): string { + const trimParts = [`start=${segment.start}`] + if (segment.end !== undefined) { + trimParts.push(`end=${segment.end}`) + } + const trim = `trim=${trimParts.join(':')}` + + // setpts=PTS-STARTPTS resets timestamps after trim. + // Dividing by speed makes it faster (speed>1) or slower (speed<1). + const setpts = + segment.speed === 1 + ? 'setpts=PTS-STARTPTS' + : `setpts=(PTS-STARTPTS)/${segment.speed}` + + return `[0:v]${trim},${setpts},fps=${frameRate},scale=${width}:${height}[v${index}]` +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +export async function concatenateVideos( + options: ConcatenateOptions, +): Promise { + const { outputDimensions, frameRate, inputFiles, outputFile, signal } = + options + + if (!outputDimensions || !frameRate || !inputFiles || !outputFile) { + throw new Error('Missing required parameters') + } + + const timerId = `concat-${inputFiles.length}-videos-${outputFile.split('/').pop()}` + console.time(timerId) + + const inputArgs = inputFiles.map((file) => { + return `-i "${file.path}"` + }).join(' ') + + const filterComplexParts: string[] = [] + const videoStreamParts: string[] = [] + + inputFiles.forEach((file, index) => { + const videoStream = `[${index}:v:0]` + let trimmedVideo = videoStream + + if (file.start !== undefined || file.end !== undefined) { + const start = file.start ?? 0 + const end = file.end ? `end=${file.end}` : '' + trimmedVideo = `${videoStream}trim=start=${start}:${end},setpts=PTS-STARTPTS` + } + + filterComplexParts.push( + `${trimmedVideo},fps=${frameRate},scale=${outputDimensions.width}:${outputDimensions.height}[v${index}]`, + ) + videoStreamParts.push(`[v${index}]`) + }) + + filterComplexParts.push( + `${videoStreamParts.join('')}concat=n=${inputFiles.length}:v=1:a=0[v_out]`, + ) + + const filterComplex = filterComplexParts.join('; ') + const command = + `ffmpeg ${inputArgs} -filter_complex "${filterComplex}" -map "[v_out]" "${outputFile}"` + + console.log('Running FFmpeg command:', command) + + try { + await runCommand({ command, signal }) + } finally { + console.timeEnd(timerId) + } +} + +/** + * Speed up (or slow down) sections of a video by timestamp ranges. + * + * Sections not covered by any SpeedSection play at normal speed. + * Uses a single ffmpeg filter_complex: trim each segment, apply setpts + * speed, normalize fps/scale, then concat — no intermediate files. + * + * @example + * ```ts + * await speedUpSections({ + * inputFile: 'recording.mp4', + * sections: [ + * { start: 10, end: 20, speed: 4 }, // 4x between 10s-20s + * { start: 30, end: 40, speed: 2 }, // 2x between 30s-40s + * ], + * }) + * // → outputs recording-fast.mp4 + * ``` + */ +export async function speedUpSections( + options: SpeedUpSectionsOptions, +): Promise { + const { inputFile, sections, signal } = options + + if (sections.length === 0) { + throw new Error('At least one speed section is required') + } + for (const s of sections) { + if (s.speed <= 0) { + throw new Error(`Speed must be > 0, got ${s.speed}`) + } + if (s.end <= s.start) { + throw new Error( + `Section end (${s.end}) must be greater than start (${s.start})`, + ) + } + } + + const outputFile = options.outputFile ?? defaultOutputPath(inputFile) + + // Probe input for defaults + const dims = options.outputDimensions + const fps = options.frameRate + const needsProbe = !dims || !fps + const probed = needsProbe ? await probeVideo(inputFile) : undefined + + const width = dims?.width ?? probed!.width + const height = dims?.height ?? probed!.height + const frameRate = fps ?? probed!.frameRate + + const timerId = `speedup-${sections.length}-sections-${outputFile.split('/').pop()}` + console.time(timerId) + + const segments = buildSegments(sections) + + const filterParts = segments.map((segment, index) => { + return buildSegmentFilter({ + segment, + index, + frameRate, + width, + height, + }) + }) + + const streamLabels = segments.map((_, i) => { + return `[v${i}]` + }).join('') + + filterParts.push( + `${streamLabels}concat=n=${segments.length}:v=1:a=0[v_out]`, + ) + + const filterComplex = filterParts.join('; ') + const command = + `ffmpeg -i "${inputFile}" -filter_complex "${filterComplex}" -map "[v_out]" "${outputFile}"` + + console.log('Running FFmpeg command:', command) + + try { + await runCommand({ command, signal }) + } finally { + console.timeEnd(timerId) + } + + return outputFile +} + +// --------------------------------------------------------------------------- +// Idle section computation +// --------------------------------------------------------------------------- + +export interface ExecutionTimestamp { + /** Start time in seconds relative to recording start */ + start: number + /** End time in seconds relative to recording start */ + end: number +} + +/** + * Compute which parts of a recording are "idle" (no execute() calls) + * and return them as SpeedSections that can be passed to speedUpSections(). + * + * A buffer of INTERACTION_BUFFER_SECONDS is kept around each execution + * at normal speed so the viewer sees context before/after each action. + * + * @example + * ```ts + * const { executionTimestamps, duration } = await stopRecording() + * const idleSections = computeIdleSections({ + * executionTimestamps, + * totalDurationMs: duration, + * }) + * await speedUpSections({ + * inputFile: recordingPath, + * sections: idleSections, + * }) + * ``` + */ +export function computeIdleSections({ + executionTimestamps, + totalDurationMs, + speed = 4, + bufferSeconds = INTERACTION_BUFFER_SECONDS, +}: { + executionTimestamps: ExecutionTimestamp[] + /** Total recording duration in milliseconds (from stopRecording result) */ + totalDurationMs: number + /** Speed multiplier for idle sections (default 4) */ + speed?: number + /** Override the default buffer around each execution (seconds) */ + bufferSeconds?: number +}): SpeedSection[] { + const totalDuration = totalDurationMs / 1000 + + if (executionTimestamps.length === 0) { + // Entire video is idle + if (totalDuration <= 0) { + return [] + } + return [{ start: 0, end: totalDuration, speed }] + } + + // Apply buffer: expand each execution range by bufferSeconds on each side, + // then merge overlapping ranges into consolidated "active" intervals. + const buffered = executionTimestamps + .map((t) => ({ + start: Math.max(0, t.start - bufferSeconds), + end: Math.min(totalDuration, t.end + bufferSeconds), + })) + .sort((a, b) => { + return a.start - b.start + }) + + // Merge overlapping/adjacent buffered ranges + const merged: Array<{ start: number; end: number }> = [] + for (const range of buffered) { + const last = merged[merged.length - 1] + if (last && range.start <= last.end) { + last.end = Math.max(last.end, range.end) + } else { + merged.push({ ...range }) + } + } + + // Gaps between merged active ranges are idle sections to speed up + const idle: SpeedSection[] = [] + let cursor = 0 + + for (const active of merged) { + if (active.start > cursor) { + idle.push({ start: cursor, end: active.start, speed }) + } + cursor = active.end + } + + // Trailing idle after last execution + if (cursor < totalDuration) { + idle.push({ start: cursor, end: totalDuration, speed }) + } + + return idle +} + +// --------------------------------------------------------------------------- +// High-level demo video creation +// --------------------------------------------------------------------------- + +export interface CreateDemoVideoOptions { + /** Path to the raw recording file */ + recordingPath: string + /** Total recording duration in milliseconds (from stopRecording result) */ + durationMs: number + /** Execution timestamps (from stopRecording result) */ + executionTimestamps: ExecutionTimestamp[] + /** Speed multiplier for idle sections (default 4) */ + speed?: number + /** Output file path (defaults to recordingPath with `-demo` suffix) */ + outputFile?: string + signal?: AbortSignal +} + +/** + * Create a demo video from a recording by speeding up idle sections + * (gaps between execute() calls) while keeping interactions at normal speed. + * + * A 1-second buffer (INTERACTION_BUFFER_SECONDS) is preserved around each + * interaction so viewers see context before and after each action. + * + * Requires `ffmpeg` and `ffprobe` installed on the system. + * + * @returns The output file path + */ +export async function createDemoVideo( + options: CreateDemoVideoOptions, +): Promise { + const { + recordingPath, + durationMs, + executionTimestamps, + speed = 4, + signal, + } = options + + const outputFile = options.outputFile ?? (() => { + const ext = path.extname(recordingPath) + const base = path.basename(recordingPath, ext) + const dir = path.dirname(recordingPath) + return path.join(dir, `${base}-demo${ext}`) + })() + + const idleSections = computeIdleSections({ + executionTimestamps, + totalDurationMs: durationMs, + speed, + }) + + if (idleSections.length === 0) { + // No idle sections, nothing to speed up — copy as-is + const { copyFile } = await import('node:fs/promises') + await copyFile(recordingPath, outputFile) + return outputFile + } + + return speedUpSections({ + inputFile: recordingPath, + outputFile, + sections: idleSections, + signal, + }) +} diff --git a/playwriter/src/relay-navigation.test.ts b/playwriter/src/relay-navigation.test.ts index 2f8f70e..015a036 100644 --- a/playwriter/src/relay-navigation.test.ts +++ b/playwriter/src/relay-navigation.test.ts @@ -599,7 +599,30 @@ describe('Relay Navigation Tests', () => { expect(stopResult.size).toBeGreaterThan(10000) expect(fs.existsSync(outputPath)).toBe(true) + // Create a sped-up demo video from the recording. + // We fake executionTimestamps since this test calls screen-recording + // directly (not via executor sandbox which tracks them automatically). + const { createDemoVideo } = await import('./ffmpeg.js') + const demoPath = await createDemoVideo({ + recordingPath: outputPath, + durationMs: stopResult.duration, + executionTimestamps: [ + // Simulate two interactions with an idle gap between them + { start: 0.5, end: 1.5 }, + { start: 3, end: 4 }, + ], + speed: 4, + }) + expect(fs.existsSync(demoPath)).toBe(true) + expect(demoPath).toContain('-demo') + + // Verify the demo video is smaller (idle sections were sped up) + const demoSize = fs.statSync(demoPath).size + expect(demoSize).toBeGreaterThan(0) + console.log(`Recording: ${stopResult.size} bytes, Demo: ${demoSize} bytes`) + await recordingPage.close() fs.unlinkSync(outputPath) + fs.unlinkSync(demoPath) }, 60000) }) diff --git a/playwriter/src/skill.md b/playwriter/src/skill.md index 983ae8b..c40f6ac 100644 --- a/playwriter/src/skill.md +++ b/playwriter/src/skill.md @@ -897,6 +897,39 @@ await cancelRecording({ page: state.page }) **Key difference from getDisplayMedia**: This approach uses `chrome.tabCapture` which runs in the extension context, not the page. The recording persists across navigations because the extension holds the `MediaRecorder`, not the page's JavaScript context. +**createDemoVideo** - create a polished demo video from a recording by automatically speeding up idle sections (time between execute() calls) while keeping interactions at normal speed. Useful for creating demo videos of agent workflows without long pauses. + +While recording is active, playwriter tracks when each `execute()` call starts and ends. `stopRecording()` returns these timestamps alongside the video file. `createDemoVideo` uses this data to identify idle gaps and speed them up with ffmpeg in a single pass. + +A 1-second buffer is preserved around each interaction so viewers see context before and after each action. + +Requires `ffmpeg` and `ffprobe` installed on the system. + +```js +// Start recording +await startRecording({ page: state.page, outputPath: './recording.mp4' }) +``` + +```js +// ... multiple execute() calls with browser interactions ... +// Each call's timing is tracked automatically while recording is active +``` + +```js +// Stop recording — executionTimestamps is included in the result +const recording = await stopRecording({ page: state.page }) + +// Create demo video — idle gaps are sped up 4x (default) +const demoPath = await createDemoVideo({ + recordingPath: recording.path, + durationMs: recording.duration, + executionTimestamps: recording.executionTimestamps, + speed: 4, // optional, default 4x for idle sections + // outputFile: './demo.mp4', // optional, defaults to recording-demo.mp4 +}) +console.log('Demo video:', demoPath) +``` + ## pinned elements Users can right-click → "Copy Playwriter Element Reference" to store elements in `globalThis.playwriterPinnedElem1` (increments for each pin). The reference is copied to clipboard: