From c9492fb4b0639a33c7ae9c3e96e27bd566af0f9b Mon Sep 17 00:00:00 2001 From: "Tommy D. Rossi" Date: Thu, 26 Feb 2026 20:31:19 +0100 Subject: [PATCH] feat: optimize ffmpeg encoding for screen recordings on social media MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add hardware encoder detection with automatic fallback: - h264_videotoolbox (macOS) → h264_nvenc/qsv/amf (Windows) → h264_nvenc/qsv (Linux) → libx264 - Probe caches result per process, 5s timeout per encoder test - h264_vaapi excluded (needs -vaapi_device + hwupload in filter graph) Add social-media-optimized encoding parameters: - libx264: CRF 18 (near visually lossless for sharp text), preset fast, deblock=-1,-1 (preserves text edges), no -tune flag (animation blurs text, zerolatency is for real-time capture only) - h264_videotoolbox: -q:v 80 (≈CRF 18 for screen content) - Common: yuv420p, maxrate 25M (X.com cap), bufsize 50M, movflags +faststart Skip redundant fps/scale filters on normal-speed passthrough segments when output dimensions and frame rate match the probed input values. Previously no -c:v or quality params were specified at all — ffmpeg defaulted to libx264 -preset medium -crf 23, which is tuned for natural video, not screen recordings, and purely CPU-based. Tested on 48s 3440x1964 screen recording: 17s encode time with VideoToolbox hardware acceleration, 5.5 Mbps output bitrate. --- playwriter/src/ffmpeg.ts | 179 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 174 insertions(+), 5 deletions(-) diff --git a/playwriter/src/ffmpeg.ts b/playwriter/src/ffmpeg.ts index f2cba9e..304131a 100644 --- a/playwriter/src/ffmpeg.ts +++ b/playwriter/src/ffmpeg.ts @@ -7,6 +7,7 @@ */ import { spawn } from 'node:child_process' +import os from 'node:os' import path from 'node:path' // --------------------------------------------------------------------------- @@ -16,6 +17,136 @@ import path from 'node:path' /** Seconds of normal-speed buffer kept before and after each execution */ export const INTERACTION_BUFFER_SECONDS = 1 +// --------------------------------------------------------------------------- +// Hardware encoder detection +// --------------------------------------------------------------------------- + +/** + * Preferred hardware encoders by platform, in priority order. + * Each is tried via `ffmpeg -f lavfi -i nullsrc -t 0.01 -c:v -f null -` + * to confirm the encoder actually works (drivers present, GPU available, etc). + */ +const HW_ENCODER_CANDIDATES: Record = { + darwin: ['h264_videotoolbox'], + win32: ['h264_nvenc', 'h264_qsv', 'h264_amf'], + // h264_vaapi excluded: requires -vaapi_device and format=nv12,hwupload in the + // filter graph, which our filter_complex pipeline doesn't set up + linux: ['h264_nvenc', 'h264_qsv'], +} + +interface EncoderInfo { + /** Encoder name for `-c:v`, e.g. "h264_videotoolbox" or "libx264" */ + codec: string + /** true when using a hardware encoder */ + isHardware: boolean +} + +/** Cache so we only probe once per process */ +let cachedEncoder: EncoderInfo | undefined + +/** + * Detect the best available H.264 encoder. + * Tries platform-specific hardware encoders first, falls back to libx264. + * Result is cached for the lifetime of the process. + */ +export async function detectEncoder(): Promise { + if (cachedEncoder) { + return cachedEncoder + } + + const platform = os.platform() + const candidates = HW_ENCODER_CANDIDATES[platform] ?? [] + + for (const codec of candidates) { + const works = await testEncoder(codec) + if (works) { + cachedEncoder = { codec, isHardware: true } + return cachedEncoder + } + } + + cachedEncoder = { codec: 'libx264', isHardware: false } + return cachedEncoder +} + +/** + * Quick probe: can this encoder produce even a single frame? + * Runs `ffmpeg -f lavfi -i nullsrc=s=64x64:d=0.01 -c:v -f null -` + * and checks exit code. Timeout 5s to avoid hanging on broken drivers. + */ +function testEncoder(codec: string): Promise { + return new Promise((resolve) => { + const child = spawn('ffmpeg', [ + '-hide_banner', '-loglevel', 'error', + '-f', 'lavfi', '-i', 'nullsrc=s=64x64:d=0.01', + '-c:v', codec, + '-f', 'null', '-', + ], { stdio: 'ignore' }) + + const timeout = setTimeout(() => { + child.kill() + resolve(false) + }, 5000) + + child.on('close', (code) => { + clearTimeout(timeout) + resolve(code === 0) + }) + + child.on('error', () => { + clearTimeout(timeout) + resolve(false) + }) + }) +} + +// --------------------------------------------------------------------------- +// Encoding quality parameters +// --------------------------------------------------------------------------- + +/** + * Build encoding args optimized for screen recordings on social media. + * + * Quality rationale (screen recordings = sharp text, flat colors, UI): + * - libx264 CRF 18 = near visually lossless, prevents text compression artifacts + * - `-preset fast` = ~4x faster than default `medium`, minimal quality loss on + * screen content. NOT `ultrafast` which compresses so poorly that platforms + * (X.com, YouTube) re-encode more aggressively, making it look worse. + * - `-x264opts deblock=-1,-1` = less deblocking preserves sharp text edges + * - No `-tune` flag: `animation` blurs text edges, `zerolatency` is for + * real-time capture only + * - h264_videotoolbox `-q:v 80` ≈ CRF 18 equivalent for screen content + * - `-maxrate 25M` = X.com (Twitter) max bitrate cap + * - `-movflags +faststart` = metadata at front for web streaming + * - `-pix_fmt yuv420p` = universal compatibility across all platforms + */ +function buildEncodingArgs(encoder: EncoderInfo): string[] { + const common = [ + '-pix_fmt', 'yuv420p', + '-maxrate', '25M', + '-bufsize', '50M', + '-movflags', '+faststart', + ] + + if (encoder.isHardware) { + // Hardware encoders: use quality-based mode where supported + // h264_videotoolbox uses -q:v (1-100, 100=best), others use -b:v + const qualityArgs = encoder.codec === 'h264_videotoolbox' + ? ['-q:v', '80'] + : ['-b:v', '15M'] // high bitrate for other HW encoders + return ['-c:v', encoder.codec, ...qualityArgs, ...common] + } + + // Software: libx264 with screen-recording-optimized settings + return [ + '-c:v', 'libx264', + '-crf', '18', + '-preset', 'fast', + '-x264opts', 'deblock=-1,-1', + ...common, + ] +} + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -247,7 +378,11 @@ function buildSegments(sections: SpeedSection[]): Segment[] { /** * Build the filter string for a single segment. - * Returns `[0:v]trim=...,setpts=...,fps=...,scale=...[vN]` + * + * For sped-up segments: `[0:v]trim=...,setpts=...,fps=...,scale=...[vN]` + * For normal-speed segments where dimensions/fps match input: just + * `[0:v]trim=...,setpts=PTS-STARTPTS[vN]` — skips fps and scale filters + * to avoid unnecessary pixel processing and frame-rate checking. */ function buildSegmentFilter({ segment, @@ -255,12 +390,20 @@ function buildSegmentFilter({ frameRate, width, height, + inputWidth, + inputHeight, + inputFrameRate, }: { segment: Segment index: number frameRate: number width: number height: number + /** Original input dimensions — when they match output, scale is skipped on passthrough */ + inputWidth?: number + inputHeight?: number + /** Original input fps — when it matches output, fps filter is skipped on passthrough */ + inputFrameRate?: number }): string { const trimParts = [`start=${segment.start}`] if (segment.end !== undefined) { @@ -275,6 +418,17 @@ function buildSegmentFilter({ ? 'setpts=PTS-STARTPTS' : `setpts=(PTS-STARTPTS)/${segment.speed}` + // For normal-speed segments where output matches input: skip fps/scale + // to avoid unnecessary pixel processing and frame-rate filtering + const isPassthrough = segment.speed === 1 + && inputWidth === width + && inputHeight === height + && inputFrameRate === frameRate + + if (isPassthrough) { + return `[0:v]${trim},${setpts}[v${index}]` + } + // Cap output FPS to the probed source FPS. This prevents sped-up sections // from producing excessive frame rates when timestamps are compressed. const fps = `fps=fps=${frameRate}:round=down` @@ -299,6 +453,9 @@ export async function concatenateVideos( const timerId = `concat-${inputFiles.length}-videos-${path.basename(outputFile)}` console.time(timerId) + const encoder = await detectEncoder() + const encodingArgs = buildEncodingArgs(encoder) + // Build argv: -i file1 -i file2 ... -filter_complex "..." -map "[v_out]" output const inputArgs = inputFiles.flatMap((file) => { return ['-i', file.path] @@ -328,7 +485,13 @@ export async function concatenateVideos( ) const filterComplex = filterComplexParts.join('; ') - const args = [...inputArgs, '-filter_complex', filterComplex, '-map', '[v_out]', outputFile] + const args = [ + ...inputArgs, + '-filter_complex', filterComplex, + '-map', '[v_out]', + ...encodingArgs, + outputFile, + ] console.log('Running FFmpeg concat:', args.join(' ')) @@ -379,16 +542,18 @@ export async function speedUpSections( const outputFile = options.outputFile ?? defaultOutputPath(inputFile) - // Probe input for defaults + // Probe input when needed for defaults or for passthrough optimization const dims = options.outputDimensions const fps = options.frameRate - const needsProbe = !dims || !fps - const probed = needsProbe ? await probeVideo(inputFile) : undefined + const probed = (!dims || !fps) ? await probeVideo(inputFile) : undefined const width = dims?.width ?? probed!.width const height = dims?.height ?? probed!.height const frameRate = fps ?? probed!.frameRate + const encoder = await detectEncoder() + const encodingArgs = buildEncodingArgs(encoder) + const timerId = `speedup-${sections.length}-sections-${path.basename(outputFile)}` console.time(timerId) @@ -401,6 +566,9 @@ export async function speedUpSections( frameRate, width, height, + inputWidth: probed?.width, + inputHeight: probed?.height, + inputFrameRate: probed?.frameRate, }) }) @@ -418,6 +586,7 @@ export async function speedUpSections( '-filter_complex', filterComplex, '-map', '[v_out]', '-r', String(frameRate), + ...encodingArgs, outputFile, ]