From c13c835c5fb9983618d6ff496b217360d389b69e Mon Sep 17 00:00:00 2001 From: "Tommy D. Rossi" Date: Thu, 26 Feb 2026 12:57:39 +0100 Subject: [PATCH] feat: add resizeImage sandbox utility for shrinking screenshots before LLM ingestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracts the sharp resize logic from screenshotWithAccessibilityLabels into a standalone resizeImage() function exposed in the executor sandbox. Agents can now shrink screenshots (or any image) to consume fewer context tokens. Default behavior: fit within 1568×1568px (Claude-optimal), overwrite input file. Also supports explicit width/height/maxDimension/fit/quality for custom resizing. Replaces the old macOS-specific sips advice in skill.md. --- playwriter/CHANGELOG.md | 19 +++++ playwriter/package.json | 2 +- playwriter/src/aria-snapshot.ts | 132 +++++++++++++++++++++++++++----- playwriter/src/executor.ts | 2 + playwriter/src/skill.md | 21 ++--- 5 files changed, 142 insertions(+), 34 deletions(-) diff --git a/playwriter/CHANGELOG.md b/playwriter/CHANGELOG.md index 5eb13b0..274d678 100644 --- a/playwriter/CHANGELOG.md +++ b/playwriter/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 0.0.78 + +### Features + +- **Add `resizeImage` sandbox utility**: Standalone function to resize images, useful for shrinking screenshots before reading them back into context. Default LLM-optimal mode fits within 1568×1568px; also supports explicit width/height/maxDimension. Available in execute sandbox alongside other utilities. + +## 0.0.77 + +### Improvements + +- **Cap speed-up output to source fps in FFmpeg pipeline**: Speed-up filters now use explicit `fps=fps=:round=down` and set output `-r` to the same probed frame rate, keeping accelerated sections bounded to the recording's native fps. + +## 0.0.76 + +### Bug Fixes + +- **Fix ultra-short/slow demo generation on variable-framerate recordings**: `probeVideo()` now prefers `avg_frame_rate` and clamps output FPS to sane bounds, avoiding accidental `fps=30000` filter chains. +- **Avoid speeding entire video when no execute timestamps exist**: `computeIdleSections()` now returns no idle sections when timestamps are empty, so `createDemoVideo()` preserves original speed instead of aggressively compressing full recordings. + ## 0.0.75 ### Improvements diff --git a/playwriter/package.json b/playwriter/package.json index f8cf766..dbe6ab4 100644 --- a/playwriter/package.json +++ b/playwriter/package.json @@ -1,7 +1,7 @@ { "name": "playwriter", "description": "", - "version": "0.0.75", + "version": "0.0.78", "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/playwriter/src/aria-snapshot.ts b/playwriter/src/aria-snapshot.ts index fc760cb..cdee731 100644 --- a/playwriter/src/aria-snapshot.ts +++ b/playwriter/src/aria-snapshot.ts @@ -84,6 +84,113 @@ export interface ScreenshotResult { labelCount: number } +// ============================================================================ +// Image Resize Utility +// ============================================================================ + +/** + * LLM-optimal max dimension. Claude auto-resizes images larger than 1568px + * on any edge, adding latency. Token cost: (width * height) / 750. + */ +export const LLM_MAX_DIMENSION = 1568 + +export interface ResizeImageOptions { + /** Input: file path or Buffer */ + input: string | Buffer + /** Max pixels on longest edge. Default 1568 (Claude-optimal). + * Ignored if explicit width/height provided. */ + maxDimension?: number + /** Explicit target width in px (aspect ratio preserved unless both width+height set) */ + width?: number + /** Explicit target height in px */ + height?: number + /** How to fit when both width+height specified. Default 'inside' (preserve aspect ratio, no crop) */ + fit?: 'inside' | 'cover' | 'contain' | 'fill' + /** JPEG quality 1-100. Default 80 */ + quality?: number + /** Output file path. Defaults to overwriting the input file (when input is a path) */ + output?: string +} + +export interface ResizeImageResult { + buffer: Buffer + mimeType: 'image/jpeg' + /** Only set if output path was provided */ + path?: string +} + +/** + * Resize an image using sharp. Useful for shrinking screenshots before reading + * them back into context so they consume fewer tokens. + * + * Default behavior (no width/height): fits within 1568×1568px, preserving + * aspect ratio, never upscales. This is optimal for Claude vision. + * + * Explicit width/height: resizes to those dimensions using the fit strategy. + */ +export async function resizeImage(options: ResizeImageOptions): Promise { + const sharp = await sharpPromise + if (!sharp) { + throw new Error('sharp is not installed — install it with: pnpm add sharp') + } + + const inputBuffer = (() => { + if (Buffer.isBuffer(options.input)) { + return options.input + } + return fs.readFileSync(options.input) + })() + + const quality = options.quality ?? 80 + const hasExplicitDimensions = options.width !== undefined || options.height !== undefined + + const fit = options.fit ?? 'inside' + const resizeOpts = (() => { + if (hasExplicitDimensions) { + return { + width: options.width, + height: options.height, + fit, + withoutEnlargement: false, + } + } + const max = options.maxDimension ?? LLM_MAX_DIMENSION + return { + width: max, + height: max, + fit, + withoutEnlargement: true, + } + })() + + const buffer = await sharp(inputBuffer) + .resize(resizeOpts) + .jpeg({ quality }) + .toBuffer() + + // Default: overwrite input file. When input is a Buffer, no file is written + // unless output is explicitly set. + const outputPath = (() => { + if (options.output) { + return options.output + } + if (typeof options.input === 'string') { + return options.input + } + return undefined + })() + + if (outputPath) { + fs.writeFileSync(outputPath, buffer) + } + + return { + buffer, + mimeType: 'image/jpeg', + ...(outputPath ? { path: outputPath } : {}), + } +} + export interface AriaSnapshotResult { snapshot: string tree: AriaSnapshotNode[] @@ -1444,16 +1551,12 @@ export async function screenshotWithAccessibilityLabels({ height: number } - // Max 1568px on any edge (larger gets auto-resized by Claude, adding latency) - // Token formula: tokens = (width * height) / 750 - const MAX_DIMENSION = 1568 - // Check if sharp is available for resizing const sharp = await sharpPromise - // Clip dimensions: if sharp unavailable, limit capture area to MAX_DIMENSION - const clipWidth = sharp ? viewport.width : Math.min(viewport.width, MAX_DIMENSION) - const clipHeight = sharp ? viewport.height : Math.min(viewport.height, MAX_DIMENSION) + // Clip dimensions: if sharp unavailable, limit capture area to LLM_MAX_DIMENSION + const clipWidth = sharp ? viewport.width : Math.min(viewport.width, LLM_MAX_DIMENSION) + const clipHeight = sharp ? viewport.height : Math.min(viewport.height, LLM_MAX_DIMENSION) // Take viewport screenshot with scale: 'css' to ignore device pixel ratio const screenshotStart = Date.now() @@ -1467,23 +1570,16 @@ export async function screenshotWithAccessibilityLabels({ log(`page.screenshot: ${Date.now() - screenshotStart}ms`) } - // Resize with sharp if available, otherwise use clipped raw buffer + // Resize with resizeImage if sharp available, otherwise use clipped raw buffer const resizeStart = Date.now() const buffer = await (async () => { if (!sharp) { - logger?.error?.('[playwriter] sharp not available, using clipped screenshot (max', MAX_DIMENSION, 'px)') + logger?.error?.('[playwriter] sharp not available, using clipped screenshot (max', LLM_MAX_DIMENSION, 'px)') return rawBuffer } try { - return await sharp(rawBuffer) - .resize({ - width: MAX_DIMENSION, - height: MAX_DIMENSION, - fit: 'inside', // Scale down to fit, preserving aspect ratio - withoutEnlargement: true, // Don't upscale small images - }) - .jpeg({ quality: 80 }) - .toBuffer() + const result = await resizeImage({ input: rawBuffer }) + return result.buffer } catch (err) { logger?.error?.('[playwriter] sharp resize failed, using raw buffer:', err) return rawBuffer diff --git a/playwriter/src/executor.ts b/playwriter/src/executor.ts index 0b0b28f..1ca674f 100644 --- a/playwriter/src/executor.ts +++ b/playwriter/src/executor.ts @@ -26,6 +26,7 @@ import { ScopedFS } from './scoped-fs.js' import { screenshotWithAccessibilityLabels, getAriaSnapshot, + resizeImage, type ScreenshotResult, type SnapshotFormat, } from './aria-snapshot.js' @@ -1029,6 +1030,7 @@ export class PlaywrightExecutor { formatStylesAsText, getReactSource: getReactSourceFn, screenshotWithAccessibilityLabels: screenshotWithAccessibilityLabelsFn, + resizeImage, ghostCursor: { show: showGhostCursor, hide: hideGhostCursor, diff --git a/playwriter/src/skill.md b/playwriter/src/skill.md index 970e61f..99a166c 100644 --- a/playwriter/src/skill.md +++ b/playwriter/src/skill.md @@ -913,23 +913,14 @@ await screenshotWithAccessibilityLabels({ page: state.page }) Labels are color-coded: yellow=links, orange=buttons, coral=inputs, pink=checkboxes, peach=sliders, salmon=menus, amber=tabs. -**resizeImage** - resize an image to consume fewer tokens when read back into context. Useful when you take a `page.screenshot()` and need to ingest the image later. By default fits within 1568×1568px (Claude-optimal, avoids server-side re-encoding). Token cost formula: `(width × height) / 750`. Always outputs JPEG. +**resizeImage** - shrink an image so it consumes fewer tokens when read back into context. Overwrites the input file by default. Fits within 1568×1568px (Claude-optimal). Always outputs JPEG. ```js -// Shrink a screenshot for LLM ingestion (default: max 1568px, JPEG q80) -await resizeImage({ input: './screenshot.png', output: './small.jpg' }) - -// Resize to a specific width (aspect ratio preserved) -await resizeImage({ input: './large.png', width: 800, output: './resized.jpg' }) - -// Resize to fit inside exact dimensions -await resizeImage({ input: './photo.png', width: 1024, height: 768, output: './fit.jpg' }) - -// Custom max dimension and quality -await resizeImage({ input: './shot.png', maxDimension: 1024, quality: 60, output: './out.jpg' }) +// Shrink screenshot in-place for LLM ingestion +await resizeImage({ input: './screenshot.png' }) ``` -Options: `input` (path or Buffer), `output` (path, optional), `maxDimension` (default 1568, ignored if width/height set), `width`, `height`, `fit` ('inside' | 'cover' | 'contain' | 'fill', default 'inside'), `quality` (1-100, default 80). Returns `{ buffer, mimeType, path? }`. +Also supports explicit dimensions: `width`, `height`, `maxDimension`, `fit` ('inside' | 'cover' | 'contain' | 'fill'), `quality` (1-100, default 80), `output` (defaults to overwriting input). **recording.start / recording.stop** - record the page as a video at native FPS (30-60fps). Uses `chrome.tabCapture` in the extension context, so **recording survives page navigation**. Video is saved as mp4. @@ -1034,10 +1025,10 @@ Always use `scale: 'css'` to avoid 2-4x larger images on high-DPI displays: await state.page.screenshot({ path: 'shot.png', scale: 'css' }) ``` -If you want to read back the image file into context, resize it first with `resizeImage` so it consumes fewer tokens: +If you want to read back the image file into context, resize it first so it consumes fewer tokens: ```js -await resizeImage({ input: './shot.png', output: './small.jpg' }) +await resizeImage({ input: './shot.png' }) ``` ## page.evaluate