diff --git a/AGENTS.md b/AGENTS.md index a953e99..b5c523c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,13 +53,18 @@ to test CLI changes without publishing: ```bash # mac/linux: kill any existing relay on 19988 - lsof -ti :19988 | xargs kill + PIDS=($(lsof -ti :19988)) + if [ ${#PIDS[@]} -gt 0 ]; then kill "${PIDS[@]}"; fi + # verify port is free (must print nothing) + lsof -ti :19988 # windows (powershell): kill any existing relay on 19988 Get-NetTCPConnection -LocalPort 19988 | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force } + # verify port is free (must print nothing) + Get-NetTCPConnection -LocalPort 19988 -ErrorAction SilentlyContinue tsx playwriter/src/cli.ts -s 1 -e "await page.goto('https://example.com')" - tsx playwriter/src/cli.ts -s 1 -e "console.log(await accessibilitySnapshot({ page }))" + tsx playwriter/src/cli.ts -s 1 -e "console.log(await snapshot({ page }))" tsx playwriter/src/cli.ts session new tsx playwriter/src/cli.ts -s 1 -e "await page.click('button')" ``` @@ -365,36 +370,35 @@ you can open files when i ask me "open in zed the line where ..." using the comm - if you encounter typescript lint errors for an npm package, read the node_modules/package/\*.d.ts files to understand the typescript types of the package. if you cannot understand them, ask me to help you with it. - NEVER silently suppress errors in catch {} blocks if they contain more than one function call - ```ts // BAD. DO NOT DO THIS -let favicon: string | undefined +let favicon: string | undefined; if (docsConfig?.favicon) { - if (typeof docsConfig.favicon === 'string') { - favicon = docsConfig.favicon + if (typeof docsConfig.favicon === "string") { + favicon = docsConfig.favicon; } else if (docsConfig.favicon?.light) { // Use light favicon as default, could be enhanced with theme detection - favicon = docsConfig.favicon.light + favicon = docsConfig.favicon.light; } } // DO THIS. use an iife. Immediately Invoked Function Expression const favicon: string = (() => { if (!docsConfig?.favicon) { - return '' + return ""; } - if (typeof docsConfig.favicon === 'string') { - return docsConfig.favicon + if (typeof docsConfig.favicon === "string") { + return docsConfig.favicon; } if (docsConfig.favicon?.light) { // Use light favicon as default, could be enhanced with theme detection - return docsConfig.favicon.light + return docsConfig.favicon.light; } - return '' -})() + return ""; +})(); // if you already know the type use it: const favicon: string = () => { // ... -} +}; ``` - when a package has to import files from another packages in the workspace never add a new tsconfig path, instead add that package as a workspace dependency using `pnpm i "package@workspace:*"` @@ -411,12 +415,12 @@ always specify the type when creating arrays, especially for empty arrays. if yo ```ts // BAD: Type will be never[] -const items = [] +const items = []; // GOOD: Specify the expected type -const items: string[] = [] -const numbers: number[] = [] -const users: User[] = [] +const items: string[] = []; +const numbers: number[] = []; +const users: User[] = []; ``` remember to always add the explicit type to avoid unexpected type inference. @@ -574,7 +578,7 @@ to understand how the code you are writing works, you should add inline snapshot - for very long snapshots you should use `toMatchFileSnapshot(filename)` instead of `toMatchInlineSnapshot()`. put the snapshot files in a snapshots/ directory and use the appropriate extension for the file based on the content -never test client react components. only React and browser independent code. +never test client react components. only React and browser independent code. most tests should be simple calls to functions with some expect calls, no mocks. test files should be called the same as the file where the tested function is being exported from. @@ -591,13 +595,12 @@ sometimes tests work directly on database data, using prisma. to run these tests never write tests yourself that call prisma or interact with database or emails. for these, ask the user to write them for you. changelogs.md - # writing docs when generating a .md or .mdx file to document things, always add a frontmatter with title and description. also add a prompt field with the exact prompt used to generate the doc. use @ to reference files and urls and provide any context necessary to be able to recreate this file from scratch using a model. if you used urls also reference them. reference all files you had to read to create the doc. use yaml | syntax to add this prompt and never go over the column width of 80 - # github + you can use the `gh` cli to do operations on github for the current repository. For example: open issues, open PRs, check actions status, read workflow logs, etc. ## creating issues and pull requests @@ -680,7 +683,6 @@ This will download the source code in ./opensrc. which should be put in .gitigno you can control the browser using the playwright mcp tools. these tools let you control the browser to get information or accomplish actions if i ask you to test something in the browser, know that the website dev server is already running at http://localhost:7664 for website and :7777 for docs-website (but docs-website needs to use the website domain specifically, for example name-hash.localhost:7777) - # zod when you need to create a complex type that comes from a prisma table, do not create a new schema that tries to recreate the prisma table structure. instead just use `z.any() as ZodType)` to get type safety but leave any in the schema. this gets most of the benefits of zod without having to define a new zod schema that can easily go out of sync. @@ -690,40 +692,17 @@ when you need to create a complex type that comes from a prisma table, do not cr you MUST use the built in zod v4 toJSONSchema and not the npm package `zod-to-json-schema` which is outdated and does not support zod v4. ```ts -import { toJSONSchema } from 'zod' +import { toJSONSchema } from "zod"; const mySchema = z.object({ id: z.string().uuid(), name: z.string().min(3).max(100), age: z.number().min(0).optional(), -}) +}); const jsonSchema = toJSONSchema(mySchema, { - removeAdditionalStrategy: 'strict', -}) + removeAdditionalStrategy: "strict", +}); ``` github.md - - - -## Source Code Reference - -Source code for dependencies is available in `opensrc/` for deeper understanding of implementation details. - -See `opensrc/sources.json` for the list of available packages and their versions. - -Use this source code when you need to understand how a package works internally, not just its types/interface. - -### Fetching Additional Source Code - -To fetch source code for a package or repository you need to understand, run: - -```bash -npx opensrc # npm package (e.g., npx opensrc zod) -npx opensrc pypi: # Python package (e.g., npx opensrc pypi:requests) -npx opensrc crates: # Rust crate (e.g., npx opensrc crates:serde) -npx opensrc / # GitHub repo (e.g., npx opensrc vercel/ai) -``` - - diff --git a/playwriter/CHANGELOG.md b/playwriter/CHANGELOG.md index 9be2032..5eb13b0 100644 --- a/playwriter/CHANGELOG.md +++ b/playwriter/CHANGELOG.md @@ -1,5 +1,46 @@ # Changelog +## 0.0.75 + +### Improvements + +- **Switch minimal cursor to triangular pointer icon**: Updated the `minimal` ghost cursor style to use a stylized triangular SVG pointer (with subtle drop shadow) instead of the circular indicator, while keeping `dot` and `screenstudio` styles available. + +## 0.0.74 + +### Improvements + +- **Switch default ghost cursor to a stylized minimal look**: Updated cursor rendering defaults to a cleaner minimal style while preserving `dot` and `screenstudio` options for explicit overrides. + +## 0.0.73 + +### Improvements + +- **Simplify recording integration in executor**: Moved ghost-cursor-aware recording wrappers out of `executor.ts` into `screen-recording.ts` via `createRecordingApi(...)`, reducing executor complexity while preserving existing `recording.*` and backward-compatible top-level recording helpers. + +## 0.0.72 + +### Improvements + +- **Reduce false "extension disconnected" on relay restarts**: `playwriter session new` now waits longer for extension reconnect and adds a short polling grace window before failing, preventing transient post-restart races from surfacing as hard disconnect errors. + +## 0.0.71 + +### Features + +- **Add `recording` and `ghostCursor` namespaces in execute context**: New `recording.start/stop/isRecording/cancel` and `ghostCursor.show/hide` APIs are now exposed for cleaner scripting while keeping `startRecording`, `stopRecording`, `isRecording`, and `cancelRecording` as backward-compatible aliases. +- **Manual cursor overlay controls**: Cursor overlay can now be shown/hidden explicitly outside recording flows for screenshot and demo generation. + +## 0.0.70 + +### Features + +- **Ghost cursor overlay during recording**: Playwriter now auto-enables a smooth in-page ghost cursor when `startRecording()` is called, driven by `page.onMouseAction` callbacks from the Playwright fork so both `page.mouse.*` and `locator.click()` actions are visualized. + +### Tests + +- **Add ghost-cursor integration coverage**: Extended `on-mouse-action.test.ts` to verify callback-driven cursor animation and teardown in real extension-connected runs. + ## 0.0.69 ### Bug Fixes diff --git a/playwriter/package.json b/playwriter/package.json index 31f588d..f8cf766 100644 --- a/playwriter/package.json +++ b/playwriter/package.json @@ -1,7 +1,7 @@ { "name": "playwriter", "description": "", - "version": "0.0.69", + "version": "0.0.75", "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/playwriter/scripts/build-client-bundles.ts b/playwriter/scripts/build-client-bundles.ts index 7c34a5c..6d7c27d 100644 --- a/playwriter/scripts/build-client-bundles.ts +++ b/playwriter/scripts/build-client-bundles.ts @@ -47,6 +47,11 @@ const BUNDLES: BundleConfig[] = [ type: 'source', entry: 'a11y-client.ts', }, + { + name: 'ghost-cursor-client', + type: 'source', + entry: 'ghost-cursor-client.ts', + }, // Wrapper bundles (npm packages → globalThis) { diff --git a/playwriter/scripts/encode-screenstudio-cursor.ts b/playwriter/scripts/encode-screenstudio-cursor.ts new file mode 100644 index 0000000..d46e46e --- /dev/null +++ b/playwriter/scripts/encode-screenstudio-cursor.ts @@ -0,0 +1,23 @@ +/** + * Encodes the extracted Screen Studio cursor SVG as a data URL TS module. + */ + +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const currentDir = path.dirname(fileURLToPath(import.meta.url)) +const sourcePath = path.join(currentDir, '..', 'src', 'assets', 'cursors', 'screen-studio', 'pointer-macos-tahoe.svg') +const outputPath = path.join(currentDir, '..', 'src', 'assets', 'cursors', 'screen-studio', 'pointer-macos-tahoe-data-url.ts') + +function main() { + const svg = fs.readFileSync(sourcePath, 'utf-8').trim() + const base64 = Buffer.from(svg, 'utf-8').toString('base64') + const dataUrl = `data:image/svg+xml;base64,${base64}` + + const output = `/**\n * Generated from pointer-macos-tahoe.svg via scripts/encode-screenstudio-cursor.ts.\n */\n\nexport const SCREENSTUDIO_POINTER_MACOS_TAHOE_DATA_URL = '${dataUrl}'\n` + fs.writeFileSync(outputPath, output) + console.log(`Wrote ${outputPath}`) +} + +main() diff --git a/playwriter/src/assets/cursors/screen-studio/pointer-macos-tahoe-data-url.ts b/playwriter/src/assets/cursors/screen-studio/pointer-macos-tahoe-data-url.ts new file mode 100644 index 0000000..db564a4 --- /dev/null +++ b/playwriter/src/assets/cursors/screen-studio/pointer-macos-tahoe-data-url.ts @@ -0,0 +1,5 @@ +/** + * Generated from pointer-macos-tahoe.svg via scripts/encode-screenstudio-cursor.ts. + */ + +export const SCREENSTUDIO_POINTER_MACOS_TAHOE_DATA_URL = 'data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNjE4IiBoZWlnaHQ9Ijk1OCIgdmlld0JveD0iMCAwIDYxOCA5NTgiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxnIGZpbHRlcj0idXJsKCNmaWx0ZXIwX2RfMzg0XzI3KSI+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNMTI3LjA2MiAzNy4wMzMxTDU0MC42OTYgNDUxLjU1NUM1OTIuNjUzIDUwMy42NiA1NTUuNzk0IDU5Mi41NzQgNDgyLjIyNiA1OTIuNTc0TDQyMS44MzEgNTkyLjU2OUw0ODEuODIxIDczNS4wNTRDNDkyLjMzMSA3NjAuMDIxIDQ5Mi40NzkgNzg3LjY1MiA0ODIuMjY1IDgxMi43NjdDNDcyLjAwMiA4MzcuOTMyIDQ1Mi41NjEgODU3LjU3IDQyNy40OTYgODY4LjA4QzQxNC44NjQgODczLjM1OSA0MDEuNjQgODc2LjAyNCAzODguMTIxIDg3Ni4wMjRDMzQ3LjExNyA4NzYuMDI0IDMxMC4zNTggODUxLjYgMjk0LjQ3IDgxMy44MDRMMjMxLjQyIDY2My45MThMMTkwLjM2OCA3MDAuMzM3QzEzNy4wMjkgNzQ3LjUwOCA1MyA3MDkuNjYzIDUzIDYzOC40MTNWNjcuNjc0NEM1MyAyOC45OTAzIDk5LjcyNjggOS42NDgyOCAxMjcuMDYyIDM3LjAzMzFaIiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBmaWxsLXJ1bGU9ImV2ZW5vZGQiIGNsaXAtcnVsZT0iZXZlbm9kZCIgZD0iTTEwMi4zMTYgOTkuNjUyQzEwMi4zMTYgOTMuMTg4MiAxMTAuMTYyIDg5LjkzMTYgMTE0LjcwMSA5NC41MjA0TDUwNC44OTcgNDg1LjU1NUM1MjYuMTY0IDUwNi44NzEgNTExLjA2NSA1NDMuMjM2IDQ4MC45NjcgNTQzLjIzNkwzNDcuNTQ2IDU0My4xNjFMNDM2LjM0MiA3NTQuMTQzQzQ0Ny41NDIgNzgwLjc4OCA0MzUuMDA5IDgxMS40MjkgNDA4LjQxNCA4MjIuNTgxQzM4MS43MiA4MzMuNzgxIDM1MS4xMjggODIxLjI5OCAzMzkuOTc3IDc5NC43MDJMMjUwLjI5MyA1ODEuMzUyTDE1OC41MTcgNjYyLjY0NEMxMzcuOTkxIDY4MC44MDEgMTA2LjMxOSA2NjguMTQ1IDEwMi42NjQgNjQyLjMyM0wxMDIuMzE2IDYzNy4zMzFWOTkuNjUyWiIgZmlsbD0iYmxhY2siLz4KPC9nPgo8ZGVmcz4KPGZpbHRlciBpZD0iZmlsdGVyMF9kXzM4NF8yNyIgeD0iMC4zNCIgeT0iMC43OTkyMTkiIHdpZHRoPSI2MTcuMzIiIGhlaWdodD0iOTU3LjE0NCIgZmlsdGVyVW5pdHM9InVzZXJTcGFjZU9uVXNlIiBjb2xvci1pbnRlcnBvbGF0aW9uLWZpbHRlcnM9InNSR0IiPgo8ZmVGbG9vZCBmbG9vZC1vcGFjaXR5PSIwIiByZXN1bHQ9IkJhY2tncm91bmRJbWFnZUZpeCIvPgo8ZmVDb2xvck1hdHJpeCBpbj0iU291cmNlQWxwaGEiIHR5cGU9Im1hdHJpeCIgdmFsdWVzPSIwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAxMjcgMCIgcmVzdWx0PSJoYXJkQWxwaGEiLz4KPGZlT2Zmc2V0IGR5PSIyOS4yNiIvPgo8ZmVHYXVzc2lhbkJsdXIgc3RkRGV2aWF0aW9uPSIyNi4zMyIvPgo8ZmVDb21wb3NpdGUgaW4yPSJoYXJkQWxwaGEiIG9wZXJhdG9yPSJvdXQiLz4KPGZlQ29sb3JNYXRyaXggdHlwZT0ibWF0cml4IiB2YWx1ZXM9IjAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAuNjUgMCIvPgo8ZmVCbGVuZCBtb2RlPSJub3JtYWwiIGluMj0iQmFja2dyb3VuZEltYWdlRml4IiByZXN1bHQ9ImVmZmVjdDFfZHJvcFNoYWRvd18zODRfMjciLz4KPGZlQmxlbmQgbW9kZT0ibm9ybWFsIiBpbj0iU291cmNlR3JhcGhpYyIgaW4yPSJlZmZlY3QxX2Ryb3BTaGFkb3dfMzg0XzI3IiByZXN1bHQ9InNoYXBlIi8+CjwvZmlsdGVyPgo8L2RlZnM+Cjwvc3ZnPg==' diff --git a/playwriter/src/assets/cursors/screen-studio/pointer-macos-tahoe.svg b/playwriter/src/assets/cursors/screen-studio/pointer-macos-tahoe.svg new file mode 100644 index 0000000..2acf2f6 --- /dev/null +++ b/playwriter/src/assets/cursors/screen-studio/pointer-macos-tahoe.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/playwriter/src/cli.ts b/playwriter/src/cli.ts index ebbfb8a..2fb904e 100644 --- a/playwriter/src/cli.ts +++ b/playwriter/src/cli.ts @@ -5,6 +5,7 @@ import path from 'node:path' import util from 'node:util' import { fileURLToPath } from 'node:url' import { cac } from '@xmorse/cac' +import pc from 'picocolors' // Prevent Buffers from dumping hex bytes in util.inspect output. Buffer.prototype[util.inspect.custom] = function () { @@ -15,9 +16,10 @@ import { VERSION, LOG_FILE_PATH, LOG_CDP_FILE_PATH, parseRelayHost } from './uti import { ensureRelayServer, RELAY_PORT, - waitForExtension, + waitForConnectedExtensions, getExtensionOutdatedWarning, getExtensionStatus, + type ExtensionStatus, } from './relay-client.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -26,15 +28,6 @@ const cliRelayEnv = { PLAYWRITER_AUTO_ENABLE: '1' } const cli = cac('playwriter') -type ExtensionStatus = { - extensionId: string - stableKey?: string - browser: string | null - profile: { email: string; id: string } | null - activeTargets: number - playwriterVersion?: string | null -} - cli .command('', 'Start the MCP server or controls the browser with -e') .option('--host ', 'Remote relay server host to connect to (or use PLAYWRITER_HOST env var)') @@ -136,8 +129,12 @@ async function executeCode(options: { if (!host && !process.env.PLAYWRITER_HOST) { const restarted = await ensureRelayServer({ logger: console, env: cliRelayEnv }) if (restarted) { - const connected = await waitForExtension({ logger: console, timeoutMs: 10000 }) - if (!connected) { + const connectedExtensions = await waitForConnectedExtensions({ + logger: console, + timeoutMs: 10000, + pollIntervalMs: 250, + }) + if (connectedExtensions.length === 0) { console.error('Warning: Extension not connected. Commands may fail.') } } @@ -212,15 +209,30 @@ cli .option('--host ', 'Remote relay server host') .option('--browser ', 'Stable browser key when multiple browsers are connected') .action(async (options: { host?: string; browser?: string }) => { - if (!options.host && !process.env.PLAYWRITER_HOST) { + const isLocal = !options.host && !process.env.PLAYWRITER_HOST + + let extensions: ExtensionStatus[] = [] + + if (isLocal) { await ensureRelayServer({ logger: console, env: cliRelayEnv }) - await waitForExtension({ - timeoutMs: 3000, + extensions = await waitForConnectedExtensions({ + timeoutMs: 12000, + pollIntervalMs: 250, logger: console, }) + + if (extensions.length === 0) { + console.log(pc.dim('Waiting briefly for extension to reconnect...')) + extensions = await waitForConnectedExtensions({ + timeoutMs: 10000, + pollIntervalMs: 250, + logger: console, + }) + } + } else { + extensions = await fetchExtensionsStatus(options.host) } - const extensions = await fetchExtensionsStatus(options.host) if (extensions.length === 0) { console.error('No connected browsers detected. Click the Playwriter extension icon.') process.exit(1) diff --git a/playwriter/src/executor.ts b/playwriter/src/executor.ts index ce0054a..0b0b28f 100644 --- a/playwriter/src/executor.ts +++ b/playwriter/src/executor.ts @@ -33,8 +33,10 @@ import { createGhostBrowserChrome, type GhostBrowserCommandResult } from './ghos export type { SnapshotFormat } import { getCleanHTML, type GetCleanHTMLOptions } from './clean-html.js' import { getPageMarkdown, type GetPageMarkdownOptions } from './page-markdown.js' -import { startRecording, stopRecording, isRecording, cancelRecording } from './screen-recording.js' +import { createRecordingApi } from './screen-recording.js' import { createDemoVideo } from './ffmpeg.js' +import { type GhostCursorClientOptions } from './ghost-cursor.js' +import { RecordingGhostCursorController } from './recording-ghost-cursor.js' const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) @@ -948,18 +950,51 @@ export class PlaywrightExecutor { // Recording uses chrome.tabCapture which requires activeTab permission. // This permission is granted when the user clicks the Playwriter extension icon on a tab. const relayPort = this.cdpConfig.port || 19988 - // Recording will work on any tab where the user has clicked the icon. - const withRecordingDefaults = ( - fn: (opts: T & { relayPort: number; sessionId?: string }) => Promise, - ) => { - return async (options: T = {} as T) => { - const targetPage = options.page || page - // Use Playwright's exposed tab session ID directly. - const sessionId = options.sessionId || targetPage.sessionId() || undefined - return fn({ page: targetPage, sessionId, relayPort, ...options }) - } - } const self = this + const recordingGhostCursor = new RecordingGhostCursorController({ + logger: { + error: (...args: unknown[]) => { + self.logger.error(...args) + }, + }, + }) + + const showGhostCursor = async (options?: ({ page?: Page } & GhostCursorClientOptions)) => { + const targetPage = options?.page || page + const cursorOptions: GhostCursorClientOptions | undefined = (() => { + if (!options) { + return undefined + } + + const { page: _ignoredPage, ...rest } = options + return rest + })() + + await recordingGhostCursor.show({ page: targetPage, cursorOptions }) + } + + const hideGhostCursor = async (options?: { page?: Page }) => { + const targetPage = options?.page || page + await recordingGhostCursor.hide({ page: targetPage }) + } + + const recordingApi = createRecordingApi({ + context, + defaultPage: page, + relayPort, + ghostCursorController: recordingGhostCursor, + onStart: () => { + self.recordingStartedAt = Date.now() + self.executionTimestamps = [] + }, + onFinish: () => { + self.recordingStartedAt = null + self.executionTimestamps = [] + }, + getExecutionTimestamps: () => { + return self.executionTimestamps + }, + }) // Ghost Browser API - creates chrome object that mirrors Ghost Browser's APIs // See extension/src/ghost-browser-api.d.ts for full API documentation @@ -994,26 +1029,21 @@ export class PlaywrightExecutor { formatStylesAsText, getReactSource: getReactSourceFn, screenshotWithAccessibilityLabels: screenshotWithAccessibilityLabelsFn, - startRecording: async (opts?: Parameters[0]) => { - const result = await withRecordingDefaults(startRecording)(opts) - self.recordingStartedAt = Date.now() - self.executionTimestamps = [] - return result + ghostCursor: { + show: showGhostCursor, + hide: hideGhostCursor, }, - stopRecording: async (opts?: Parameters[0]) => { - const result = await withRecordingDefaults(stopRecording)(opts) - const executionTimestamps = [...self.executionTimestamps] - self.recordingStartedAt = null - self.executionTimestamps = [] - return { ...result, executionTimestamps } - }, - isRecording: withRecordingDefaults(isRecording), - cancelRecording: async (opts?: Parameters[0]) => { - const result = await withRecordingDefaults(cancelRecording)(opts) - self.recordingStartedAt = null - self.executionTimestamps = [] - return result + recording: { + start: recordingApi.start, + stop: recordingApi.stop, + isRecording: recordingApi.isRecording, + cancel: recordingApi.cancel, }, + // Backward-compatible aliases + startRecording: recordingApi.start, + stopRecording: recordingApi.stop, + isRecording: recordingApi.isRecording, + cancelRecording: recordingApi.cancel, createDemoVideo, resetPlaywright: async () => { const { page: newPage, context: newContext } = await self.reset() diff --git a/playwriter/src/ghost-cursor-client.ts b/playwriter/src/ghost-cursor-client.ts new file mode 100644 index 0000000..2612a8c --- /dev/null +++ b/playwriter/src/ghost-cursor-client.ts @@ -0,0 +1,367 @@ +/** + * Browser-side ghost cursor renderer. + * Injected into the page to visualize automated mouse actions with smooth easing. + */ + +import { SCREENSTUDIO_POINTER_MACOS_TAHOE_DATA_URL } from './assets/cursors/screen-studio/pointer-macos-tahoe-data-url.js' + +type GhostCursorActionType = 'move' | 'down' | 'up' | 'wheel' +type GhostCursorButton = 'left' | 'right' | 'middle' | 'none' +type GhostCursorStyle = 'minimal' | 'dot' | 'screenstudio' + +interface GhostCursorAction { + type: GhostCursorActionType + x: number + y: number + button: GhostCursorButton +} + +export interface GhostCursorClientOptions { + style?: GhostCursorStyle + color?: string + size?: number + zIndex?: number + easing?: string + minDurationMs?: number + maxDurationMs?: number + speedPxPerMs?: number +} + +interface GhostCursorRuntimeOptions { + style: GhostCursorStyle + color: string + size: number + zIndex: number + easing: string + minDurationMs: number + maxDurationMs: number + speedPxPerMs: number +} + +interface GhostCursorRuntimeState { + cursorElement: ReturnType | null + options: GhostCursorRuntimeOptions + x: number + y: number + scale: number + hasPosition: boolean + enabled: boolean +} + +interface GhostCursorApi { + enable: (options?: GhostCursorClientOptions) => void + disable: () => void + applyMouseAction: (action: GhostCursorAction) => void + isEnabled: () => boolean +} + +declare global { + var __playwriterGhostCursor: GhostCursorApi | undefined +} + +const CURSOR_ID = '__playwriter_ghost_cursor__' +const SCREENSTUDIO_POINTER_ASPECT_RATIO = 618 / 958 +const SCREENSTUDIO_HOTSPOT_X_RATIO = 0.14 +const SCREENSTUDIO_HOTSPOT_Y_RATIO = 0.06 +const MINIMAL_TRIANGLE_HOTSPOT_X_RATIO = 0.07 +const MINIMAL_TRIANGLE_HOTSPOT_Y_RATIO = 0.06 + +const DEFAULT_OPTIONS: GhostCursorRuntimeOptions = { + style: 'minimal', + color: '#111827', + size: 22, + zIndex: 2147483647, + easing: 'cubic-bezier(0.22, 1, 0.36, 1)', + minDurationMs: 40, + maxDurationMs: 450, + speedPxPerMs: 2.2, +} + +const runtime: GhostCursorRuntimeState = { + cursorElement: null, + options: DEFAULT_OPTIONS, + x: 0, + y: 0, + scale: 1, + hasPosition: false, + enabled: false, +} + +function clamp(options: { value: number; min: number; max: number }): number { + const { value, min, max } = options + return Math.min(max, Math.max(min, value)) +} + +function mergeOptions(options?: GhostCursorClientOptions): GhostCursorRuntimeOptions { + if (!options) { + return DEFAULT_OPTIONS + } + + return { + style: options.style ?? DEFAULT_OPTIONS.style, + color: options.color ?? DEFAULT_OPTIONS.color, + size: options.size ?? DEFAULT_OPTIONS.size, + zIndex: options.zIndex ?? DEFAULT_OPTIONS.zIndex, + easing: options.easing ?? DEFAULT_OPTIONS.easing, + minDurationMs: options.minDurationMs ?? DEFAULT_OPTIONS.minDurationMs, + maxDurationMs: options.maxDurationMs ?? DEFAULT_OPTIONS.maxDurationMs, + speedPxPerMs: options.speedPxPerMs ?? DEFAULT_OPTIONS.speedPxPerMs, + } +} + +function getCursorDimensions(): { width: number; height: number } { + if (runtime.options.style === 'screenstudio') { + const height = runtime.options.size + const width = Math.max(10, Math.round(height * SCREENSTUDIO_POINTER_ASPECT_RATIO)) + return { width, height } + } + + if (runtime.options.style === 'minimal') { + const size = Math.max(12, runtime.options.size) + return { width: size, height: size } + } + + return { width: runtime.options.size, height: runtime.options.size } +} + +function getHotspotOffsetPx(): { x: number; y: number } { + const dimensions = getCursorDimensions() + + if (runtime.options.style === 'screenstudio') { + return { + x: Math.round(dimensions.width * SCREENSTUDIO_HOTSPOT_X_RATIO), + y: Math.round(dimensions.height * SCREENSTUDIO_HOTSPOT_Y_RATIO), + } + } + + if (runtime.options.style === 'minimal') { + return { + x: Math.round(dimensions.width * MINIMAL_TRIANGLE_HOTSPOT_X_RATIO), + y: Math.round(dimensions.height * MINIMAL_TRIANGLE_HOTSPOT_Y_RATIO), + } + } + + return { + x: Math.round(dimensions.width / 2), + y: Math.round(dimensions.height / 2), + } +} + +function getBaseOpacity(): string { + if (runtime.options.style === 'screenstudio') { + return '0.95' + } + + if (runtime.options.style === 'minimal') { + return '1' + } + + return '0.72' +} + +function applyTransform(): void { + if (!runtime.cursorElement) { + return + } + + const hotspot = getHotspotOffsetPx() + runtime.cursorElement.style.transform = `translate3d(${runtime.x - hotspot.x}px, ${runtime.y - hotspot.y}px, 0) scale(${runtime.scale})` +} + +function computeDurationMs(options: { targetX: number; targetY: number }): number { + if (!runtime.hasPosition) { + return 0 + } + + const dx = options.targetX - runtime.x + const dy = options.targetY - runtime.y + const distance = Math.hypot(dx, dy) + const rawDurationMs = distance / runtime.options.speedPxPerMs + + return clamp({ + value: rawDurationMs, + min: runtime.options.minDurationMs, + max: runtime.options.maxDurationMs, + }) +} + +function createCursorElement() { + const element = document.createElement('div') + element.id = CURSOR_ID + element.setAttribute('aria-hidden', 'true') + element.style.position = 'fixed' + element.style.left = '0' + element.style.top = '0' + element.style.pointerEvents = 'none' + element.style.zIndex = `${runtime.options.zIndex}` + element.style.opacity = getBaseOpacity() + element.style.transitionProperty = 'transform, opacity' + element.style.transitionTimingFunction = runtime.options.easing + element.style.transitionDuration = '0ms' + element.style.willChange = 'transform' + + runtime.cursorElement = element + applyRuntimeVisualOptions() + + return element +} + +function ensureCursorElement() { + const existing = document.getElementById(CURSOR_ID) + if (existing) { + runtime.cursorElement = existing + return existing + } + + const element = createCursorElement() + runtime.cursorElement = element + const root = document.documentElement || document.body + root.appendChild(element) + return element +} + +function applyRuntimeVisualOptions(): void { + if (!runtime.cursorElement) { + return + } + + const dimensions = getCursorDimensions() + runtime.cursorElement.style.width = `${dimensions.width}px` + runtime.cursorElement.style.height = `${dimensions.height}px` + runtime.cursorElement.style.zIndex = `${runtime.options.zIndex}` + runtime.cursorElement.style.transitionTimingFunction = runtime.options.easing + + if (runtime.options.style === 'screenstudio') { + runtime.cursorElement.style.borderRadius = '0' + runtime.cursorElement.style.border = 'none' + runtime.cursorElement.style.backgroundColor = 'transparent' + runtime.cursorElement.style.backgroundImage = `url("${SCREENSTUDIO_POINTER_MACOS_TAHOE_DATA_URL}")` + runtime.cursorElement.style.backgroundRepeat = 'no-repeat' + runtime.cursorElement.style.backgroundPosition = 'left top' + runtime.cursorElement.style.backgroundSize = 'contain' + runtime.cursorElement.style.backdropFilter = 'none' + runtime.cursorElement.style.filter = 'none' + runtime.cursorElement.style.boxShadow = 'none' + runtime.cursorElement.style.opacity = getBaseOpacity() + return + } + + if (runtime.options.style === 'minimal') { + const triangleSvg = `` + const triangleDataUrl = `url("data:image/svg+xml,${encodeURIComponent(triangleSvg)}")` + runtime.cursorElement.style.borderRadius = '0' + runtime.cursorElement.style.border = 'none' + runtime.cursorElement.style.backgroundColor = 'transparent' + runtime.cursorElement.style.backgroundImage = triangleDataUrl + runtime.cursorElement.style.backgroundRepeat = 'no-repeat' + runtime.cursorElement.style.backgroundSize = 'contain' + runtime.cursorElement.style.backgroundPosition = 'left top' + runtime.cursorElement.style.backdropFilter = 'none' + runtime.cursorElement.style.boxShadow = 'none' + runtime.cursorElement.style.filter = 'drop-shadow(0 1px 1px rgba(0, 0, 0, 0.3))' + runtime.cursorElement.style.opacity = getBaseOpacity() + return + } + + runtime.cursorElement.style.borderRadius = '999px' + runtime.cursorElement.style.border = 'none' + runtime.cursorElement.style.backgroundColor = runtime.options.color + runtime.cursorElement.style.backgroundImage = 'none' + runtime.cursorElement.style.backdropFilter = 'none' + runtime.cursorElement.style.filter = 'none' + runtime.cursorElement.style.boxShadow = '0 2px 10px rgba(0, 0, 0, 0.18), inset 0 0 0 2px rgba(255, 255, 255, 0.55)' + runtime.cursorElement.style.opacity = getBaseOpacity() +} + +function moveCursor(options: { x: number; y: number }): void { + if (!runtime.enabled) { + return + } + + const element = ensureCursorElement() + const durationMs = computeDurationMs({ targetX: options.x, targetY: options.y }) + element.style.transitionDuration = `${Math.round(durationMs)}ms` + + runtime.x = options.x + runtime.y = options.y + runtime.hasPosition = true + applyTransform() +} + +function setPressed(options: { pressed: boolean }): void { + if (!runtime.enabled) { + return + } + + const element = ensureCursorElement() + runtime.scale = options.pressed + ? runtime.options.style === 'screenstudio' + ? 0.94 + : runtime.options.style === 'minimal' + ? 0.93 + : 0.82 + : 1 + element.style.opacity = options.pressed ? '1' : getBaseOpacity() + applyTransform() +} + +function enable(options?: GhostCursorClientOptions): void { + runtime.options = mergeOptions(options) + runtime.enabled = true + ensureCursorElement() + applyRuntimeVisualOptions() + + if (!runtime.hasPosition) { + runtime.x = Math.round(window.innerWidth / 2) + runtime.y = Math.round(window.innerHeight / 2) + runtime.scale = 1 + runtime.hasPosition = true + applyTransform() + } +} + +function disable(): void { + runtime.enabled = false + runtime.scale = 1 + runtime.hasPosition = false + + if (runtime.cursorElement) { + runtime.cursorElement.remove() + runtime.cursorElement = null + } +} + +function applyMouseAction(action: GhostCursorAction): void { + if (!runtime.enabled) { + return + } + + if (action.type === 'move' || action.type === 'wheel') { + moveCursor({ x: action.x, y: action.y }) + return + } + + if (action.type === 'down') { + moveCursor({ x: action.x, y: action.y }) + setPressed({ pressed: true }) + return + } + + if (action.type === 'up') { + moveCursor({ x: action.x, y: action.y }) + setPressed({ pressed: false }) + } +} + +const api: GhostCursorApi = { + enable, + disable, + applyMouseAction, + isEnabled: () => { + return runtime.enabled + }, +} + +globalThis.__playwriterGhostCursor = api + +export {} diff --git a/playwriter/src/ghost-cursor.ts b/playwriter/src/ghost-cursor.ts new file mode 100644 index 0000000..f065d5f --- /dev/null +++ b/playwriter/src/ghost-cursor.ts @@ -0,0 +1,110 @@ +/** + * Node-side ghost cursor helpers. + * Injects the browser bundle and forwards mouse action events to the page overlay. + */ + +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import type { Page, MouseActionEvent } from '@xmorse/playwright-core' + +export interface GhostCursorClientOptions { + style?: 'minimal' | 'dot' | 'screenstudio' + color?: string + size?: number + zIndex?: number + easing?: string + minDurationMs?: number + maxDurationMs?: number + speedPxPerMs?: number +} + +interface GhostCursorBrowserApi { + enable: (options?: GhostCursorClientOptions) => void + disable: () => void + applyMouseAction: (event: MouseActionEvent) => void +} + +let ghostCursorCode: string | null = null + +function getGhostCursorCode(): string { + if (ghostCursorCode) { + return ghostCursorCode + } + + const currentDir = path.dirname(fileURLToPath(import.meta.url)) + const bundlePath = path.join(currentDir, '..', 'dist', 'ghost-cursor-client.js') + ghostCursorCode = fs.readFileSync(bundlePath, 'utf-8') + return ghostCursorCode +} + +async function ensureGhostCursorInjected(options: { page: Page }): Promise { + const { page } = options + const hasGhostCursor = await page.evaluate(() => { + return Boolean((globalThis as { __playwriterGhostCursor?: unknown }).__playwriterGhostCursor) + }) + + if (hasGhostCursor) { + return + } + + const code = getGhostCursorCode() + await page.evaluate(code) +} + +export async function enableGhostCursor(options: { + page: Page + cursorOptions?: GhostCursorClientOptions +}): Promise { + const { page, cursorOptions } = options + await ensureGhostCursorInjected({ page }) + + await page.evaluate( + ({ optionsFromNode }) => { + const api = (globalThis as { __playwriterGhostCursor?: GhostCursorBrowserApi }).__playwriterGhostCursor + api?.enable(optionsFromNode) + }, + { optionsFromNode: cursorOptions }, + ) +} + +export async function disableGhostCursor(options: { page: Page }): Promise { + const { page } = options + await page.evaluate(() => { + const api = (globalThis as { __playwriterGhostCursor?: GhostCursorBrowserApi }).__playwriterGhostCursor + api?.disable() + }) +} + +export async function applyGhostCursorMouseAction(options: { + page: Page + event: MouseActionEvent +}): Promise { + const { page, event } = options + + const applied = await page.evaluate( + ({ serializedEvent }) => { + const api = (globalThis as { __playwriterGhostCursor?: GhostCursorBrowserApi }).__playwriterGhostCursor + if (!api) { + return false + } + + api.applyMouseAction(serializedEvent) + return true + }, + { serializedEvent: event }, + ) + + if (applied) { + return + } + + await ensureGhostCursorInjected({ page }) + await page.evaluate( + ({ serializedEvent }) => { + const api = (globalThis as { __playwriterGhostCursor?: GhostCursorBrowserApi }).__playwriterGhostCursor + api?.applyMouseAction(serializedEvent) + }, + { serializedEvent: event }, + ) +} diff --git a/playwriter/src/on-mouse-action.test.ts b/playwriter/src/on-mouse-action.test.ts index 402058a..c74cdbb 100644 --- a/playwriter/src/on-mouse-action.test.ts +++ b/playwriter/src/on-mouse-action.test.ts @@ -4,9 +4,10 @@ * and locator-initiated actions like page.locator().click(). */ import { chromium } from '@xmorse/playwright-core' -import type { MouseActionEvent } from '@xmorse/playwright-core' +import type { MouseActionEvent, Page } from '@xmorse/playwright-core' import { describe, it, expect, beforeAll, afterAll } from 'vitest' import { getCdpUrl } from './utils.js' +import { enableGhostCursor, applyGhostCursorMouseAction, disableGhostCursor } from './ghost-cursor.js' import { setupTestContext, cleanupTestContext, @@ -113,6 +114,62 @@ describe('onMouseAction callback', () => { await safeCloseCDPBrowser(directBrowser) }, 30000) + it('should animate ghost cursor from onMouseAction callback', async () => { + const browserContext = testCtx!.browserContext + + const directBrowser = await chromium.connectOverCDP(getCdpUrl({ port: TEST_PORT })) + let targetPage: Page | null = null + try { + const contexts = directBrowser.contexts() + const pages = contexts[0].pages() + targetPage = pages.find((p) => p.url().startsWith('data:'))! + expect(targetPage).toBeDefined() + const pageForTest = targetPage + + await enableGhostCursor({ page: pageForTest }) + pageForTest.onMouseAction = async (event) => { + await applyGhostCursorMouseAction({ page: pageForTest, event }) + } + + await pageForTest.mouse.click(140, 120) + + const cursorState = await pageForTest.evaluate(() => { + const cursorElement = document.getElementById('__playwriter_ghost_cursor__') + if (!cursorElement) { + return { exists: false, transform: '' } + } + return { + exists: true, + transform: cursorElement.getAttribute('style') || '', + } + }) + + expect(cursorState.exists).toBe(true) + const translateMatch = cursorState.transform.match(/translate3d\(([-\d.]+)px, ([-\d.]+)px, 0px\)/) + expect(translateMatch).toBeTruthy() + const translateX = Number(translateMatch![1]) + const translateY = Number(translateMatch![2]) + // Screen Studio cursor uses a hotspot offset, so CSS position is slightly above/left of click target. + expect(translateX).toBeLessThanOrEqual(140) + expect(translateY).toBeLessThanOrEqual(120) + expect(translateX).toBeGreaterThan(120) + expect(translateY).toBeGreaterThan(100) + + await disableGhostCursor({ page: pageForTest }) + + const hasGhostCursor = await pageForTest.evaluate(() => { + return Boolean(document.getElementById('__playwriter_ghost_cursor__')) + }) + expect(hasGhostCursor).toBe(false) + } finally { + if (targetPage) { + targetPage.onMouseAction = null + await disableGhostCursor({ page: targetPage }) + } + await safeCloseCDPBrowser(directBrowser) + } + }, 30000) + it('should not fire when onMouseAction is set to null', async () => { const browserContext = testCtx!.browserContext diff --git a/playwriter/src/recording-ghost-cursor.ts b/playwriter/src/recording-ghost-cursor.ts new file mode 100644 index 0000000..ef3adad --- /dev/null +++ b/playwriter/src/recording-ghost-cursor.ts @@ -0,0 +1,99 @@ +/** + * Encapsulates ghost cursor lifecycle for recording sessions. + * Keeps onMouseAction chaining/restoration isolated from executor logic. + */ + +import type { BrowserContext, Page } from '@xmorse/playwright-core' +import { applyGhostCursorMouseAction, disableGhostCursor, enableGhostCursor, type GhostCursorClientOptions } from './ghost-cursor.js' + +interface RecordingGhostCursorLogger { + error: (...args: unknown[]) => void +} + +interface RecordingTargetOptions { + page?: Page + sessionId?: string +} + +export class RecordingGhostCursorController { + private readonly previousMouseActionByPage = new WeakMap() + private readonly logger: RecordingGhostCursorLogger + + constructor(options: { logger: RecordingGhostCursorLogger }) { + this.logger = options.logger + } + + resolveRecordingTargetPage(options: { + context: BrowserContext + defaultPage: Page + target?: RecordingTargetOptions + }): Page { + const { context, defaultPage, target } = options + + if (target?.page) { + return target.page + } + + if (target?.sessionId) { + const pageForSession = context.pages().find((candidatePage) => { + return candidatePage.sessionId() === target.sessionId + }) + + if (pageForSession) { + return pageForSession + } + } + + return defaultPage + } + + async enableForRecording(options: { page: Page }): Promise { + const { page } = options + + try { + await enableGhostCursor({ page }) + if (!this.previousMouseActionByPage.has(page)) { + this.previousMouseActionByPage.set(page, page.onMouseAction) + } + + const previousMouseAction = this.previousMouseActionByPage.get(page) + page.onMouseAction = async (event) => { + void applyGhostCursorMouseAction({ page, event }).catch((error) => { + this.logger.error('[playwriter] Failed to apply ghost cursor action', error) + }) + + if (!previousMouseAction) { + return + } + + await previousMouseAction(event) + } + } catch (error) { + page.onMouseAction = this.previousMouseActionByPage.get(page) ?? null + this.previousMouseActionByPage.delete(page) + this.logger.error('[playwriter] Failed to enable ghost cursor', error) + } + } + + async disableForRecording(options: { page: Page }): Promise { + const { page } = options + page.onMouseAction = this.previousMouseActionByPage.get(page) ?? null + this.previousMouseActionByPage.delete(page) + + try { + await disableGhostCursor({ page }) + } catch (error) { + this.logger.error('[playwriter] Failed to disable ghost cursor', error) + } + } + + async show(options: { page: Page; cursorOptions?: GhostCursorClientOptions }): Promise { + const { page, cursorOptions } = options + await enableGhostCursor({ page, cursorOptions }) + } + + async hide(options: { page: Page }): Promise { + const { page } = options + await disableGhostCursor({ page }) + } +} diff --git a/playwriter/src/relay-client.ts b/playwriter/src/relay-client.ts index 9a4411b..ad65419 100644 --- a/playwriter/src/relay-client.ts +++ b/playwriter/src/relay-client.ts @@ -15,6 +15,15 @@ const __dirname = path.dirname(__filename) export const RELAY_PORT = Number(process.env.PLAYWRITER_PORT) || 19988 +export type ExtensionStatus = { + extensionId: string + stableKey?: string + browser: string | null + profile: { email: string; id: string } | null + activeTargets: number + playwriterVersion: string | null +} + export async function getRelayServerVersion(port: number = RELAY_PORT): Promise { try { const response = await fetch(`http://127.0.0.1:${port}/version`, { @@ -46,33 +55,81 @@ export async function getExtensionStatus( } } +export async function getExtensionsStatus(port: number = RELAY_PORT): Promise { + try { + const response = await fetch(`http://127.0.0.1:${port}/extensions/status`, { + signal: AbortSignal.timeout(2000), + }) + if (!response.ok) { + const fallback = await fetch(`http://127.0.0.1:${port}/extension/status`, { + signal: AbortSignal.timeout(2000), + }) + if (!fallback.ok) { + return [] + } + + const fallbackData = (await fallback.json()) as { + connected: boolean + activeTargets: number + browser: string | null + profile: { email: string; id: string } | null + playwriterVersion?: string | null + } + + if (!fallbackData?.connected) { + return [] + } + + return [ + { + extensionId: 'default', + stableKey: undefined, + browser: fallbackData.browser, + profile: fallbackData.profile, + activeTargets: fallbackData.activeTargets, + playwriterVersion: fallbackData.playwriterVersion || null, + }, + ] + } + + const data = (await response.json()) as { + extensions: ExtensionStatus[] + } + + return data.extensions || [] + } catch { + return [] + } +} + /** - * Wait for the extension to connect to the relay server. - * Returns true if connected within timeout, false otherwise. + * Wait for at least one extension to appear in extensions status. + * Returns connected extension entries, or [] on timeout. */ -export async function waitForExtension( +export async function waitForConnectedExtensions( options: { port?: number timeoutMs?: number + pollIntervalMs?: number logger?: { log: (...args: any[]) => void } } = {}, -): Promise { - const { port = RELAY_PORT, timeoutMs = 5000, logger } = options +): Promise { + const { port = RELAY_PORT, timeoutMs = 5000, pollIntervalMs = 200, logger } = options const startTime = Date.now() logger?.log(pc.dim('Waiting for extension to connect...')) while (Date.now() - startTime < timeoutMs) { - const status = await getExtensionStatus(port) - if (status?.connected) { + const extensions = await getExtensionsStatus(port) + if (extensions.length > 0) { logger?.log(pc.green('Extension connected')) - return true + return extensions } - await sleep(200) + await sleep(pollIntervalMs) } logger?.log(pc.yellow('Extension did not connect within timeout')) - return false + return [] } async function killRelayServer(options: { port: number; waitForFreeMs?: number }): Promise { diff --git a/playwriter/src/screen-recording.ts b/playwriter/src/screen-recording.ts index 3a3da2f..3004fd9 100644 --- a/playwriter/src/screen-recording.ts +++ b/playwriter/src/screen-recording.ts @@ -8,7 +8,7 @@ import os from 'node:os' import path from 'node:path' -import type { Page } from '@xmorse/playwright-core' +import type { BrowserContext, Page } from '@xmorse/playwright-core' import type { StartRecordingResult, StopRecordingResult, @@ -16,6 +16,7 @@ import type { CancelRecordingResult, } from './protocol.js' import { EXTENSION_IDS } from './utils.js' +import { RecordingGhostCursorController } from './recording-ghost-cursor.js' /** * Generate a shell command to quit and restart Chrome with flags that allow automatic tab capture. @@ -86,6 +87,132 @@ export interface RecordingState { tabId?: number } +export interface ExecutionTimestamp { + start: number + end: number +} + +interface RecordingTargetOptions { + page?: Page + sessionId?: string +} + +interface CreateRecordingApiOptions { + context: BrowserContext + defaultPage: Page + relayPort: number + ghostCursorController: RecordingGhostCursorController + onStart: () => void + onFinish: () => void + getExecutionTimestamps: () => ExecutionTimestamp[] +} + +interface StartRecordingWithDefaultsOptions extends Omit {} +interface StopRecordingWithDefaultsOptions extends Omit {} +interface IsRecordingWithDefaultsOptions { + page?: Page + sessionId?: string +} +interface CancelRecordingWithDefaultsOptions { + page?: Page + sessionId?: string +} + +function resolveRecordingTargetPage(options: { + context: BrowserContext + defaultPage: Page + ghostCursorController: RecordingGhostCursorController + target?: RecordingTargetOptions +}): Page { + return options.ghostCursorController.resolveRecordingTargetPage({ + context: options.context, + defaultPage: options.defaultPage, + target: options.target, + }) +} + +function withRecordingDefaults(options: { + relayPort: number + defaultPage: Page + fn: (opts: T & { relayPort: number; sessionId?: string }) => Promise +}): (input?: T) => Promise { + const { relayPort, defaultPage, fn } = options + return async (input: T = {} as T) => { + const targetPage = input.page || defaultPage + const sessionId = input.sessionId || targetPage.sessionId() || undefined + return fn({ page: targetPage, sessionId, relayPort, ...input }) + } +} + +export function createRecordingApi(options: CreateRecordingApiOptions): { + start: (opts?: StartRecordingWithDefaultsOptions) => Promise + stop: (opts?: StopRecordingWithDefaultsOptions) => Promise<{ path: string; duration: number; size: number; executionTimestamps: ExecutionTimestamp[] }> + isRecording: (opts?: IsRecordingWithDefaultsOptions) => Promise + cancel: (opts?: CancelRecordingWithDefaultsOptions) => Promise +} { + const { context, defaultPage, relayPort, ghostCursorController, onStart, onFinish, getExecutionTimestamps } = options + + const startWithDefaults = withRecordingDefaults({ + relayPort, + defaultPage, + fn: startRecording, + }) + const stopWithDefaults = withRecordingDefaults({ + relayPort, + defaultPage, + fn: stopRecording, + }) + const isRecordingWithDefaults = async (opts: IsRecordingWithDefaultsOptions = {}): Promise => { + const targetPage = opts.page || defaultPage + const sessionId = opts.sessionId || targetPage.sessionId() || undefined + return isRecording({ page: targetPage, sessionId, relayPort }) + } + + const cancelWithDefaults = async (opts: CancelRecordingWithDefaultsOptions = {}): Promise => { + const targetPage = opts.page || defaultPage + const sessionId = opts.sessionId || targetPage.sessionId() || undefined + await cancelRecording({ page: targetPage, sessionId, relayPort }) + } + + const start = async (opts?: StartRecordingWithDefaultsOptions): Promise => { + const targetPage = resolveRecordingTargetPage({ context, defaultPage, ghostCursorController, target: opts }) + const result = await startWithDefaults(opts) + onStart() + await ghostCursorController.enableForRecording({ page: targetPage }) + return result + } + + const stop = async ( + opts?: StopRecordingWithDefaultsOptions, + ): Promise<{ path: string; duration: number; size: number; executionTimestamps: ExecutionTimestamp[] }> => { + const targetPage = resolveRecordingTargetPage({ context, defaultPage, ghostCursorController, target: opts }) + try { + const result = await stopWithDefaults(opts) + return { ...result, executionTimestamps: [...getExecutionTimestamps()] } + } finally { + onFinish() + await ghostCursorController.disableForRecording({ page: targetPage }) + } + } + + const cancel = async (opts?: CancelRecordingWithDefaultsOptions): Promise => { + const targetPage = resolveRecordingTargetPage({ context, defaultPage, ghostCursorController, target: opts }) + try { + await cancelWithDefaults(opts) + } finally { + onFinish() + await ghostCursorController.disableForRecording({ page: targetPage }) + } + } + + return { + start, + stop, + isRecording: isRecordingWithDefaults, + cancel, + } +} + /** * Start recording the page. * The recording is handled by the extension, so it survives page navigation. diff --git a/playwriter/src/skill.md b/playwriter/src/skill.md index 6a61437..4afa9bf 100644 --- a/playwriter/src/skill.md +++ b/playwriter/src/skill.md @@ -861,13 +861,15 @@ await screenshotWithAccessibilityLabels({ page: state.page }) Labels are color-coded: yellow=links, orange=buttons, coral=inputs, pink=checkboxes, peach=sliders, salmon=menus, amber=tabs. -**startRecording / stopRecording** - record the page as a video at native FPS (30-60fps). Uses `chrome.tabCapture` in the extension context, so **recording survives page navigation**. Video is saved as mp4. +**recording.start / recording.stop** - record the page as a video at native FPS (30-60fps). Uses `chrome.tabCapture` in the extension context, so **recording survives page navigation**. Video is saved as mp4. + +While recording is active, Playwriter automatically overlays a smooth ghost cursor that follows automated mouse actions (`page.mouse.*`, `locator.click()`, hover flows) using `page.onMouseAction` from the Playwright fork. **Note**: Recording requires the user to have clicked the Playwriter extension icon on the tab. This grants `activeTab` permission needed for `chrome.tabCapture`. Recording works on tabs where the icon was clicked - if you need to record a new tab, ask the user to click the icon on it first. ```js // Start recording - outputPath must be specified upfront -await startRecording({ +await recording.start({ page: state.page, outputPath: './recording.mp4', frameRate: 30, // default: 30 @@ -881,7 +883,7 @@ await state.page.waitForLoadState('domcontentloaded') await state.page.goBack() // Stop and get result -const { path, duration, size } = await stopRecording({ page: state.page }) +const { path, duration, size } = await recording.stop({ page: state.page }) console.log(`Saved ${size} bytes, duration: ${duration}ms`) ``` @@ -889,17 +891,32 @@ Additional recording utilities: ```js // Check if recording is active -const { isRecording, startedAt } = await isRecording({ page: state.page }) +const { isRecording, startedAt } = await recording.isRecording({ page: state.page }) // Cancel recording without saving -await cancelRecording({ page: state.page }) +await recording.cancel({ page: state.page }) ``` +**ghostCursor.show / ghostCursor.hide** - manually show or hide the in-page cursor overlay. Useful for screenshots and demos even when recording is not running. + +```js +// Show cursor in the center (or keep current position if already visible) +await ghostCursor.show({ page: state.page }) + +// Optional styles: 'minimal' (default triangular pointer), 'dot', 'screenstudio' +await ghostCursor.show({ page: state.page, style: 'minimal' }) + +// Hide cursor overlay +await ghostCursor.hide({ page: state.page }) +``` + +`startRecording`, `stopRecording`, `isRecording`, and `cancelRecording` remain available as backward-compatible aliases. + **Key difference from getDisplayMedia**: This approach uses `chrome.tabCapture` which runs in the extension context, not the page. The recording persists across navigations because the extension holds the `MediaRecorder`, not the page's JavaScript context. **createDemoVideo** - create a polished demo video from a recording by automatically speeding up idle sections (time between execute() calls) while keeping interactions at normal speed. Useful for creating demo videos of agent workflows without long pauses. -While recording is active, playwriter tracks when each `execute()` call starts and ends. `stopRecording()` returns these timestamps alongside the video file. `createDemoVideo` uses this data to identify idle gaps and speed them up with ffmpeg in a single pass. +While recording is active, playwriter tracks when each `execute()` call starts and ends. `recording.stop()` returns these timestamps alongside the video file. `createDemoVideo` uses this data to identify idle gaps and speed them up with ffmpeg in a single pass. A 1-second buffer is preserved around each interaction so viewers see context before and after each action. @@ -907,7 +924,7 @@ Requires `ffmpeg` and `ffprobe` installed on the system. ```js // Start recording -await startRecording({ page: state.page, outputPath: './recording.mp4' }) +await recording.start({ page: state.page, outputPath: './recording.mp4' }) ``` ```js @@ -917,13 +934,13 @@ await startRecording({ page: state.page, outputPath: './recording.mp4' }) ```js // Stop recording — executionTimestamps is included in the result -const recording = await stopRecording({ page: state.page }) +const recordingResult = await recording.stop({ page: state.page }) // Create demo video — idle gaps are sped up 4x (default) const demoPath = await createDemoVideo({ - recordingPath: recording.path, - durationMs: recording.duration, - executionTimestamps: recording.executionTimestamps, + recordingPath: recordingResult.path, + durationMs: recordingResult.duration, + executionTimestamps: recordingResult.executionTimestamps, speed: 5, // optional, default 5x for idle sections // outputFile: './demo.mp4', // optional, defaults to recording-demo.mp4 }) diff --git a/playwriter/tsconfig.json b/playwriter/tsconfig.json index a077fd5..0bfef53 100644 --- a/playwriter/tsconfig.json +++ b/playwriter/tsconfig.json @@ -6,5 +6,5 @@ "types": ["node", "chrome"] }, "include": ["src"], - "exclude": ["src/a11y-client.ts"] + "exclude": ["src/a11y-client.ts", "src/ghost-cursor-client.ts"] } diff --git a/website/public/SKILL.md b/website/public/SKILL.md index f93bdcb..4afa9bf 100644 --- a/website/public/SKILL.md +++ b/website/public/SKILL.md @@ -903,6 +903,9 @@ await recording.cancel({ page: state.page }) // Show cursor in the center (or keep current position if already visible) await ghostCursor.show({ page: state.page }) +// Optional styles: 'minimal' (default triangular pointer), 'dot', 'screenstudio' +await ghostCursor.show({ page: state.page, style: 'minimal' }) + // Hide cursor overlay await ghostCursor.hide({ page: state.page }) ```