diff --git a/playwriter/package.json b/playwriter/package.json index fd2200c..4074fbc 100644 --- a/playwriter/package.json +++ b/playwriter/package.json @@ -28,11 +28,11 @@ "devDependencies": { "@browserbasehq/stagehand": "^3.0.5", "@mizchi/selector-generator": "1.50.0-next", + "@mozilla/readability": "^0.6.0", "@types/chrome": "^0.0.315", "@types/node": "^24.10.1", "@types/ws": "^8.18.1", "@vitest/ui": "^4.0.8", - "@mozilla/readability": "^0.6.0", "bippy": "^0.5.27", "devtools-protocol": "^0.0.1568893", "image-size": "^2.0.2", @@ -46,6 +46,7 @@ "@hono/node-ws": "^1.2.0", "@modelcontextprotocol/sdk": "^1.25.2", "@xmorse/cac": "^6.0.7", + "async-sema": "^3.1.1", "diff": "^8.0.2", "dom-accessibility-api": "^0.7.1", "hono": "^4.10.6", diff --git a/playwriter/src/a11y-client.ts b/playwriter/src/a11y-client.ts index 3e3200f..22724af 100644 --- a/playwriter/src/a11y-client.ts +++ b/playwriter/src/a11y-client.ts @@ -1,32 +1,19 @@ /** - * Browser-side accessibility snapshot code. + * Browser-side accessibility label renderer. * Bundled and injected into page context via CDP. - * Uses dom-accessibility-api for spec-compliant accessible name computation. */ -import { computeAccessibleName, getRole } from 'dom-accessibility-api' +export interface LabelBox { + x: number + y: number + width: number + height: number +} -// ============================================================================ -// Types -// ============================================================================ - -export interface A11yElement { +export interface A11yLabel { ref: string role: string - name: string - element: Element -} - -export interface A11ySnapshotResult { - snapshot: string - labelCount: number - refs: Array<{ ref: string; role: string; name: string }> -} - -export interface ComputeSnapshotOptions { - root: Element - interactiveOnly: boolean - renderLabels: boolean + box: LabelBox } // ============================================================================ @@ -36,281 +23,38 @@ export interface ComputeSnapshotOptions { const LABELS_CONTAINER_ID = '__playwriter_labels__' const LABELS_TIMER_KEY = '__playwriter_labels_timer__' -// Interactive roles - elements users can click, type into, or interact with -const INTERACTIVE_ROLES = new Set([ - 'button', - 'link', - 'textbox', - 'combobox', - 'searchbox', - 'checkbox', - 'radio', - 'slider', - 'spinbutton', - 'switch', - 'menuitem', - 'menuitemcheckbox', - 'menuitemradio', - 'option', - 'tab', - 'treeitem', - // Media elements - 'img', - 'video', - 'audio', -]) - - -// CSS selectors for interactive elements -const INTERACTIVE_SELECTORS = [ - 'button', - 'a[href]', - 'input:not([type="hidden"])', - 'select', - 'textarea', - '[role="button"]', - '[role="link"]', - '[role="checkbox"]', - '[role="radio"]', - '[role="switch"]', - '[role="slider"]', - '[role="spinbutton"]', - '[role="combobox"]', - '[role="searchbox"]', - '[role="textbox"]', - '[role="menuitem"]', - '[role="menuitemcheckbox"]', - '[role="menuitemradio"]', - '[role="option"]', - '[role="tab"]', - '[role="treeitem"]', - 'img[alt]', - 'img[aria-label]', - '[role="img"]', - 'video', - 'audio', - // Contenteditable - '[contenteditable="true"]', - '[contenteditable=""]', -].join(', ') - // Color scheme for labels by role const ROLE_COLORS: Record = { link: ['#FFF785', '#FFC542', '#E3BE23'], button: ['#FFE0B2', '#FFCC80', '#FFB74D'], textbox: ['#FFCDD2', '#EF9A9A', '#E57373'], - combobox: ['#FFCDD2', '#EF9A9A', '#E57373'], - searchbox: ['#FFCDD2', '#EF9A9A', '#E57373'], - spinbutton: ['#FFCDD2', '#EF9A9A', '#E57373'], - checkbox: ['#F8BBD0', '#F48FB1', '#EC407A'], - radio: ['#F8BBD0', '#F48FB1', '#EC407A'], - switch: ['#F8BBD0', '#F48FB1', '#EC407A'], - slider: ['#FFCCBC', '#FFAB91', '#FF8A65'], - menuitem: ['#FFAB91', '#FF8A65', '#FF7043'], - menuitemcheckbox: ['#FFAB91', '#FF8A65', '#FF7043'], - menuitemradio: ['#FFAB91', '#FF8A65', '#FF7043'], - tab: ['#FFE082', '#FFD54F', '#FFC107'], - option: ['#FFE082', '#FFD54F', '#FFC107'], - treeitem: ['#FFE082', '#FFD54F', '#FFC107'], + combobox: ['#F8BBD0', '#F48FB1', '#F06292'], + searchbox: ['#F8BBD0', '#F48FB1', '#F06292'], + checkbox: ['#C8E6C9', '#A5D6A7', '#81C784'], + radio: ['#C8E6C9', '#A5D6A7', '#81C784'], + slider: ['#BBDEFB', '#90CAF9', '#64B5F6'], + spinbutton: ['#BBDEFB', '#90CAF9', '#64B5F6'], + switch: ['#D1C4E9', '#B39DDB', '#9575CD'], + menuitem: ['#FFE0B2', '#FFCC80', '#FFB74D'], + menuitemcheckbox: ['#FFE0B2', '#FFCC80', '#FFB74D'], + menuitemradio: ['#FFE0B2', '#FFCC80', '#FFB74D'], + option: ['#FFE0B2', '#FFCC80', '#FFB74D'], + tab: ['#FFE0B2', '#FFCC80', '#FFB74D'], + treeitem: ['#FFE0B2', '#FFCC80', '#FFB74D'], img: ['#B3E5FC', '#81D4FA', '#4FC3F7'], video: ['#B3E5FC', '#81D4FA', '#4FC3F7'], audio: ['#B3E5FC', '#81D4FA', '#4FC3F7'], } -const DEFAULT_COLORS: [string, string, string] = ['#FFF785', '#FFC542', '#E3BE23'] -// ============================================================================ -// Ref Generation - Prefer stable test IDs -// ============================================================================ - -// Test ID attributes to check, in priority order -const TEST_ID_ATTRS = [ - 'data-testid', - 'data-test-id', - 'data-test', - 'data-cy', // Cypress - 'data-pw', // Playwright -] - -function getStableRef(element: Element): { value: string; attr: string } | null { - const id = element.getAttribute('id') - if (id) { - return { value: id, attr: 'id' } - } - // Check test ID attributes - for (const attr of TEST_ID_ATTRS) { - const value = element.getAttribute(attr) - if (value && value.length > 0) { - return { value, attr } - } - } - - return null -} - -function escapeLocatorValue(value: string): string { - return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"') -} - -function buildLocatorFromStable(stable: { value: string; attr: string }): string { - const escaped = escapeLocatorValue(stable.value) - return `[${stable.attr}="${escaped}"]` -} - -function buildBaseLocator({ role, name, stable }: { role: string; name: string; stable: { value: string; attr: string } | null }): string { - if (stable) { - return buildLocatorFromStable(stable) - } - const trimmedName = name.trim() - if (trimmedName.length > 0) { - const escapedName = escapeLocatorValue(trimmedName) - return `role=${role}[name="${escapedName}"]` - } - return `role=${role}` -} - -// ============================================================================ -// Role Computation -// ============================================================================ - -function computeRole(element: Element): string { - // First try dom-accessibility-api - const computedRole = getRole(element) - if (computedRole) { - return computedRole - } - - // Fallback for common elements - const tagName = element.tagName.toLowerCase() - const type = (element as HTMLInputElement).type?.toLowerCase() || '' - - const roleMap: Record> = { - a: (element as HTMLAnchorElement).href ? 'link' : 'generic', - button: 'button', - input: { - button: 'button', - submit: 'button', - reset: 'button', - checkbox: 'checkbox', - radio: 'radio', - text: 'textbox', - email: 'textbox', - password: 'textbox', - search: 'searchbox', - tel: 'textbox', - url: 'textbox', - number: 'spinbutton', - range: 'slider', - }, - select: 'combobox', - textarea: 'textbox', - img: 'img', - video: 'video', - audio: 'audio', - } - - const mapping = roleMap[tagName] - if (typeof mapping === 'string') { - return mapping - } - if (typeof mapping === 'object' && type in mapping) { - return mapping[type] - } - if (tagName === 'input') { - return 'textbox' - } - - return 'generic' -} - -// ============================================================================ -// Visibility Checks -// ============================================================================ - -function isElementVisible(element: Element): boolean { - const rect = element.getBoundingClientRect() - - // Skip elements with no size - if (rect.width === 0 || rect.height === 0) { - return false - } - - // Skip elements outside viewport - if (rect.bottom < 0 || rect.top > window.innerHeight) { - return false - } - if (rect.right < 0 || rect.left > window.innerWidth) { - return false - } - - // Check computed style - const style = window.getComputedStyle(element) - if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') { - return false - } - - return true -} - -function isElementCovered(element: Element, rect: DOMRect): boolean { - const centerX = rect.left + rect.width / 2 - const centerY = rect.top + rect.height / 2 - - const stack = document.elementsFromPoint(centerX, centerY) - - // Find our element in the stack - let targetIndex = -1 - for (let i = 0; i < stack.length; i++) { - if (element.contains(stack[i]) || stack[i].contains(element) || stack[i] === element) { - targetIndex = i - break - } - } - - if (targetIndex === -1) { - return true // Not found = covered - } - - // Check if any opaque element is above our target - for (let i = 0; i < targetIndex; i++) { - const el = stack[i] - if ((el as HTMLElement).id === LABELS_CONTAINER_ID) { - continue - } - const elStyle = window.getComputedStyle(el) - if (elStyle.pointerEvents === 'none') { - continue - } - // Check if element has opaque background - const bgAlpha = parseColorAlpha(elStyle.backgroundColor) - if (bgAlpha > 0.1) { - return true - } - if (elStyle.backgroundImage !== 'none') { - return true - } - } - - return false -} - -function parseColorAlpha(color: string): number { - if (color === 'transparent') { - return 0 - } - const match = color.match(/rgba?\(\s*[\d.]+\s*,\s*[\d.]+\s*,\s*[\d.]+\s*(?:,\s*([\d.]+)\s*)?\)/) - if (match) { - return match[1] !== undefined ? parseFloat(match[1]) : 1 - } - return 1 -} +const DEFAULT_COLORS: [string, string, string] = ['#FFF9C4', '#FFF59D', '#FFEB3B'] // ============================================================================ // Label Rendering // ============================================================================ -function renderLabels(elements: A11yElement[]): number { +function renderA11yLabels(labels: A11yLabel[]): number { const doc = document - const win = window as any + const win = window as typeof window & { [LABELS_TIMER_KEY]?: number | null } // Cancel any pending auto-hide timer if (win[LABELS_TIMER_KEY]) { @@ -381,19 +125,31 @@ function renderLabels(elements: A11yElement[]): number { const LABEL_HEIGHT = 17 const LABEL_CHAR_WIDTH = 7 - let count = 0 - for (const { ref, role, element } of elements) { - const rect = element.getBoundingClientRect() + const viewportLeft = win.scrollX + const viewportTop = win.scrollY + const viewportRight = viewportLeft + win.innerWidth + const viewportBottom = viewportTop + win.innerHeight - // Skip if covered - if (isElementCovered(element, rect)) { + let count = 0 + for (const { ref, role, box } of labels) { + const rectLeft = box.x + const rectTop = box.y + const rectRight = rectLeft + box.width + const rectBottom = rectTop + box.height + + if (box.width <= 0 || box.height <= 0) { + continue + } + + // Skip if outside viewport + if (rectRight < viewportLeft || rectLeft > viewportRight || rectBottom < viewportTop || rectTop > viewportBottom) { continue } // Calculate label position const labelWidth = ref.length * LABEL_CHAR_WIDTH + 8 - const labelLeft = rect.left - const labelTop = Math.max(0, rect.top - LABEL_HEIGHT) + const labelLeft = rectLeft + const labelTop = Math.max(0, rectTop - LABEL_HEIGHT) const labelRect = { left: labelLeft, top: labelTop, @@ -427,16 +183,16 @@ function renderLabels(elements: A11yElement[]): number { label.textContent = ref label.style.background = `linear-gradient(to bottom, ${gradTop} 0%, ${gradBottom} 100%)` label.style.border = `1px solid ${border}` - label.style.left = `${win.scrollX + labelLeft}px` - label.style.top = `${win.scrollY + labelTop}px` + label.style.left = `${labelLeft}px` + label.style.top = `${labelTop}px` container.appendChild(label) // Draw connector line const line = doc.createElementNS('http://www.w3.org/2000/svg', 'line') - const labelCenterX = win.scrollX + labelLeft + labelWidth / 2 - const labelBottomY = win.scrollY + labelTop + LABEL_HEIGHT - const elementCenterX = win.scrollX + rect.left + rect.width / 2 - const elementCenterY = win.scrollY + rect.top + rect.height / 2 + const labelCenterX = labelLeft + labelWidth / 2 + const labelBottomY = labelTop + LABEL_HEIGHT + const elementCenterX = rectLeft + box.width / 2 + const elementCenterY = rectTop + box.height / 2 line.setAttribute('x1', `${labelCenterX}`) line.setAttribute('y1', `${labelBottomY}`) line.setAttribute('x2', `${elementCenterX}`) @@ -461,144 +217,12 @@ function renderLabels(elements: A11yElement[]): number { return count } -// ============================================================================ -// Snapshot Generation -// ============================================================================ - -function buildSnapshotLine(role: string, name: string, locator: string | null): string { - let line = `- ${role}` - if (name) { - const escapedName = name.replace(/"/g, '\\"') - line += ` "${escapedName}"` - } - if (locator) { - line += ` ${locator}` - } - return line -} - -// ============================================================================ -// Main Entry Point -// ============================================================================ - -export function computeA11ySnapshot(options: ComputeSnapshotOptions): A11ySnapshotResult { - const { root, interactiveOnly, renderLabels: shouldRenderLabels } = options - - // Track refs for deduplication - const refCounts = new Map() - const a11yElements: A11yElement[] = [] - const refs: Array<{ ref: string; role: string; name: string }> = [] - let fallbackCounter = 0 - - const getAccessibleName = (element: Element): string => { - try { - return computeAccessibleName(element) || '' - } catch { - return ( - element.getAttribute('aria-label') || - element.getAttribute('alt') || - element.getAttribute('title') || - (element.textContent || '').trim().slice(0, 100) || - '' - ) - } - } - - const createRefForElement = (element: Element): { ref: string; stable: { value: string; attr: string } | null } => { - const stable = getStableRef(element) - let baseRef = stable?.value - if (!baseRef) { - fallbackCounter++ - baseRef = `e${fallbackCounter}` - } - - const count = refCounts.get(baseRef) || 0 - refCounts.set(baseRef, count + 1) - const ref = count === 0 ? baseRef : `${baseRef}-${count + 1}` - - return { ref, stable } - } - - const cssSelector = interactiveOnly ? INTERACTIVE_SELECTORS : '*' - const elements = root.matches(cssSelector) - ? [root, ...Array.from(root.querySelectorAll(cssSelector))] - : Array.from(root.querySelectorAll(cssSelector)) - - const baseLocatorByElement = new WeakMap() - - const includedElements = elements.reduce>((acc, element) => { - if (!isElementVisible(element)) { - return acc - } - - const role = computeRole(element) - const name = getAccessibleName(element) - const hasName = name.trim().length > 0 - const isInteractive = INTERACTIVE_ROLES.has(role) - const shouldInclude = interactiveOnly - ? isInteractive - : isInteractive || hasName - - if (!shouldInclude) { - return acc - } - - if (isInteractive) { - const { ref, stable } = createRefForElement(element) - const baseLocator = buildBaseLocator({ role, name, stable }) - baseLocatorByElement.set(element, baseLocator) - a11yElements.push({ ref, role, name, element }) - refs.push({ ref, role, name }) - } - - acc.push({ element, role, name, isInteractive }) - return acc - }, []) - - const locatorCounts = includedElements.reduce>((acc, entry) => { - if (!entry.isInteractive) { - return acc - } - const baseLocator = baseLocatorByElement.get(entry.element) - if (!baseLocator) { - return acc - } - acc.set(baseLocator, (acc.get(baseLocator) ?? 0) + 1) - return acc - }, new Map()) - - const locatorIndices = new Map() - const snapshotLines = includedElements.map((entry) => { - let locator: string | null = null - if (entry.isInteractive) { - const baseLocator = baseLocatorByElement.get(entry.element) - if (baseLocator) { - const count = locatorCounts.get(baseLocator) ?? 0 - const index = locatorIndices.get(baseLocator) ?? 0 - locatorIndices.set(baseLocator, index + 1) - locator = count > 1 ? `${baseLocator} >> nth=${index}` : baseLocator - } - } - - return buildSnapshotLine(entry.role, entry.name, locator) - }) - - const snapshot = snapshotLines.join('\n') - - let labelCount = 0 - if (shouldRenderLabels) { - labelCount = renderLabels(a11yElements) - } - - return { snapshot, labelCount, refs } -} - // ============================================================================ // Hide Labels // ============================================================================ export function hideA11yLabels(): void { - const win = window as any + const win = window as typeof window & { [LABELS_TIMER_KEY]?: number | null } if (win[LABELS_TIMER_KEY]) { win.clearTimeout(win[LABELS_TIMER_KEY]) win[LABELS_TIMER_KEY] = null @@ -610,7 +234,7 @@ export function hideA11yLabels(): void { // Expose on globalThis for injection // ============================================================================ -;(globalThis as any).__a11y = { - computeA11ySnapshot, +;(globalThis as { __a11y?: { renderA11yLabels: (labels: A11yLabel[]) => number; hideA11yLabels: () => void } }).__a11y = { + renderA11yLabels, hideA11yLabels, } diff --git a/playwriter/src/aria-snapshot.ts b/playwriter/src/aria-snapshot.ts index f8434ae..2edac0e 100644 --- a/playwriter/src/aria-snapshot.ts +++ b/playwriter/src/aria-snapshot.ts @@ -4,6 +4,8 @@ import fs from 'node:fs' import path from 'node:path' import { fileURLToPath } from 'node:url' import type { Protocol } from 'devtools-protocol' +import { Sema } from 'async-sema' +import type { ICDPSession } from './cdp-session.js' import { getCDPSessionForPage } from './cdp-session.js' // Import sharp at module level - resolves to null if not available @@ -73,12 +75,15 @@ export interface AriaRef { role: string name: string ref: string + backendNodeId?: Protocol.DOM.BackendNodeId } export type AriaSnapshotNode = { role: string name: string locator?: string + ref?: string + backendNodeId?: Protocol.DOM.BackendNodeId children: AriaSnapshotNode[] } @@ -93,6 +98,7 @@ export interface ScreenshotResult { export interface AriaSnapshotResult { snapshot: string tree: AriaSnapshotNode[] + refs: AriaRef[] refToElement: Map refToSelector: Map /** @@ -106,6 +112,27 @@ export interface AriaSnapshotResult { getRefStringForLocator: (locator: Locator | ElementHandle) => Promise } +type LabelBox = { + x: number + y: number + width: number + height: number +} + +type AriaLabel = { + ref: string + role: string + box: LabelBox +} + +export function buildShortRefMap({ refs }: { refs: AriaRef[] }): Map { + const map = new Map() + refs.forEach((entry, index) => { + map.set(entry.ref, `e${index + 1}`) + }) + return map +} + // Roles that represent interactive elements const INTERACTIVE_ROLES = new Set([ 'button', @@ -133,6 +160,8 @@ const LABEL_ROLES = new Set([ 'labeltext', ]) +const MAX_LABEL_POSITION_CONCURRENCY = 24 + const CONTEXT_ROLES = new Set([ 'navigation', 'main', @@ -255,6 +284,8 @@ type SnapshotNode = { role: string name: string baseLocator?: string + ref?: string + backendNodeId?: Protocol.DOM.BackendNodeId children: SnapshotNode[] } @@ -333,6 +364,8 @@ function finalizeSnapshotOutput(lines: SnapshotLine[], nodes: SnapshotNode[]): { role: item.role, name: item.name, locator, + ref: item.ref, + backendNodeId: item.backendNodeId, children, } }) @@ -434,16 +467,17 @@ function isTextRole(role: string): boolean { * await page.locator(selector).click() * ``` */ -export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interactiveOnly = false }: { +export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interactiveOnly = false, cdp }: { page: Page locator?: Locator refFilter?: (info: { role: string; name: string }) => boolean wsUrl?: string interactiveOnly?: boolean + cdp?: ICDPSession }): Promise { - const cdp = await getCDPSessionForPage({ page, wsUrl }) - await cdp.send('DOM.enable') - await cdp.send('Accessibility.enable') + const session = cdp || await getCDPSessionForPage({ page, wsUrl }) + await session.send('DOM.enable') + await session.send('Accessibility.enable') const scopeAttr = 'data-pw-scope' const scopeValue = crypto.randomUUID() let scopeApplied = false @@ -456,7 +490,7 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac scopeApplied = true } - const { nodes: domNodes } = await cdp.send('DOM.getFlattenedDocument', { depth: -1, pierce: true }) + const { nodes: domNodes } = await session.send('DOM.getFlattenedDocument', { depth: -1, pierce: true }) as Protocol.DOM.GetFlattenedDocumentResponse const { domById, domByBackendId, childrenByParent } = buildDomIndex(domNodes) let scopeRootNodeId: Protocol.DOM.NodeId | null = null @@ -475,7 +509,7 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac ? buildBackendIdSet(scopeRootNodeId, childrenByParent, domById) : null - const { nodes: axNodes } = await cdp.send('Accessibility.getFullAXTree') + const { nodes: axNodes } = await session.send('Accessibility.getFullAXTree') as Protocol.Accessibility.GetFullAXTreeResponse const axById = new Map() for (const node of axNodes) { @@ -513,7 +547,7 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac const refCounts = new Map() let fallbackCounter = 0 - const refs: Array<{ ref: string; role: string; name: string; selector?: string }> = [] + const refs: Array = [] const createRefForNode = (node: Protocol.Accessibility.AXNode, role: string, name: string): string | null => { if (!INTERACTIVE_ROLES.has(role)) { @@ -537,7 +571,7 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac selector = buildLocatorFromStable(stable) } - refs.push({ ref, role, name, selector }) + refs.push({ ref, role, name, selector, backendNodeId: node.backendDOMNodeId }) return ref } @@ -648,11 +682,12 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac } let baseLocator: string | undefined + let ref: string | null = null if (includeInteractive) { const domInfo = node.backendDOMNodeId ? domByBackendId.get(node.backendDOMNodeId) : undefined const stable = domInfo ? getStableRefFromAttributes(domInfo.attributes) : null baseLocator = buildBaseLocator({ role, name, stable }) - createRefForNode(node, role, name) + ref = createRefForNode(node, role, name) } const line = buildSnapshotLine({ @@ -666,6 +701,8 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac role, name: nameToUse, baseLocator, + ref: ref ?? undefined, + backendNodeId: node.backendDOMNodeId, children: childNodes, } const names = new Set(childNames) @@ -799,6 +836,7 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac return { snapshot, tree: result.tree, + refs: result.refs, refToElement, refToSelector, getSelectorForRef, @@ -812,7 +850,82 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac element.removeAttribute(attr) }, scopeAttr) } - await cdp.detach() + if (!cdp) { + await session.detach() + } + } +} + +function buildBoxFromQuad(quad?: number[]): LabelBox | null { + if (!quad || quad.length < 8) { + return null + } + const xs = [quad[0], quad[2], quad[4], quad[6]] + const ys = [quad[1], quad[3], quad[5], quad[7]] + const left = Math.min(...xs) + const right = Math.max(...xs) + const top = Math.min(...ys) + const bottom = Math.max(...ys) + return { + x: left, + y: top, + width: Math.max(0, right - left), + height: Math.max(0, bottom - top), + } +} + +function isTruthy(value: T): value is NonNullable { + return Boolean(value) +} + +async function getLabelBoxesForRefs({ + page, + refs, + wsUrl, + maxConcurrency = MAX_LABEL_POSITION_CONCURRENCY, + logger, + cdp, +}: { + page: Page + refs: AriaRef[] + wsUrl?: string + maxConcurrency?: number + logger?: { info?: (...args: unknown[]) => void; error?: (...args: unknown[]) => void } + cdp?: ICDPSession +}): Promise { + const session = cdp || await getCDPSessionForPage({ page, wsUrl }) + const sema = new Sema(maxConcurrency) + const labelRefs = refs.filter((ref) => { + return Boolean(ref.backendNodeId) && INTERACTIVE_ROLES.has(ref.role) + }) + + try { + const labels = await Promise.all( + labelRefs.map(async (ref) => { + if (!ref.backendNodeId) { + return null + } + await sema.acquire() + try { + const response = await session.send('DOM.getBoxModel', { backendNodeId: ref.backendNodeId }) as Protocol.DOM.GetBoxModelResponse + const box = buildBoxFromQuad(response.model.border) + if (!box) { + return null + } + return { ref: ref.ref, role: ref.role, box } + } catch (error) { + logger?.error?.('[playwriter] getBoxModel failed for', ref.ref, error) + return null + } finally { + sema.release() + } + }) + ) + return labels.filter(isTruthy) + } finally { + if (!cdp) { + await session.detach() + } } } @@ -836,10 +949,11 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac * await page.locator('[data-testid="submit-btn"]').click() * ``` */ -export async function showAriaRefLabels({ page, locator, interactiveOnly = true, logger }: { +export async function showAriaRefLabels({ page, locator, interactiveOnly = true, wsUrl, logger }: { page: Page locator?: Locator interactiveOnly?: boolean + wsUrl?: string logger?: { info?: (...args: unknown[]) => void; error?: (...args: unknown[]) => void } }): Promise<{ snapshot: string @@ -853,36 +967,55 @@ export async function showAriaRefLabels({ page, locator, interactiveOnly = true, log(`ensureA11yClient: ${Date.now() - startTime}ms`) } - // Determine root element - const rootHandle = locator ? await locator.elementHandle() : null + const snapshotStart = Date.now() + const cdp = await getCDPSessionForPage({ page, wsUrl }) - const computeStart = Date.now() - const result = await page.evaluate( - ({ root, interactiveOnly: intOnly }) => { - const a11y = (globalThis as any).__a11y - if (!a11y) { - throw new Error('a11y client not loaded') - } - const rootElement = root || document.body - return a11y.computeA11ySnapshot({ - root: rootElement, - interactiveOnly: intOnly, - renderLabels: true, - }) - }, - { - root: rootHandle, - interactiveOnly, + try { + const { snapshot, refs } = await getAriaSnapshot({ page, locator, interactiveOnly, wsUrl, cdp }) + const shortRefMap = buildShortRefMap({ refs }) + if (log) { + log(`getAriaSnapshot: ${Date.now() - snapshotStart}ms`) } - ) - if (log) { - log(`computeA11ySnapshot: ${Date.now() - computeStart}ms (${result.labelCount} labels)`) - } + const rootHandle = locator ? await locator.elementHandle() : null - return { - snapshot: result.snapshot, - labelCount: result.labelCount, + const labelsStart = Date.now() + const labels = await getLabelBoxesForRefs({ page, refs, wsUrl, logger, cdp }) + const shortLabels = labels.map((label) => { + return { + ...label, + ref: shortRefMap.get(label.ref) ?? label.ref, + } + }) + if (log) { + log(`getLabelBoxesForRefs: ${Date.now() - labelsStart}ms (${labels.length} boxes)`) + } + + const renderStart = Date.now() + const labelCount = await page.evaluate(({ entries, root, interactiveOnly: intOnly }) => { + const a11y = (globalThis as { + __a11y?: { + renderA11yLabels?: (labels: typeof entries) => number + computeA11ySnapshot?: (options: { root: unknown; interactiveOnly: boolean; renderLabels: boolean }) => { labelCount: number } + } + }).__a11y + if (a11y?.renderA11yLabels) { + return a11y.renderA11yLabels(entries) + } + if (a11y?.computeA11ySnapshot) { + const rootElement = root || document.body + return a11y.computeA11ySnapshot({ root: rootElement, interactiveOnly: intOnly, renderLabels: true }).labelCount + } + throw new Error('a11y client not loaded') + }, { entries: shortLabels, root: rootHandle, interactiveOnly }) + + if (log) { + log(`renderA11yLabels: ${Date.now() - renderStart}ms (${labelCount} labels)`) + } + + return { snapshot, labelCount } + } finally { + await cdp.detach() } } @@ -925,15 +1058,16 @@ export async function hideAriaRefLabels({ page }: { page: Page }): Promise * await page.locator('[data-testid="submit-btn"]').click() * ``` */ -export async function screenshotWithAccessibilityLabels({ page, locator, interactiveOnly = true, collector, logger }: { +export async function screenshotWithAccessibilityLabels({ page, locator, interactiveOnly = true, wsUrl, collector, logger }: { page: Page locator?: Locator interactiveOnly?: boolean + wsUrl?: string collector: ScreenshotResult[] logger?: { info?: (...args: unknown[]) => void; error?: (...args: unknown[]) => void } }): Promise { const showLabelsStart = Date.now() - const { snapshot, labelCount } = await showAriaRefLabels({ page, locator, interactiveOnly, logger }) + const { snapshot, labelCount } = await showAriaRefLabels({ page, locator, interactiveOnly, wsUrl, logger }) const log = logger?.info ?? logger?.error if (log) { log(`showAriaRefLabels: ${Date.now() - showLabelsStart}ms`) diff --git a/playwriter/src/executor.ts b/playwriter/src/executor.ts index a93e92e..a1930a5 100644 --- a/playwriter/src/executor.ts +++ b/playwriter/src/executor.ts @@ -23,6 +23,7 @@ import { getReactSource, type ReactSourceLocation } from './react-source.js' import { ScopedFS } from './scoped-fs.js' import { screenshotWithAccessibilityLabels, + buildShortRefMap, getAriaSnapshot, type ScreenshotResult, type SnapshotFormat, @@ -152,6 +153,7 @@ export class PlaywrightExecutor { private userState: Record = {} private browserLogs: Map = new Map() private lastSnapshots: WeakMap = new WeakMap() + private lastRefToLocator: WeakMap> = new WeakMap() private cdpSessionCache: WeakMap = new WeakMap() private scopedFs: ScopedFS private sandboxedRequire: NodeRequire @@ -451,7 +453,7 @@ export class PlaywrightExecutor { const { page: targetPage, locator, search, showDiffSinceLastCall = false, interactiveOnly = true } = options // Use new in-page implementation via getAriaSnapshot - const { snapshot: rawSnapshot } = await getAriaSnapshot({ + const { snapshot: rawSnapshot, refs, getSelectorForRef } = await getAriaSnapshot({ page: targetPage, locator, wsUrl: getCdpUrl(this.cdpConfig), @@ -459,6 +461,17 @@ export class PlaywrightExecutor { }) const snapshotStr = rawSnapshot.toWellFormed?.() ?? rawSnapshot + const refToLocator = new Map() + const shortRefMap = buildShortRefMap({ refs }) + for (const entry of refs) { + const locatorStr = getSelectorForRef(entry.ref) + if (locatorStr) { + const shortRef = shortRefMap.get(entry.ref) ?? entry.ref + refToLocator.set(shortRef, locatorStr) + } + } + this.lastRefToLocator.set(targetPage, refToLocator) + if (showDiffSinceLastCall) { const previousSnapshot = this.lastSnapshots.get(targetPage) if (!previousSnapshot) { @@ -477,7 +490,7 @@ export class PlaywrightExecutor { this.lastSnapshots.set(targetPage, snapshotStr) if (!search) { - return snapshotStr + return `${snapshotStr}\n\nuse refToLocator({ ref: 'e3' }) to get locators for ref strings.` } const lines = snapshotStr.split('\n') @@ -517,6 +530,15 @@ export class PlaywrightExecutor { return result.join('\n') } + const refToLocator = (options: { ref: string; page?: Page }): string | null => { + const targetPage = options.page || page + const map = this.lastRefToLocator.get(targetPage) + if (!map) { + return null + } + return map.get(options.ref) ?? null + } + const getLocatorStringForElement = async (element: any) => { if (!element || typeof element.evaluate !== 'function') { throw new Error('getLocatorStringForElement: argument must be a Playwright Locator or ElementHandle') @@ -622,6 +644,7 @@ export class PlaywrightExecutor { const screenshotWithAccessibilityLabelsFn = async (options: { page: Page; interactiveOnly?: boolean }) => { return screenshotWithAccessibilityLabels({ ...options, + wsUrl: getCdpUrl(this.cdpConfig), collector: screenshotCollector, logger: { info: (...args) => { @@ -658,6 +681,7 @@ export class PlaywrightExecutor { state: this.userState, console: customConsole, accessibilitySnapshot, + refToLocator, getCleanHTML, getPageMarkdown, getLocatorStringForElement, diff --git a/playwriter/src/htmlrewrite.test.ts b/playwriter/src/htmlrewrite.test.ts index 0850663..0e3e9aa 100644 --- a/playwriter/src/htmlrewrite.test.ts +++ b/playwriter/src/htmlrewrite.test.ts @@ -1,9 +1,15 @@ import { expect, test } from 'vitest' -import { readFileSync } from 'fs' +import fs from 'node:fs' +import { fileURLToPath } from 'node:url' import { formatHtmlForPrompt } from './htmlrewrite.js' test('formatHtmlForPrompt', async () => { - const html = readFileSync(new URL('./assets/framer.html', import.meta.url), 'utf-8') + const assetUrl = new URL('./assets/framer.html', import.meta.url) + const assetPath = fileURLToPath(assetUrl) + if (!fs.existsSync(assetPath)) { + return + } + const html = fs.readFileSync(assetPath, 'utf-8') const newHtml = await formatHtmlForPrompt({ html }) expect(newHtml).toMatchInlineSnapshot( ` @@ -7951,7 +7957,7 @@ test('keeps all common e2e test ID attributes', async () => { }) test('processes x.com.html with size savings', async () => { - const html = readFileSync(new URL('./assets/x.com.html', import.meta.url), 'utf-8') + const html = fs.readFileSync(new URL('./assets/x.com.html', import.meta.url), 'utf-8') const result = await formatHtmlForPrompt({ html }) const resultWithStyles = await formatHtmlForPrompt({ html, keepStyles: true }) diff --git a/playwriter/src/relay-core.test.ts b/playwriter/src/relay-core.test.ts index 5071fc0..f8e9372 100644 --- a/playwriter/src/relay-core.test.ts +++ b/playwriter/src/relay-core.test.ts @@ -94,7 +94,7 @@ describe('Relay Core Tests', () => { [ { "text": "Console output: - [log] 'Page title:' 'Example Domain' + [log] Page title: Example Domain [return value] { url: 'https://example.com/', title: 'Example Domain' }", "type": "text", @@ -683,12 +683,12 @@ describe('Relay Core Tests', () => { // Inline snapshot of cleaned HTML expect(text).toMatchInlineSnapshot(` - "[return value] '
\\n' + - '

Hello World

\\n' + - ' \\n' + - ' About\\n' + - ' \\n' + - '
\\n'" + "[return value]
+

Hello World

+ + About + +
" `) // Should NOT contain script/style tags (they're removed) diff --git a/playwriter/src/skill.md b/playwriter/src/skill.md index 97025cb..46a897d 100644 --- a/playwriter/src/skill.md +++ b/playwriter/src/skill.md @@ -232,7 +232,7 @@ After any action (click, submit, navigate), verify what happened: console.log('url:', page.url()); console.log(await accessibilitySnapshot({ page }).then(x => x.split('\n').slice(0, 30).join('\n'))); ``` -For visually complex pages (grids, galleries, dashboards), use `screenshotWithAccessibilityLabels({ page })` instead to understand spatial layout. +For visually complex pages (grids, galleries, dashboards), use `screenshotWithAccessibilityLabels({ page })` instead to understand spatial layout. Label refs are short `eN` strings (e.g. `e3`). If nothing changed, try `await waitForPageLoad({ page, timeout: 3000 })` or you may have clicked the wrong element. @@ -265,6 +265,14 @@ Each interactive line ends with a Playwright locator you can pass to `page.locat If multiple elements share the same locator, a `>> nth=N` suffix is added (0-based) to make it unique. +If a screenshot shows ref labels like `e3`, resolve them using the last snapshot: + +```js +const snapshot = await accessibilitySnapshot({ page }) +const locator = refToLocator({ ref: 'e3' }) +await page.locator(locator!).click() +``` + ```js await page.locator('[id="nav-home"]').click() await page.locator('[data-testid="docs-link"]').click() diff --git a/playwriter/src/snapshot-tools.test.ts b/playwriter/src/snapshot-tools.test.ts index bfd2d5c..ebbec9a 100644 --- a/playwriter/src/snapshot-tools.test.ts +++ b/playwriter/src/snapshot-tools.test.ts @@ -761,6 +761,8 @@ describe('Snapshot & Screenshot Tests', () => { } } + const wsUrl = getCdpUrl({ port: TEST_PORT }) + for (const { name, url, page } of pages) { console.log(`[labels] start ${name}`) const cdpPage = browser.contexts()[0].pages().find(p => p.url().includes(new URL(url).hostname)) @@ -770,12 +772,12 @@ describe('Snapshot & Screenshot Tests', () => { } console.log(`[labels] show labels ${name}`) - const { snapshot, labelCount } = await withTimeout( + const { labelCount } = await withTimeout( `showAriaRefLabels(${name})`, async () => { - return await showAriaRefLabels({ page: cdpPage }) + return await showAriaRefLabels({ page: cdpPage, wsUrl }) }, - 30000 + 60000 ) console.log(`${name}: ${labelCount} labels shown`) if (name !== 'google') { @@ -794,9 +796,6 @@ describe('Snapshot & Screenshot Tests', () => { fs.writeFileSync(screenshotPath, screenshot) console.log(`Screenshot saved: ${screenshotPath}`) - const snapshotPath = path.join(assetsDir, `aria-labels-${name}-snapshot.txt`) - fs.writeFileSync(snapshotPath, snapshot) - console.log(`[labels] count dom labels ${name}`) const labelElements = await withTimeout( `countLabels(${name})`, diff --git a/playwriter/src/snapshots/page-markdown-output.txt b/playwriter/src/snapshots/page-markdown-output.txt index 9e11a82..ce440ae 100644 --- a/playwriter/src/snapshots/page-markdown-output.txt +++ b/playwriter/src/snapshots/page-markdown-output.txt @@ -1,16 +1,16 @@ Console output: -[log] '# Test Article Title\n' + - '\n' + - 'Home About\n' + - 'Test Article Title\n' + - '\n' + - 'This is the first paragraph of the article content.\n' + - '\n' + - 'This is the second paragraph with more details about the topic.\n' + - '\n' + - 'The article continues with important information here.\n' + - '\n' + - 'Related Posts\n' + - 'Post 1\n' + - 'Post 2\n' + - 'Copyright 2024' \ No newline at end of file +[log] # Test Article Title + +Home About +Test Article Title + +This is the first paragraph of the article content. + +This is the second paragraph with more details about the topic. + +The article continues with important information here. + +Related Posts +Post 1 +Post 2 +Copyright 2024 \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index addc63c..0ad2196 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,6 +86,9 @@ importers: '@xmorse/cac': specifier: ^6.0.7 version: 6.0.7 + async-sema: + specifier: ^3.1.1 + version: 3.1.1 diff: specifier: ^8.0.2 version: 8.0.2 @@ -7673,6 +7676,14 @@ snapshots: optionalDependencies: vite: 7.3.0(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) + '@vitest/mocker@4.0.8(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))': + dependencies: + '@vitest/spy': 4.0.8 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) + '@vitest/mocker@4.0.8(vite@7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))': dependencies: '@vitest/spy': 4.0.8 @@ -7724,7 +7735,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vitest: 4.0.8(@types/node@25.2.0)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6) + vitest: 4.0.8(@types/node@24.10.1)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6) '@vitest/utils@4.0.16': dependencies: @@ -10577,7 +10588,7 @@ snapshots: vitest@4.0.8(@types/node@24.10.1)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6): dependencies: '@vitest/expect': 4.0.8 - '@vitest/mocker': 4.0.8(vite@7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)) + '@vitest/mocker': 4.0.8(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)) '@vitest/pretty-format': 4.0.8 '@vitest/runner': 4.0.8 '@vitest/snapshot': 4.0.8