feat: shorten a11y labels and add ref resolver

playwriter/package.json

@@ -28,11 +28,11 @@
   "devDependencies": {
     "@browserbasehq/stagehand": "^3.0.5",
     "@mizchi/selector-generator": "1.50.0-next",
+    "@mozilla/readability": "^0.6.0",
     "@types/chrome": "^0.0.315",
     "@types/node": "^24.10.1",
     "@types/ws": "^8.18.1",
     "@vitest/ui": "^4.0.8",
-    "@mozilla/readability": "^0.6.0",
     "bippy": "^0.5.27",
     "devtools-protocol": "^0.0.1568893",
     "image-size": "^2.0.2",
@@ -46,6 +46,7 @@
     "@hono/node-ws": "^1.2.0",
     "@modelcontextprotocol/sdk": "^1.25.2",
     "@xmorse/cac": "^6.0.7",
+    "async-sema": "^3.1.1",
     "diff": "^8.0.2",
     "dom-accessibility-api": "^0.7.1",
     "hono": "^4.10.6",

playwriter/src/a11y-client.ts

@@ -1,32 +1,19 @@
 /**
- * Browser-side accessibility snapshot code.
+ * Browser-side accessibility label renderer.
  * Bundled and injected into page context via CDP.
- * Uses dom-accessibility-api for spec-compliant accessible name computation.
  */
 
-import { computeAccessibleName, getRole } from 'dom-accessibility-api'
+export interface LabelBox {
+  x: number
+  y: number
+  width: number
+  height: number
+}
 
-// ============================================================================
-// Types
-// ============================================================================
-
-export interface A11yElement {
+export interface A11yLabel {
   ref: string
   role: string
-  name: string
-  element: Element
-}
-
-export interface A11ySnapshotResult {
-  snapshot: string
-  labelCount: number
-  refs: Array<{ ref: string; role: string; name: string }>
-}
-
-export interface ComputeSnapshotOptions {
-  root: Element
-  interactiveOnly: boolean
-  renderLabels: boolean
+  box: LabelBox
 }
 
 // ============================================================================
@@ -36,281 +23,38 @@ export interface ComputeSnapshotOptions {
 const LABELS_CONTAINER_ID = '__playwriter_labels__'
 const LABELS_TIMER_KEY = '__playwriter_labels_timer__'
 
-// Interactive roles - elements users can click, type into, or interact with
-const INTERACTIVE_ROLES = new Set([
-  'button',
-  'link',
-  'textbox',
-  'combobox',
-  'searchbox',
-  'checkbox',
-  'radio',
-  'slider',
-  'spinbutton',
-  'switch',
-  'menuitem',
-  'menuitemcheckbox',
-  'menuitemradio',
-  'option',
-  'tab',
-  'treeitem',
-  // Media elements
-  'img',
-  'video',
-  'audio',
-])
-
-
-// CSS selectors for interactive elements
-const INTERACTIVE_SELECTORS = [
-  'button',
-  'a[href]',
-  'input:not([type="hidden"])',
-  'select',
-  'textarea',
-  '[role="button"]',
-  '[role="link"]',
-  '[role="checkbox"]',
-  '[role="radio"]',
-  '[role="switch"]',
-  '[role="slider"]',
-  '[role="spinbutton"]',
-  '[role="combobox"]',
-  '[role="searchbox"]',
-  '[role="textbox"]',
-  '[role="menuitem"]',
-  '[role="menuitemcheckbox"]',
-  '[role="menuitemradio"]',
-  '[role="option"]',
-  '[role="tab"]',
-  '[role="treeitem"]',
-  'img[alt]',
-  'img[aria-label]',
-  '[role="img"]',
-  'video',
-  'audio',
-  // Contenteditable
-  '[contenteditable="true"]',
-  '[contenteditable=""]',
-].join(', ')
-
 // Color scheme for labels by role
 const ROLE_COLORS: Record<string, [string, string, string]> = {
   link: ['#FFF785', '#FFC542', '#E3BE23'],
   button: ['#FFE0B2', '#FFCC80', '#FFB74D'],
   textbox: ['#FFCDD2', '#EF9A9A', '#E57373'],
-  combobox: ['#FFCDD2', '#EF9A9A', '#E57373'],
-  searchbox: ['#FFCDD2', '#EF9A9A', '#E57373'],
-  spinbutton: ['#FFCDD2', '#EF9A9A', '#E57373'],
-  checkbox: ['#F8BBD0', '#F48FB1', '#EC407A'],
-  radio: ['#F8BBD0', '#F48FB1', '#EC407A'],
-  switch: ['#F8BBD0', '#F48FB1', '#EC407A'],
-  slider: ['#FFCCBC', '#FFAB91', '#FF8A65'],
-  menuitem: ['#FFAB91', '#FF8A65', '#FF7043'],
-  menuitemcheckbox: ['#FFAB91', '#FF8A65', '#FF7043'],
-  menuitemradio: ['#FFAB91', '#FF8A65', '#FF7043'],
-  tab: ['#FFE082', '#FFD54F', '#FFC107'],
-  option: ['#FFE082', '#FFD54F', '#FFC107'],
-  treeitem: ['#FFE082', '#FFD54F', '#FFC107'],
+  combobox: ['#F8BBD0', '#F48FB1', '#F06292'],
+  searchbox: ['#F8BBD0', '#F48FB1', '#F06292'],
+  checkbox: ['#C8E6C9', '#A5D6A7', '#81C784'],
+  radio: ['#C8E6C9', '#A5D6A7', '#81C784'],
+  slider: ['#BBDEFB', '#90CAF9', '#64B5F6'],
+  spinbutton: ['#BBDEFB', '#90CAF9', '#64B5F6'],
+  switch: ['#D1C4E9', '#B39DDB', '#9575CD'],
+  menuitem: ['#FFE0B2', '#FFCC80', '#FFB74D'],
+  menuitemcheckbox: ['#FFE0B2', '#FFCC80', '#FFB74D'],
+  menuitemradio: ['#FFE0B2', '#FFCC80', '#FFB74D'],
+  option: ['#FFE0B2', '#FFCC80', '#FFB74D'],
+  tab: ['#FFE0B2', '#FFCC80', '#FFB74D'],
+  treeitem: ['#FFE0B2', '#FFCC80', '#FFB74D'],
   img: ['#B3E5FC', '#81D4FA', '#4FC3F7'],
   video: ['#B3E5FC', '#81D4FA', '#4FC3F7'],
   audio: ['#B3E5FC', '#81D4FA', '#4FC3F7'],
 }
-const DEFAULT_COLORS: [string, string, string] = ['#FFF785', '#FFC542', '#E3BE23']
 
-// ============================================================================
-// Ref Generation - Prefer stable test IDs
-// ============================================================================
-
-// Test ID attributes to check, in priority order
-const TEST_ID_ATTRS = [
-  'data-testid',
-  'data-test-id',
-  'data-test',
-  'data-cy', // Cypress
-  'data-pw', // Playwright
-]
-
-function getStableRef(element: Element): { value: string; attr: string } | null {
-  const id = element.getAttribute('id')
-  if (id) {
-    return { value: id, attr: 'id' }
-  }
-  // Check test ID attributes
-  for (const attr of TEST_ID_ATTRS) {
-    const value = element.getAttribute(attr)
-    if (value && value.length > 0) {
-      return { value, attr }
-    }
-  }
-
-  return null
-}
-
-function escapeLocatorValue(value: string): string {
-  return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
-}
-
-function buildLocatorFromStable(stable: { value: string; attr: string }): string {
-  const escaped = escapeLocatorValue(stable.value)
-  return `[${stable.attr}="${escaped}"]`
-}
-
-function buildBaseLocator({ role, name, stable }: { role: string; name: string; stable: { value: string; attr: string } | null }): string {
-  if (stable) {
-    return buildLocatorFromStable(stable)
-  }
-  const trimmedName = name.trim()
-  if (trimmedName.length > 0) {
-    const escapedName = escapeLocatorValue(trimmedName)
-    return `role=${role}[name="${escapedName}"]`
-  }
-  return `role=${role}`
-}
-
-// ============================================================================
-// Role Computation
-// ============================================================================
-
-function computeRole(element: Element): string {
-  // First try dom-accessibility-api
-  const computedRole = getRole(element)
-  if (computedRole) {
-    return computedRole
-  }
-
-  // Fallback for common elements
-  const tagName = element.tagName.toLowerCase()
-  const type = (element as HTMLInputElement).type?.toLowerCase() || ''
-
-  const roleMap: Record<string, string | Record<string, string>> = {
-    a: (element as HTMLAnchorElement).href ? 'link' : 'generic',
-    button: 'button',
-    input: {
-      button: 'button',
-      submit: 'button',
-      reset: 'button',
-      checkbox: 'checkbox',
-      radio: 'radio',
-      text: 'textbox',
-      email: 'textbox',
-      password: 'textbox',
-      search: 'searchbox',
-      tel: 'textbox',
-      url: 'textbox',
-      number: 'spinbutton',
-      range: 'slider',
-    },
-    select: 'combobox',
-    textarea: 'textbox',
-    img: 'img',
-    video: 'video',
-    audio: 'audio',
-  }
-
-  const mapping = roleMap[tagName]
-  if (typeof mapping === 'string') {
-    return mapping
-  }
-  if (typeof mapping === 'object' && type in mapping) {
-    return mapping[type]
-  }
-  if (tagName === 'input') {
-    return 'textbox'
-  }
-
-  return 'generic'
-}
-
-// ============================================================================
-// Visibility Checks
-// ============================================================================
-
-function isElementVisible(element: Element): boolean {
-  const rect = element.getBoundingClientRect()
-
-  // Skip elements with no size
-  if (rect.width === 0 || rect.height === 0) {
-    return false
-  }
-
-  // Skip elements outside viewport
-  if (rect.bottom < 0 || rect.top > window.innerHeight) {
-    return false
-  }
-  if (rect.right < 0 || rect.left > window.innerWidth) {
-    return false
-  }
-
-  // Check computed style
-  const style = window.getComputedStyle(element)
-  if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
-    return false
-  }
-
-  return true
-}
-
-function isElementCovered(element: Element, rect: DOMRect): boolean {
-  const centerX = rect.left + rect.width / 2
-  const centerY = rect.top + rect.height / 2
-
-  const stack = document.elementsFromPoint(centerX, centerY)
-
-  // Find our element in the stack
-  let targetIndex = -1
-  for (let i = 0; i < stack.length; i++) {
-    if (element.contains(stack[i]) || stack[i].contains(element) || stack[i] === element) {
-      targetIndex = i
-      break
-    }
-  }
-
-  if (targetIndex === -1) {
-    return true // Not found = covered
-  }
-
-  // Check if any opaque element is above our target
-  for (let i = 0; i < targetIndex; i++) {
-    const el = stack[i]
-    if ((el as HTMLElement).id === LABELS_CONTAINER_ID) {
-      continue
-    }
-    const elStyle = window.getComputedStyle(el)
-    if (elStyle.pointerEvents === 'none') {
-      continue
-    }
-    // Check if element has opaque background
-    const bgAlpha = parseColorAlpha(elStyle.backgroundColor)
-    if (bgAlpha > 0.1) {
-      return true
-    }
-    if (elStyle.backgroundImage !== 'none') {
-      return true
-    }
-  }
-
-  return false
-}
-
-function parseColorAlpha(color: string): number {
-  if (color === 'transparent') {
-    return 0
-  }
-  const match = color.match(/rgba?\(\s*[\d.]+\s*,\s*[\d.]+\s*,\s*[\d.]+\s*(?:,\s*([\d.]+)\s*)?\)/)
-  if (match) {
-    return match[1] !== undefined ? parseFloat(match[1]) : 1
-  }
-  return 1
-}
+const DEFAULT_COLORS: [string, string, string] = ['#FFF9C4', '#FFF59D', '#FFEB3B']
 
 // ============================================================================
 // Label Rendering
 // ============================================================================
 
-function renderLabels(elements: A11yElement[]): number {
+function renderA11yLabels(labels: A11yLabel[]): number {
   const doc = document
-  const win = window as any
+  const win = window as typeof window & { [LABELS_TIMER_KEY]?: number | null }
 
   // Cancel any pending auto-hide timer
   if (win[LABELS_TIMER_KEY]) {
@@ -381,19 +125,31 @@ function renderLabels(elements: A11yElement[]): number {
   const LABEL_HEIGHT = 17
   const LABEL_CHAR_WIDTH = 7
 
-  let count = 0
-  for (const { ref, role, element } of elements) {
-    const rect = element.getBoundingClientRect()
+  const viewportLeft = win.scrollX
+  const viewportTop = win.scrollY
+  const viewportRight = viewportLeft + win.innerWidth
+  const viewportBottom = viewportTop + win.innerHeight
 
-    // Skip if covered
-    if (isElementCovered(element, rect)) {
+  let count = 0
+  for (const { ref, role, box } of labels) {
+    const rectLeft = box.x
+    const rectTop = box.y
+    const rectRight = rectLeft + box.width
+    const rectBottom = rectTop + box.height
+
+    if (box.width <= 0 || box.height <= 0) {
+      continue
+    }
+
+    // Skip if outside viewport
+    if (rectRight < viewportLeft || rectLeft > viewportRight || rectBottom < viewportTop || rectTop > viewportBottom) {
       continue
     }
 
     // Calculate label position
     const labelWidth = ref.length * LABEL_CHAR_WIDTH + 8
-    const labelLeft = rect.left
-    const labelTop = Math.max(0, rect.top - LABEL_HEIGHT)
+    const labelLeft = rectLeft
+    const labelTop = Math.max(0, rectTop - LABEL_HEIGHT)
     const labelRect = {
       left: labelLeft,
       top: labelTop,
@@ -427,16 +183,16 @@ function renderLabels(elements: A11yElement[]): number {
     label.textContent = ref
     label.style.background = `linear-gradient(to bottom, ${gradTop} 0%, ${gradBottom} 100%)`
     label.style.border = `1px solid ${border}`
-    label.style.left = `${win.scrollX + labelLeft}px`
-    label.style.top = `${win.scrollY + labelTop}px`
+    label.style.left = `${labelLeft}px`
+    label.style.top = `${labelTop}px`
     container.appendChild(label)
 
     // Draw connector line
     const line = doc.createElementNS('http://www.w3.org/2000/svg', 'line')
-    const labelCenterX = win.scrollX + labelLeft + labelWidth / 2
-    const labelBottomY = win.scrollY + labelTop + LABEL_HEIGHT
-    const elementCenterX = win.scrollX + rect.left + rect.width / 2
-    const elementCenterY = win.scrollY + rect.top + rect.height / 2
+    const labelCenterX = labelLeft + labelWidth / 2
+    const labelBottomY = labelTop + LABEL_HEIGHT
+    const elementCenterX = rectLeft + box.width / 2
+    const elementCenterY = rectTop + box.height / 2
     line.setAttribute('x1', `${labelCenterX}`)
     line.setAttribute('y1', `${labelBottomY}`)
     line.setAttribute('x2', `${elementCenterX}`)
@@ -461,144 +217,12 @@ function renderLabels(elements: A11yElement[]): number {
   return count
 }
 
-// ============================================================================
-// Snapshot Generation
-// ============================================================================
-
-function buildSnapshotLine(role: string, name: string, locator: string | null): string {
-  let line = `- ${role}`
-  if (name) {
-    const escapedName = name.replace(/"/g, '\\"')
-    line += ` "${escapedName}"`
-  }
-  if (locator) {
-    line += ` ${locator}`
-  }
-  return line
-}
-
-// ============================================================================
-// Main Entry Point
-// ============================================================================
-
-export function computeA11ySnapshot(options: ComputeSnapshotOptions): A11ySnapshotResult {
-  const { root, interactiveOnly, renderLabels: shouldRenderLabels } = options
-
-  // Track refs for deduplication
-  const refCounts = new Map<string, number>()
-  const a11yElements: A11yElement[] = []
-  const refs: Array<{ ref: string; role: string; name: string }> = []
-  let fallbackCounter = 0
-
-  const getAccessibleName = (element: Element): string => {
-    try {
-      return computeAccessibleName(element) || ''
-    } catch {
-      return (
-        element.getAttribute('aria-label') ||
-        element.getAttribute('alt') ||
-        element.getAttribute('title') ||
-        (element.textContent || '').trim().slice(0, 100) ||
-        ''
-      )
-    }
-  }
-
-  const createRefForElement = (element: Element): { ref: string; stable: { value: string; attr: string } | null } => {
-    const stable = getStableRef(element)
-    let baseRef = stable?.value
-    if (!baseRef) {
-      fallbackCounter++
-      baseRef = `e${fallbackCounter}`
-    }
-
-    const count = refCounts.get(baseRef) || 0
-    refCounts.set(baseRef, count + 1)
-    const ref = count === 0 ? baseRef : `${baseRef}-${count + 1}`
-
-    return { ref, stable }
-  }
-
-  const cssSelector = interactiveOnly ? INTERACTIVE_SELECTORS : '*'
-  const elements = root.matches(cssSelector)
-    ? [root, ...Array.from(root.querySelectorAll(cssSelector))]
-    : Array.from(root.querySelectorAll(cssSelector))
-
-  const baseLocatorByElement = new WeakMap<Element, string>()
-
-  const includedElements = elements.reduce<Array<{ element: Element; role: string; name: string; isInteractive: boolean }>>((acc, element) => {
-    if (!isElementVisible(element)) {
-      return acc
-    }
-
-    const role = computeRole(element)
-    const name = getAccessibleName(element)
-    const hasName = name.trim().length > 0
-    const isInteractive = INTERACTIVE_ROLES.has(role)
-    const shouldInclude = interactiveOnly
-      ? isInteractive
-      : isInteractive || hasName
-
-    if (!shouldInclude) {
-      return acc
-    }
-
-    if (isInteractive) {
-      const { ref, stable } = createRefForElement(element)
-      const baseLocator = buildBaseLocator({ role, name, stable })
-      baseLocatorByElement.set(element, baseLocator)
-      a11yElements.push({ ref, role, name, element })
-      refs.push({ ref, role, name })
-    }
-
-    acc.push({ element, role, name, isInteractive })
-    return acc
-  }, [])
-
-  const locatorCounts = includedElements.reduce<Map<string, number>>((acc, entry) => {
-    if (!entry.isInteractive) {
-      return acc
-    }
-    const baseLocator = baseLocatorByElement.get(entry.element)
-    if (!baseLocator) {
-      return acc
-    }
-    acc.set(baseLocator, (acc.get(baseLocator) ?? 0) + 1)
-    return acc
-  }, new Map<string, number>())
-
-  const locatorIndices = new Map<string, number>()
-  const snapshotLines = includedElements.map((entry) => {
-    let locator: string | null = null
-    if (entry.isInteractive) {
-      const baseLocator = baseLocatorByElement.get(entry.element)
-      if (baseLocator) {
-        const count = locatorCounts.get(baseLocator) ?? 0
-        const index = locatorIndices.get(baseLocator) ?? 0
-        locatorIndices.set(baseLocator, index + 1)
-        locator = count > 1 ? `${baseLocator} >> nth=${index}` : baseLocator
-      }
-    }
-
-    return buildSnapshotLine(entry.role, entry.name, locator)
-  })
-
-  const snapshot = snapshotLines.join('\n')
-
-  let labelCount = 0
-  if (shouldRenderLabels) {
-    labelCount = renderLabels(a11yElements)
-  }
-
-  return { snapshot, labelCount, refs }
-}
-
 // ============================================================================
 // Hide Labels
 // ============================================================================
 
 export function hideA11yLabels(): void {
-  const win = window as any
+  const win = window as typeof window & { [LABELS_TIMER_KEY]?: number | null }
   if (win[LABELS_TIMER_KEY]) {
     win.clearTimeout(win[LABELS_TIMER_KEY])
     win[LABELS_TIMER_KEY] = null
@@ -610,7 +234,7 @@ export function hideA11yLabels(): void {
 // Expose on globalThis for injection
 // ============================================================================
 
-;(globalThis as any).__a11y = {
-  computeA11ySnapshot,
+;(globalThis as { __a11y?: { renderA11yLabels: (labels: A11yLabel[]) => number; hideA11yLabels: () => void } }).__a11y = {
+  renderA11yLabels,
   hideA11yLabels,
 }

playwriter/src/aria-snapshot.ts

@@ -4,6 +4,8 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import type { Protocol } from 'devtools-protocol'
+import { Sema } from 'async-sema'
+import type { ICDPSession } from './cdp-session.js'
 import { getCDPSessionForPage } from './cdp-session.js'
 
 // Import sharp at module level - resolves to null if not available
@@ -73,12 +75,15 @@ export interface AriaRef {
   role: string
   name: string
   ref: string
+  backendNodeId?: Protocol.DOM.BackendNodeId
 }
 
 export type AriaSnapshotNode = {
   role: string
   name: string
   locator?: string
+  ref?: string
+  backendNodeId?: Protocol.DOM.BackendNodeId
   children: AriaSnapshotNode[]
 }
 
@@ -93,6 +98,7 @@ export interface ScreenshotResult {
 export interface AriaSnapshotResult {
   snapshot: string
   tree: AriaSnapshotNode[]
+  refs: AriaRef[]
   refToElement: Map<string, { role: string; name: string }>
   refToSelector: Map<string, string>
   /**
@@ -106,6 +112,27 @@ export interface AriaSnapshotResult {
   getRefStringForLocator: (locator: Locator | ElementHandle) => Promise<string | null>
 }
 
+type LabelBox = {
+  x: number
+  y: number
+  width: number
+  height: number
+}
+
+type AriaLabel = {
+  ref: string
+  role: string
+  box: LabelBox
+}
+
+export function buildShortRefMap({ refs }: { refs: AriaRef[] }): Map<string, string> {
+  const map = new Map<string, string>()
+  refs.forEach((entry, index) => {
+    map.set(entry.ref, `e${index + 1}`)
+  })
+  return map
+}
+
 // Roles that represent interactive elements
 const INTERACTIVE_ROLES = new Set([
   'button',
@@ -133,6 +160,8 @@ const LABEL_ROLES = new Set([
   'labeltext',
 ])
 
+const MAX_LABEL_POSITION_CONCURRENCY = 24
+
 const CONTEXT_ROLES = new Set([
   'navigation',
   'main',
@@ -255,6 +284,8 @@ type SnapshotNode = {
   role: string
   name: string
   baseLocator?: string
+  ref?: string
+  backendNodeId?: Protocol.DOM.BackendNodeId
   children: SnapshotNode[]
 }
 
@@ -333,6 +364,8 @@ function finalizeSnapshotOutput(lines: SnapshotLine[], nodes: SnapshotNode[]): {
         role: item.role,
         name: item.name,
         locator,
+        ref: item.ref,
+        backendNodeId: item.backendNodeId,
         children,
       }
     })
@@ -434,16 +467,17 @@ function isTextRole(role: string): boolean {
  * await page.locator(selector).click()
  * ```
  */
-export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interactiveOnly = false }: {
+export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interactiveOnly = false, cdp }: {
   page: Page
   locator?: Locator
   refFilter?: (info: { role: string; name: string }) => boolean
   wsUrl?: string
   interactiveOnly?: boolean
+  cdp?: ICDPSession
 }): Promise<AriaSnapshotResult> {
-  const cdp = await getCDPSessionForPage({ page, wsUrl })
-  await cdp.send('DOM.enable')
-  await cdp.send('Accessibility.enable')
+  const session = cdp || await getCDPSessionForPage({ page, wsUrl })
+  await session.send('DOM.enable')
+  await session.send('Accessibility.enable')
   const scopeAttr = 'data-pw-scope'
   const scopeValue = crypto.randomUUID()
   let scopeApplied = false
@@ -456,7 +490,7 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac
       scopeApplied = true
     }
 
-    const { nodes: domNodes } = await cdp.send('DOM.getFlattenedDocument', { depth: -1, pierce: true })
+    const { nodes: domNodes } = await session.send('DOM.getFlattenedDocument', { depth: -1, pierce: true }) as Protocol.DOM.GetFlattenedDocumentResponse
     const { domById, domByBackendId, childrenByParent } = buildDomIndex(domNodes)
 
     let scopeRootNodeId: Protocol.DOM.NodeId | null = null
@@ -475,7 +509,7 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac
       ? buildBackendIdSet(scopeRootNodeId, childrenByParent, domById)
       : null
 
-    const { nodes: axNodes } = await cdp.send('Accessibility.getFullAXTree')
+    const { nodes: axNodes } = await session.send('Accessibility.getFullAXTree') as Protocol.Accessibility.GetFullAXTreeResponse
 
     const axById = new Map<Protocol.Accessibility.AXNodeId, Protocol.Accessibility.AXNode>()
     for (const node of axNodes) {
@@ -513,7 +547,7 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac
 
     const refCounts = new Map<string, number>()
     let fallbackCounter = 0
-    const refs: Array<{ ref: string; role: string; name: string; selector?: string }> = []
+    const refs: Array<AriaRef & { selector?: string }> = []
 
     const createRefForNode = (node: Protocol.Accessibility.AXNode, role: string, name: string): string | null => {
       if (!INTERACTIVE_ROLES.has(role)) {
@@ -537,7 +571,7 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac
         selector = buildLocatorFromStable(stable)
       }
 
-      refs.push({ ref, role, name, selector })
+      refs.push({ ref, role, name, selector, backendNodeId: node.backendDOMNodeId })
       return ref
     }
 
@@ -648,11 +682,12 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac
       }
 
       let baseLocator: string | undefined
+      let ref: string | null = null
       if (includeInteractive) {
         const domInfo = node.backendDOMNodeId ? domByBackendId.get(node.backendDOMNodeId) : undefined
         const stable = domInfo ? getStableRefFromAttributes(domInfo.attributes) : null
         baseLocator = buildBaseLocator({ role, name, stable })
-        createRefForNode(node, role, name)
+        ref = createRefForNode(node, role, name)
       }
 
       const line = buildSnapshotLine({
@@ -666,6 +701,8 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac
         role,
         name: nameToUse,
         baseLocator,
+        ref: ref ?? undefined,
+        backendNodeId: node.backendDOMNodeId,
         children: childNodes,
       }
       const names = new Set(childNames)
@@ -799,6 +836,7 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac
     return {
       snapshot,
       tree: result.tree,
+      refs: result.refs,
       refToElement,
       refToSelector,
       getSelectorForRef,
@@ -812,7 +850,82 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac
         element.removeAttribute(attr)
       }, scopeAttr)
     }
-    await cdp.detach()
+    if (!cdp) {
+      await session.detach()
+    }
+  }
+}
+
+function buildBoxFromQuad(quad?: number[]): LabelBox | null {
+  if (!quad || quad.length < 8) {
+    return null
+  }
+  const xs = [quad[0], quad[2], quad[4], quad[6]]
+  const ys = [quad[1], quad[3], quad[5], quad[7]]
+  const left = Math.min(...xs)
+  const right = Math.max(...xs)
+  const top = Math.min(...ys)
+  const bottom = Math.max(...ys)
+  return {
+    x: left,
+    y: top,
+    width: Math.max(0, right - left),
+    height: Math.max(0, bottom - top),
+  }
+}
+
+function isTruthy<T>(value: T): value is NonNullable<T> {
+  return Boolean(value)
+}
+
+async function getLabelBoxesForRefs({
+  page,
+  refs,
+  wsUrl,
+  maxConcurrency = MAX_LABEL_POSITION_CONCURRENCY,
+  logger,
+  cdp,
+}: {
+  page: Page
+  refs: AriaRef[]
+  wsUrl?: string
+  maxConcurrency?: number
+  logger?: { info?: (...args: unknown[]) => void; error?: (...args: unknown[]) => void }
+  cdp?: ICDPSession
+}): Promise<AriaLabel[]> {
+  const session = cdp || await getCDPSessionForPage({ page, wsUrl })
+  const sema = new Sema(maxConcurrency)
+  const labelRefs = refs.filter((ref) => {
+    return Boolean(ref.backendNodeId) && INTERACTIVE_ROLES.has(ref.role)
+  })
+
+  try {
+    const labels = await Promise.all(
+      labelRefs.map(async (ref) => {
+        if (!ref.backendNodeId) {
+          return null
+        }
+        await sema.acquire()
+        try {
+          const response = await session.send('DOM.getBoxModel', { backendNodeId: ref.backendNodeId }) as Protocol.DOM.GetBoxModelResponse
+          const box = buildBoxFromQuad(response.model.border)
+          if (!box) {
+            return null
+          }
+          return { ref: ref.ref, role: ref.role, box }
+        } catch (error) {
+          logger?.error?.('[playwriter] getBoxModel failed for', ref.ref, error)
+          return null
+        } finally {
+          sema.release()
+        }
+      })
+    )
+    return labels.filter(isTruthy)
+  } finally {
+    if (!cdp) {
+      await session.detach()
+    }
   }
 }
 
@@ -836,10 +949,11 @@ export async function getAriaSnapshot({ page, locator, refFilter, wsUrl, interac
  * await page.locator('[data-testid="submit-btn"]').click()
  * ```
  */
-export async function showAriaRefLabels({ page, locator, interactiveOnly = true, logger }: {
+export async function showAriaRefLabels({ page, locator, interactiveOnly = true, wsUrl, logger }: {
   page: Page
   locator?: Locator
   interactiveOnly?: boolean
+  wsUrl?: string
   logger?: { info?: (...args: unknown[]) => void; error?: (...args: unknown[]) => void }
 }): Promise<{
   snapshot: string
@@ -853,36 +967,55 @@ export async function showAriaRefLabels({ page, locator, interactiveOnly = true,
     log(`ensureA11yClient: ${Date.now() - startTime}ms`)
   }
 
-  // Determine root element
-  const rootHandle = locator ? await locator.elementHandle() : null
+  const snapshotStart = Date.now()
+  const cdp = await getCDPSessionForPage({ page, wsUrl })
 
-  const computeStart = Date.now()
-  const result = await page.evaluate(
-    ({ root, interactiveOnly: intOnly }) => {
-      const a11y = (globalThis as any).__a11y
-      if (!a11y) {
-        throw new Error('a11y client not loaded')
-      }
-      const rootElement = root || document.body
-      return a11y.computeA11ySnapshot({
-        root: rootElement,
-        interactiveOnly: intOnly,
-        renderLabels: true,
-      })
-    },
-    {
-      root: rootHandle,
-      interactiveOnly,
+  try {
+    const { snapshot, refs } = await getAriaSnapshot({ page, locator, interactiveOnly, wsUrl, cdp })
+    const shortRefMap = buildShortRefMap({ refs })
+    if (log) {
+      log(`getAriaSnapshot: ${Date.now() - snapshotStart}ms`)
     }
-  )
 
-  if (log) {
-    log(`computeA11ySnapshot: ${Date.now() - computeStart}ms (${result.labelCount} labels)`)
-  }
+    const rootHandle = locator ? await locator.elementHandle() : null
 
-  return {
-    snapshot: result.snapshot,
-    labelCount: result.labelCount,
+    const labelsStart = Date.now()
+    const labels = await getLabelBoxesForRefs({ page, refs, wsUrl, logger, cdp })
+    const shortLabels = labels.map((label) => {
+      return {
+        ...label,
+        ref: shortRefMap.get(label.ref) ?? label.ref,
+      }
+    })
+    if (log) {
+      log(`getLabelBoxesForRefs: ${Date.now() - labelsStart}ms (${labels.length} boxes)`)
+    }
+
+    const renderStart = Date.now()
+    const labelCount = await page.evaluate(({ entries, root, interactiveOnly: intOnly }) => {
+      const a11y = (globalThis as {
+        __a11y?: {
+          renderA11yLabels?: (labels: typeof entries) => number
+          computeA11ySnapshot?: (options: { root: unknown; interactiveOnly: boolean; renderLabels: boolean }) => { labelCount: number }
+        }
+      }).__a11y
+      if (a11y?.renderA11yLabels) {
+        return a11y.renderA11yLabels(entries)
+      }
+      if (a11y?.computeA11ySnapshot) {
+        const rootElement = root || document.body
+        return a11y.computeA11ySnapshot({ root: rootElement, interactiveOnly: intOnly, renderLabels: true }).labelCount
+      }
+      throw new Error('a11y client not loaded')
+    }, { entries: shortLabels, root: rootHandle, interactiveOnly })
+
+    if (log) {
+      log(`renderA11yLabels: ${Date.now() - renderStart}ms (${labelCount} labels)`)
+    }
+
+    return { snapshot, labelCount }
+  } finally {
+    await cdp.detach()
   }
 }
 
@@ -925,15 +1058,16 @@ export async function hideAriaRefLabels({ page }: { page: Page }): Promise<void>
  * await page.locator('[data-testid="submit-btn"]').click()
  * ```
  */
-export async function screenshotWithAccessibilityLabels({ page, locator, interactiveOnly = true, collector, logger }: {
+export async function screenshotWithAccessibilityLabels({ page, locator, interactiveOnly = true, wsUrl, collector, logger }: {
   page: Page
   locator?: Locator
   interactiveOnly?: boolean
+  wsUrl?: string
   collector: ScreenshotResult[]
   logger?: { info?: (...args: unknown[]) => void; error?: (...args: unknown[]) => void }
 }): Promise<void> {
   const showLabelsStart = Date.now()
-  const { snapshot, labelCount } = await showAriaRefLabels({ page, locator, interactiveOnly, logger })
+  const { snapshot, labelCount } = await showAriaRefLabels({ page, locator, interactiveOnly, wsUrl, logger })
   const log = logger?.info ?? logger?.error
   if (log) {
     log(`showAriaRefLabels: ${Date.now() - showLabelsStart}ms`)

playwriter/src/executor.ts

@@ -23,6 +23,7 @@ import { getReactSource, type ReactSourceLocation } from './react-source.js'
 import { ScopedFS } from './scoped-fs.js'
 import {
   screenshotWithAccessibilityLabels,
+  buildShortRefMap,
   getAriaSnapshot,
   type ScreenshotResult,
   type SnapshotFormat,
@@ -152,6 +153,7 @@ export class PlaywrightExecutor {
   private userState: Record<string, any> = {}
   private browserLogs: Map<string, string[]> = new Map()
   private lastSnapshots: WeakMap<Page, string> = new WeakMap()
+  private lastRefToLocator: WeakMap<Page, Map<string, string>> = new WeakMap()
   private cdpSessionCache: WeakMap<Page, CDPSession> = new WeakMap()
   private scopedFs: ScopedFS
   private sandboxedRequire: NodeRequire
@@ -451,7 +453,7 @@ export class PlaywrightExecutor {
         const { page: targetPage, locator, search, showDiffSinceLastCall = false, interactiveOnly = true } = options
 
         // Use new in-page implementation via getAriaSnapshot
-        const { snapshot: rawSnapshot } = await getAriaSnapshot({
+        const { snapshot: rawSnapshot, refs, getSelectorForRef } = await getAriaSnapshot({
           page: targetPage,
           locator,
           wsUrl: getCdpUrl(this.cdpConfig),
@@ -459,6 +461,17 @@ export class PlaywrightExecutor {
         })
         const snapshotStr = rawSnapshot.toWellFormed?.() ?? rawSnapshot
 
+        const refToLocator = new Map<string, string>()
+        const shortRefMap = buildShortRefMap({ refs })
+        for (const entry of refs) {
+          const locatorStr = getSelectorForRef(entry.ref)
+          if (locatorStr) {
+            const shortRef = shortRefMap.get(entry.ref) ?? entry.ref
+            refToLocator.set(shortRef, locatorStr)
+          }
+        }
+        this.lastRefToLocator.set(targetPage, refToLocator)
+
         if (showDiffSinceLastCall) {
           const previousSnapshot = this.lastSnapshots.get(targetPage)
           if (!previousSnapshot) {
@@ -477,7 +490,7 @@ export class PlaywrightExecutor {
         this.lastSnapshots.set(targetPage, snapshotStr)
 
         if (!search) {
-          return snapshotStr
+          return `${snapshotStr}\n\nuse refToLocator({ ref: 'e3' }) to get locators for ref strings.`
         }
 
         const lines = snapshotStr.split('\n')
@@ -517,6 +530,15 @@ export class PlaywrightExecutor {
         return result.join('\n')
       }
 
+      const refToLocator = (options: { ref: string; page?: Page }): string | null => {
+        const targetPage = options.page || page
+        const map = this.lastRefToLocator.get(targetPage)
+        if (!map) {
+          return null
+        }
+        return map.get(options.ref) ?? null
+      }
+
       const getLocatorStringForElement = async (element: any) => {
         if (!element || typeof element.evaluate !== 'function') {
           throw new Error('getLocatorStringForElement: argument must be a Playwright Locator or ElementHandle')
@@ -622,6 +644,7 @@ export class PlaywrightExecutor {
       const screenshotWithAccessibilityLabelsFn = async (options: { page: Page; interactiveOnly?: boolean }) => {
         return screenshotWithAccessibilityLabels({
           ...options,
+          wsUrl: getCdpUrl(this.cdpConfig),
           collector: screenshotCollector,
           logger: {
             info: (...args) => {
@@ -658,6 +681,7 @@ export class PlaywrightExecutor {
         state: this.userState,
         console: customConsole,
         accessibilitySnapshot,
+        refToLocator,
         getCleanHTML,
         getPageMarkdown,
         getLocatorStringForElement,

playwriter/src/htmlrewrite.test.ts

@@ -1,9 +1,15 @@
 import { expect, test } from 'vitest'
-import { readFileSync } from 'fs'
+import fs from 'node:fs'
+import { fileURLToPath } from 'node:url'
 import { formatHtmlForPrompt } from './htmlrewrite.js'
 
 test('formatHtmlForPrompt', async () => {
-  const html = readFileSync(new URL('./assets/framer.html', import.meta.url), 'utf-8')
+  const assetUrl = new URL('./assets/framer.html', import.meta.url)
+  const assetPath = fileURLToPath(assetUrl)
+  if (!fs.existsSync(assetPath)) {
+    return
+  }
+  const html = fs.readFileSync(assetPath, 'utf-8')
   const newHtml = await formatHtmlForPrompt({ html })
   expect(newHtml).toMatchInlineSnapshot(
     `
@@ -7951,7 +7957,7 @@ test('keeps all common e2e test ID attributes', async () => {
 })
 
 test('processes x.com.html with size savings', async () => {
-  const html = readFileSync(new URL('./assets/x.com.html', import.meta.url), 'utf-8')
+  const html = fs.readFileSync(new URL('./assets/x.com.html', import.meta.url), 'utf-8')
 
   const result = await formatHtmlForPrompt({ html })
   const resultWithStyles = await formatHtmlForPrompt({ html, keepStyles: true })

playwriter/src/relay-core.test.ts

@@ -94,7 +94,7 @@ describe('Relay Core Tests', () => {
           [
             {
               "text": "Console output:
-          [log] 'Page title:' 'Example Domain'
+          [log] Page title: Example Domain
 
           [return value] { url: 'https://example.com/', title: 'Example Domain' }",
               "type": "text",
@@ -683,12 +683,12 @@ describe('Relay Core Tests', () => {
 
         // Inline snapshot of cleaned HTML
         expect(text).toMatchInlineSnapshot(`
-          "[return value] '<div data-testid="main">\\n' +
-            ' <h1>Hello World</h1>\\n' +
-            ' <button aria-label="Click me">Submit</button>\\n' +
-            ' <a href="/about" title="About page">About</a>\\n' +
-            ' <input type="text" placeholder="Enter name">\\n' +
-            '</div>\\n'"
+          "[return value] <div data-testid="main">
+           <h1>Hello World</h1>
+           <button aria-label="Click me">Submit</button>
+           <a href="/about" title="About page">About</a>
+           <input type="text" placeholder="Enter name">
+          </div>"
         `)
 
         // Should NOT contain script/style tags (they're removed)

playwriter/src/skill.md

@@ -232,7 +232,7 @@ After any action (click, submit, navigate), verify what happened:
 console.log('url:', page.url()); console.log(await accessibilitySnapshot({ page }).then(x => x.split('\n').slice(0, 30).join('\n')));
 ```
 
-For visually complex pages (grids, galleries, dashboards), use `screenshotWithAccessibilityLabels({ page })` instead to understand spatial layout.
+For visually complex pages (grids, galleries, dashboards), use `screenshotWithAccessibilityLabels({ page })` instead to understand spatial layout. Label refs are short `eN` strings (e.g. `e3`).
 
 If nothing changed, try `await waitForPageLoad({ page, timeout: 3000 })` or you may have clicked the wrong element.
 
@@ -265,6 +265,14 @@ Each interactive line ends with a Playwright locator you can pass to `page.locat
 If multiple elements share the same locator, a `>> nth=N` suffix is added (0-based)
 to make it unique.
 
+If a screenshot shows ref labels like `e3`, resolve them using the last snapshot:
+
+```js
+const snapshot = await accessibilitySnapshot({ page })
+const locator = refToLocator({ ref: 'e3' })
+await page.locator(locator!).click()
+```
+
 ```js
 await page.locator('[id="nav-home"]').click()
 await page.locator('[data-testid="docs-link"]').click()

playwriter/src/snapshot-tools.test.ts

@@ -761,6 +761,8 @@ describe('Snapshot & Screenshot Tests', () => {
             }
         }
 
+        const wsUrl = getCdpUrl({ port: TEST_PORT })
+
         for (const { name, url, page } of pages) {
             console.log(`[labels] start ${name}`)
             const cdpPage = browser.contexts()[0].pages().find(p => p.url().includes(new URL(url).hostname))
@@ -770,12 +772,12 @@ describe('Snapshot & Screenshot Tests', () => {
             }
 
             console.log(`[labels] show labels ${name}`)
-            const { snapshot, labelCount } = await withTimeout(
+            const { labelCount } = await withTimeout(
                 `showAriaRefLabels(${name})`,
                 async () => {
-                    return await showAriaRefLabels({ page: cdpPage })
+                    return await showAriaRefLabels({ page: cdpPage, wsUrl })
                 },
-                30000
+                60000
             )
             console.log(`${name}: ${labelCount} labels shown`)
             if (name !== 'google') {
@@ -794,9 +796,6 @@ describe('Snapshot & Screenshot Tests', () => {
             fs.writeFileSync(screenshotPath, screenshot)
             console.log(`Screenshot saved: ${screenshotPath}`)
 
-            const snapshotPath = path.join(assetsDir, `aria-labels-${name}-snapshot.txt`)
-            fs.writeFileSync(snapshotPath, snapshot)
-
             console.log(`[labels] count dom labels ${name}`)
             const labelElements = await withTimeout(
                 `countLabels(${name})`,

playwriter/src/snapshots/page-markdown-output.txt

@@ -1,16 +1,16 @@
 Console output:
-[log] '# Test Article Title\n' +
-  '\n' +
-  'Home About\n' +
-  'Test Article Title\n' +
-  '\n' +
-  'This is the first paragraph of the article content.\n' +
-  '\n' +
-  'This is the second paragraph with more details about the topic.\n' +
-  '\n' +
-  'The article continues with important information here.\n' +
-  '\n' +
-  'Related Posts\n' +
-  'Post 1\n' +
-  'Post 2\n' +
-  'Copyright 2024'
+[log] # Test Article Title
+
+Home About
+Test Article Title
+
+This is the first paragraph of the article content.
+
+This is the second paragraph with more details about the topic.
+
+The article continues with important information here.
+
+Related Posts
+Post 1
+Post 2
+Copyright 2024

pnpm-lock.yaml

@@ -86,6 +86,9 @@ importers:
       '@xmorse/cac':
         specifier: ^6.0.7
         version: 6.0.7
+      async-sema:
+        specifier: ^3.1.1
+        version: 3.1.1
       diff:
         specifier: ^8.0.2
         version: 8.0.2
@@ -7673,6 +7676,14 @@ snapshots:
     optionalDependencies:
       vite: 7.3.0(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)
 
+  '@vitest/mocker@4.0.8(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))':
+    dependencies:
+      '@vitest/spy': 4.0.8
+      estree-walker: 3.0.3
+      magic-string: 0.30.21
+    optionalDependencies:
+      vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)
+
   '@vitest/mocker@4.0.8(vite@7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))':
     dependencies:
       '@vitest/spy': 4.0.8
@@ -7724,7 +7735,7 @@ snapshots:
       sirv: 3.0.2
       tinyglobby: 0.2.15
       tinyrainbow: 3.0.3
-      vitest: 4.0.8(@types/node@25.2.0)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6)
+      vitest: 4.0.8(@types/node@24.10.1)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6)
 
   '@vitest/utils@4.0.16':
     dependencies:
@@ -10577,7 +10588,7 @@ snapshots:
   vitest@4.0.8(@types/node@24.10.1)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6):
     dependencies:
       '@vitest/expect': 4.0.8
-      '@vitest/mocker': 4.0.8(vite@7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))
+      '@vitest/mocker': 4.0.8(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))
       '@vitest/pretty-format': 4.0.8
       '@vitest/runner': 4.0.8
       '@vitest/snapshot': 4.0.8