slopping around
This commit is contained in:
Tommy D. Rossi
@@ -1,352 +0,0 @@
|
||||
# Plan: Visual Aria Ref Labels (Vimium-style)
|
||||
|
||||
## Goal
|
||||
|
||||
Add functions to overlay visual ref labels on interactive elements, allowing agents to:
|
||||
1. Take a screenshot with visible ref labels (e.g., `[e1]`, `[e2]`, `[e3]`)
|
||||
2. See which elements are interactive and their ref IDs
|
||||
3. Use `page.locator('aria-ref=e5')` to interact with labeled elements
|
||||
|
||||
## Reference Implementation
|
||||
|
||||
Based on `/Users/morse/Documents/GitHub/unframer-private/autofill-extension/src/content/`:
|
||||
|
||||
### gui.js - Container setup
|
||||
```js
|
||||
const guiRoot = document.createElement('div')
|
||||
guiRoot.style = `
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
z-index: 2147483647; // Max z-index to be on top of everything
|
||||
pointer-events: none; // Don't block clicks on underlying elements
|
||||
`
|
||||
document.documentElement.appendChild(guiRoot)
|
||||
```
|
||||
|
||||
### HintRenderer.tsx - Label positioning
|
||||
```tsx
|
||||
// Position label at element's top-left corner, accounting for scroll
|
||||
left: `${window.scrollX + rect.left - 16}px`
|
||||
top: `${window.scrollY + rect.top - 16}px` // 16px above element
|
||||
```
|
||||
|
||||
### Vimium-style CSS
|
||||
```css
|
||||
font-family: Helvetica, Arial, sans-serif;
|
||||
font-weight: bold;
|
||||
font-size: 12px;
|
||||
padding: 0px 2px;
|
||||
background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#FFF785), color-stop(100%,#FFC542));
|
||||
border: 1px solid #E3BE23;
|
||||
border-radius: 4px;
|
||||
color: black;
|
||||
text-shadow: rgba(255, 255, 255, 0.6) 0px 1px 0px;
|
||||
```
|
||||
|
||||
## Existing Code Context
|
||||
|
||||
### `getAriaSnapshot()` in `playwriter/src/aria-snapshot.ts`
|
||||
|
||||
Already implemented - discovers refs and provides lookup:
|
||||
|
||||
```ts
|
||||
export async function getAriaSnapshot({ page }: { page: Page }): Promise<AriaSnapshotResult> {
|
||||
// 1. Calls page._snapshotForAI() to populate Playwright's internal ref tracking
|
||||
// 2. Probes aria-ref=e1, e2, e3... to discover all refs
|
||||
// 3. Caches ElementHandles in refHandles array
|
||||
// 4. Returns { snapshot, refToElement, refHandles, getRefsForLocators, ... }
|
||||
}
|
||||
```
|
||||
|
||||
The `refHandles` array contains `{ ref: string, handle: ElementHandle }` for each interactive element. We'll use these handles to get bounding boxes and create labels.
|
||||
|
||||
### Which elements get refs?
|
||||
|
||||
Playwright's `_snapshotForAI()` uses `mode: 'ai'` which sets `refs: 'interactable'`:
|
||||
|
||||
```ts
|
||||
// From playwright/packages/injected/src/ariaSnapshot.ts
|
||||
function computeAriaRef(ariaNode, options) {
|
||||
if (options.refs === 'interactable' && (!ariaNode.box.visible || !ariaNode.receivesPointerEvents))
|
||||
return; // Skip non-interactive elements
|
||||
// ... assign ref like 'e1', 'e2', etc.
|
||||
}
|
||||
```
|
||||
|
||||
**Included:** buttons, links, inputs, textareas, selects, checkboxes, radios, sliders, any clickable element
|
||||
**Excluded:** hidden elements, static text, decorative images, elements behind overlays
|
||||
|
||||
## API Design
|
||||
|
||||
```ts
|
||||
// Show Vimium-style labels on all interactive elements
|
||||
export async function showAriaRefLabels({ page }: { page: Page }): Promise<{
|
||||
snapshot: string // The accessibility snapshot text
|
||||
labelCount: number // Number of labels shown
|
||||
}>
|
||||
|
||||
// Remove all labels from the page
|
||||
export async function hideAriaRefLabels({ page }: { page: Page }): Promise<void>
|
||||
```
|
||||
|
||||
## Implementation
|
||||
|
||||
Add to `playwriter/src/aria-snapshot.ts`:
|
||||
|
||||
```ts
|
||||
const LABELS_CONTAINER_ID = '__playwriter_labels__'
|
||||
|
||||
// Use css`` tagged template for IDE syntax highlighting (identity function)
|
||||
const css = String.raw
|
||||
|
||||
const LABEL_STYLES = css`
|
||||
.__pw_label__ {
|
||||
position: absolute;
|
||||
font: bold 11px Helvetica, Arial, sans-serif;
|
||||
padding: 1px 4px;
|
||||
background: linear-gradient(to bottom, #FFF785 0%, #FFC542 100%);
|
||||
border: 1px solid #E3BE23;
|
||||
border-radius: 3px;
|
||||
color: black;
|
||||
text-shadow: 0 1px 0 rgba(255, 255, 255, 0.6);
|
||||
white-space: nowrap;
|
||||
}
|
||||
`
|
||||
|
||||
const CONTAINER_STYLES = css`
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 0;
|
||||
z-index: 2147483647;
|
||||
pointer-events: none;
|
||||
`
|
||||
|
||||
export async function showAriaRefLabels({ page }: { page: Page }) {
|
||||
// Get snapshot and cached element handles
|
||||
const { snapshot, refHandles } = await getAriaSnapshot({ page })
|
||||
|
||||
// Single evaluate call: create container, styles, and all labels
|
||||
// Pass ElementHandles which get unwrapped to DOM elements in browser context
|
||||
const labelCount = await page.evaluate(
|
||||
({ refs, containerId, containerStyles, labelStyles }) => {
|
||||
// Remove existing labels if present (idempotent)
|
||||
document.getElementById(containerId)?.remove()
|
||||
|
||||
// Create container - absolute positioned, max z-index, no pointer events
|
||||
const container = document.createElement('div')
|
||||
container.id = containerId
|
||||
container.style.cssText = containerStyles
|
||||
|
||||
// Inject Vimium-style CSS
|
||||
const style = document.createElement('style')
|
||||
style.textContent = labelStyles
|
||||
container.appendChild(style)
|
||||
|
||||
// Create label for each interactive element
|
||||
let count = 0
|
||||
for (const { ref, element } of refs) {
|
||||
const rect = element.getBoundingClientRect()
|
||||
|
||||
// Skip elements with no size (hidden)
|
||||
if (rect.width === 0 || rect.height === 0) continue
|
||||
|
||||
const label = document.createElement('div')
|
||||
label.className = '__pw_label__'
|
||||
label.textContent = ref
|
||||
|
||||
// Position above element, accounting for scroll
|
||||
// Use scrollX/scrollY so labels scroll with the page
|
||||
label.style.left = `${scrollX + rect.left}px`
|
||||
label.style.top = `${scrollY + Math.max(0, rect.top - 16)}px`
|
||||
|
||||
container.appendChild(label)
|
||||
count++
|
||||
}
|
||||
|
||||
document.documentElement.appendChild(container)
|
||||
return count
|
||||
},
|
||||
{
|
||||
refs: refHandles.map(({ ref, handle }) => ({ ref, element: handle })),
|
||||
containerId: LABELS_CONTAINER_ID,
|
||||
containerStyles: CONTAINER_STYLES,
|
||||
labelStyles: LABEL_STYLES,
|
||||
}
|
||||
)
|
||||
|
||||
return { snapshot, labelCount }
|
||||
}
|
||||
|
||||
export async function hideAriaRefLabels({ page }: { page: Page }) {
|
||||
await page.evaluate(
|
||||
(id) => document.getElementById(id)?.remove(),
|
||||
LABELS_CONTAINER_ID
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
## Scroll Behavior
|
||||
|
||||
Labels use `position: absolute` with `scrollX/scrollY` offsets:
|
||||
- Labels stay positioned on their elements when user scrolls
|
||||
- Labels move with the document (not fixed to viewport)
|
||||
- This matches Vimium's behavior
|
||||
|
||||
## Export from index.ts
|
||||
|
||||
```ts
|
||||
export { getAriaSnapshot, showAriaRefLabels, hideAriaRefLabels } from './aria-snapshot.js'
|
||||
```
|
||||
|
||||
## Test Implementation
|
||||
|
||||
Add to `playwriter/src/mcp.test.ts`:
|
||||
|
||||
```ts
|
||||
it('should show aria ref labels and capture in screenshot', async () => {
|
||||
const browserContext = getBrowserContext()
|
||||
const serviceWorker = await getExtensionServiceWorker(browserContext)
|
||||
|
||||
// Create page with interactive elements
|
||||
const page = await browserContext.newPage()
|
||||
await page.setContent(`
|
||||
<html>
|
||||
<body style="padding: 50px; font-family: sans-serif;">
|
||||
<h1>Test Page</h1>
|
||||
<button id="submit-btn">Submit Form</button>
|
||||
<a href="/about" style="margin-left: 20px;">About Us</a>
|
||||
<input type="text" placeholder="Enter your name" style="margin-left: 20px;" />
|
||||
<select style="margin-left: 20px;">
|
||||
<option>Option 1</option>
|
||||
<option>Option 2</option>
|
||||
</select>
|
||||
</body>
|
||||
</html>
|
||||
`)
|
||||
await page.bringToFront()
|
||||
|
||||
// Enable extension for this tab
|
||||
await serviceWorker.evaluate(async () => {
|
||||
await globalThis.toggleExtensionForActiveTab()
|
||||
})
|
||||
await new Promise(r => setTimeout(r, 400))
|
||||
|
||||
// Connect via CDP
|
||||
const browser = await chromium.connectOverCDP(getCdpUrl({ port: TEST_PORT }))
|
||||
const cdpPage = browser.contexts()[0].pages().find(p => p.url().startsWith('about:'))
|
||||
expect(cdpPage).toBeDefined()
|
||||
|
||||
// Import and show labels
|
||||
const { showAriaRefLabels, hideAriaRefLabels } = await import('./aria-snapshot.js')
|
||||
const { snapshot, labelCount } = await showAriaRefLabels({ page: cdpPage! })
|
||||
|
||||
console.log('Snapshot:', snapshot.slice(0, 200))
|
||||
console.log('Label count:', labelCount)
|
||||
|
||||
expect(labelCount).toBeGreaterThan(0)
|
||||
expect(snapshot).toContain('button')
|
||||
expect(snapshot).toContain('link')
|
||||
|
||||
// Verify labels are in DOM
|
||||
const labelElements = await cdpPage!.evaluate(() =>
|
||||
document.querySelectorAll('.__pw_label__').length
|
||||
)
|
||||
expect(labelElements).toBe(labelCount)
|
||||
|
||||
// Verify label text matches refs
|
||||
const labelTexts = await cdpPage!.evaluate(() =>
|
||||
Array.from(document.querySelectorAll('.__pw_label__')).map(el => el.textContent)
|
||||
)
|
||||
expect(labelTexts.every(t => /^e\d+$/.test(t!))).toBe(true)
|
||||
console.log('Label texts:', labelTexts)
|
||||
|
||||
// Take screenshot with labels visible
|
||||
const screenshot = await cdpPage!.screenshot({ type: 'png' })
|
||||
expect(screenshot.length).toBeGreaterThan(5000) // Not empty
|
||||
|
||||
// Save screenshot for visual verification
|
||||
const fs = await import('node:fs')
|
||||
const path = await import('node:path')
|
||||
const snapshotsDir = path.join(path.dirname(new URL(import.meta.url).pathname), 'snapshots')
|
||||
if (!fs.existsSync(snapshotsDir)) fs.mkdirSync(snapshotsDir, { recursive: true })
|
||||
fs.writeFileSync(path.join(snapshotsDir, 'aria-labels-screenshot.png'), screenshot)
|
||||
console.log('Screenshot saved to snapshots/aria-labels-screenshot.png')
|
||||
|
||||
// Test that labels can be used to find elements
|
||||
const buttonLabel = labelTexts.find(t => t === 'e2') // Typically button is e2
|
||||
if (buttonLabel) {
|
||||
const buttonViaRef = cdpPage!.locator(`aria-ref=${buttonLabel}`)
|
||||
const buttonText = await buttonViaRef.textContent()
|
||||
console.log(`Button via ${buttonLabel}:`, buttonText)
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
await hideAriaRefLabels({ page: cdpPage! })
|
||||
|
||||
// Verify labels removed
|
||||
const labelsAfterHide = await cdpPage!.evaluate(() =>
|
||||
document.getElementById('__playwriter_labels__')
|
||||
)
|
||||
expect(labelsAfterHide).toBeNull()
|
||||
|
||||
await browser.close()
|
||||
await page.close()
|
||||
}, 60000)
|
||||
```
|
||||
|
||||
## Visual Result
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ │
|
||||
│ Test Page │
|
||||
│ │
|
||||
│ [e2] [e3] [e4] [e5] │
|
||||
│ ┌──────────────┐ ┌──────────┐ ┌───────────────┐ ┌────────┐ │
|
||||
│ │ Submit Form │ │ About Us │ │ Enter name... │ │Option 1│ │
|
||||
│ └──────────────┘ └──────────┘ └───────────────┘ └────────┘ │
|
||||
│ button link textbox combobox │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Yellow Vimium-style badges positioned 16px above each interactive element.
|
||||
Agent sees screenshot → identifies `[e3]` is the "About Us" link → uses `page.locator('aria-ref=e3').click()`
|
||||
|
||||
## Files to Modify
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `playwriter/src/aria-snapshot.ts` | Add `showAriaRefLabels()`, `hideAriaRefLabels()`, `LABELS_CONTAINER_ID` constant |
|
||||
| `playwriter/src/index.ts` | Add exports: `showAriaRefLabels`, `hideAriaRefLabels` |
|
||||
| `playwriter/src/mcp.test.ts` | Add screenshot test |
|
||||
| `playwriter/src/snapshots/` | Directory for test screenshots (gitignored) |
|
||||
|
||||
## Usage Example (Agent Workflow)
|
||||
|
||||
```ts
|
||||
// 1. Show labels on page
|
||||
const { snapshot, labelCount } = await showAriaRefLabels({ page })
|
||||
console.log(`Found ${labelCount} interactive elements`)
|
||||
|
||||
// 2. Take screenshot - labels are visible
|
||||
const screenshot = await page.screenshot()
|
||||
// Agent analyzes screenshot, sees [e5] on "Submit" button
|
||||
|
||||
// 3. Interact using ref
|
||||
await page.locator('aria-ref=e5').click()
|
||||
|
||||
// 4. Hide labels when done
|
||||
await hideAriaRefLabels({ page })
|
||||
```
|
||||
|
||||
## Edge Cases to Handle
|
||||
|
||||
1. **Elements near top edge** - Label would go off-screen, position below element instead
|
||||
2. **Overlapping labels** - Could add offset, but keep simple for now
|
||||
3. **Very long pages** - Labels use absolute positioning, works with scroll
|
||||
4. **Dynamic content** - Call `showAriaRefLabels` again to refresh after DOM changes
|
||||
5. **Iframes** - Current implementation only handles main frame (could extend later)
|
||||
@@ -0,0 +1,330 @@
|
||||
# Aria Snapshot Implementation Analysis
|
||||
|
||||
## Overview
|
||||
|
||||
The aria snapshot feature in playwriter extracts an accessibility tree from the browser using Chrome DevTools Protocol (CDP). This document details how it works, its CDP interactions, and frame/iframe handling capabilities.
|
||||
|
||||
## Main Implementation File
|
||||
|
||||
**File:** `playwriter/src/aria-snapshot.ts` (1358 lines)
|
||||
|
||||
## How It Works
|
||||
|
||||
### High-Level Pipeline
|
||||
|
||||
```
|
||||
1. Get CDP Session for Page
|
||||
2. Enable DOM and Accessibility domains
|
||||
3. Fetch DOM tree (with pierce: true for iframes)
|
||||
4. Fetch Accessibility tree
|
||||
5. Build raw snapshot tree
|
||||
6. Filter tree (interactive-only or full)
|
||||
7. Generate locators and refs
|
||||
8. Return formatted snapshot + utilities
|
||||
```
|
||||
|
||||
### Key Functions
|
||||
|
||||
#### `getAriaSnapshot()`
|
||||
**Location:** Line 749-1033 in `aria-snapshot.ts`
|
||||
|
||||
Main entry point that returns `AriaSnapshotResult` containing:
|
||||
- `snapshot`: String representation of the accessibility tree
|
||||
- `tree`: Structured tree with nodes
|
||||
- `refs`: Array of references to interactive elements
|
||||
- `getSelectorForRef()`: Get CSS selector for a ref
|
||||
- `getRefsForLocators()`: Get refs for Playwright locators
|
||||
|
||||
**Signature:**
|
||||
```typescript
|
||||
export async function getAriaSnapshot({
|
||||
page,
|
||||
locator,
|
||||
refFilter,
|
||||
wsUrl,
|
||||
interactiveOnly = false,
|
||||
cdp
|
||||
}: {
|
||||
page: Page
|
||||
locator?: Locator
|
||||
refFilter?: (info: { role: string; name: string }) => boolean
|
||||
wsUrl?: string
|
||||
interactiveOnly?: boolean
|
||||
cdp?: ICDPSession
|
||||
}): Promise<AriaSnapshotResult>
|
||||
```
|
||||
|
||||
## CDP Commands Used
|
||||
|
||||
### 1. DOM Domain
|
||||
|
||||
**Command:** `DOM.getFlattenedDocument`
|
||||
**Location:** Line 772
|
||||
```typescript
|
||||
const { nodes: domNodes } = await session.send('DOM.getFlattenedDocument', {
|
||||
depth: -1,
|
||||
pierce: true
|
||||
}) as Protocol.DOM.GetFlattenedDocumentResponse
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `depth: -1` - Get entire subtree
|
||||
- `pierce: true` - **Traverses iframes and shadow roots**
|
||||
|
||||
**Purpose:** Get all DOM nodes to map accessibility nodes to their attributes (test IDs, etc.)
|
||||
|
||||
### 2. Accessibility Domain
|
||||
|
||||
**Command:** `Accessibility.getFullAXTree`
|
||||
**Location:** Line 791
|
||||
```typescript
|
||||
const { nodes: axNodes } = await session.send('Accessibility.getFullAXTree')
|
||||
as Protocol.Accessibility.GetFullAXTreeResponse
|
||||
```
|
||||
|
||||
**Parameters:** None specified (uses defaults)
|
||||
- Default: Returns AX tree for root frame
|
||||
- **Has optional `frameId` parameter** (not currently used)
|
||||
|
||||
**Purpose:** Get the complete accessibility tree with roles, names, and relationships
|
||||
|
||||
## Frame/Iframe Support
|
||||
|
||||
### Current Implementation
|
||||
|
||||
**DOM Level:** ✅ **FULL SUPPORT**
|
||||
- `DOM.getFlattenedDocument` with `pierce: true` traverses all iframes and shadow roots
|
||||
- All DOM nodes from all frames are included in the flattened document
|
||||
|
||||
**Accessibility Level:** ⚠️ **LIMITED**
|
||||
- `Accessibility.getFullAXTree` is called **without `frameId` parameter**
|
||||
- According to CDP spec, when `frameId` is omitted, **only the root frame is used**
|
||||
- Cross-origin iframes may have additional restrictions
|
||||
|
||||
### CDP Spec Details
|
||||
|
||||
From `Accessibility.pdl`:
|
||||
```
|
||||
experimental command getFullAXTree
|
||||
parameters
|
||||
# The maximum depth at which descendants of the root node should be retrieved.
|
||||
# If omitted, the full tree is returned.
|
||||
optional integer depth
|
||||
# The frame for whose document the AX tree should be retrieved.
|
||||
# If omitted, the root frame is used.
|
||||
optional Page.FrameId frameId
|
||||
returns
|
||||
array of AXNode nodes
|
||||
```
|
||||
|
||||
### Implications
|
||||
|
||||
1. **Same-origin iframes:** May be included in the root frame's AX tree
|
||||
2. **Cross-origin iframes:** Likely **NOT included** due to security restrictions
|
||||
3. **Shadow DOM:** Likely included (as `pierce: true` affects DOM and shadow roots are part of the same document)
|
||||
|
||||
### Evidence from Code
|
||||
|
||||
**Scope handling (Line 760-789):**
|
||||
- Uses `data-pw-scope` attribute to scope snapshots to a locator
|
||||
- Builds `allowedBackendIds` set from DOM tree traversal
|
||||
- Filters AX nodes based on `backendDOMNodeId` membership in this set
|
||||
|
||||
**Node mapping:**
|
||||
- Each AX node has `backendDOMNodeId` property linking to DOM node
|
||||
- DOM nodes fetched with `pierce: true` include iframe contents
|
||||
- But AX tree without `frameId` may not cover all frames
|
||||
|
||||
## Key Data Structures
|
||||
|
||||
### AriaSnapshotNode
|
||||
```typescript
|
||||
type AriaSnapshotNode = {
|
||||
role: string
|
||||
name: string
|
||||
locator?: string
|
||||
ref?: string
|
||||
shortRef?: string
|
||||
backendNodeId?: Protocol.DOM.BackendNodeId
|
||||
children: AriaSnapshotNode[]
|
||||
}
|
||||
```
|
||||
|
||||
### AriaRef
|
||||
```typescript
|
||||
interface AriaRef {
|
||||
role: string
|
||||
name: string
|
||||
ref: string // Full ref (testid or e1, e2, e3...)
|
||||
shortRef: string // Short ref (e1, e2, e3...)
|
||||
backendNodeId?: Protocol.DOM.BackendNodeId
|
||||
}
|
||||
```
|
||||
|
||||
## Locator Generation
|
||||
|
||||
**File:** Lines 209-242
|
||||
|
||||
Generates Playwright-compatible locators:
|
||||
|
||||
1. **Stable refs** (preferred):
|
||||
- `[data-testid="submit-btn"]`
|
||||
- `[id="login"]`
|
||||
- Test ID attributes: `data-testid`, `data-test-id`, `data-test`, `data-cy`, `data-pw`, `data-qa`, `data-e2e`, `data-automation-id`
|
||||
|
||||
2. **Role-based fallback**:
|
||||
- `role=button[name="Submit"]`
|
||||
- `role=link[name="Learn More"]`
|
||||
|
||||
3. **Nth handling**:
|
||||
- Duplicates get `>> nth=0`, `>> nth=1`, etc.
|
||||
|
||||
## Interactive Roles
|
||||
|
||||
**Location:** Lines 123-143
|
||||
|
||||
Only these roles get refs in interactive mode:
|
||||
```typescript
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
'button', 'link', 'textbox', 'combobox', 'searchbox',
|
||||
'checkbox', 'radio', 'slider', 'spinbutton', 'switch',
|
||||
'menuitem', 'menuitemcheckbox', 'menuitemradio',
|
||||
'option', 'tab', 'treeitem', 'img', 'video', 'audio',
|
||||
])
|
||||
```
|
||||
|
||||
## Executor Integration
|
||||
|
||||
**File:** `playwriter/src/executor.ts` (Lines 533-619)
|
||||
|
||||
The `accessibilitySnapshot()` function exposed to agents:
|
||||
|
||||
```typescript
|
||||
const accessibilitySnapshot = async (options: {
|
||||
page: Page
|
||||
locator?: Locator
|
||||
search?: string | RegExp
|
||||
showDiffSinceLastCall?: boolean
|
||||
format?: SnapshotFormat
|
||||
interactiveOnly?: boolean
|
||||
}) => {
|
||||
const { snapshot, refs, getSelectorForRef } = await getAriaSnapshot({
|
||||
page: targetPage,
|
||||
locator,
|
||||
wsUrl: getCdpUrl(this.cdpConfig),
|
||||
interactiveOnly,
|
||||
})
|
||||
|
||||
// Store refs for refToLocator() function
|
||||
// Handle search filtering
|
||||
// Handle diff mode
|
||||
// Return formatted snapshot
|
||||
}
|
||||
```
|
||||
|
||||
## Screenshot with Labels
|
||||
|
||||
**Function:** `screenshotWithAccessibilityLabels()`
|
||||
**Location:** Lines 1258-1354
|
||||
|
||||
Takes a screenshot with Vimium-style labels overlaid on interactive elements:
|
||||
|
||||
1. Calls `showAriaRefLabels()` to render labels
|
||||
2. Takes screenshot with viewport clipping
|
||||
3. Resizes to max 1568px (Claude token optimization)
|
||||
4. Hides labels
|
||||
5. Returns screenshot with snapshot
|
||||
|
||||
## Tests
|
||||
|
||||
**File:** `playwriter/src/aria-snapshot.test.ts`
|
||||
|
||||
Tests against real websites:
|
||||
- Hacker News
|
||||
- GitHub
|
||||
|
||||
**Coverage:**
|
||||
- Snapshot generation
|
||||
- Interactive-only mode
|
||||
- Locator format validation
|
||||
- No wrapper roles in output
|
||||
|
||||
**Note:** No explicit iframe tests found
|
||||
|
||||
## Limitations & Findings
|
||||
|
||||
### Frame Handling Limitations
|
||||
|
||||
1. **Root frame only:** `Accessibility.getFullAXTree()` without `frameId` only returns root frame AX tree
|
||||
2. **No iframe iteration:** Code doesn't enumerate frames and call `getFullAXTree()` per frame
|
||||
3. **Cross-origin restrictions:** Even if frames were enumerated, cross-origin iframes would be blocked
|
||||
|
||||
### Potential Issues
|
||||
|
||||
1. **Elements in iframes may not appear** in accessibility snapshot
|
||||
2. **DOM includes iframe content** (via `pierce: true`) but AX tree may not
|
||||
3. **Inconsistency:** `backendNodeId` from DOM may not have matching AX node for iframe elements
|
||||
|
||||
### Evidence of Limitation
|
||||
|
||||
- No `frameId` parameter passed to `Accessibility.getFullAXTree()`
|
||||
- No frame enumeration logic
|
||||
- No tests for iframe scenarios
|
||||
- CDP spec clearly states "If omitted, the root frame is used"
|
||||
|
||||
## Possible Enhancements
|
||||
|
||||
To support iframes properly:
|
||||
|
||||
1. **Enumerate frames:**
|
||||
```typescript
|
||||
const { frameTree } = await session.send('Page.getFrameTree')
|
||||
// Recursively collect all frame IDs
|
||||
```
|
||||
|
||||
2. **Get AX tree per frame:**
|
||||
```typescript
|
||||
for (const frameId of frameIds) {
|
||||
const { nodes } = await session.send('Accessibility.getFullAXTree', { frameId })
|
||||
// Merge nodes
|
||||
}
|
||||
```
|
||||
|
||||
3. **Handle cross-origin frames:**
|
||||
- Detect cross-origin frames
|
||||
- Skip or show placeholder for restricted frames
|
||||
- Document limitation in output
|
||||
|
||||
4. **Test with iframes:**
|
||||
- Add test cases with same-origin iframes
|
||||
- Add test cases with cross-origin iframes
|
||||
- Validate behavior
|
||||
|
||||
## Summary
|
||||
|
||||
**File Paths:**
|
||||
- Main implementation: `playwriter/src/aria-snapshot.ts`
|
||||
- Executor integration: `playwriter/src/executor.ts` (lines 533-619)
|
||||
- CDP session: `playwriter/src/cdp-session.ts`
|
||||
- Tests: `playwriter/src/aria-snapshot.test.ts`
|
||||
|
||||
**CDP Commands:**
|
||||
- `DOM.enable` - Enable DOM domain
|
||||
- `DOM.getFlattenedDocument({ depth: -1, pierce: true })` - Get all DOM nodes including iframes
|
||||
- `Accessibility.enable` - Enable accessibility domain
|
||||
- `Accessibility.getFullAXTree()` - Get AX tree (root frame only by default)
|
||||
- `DOM.getBoxModel({ backendNodeId })` - Get element positions for labels
|
||||
|
||||
**Frame Handling:**
|
||||
- ✅ DOM tree includes iframe content (`pierce: true`)
|
||||
- ⚠️ Accessibility tree likely **only root frame** (no `frameId` parameter)
|
||||
- ❌ No frame enumeration or per-frame AX tree fetching
|
||||
- ❌ No explicit iframe tests
|
||||
|
||||
**Key Insight:**
|
||||
The current implementation may miss interactive elements inside iframes because:
|
||||
1. `Accessibility.getFullAXTree()` without `frameId` only returns root frame
|
||||
2. No iteration over child frames to collect their AX trees
|
||||
3. Cross-origin iframes would be blocked anyway for security
|
||||
|
||||
For multi-browser support with Firefox/WebKit, this limitation would need to be addressed as those browsers may handle iframes differently in their accessibility trees.
|
||||
@@ -1,115 +0,0 @@
|
||||
# Plan: Add Deterministic Page Waiting to mcp.ts
|
||||
|
||||
## Current Problem
|
||||
|
||||
In `mcp.ts`, both `connect()` and `reconnect()` functions:
|
||||
|
||||
```typescript
|
||||
const browser = await chromium.connectOverCDP(cdpEndpoint)
|
||||
const context = contexts[0]
|
||||
const pages = context.pages() // May be empty!
|
||||
if (pages.length === 0) {
|
||||
throw new Error(NO_TABS_ERROR)
|
||||
}
|
||||
```
|
||||
|
||||
This fails when:
|
||||
1. Extension has tabs connected, but...
|
||||
2. Playwright hasn't finished processing `Target.attachedToTarget` + `Runtime.executionContextCreated` events
|
||||
|
||||
## Solution
|
||||
|
||||
Use `context.waitForEvent('page')` with a timeout, similar to our test fix.
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Step 1: Create a helper function
|
||||
|
||||
```typescript
|
||||
async function waitForPages(context: BrowserContext, options?: {
|
||||
timeout?: number
|
||||
minPages?: number
|
||||
}): Promise<Page[]> {
|
||||
const { timeout = 5000, minPages = 1 } = options || {}
|
||||
|
||||
// Check if pages already exist
|
||||
let pages = context.pages()
|
||||
if (pages.length >= minPages) {
|
||||
return pages
|
||||
}
|
||||
|
||||
// Wait for page events
|
||||
const startTime = Date.now()
|
||||
while (pages.length < minPages && Date.now() - startTime < timeout) {
|
||||
try {
|
||||
await context.waitForEvent('page', { timeout: timeout - (Date.now() - startTime) })
|
||||
pages = context.pages()
|
||||
} catch {
|
||||
// Timeout - check one more time
|
||||
pages = context.pages()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return pages
|
||||
}
|
||||
```
|
||||
|
||||
### Step 2: Update `connect()` function (line ~305)
|
||||
|
||||
```typescript
|
||||
// Before:
|
||||
const pages = context.pages()
|
||||
if (pages.length === 0) {
|
||||
throw new Error(NO_TABS_ERROR)
|
||||
}
|
||||
|
||||
// After:
|
||||
const pages = await waitForPages(context, { timeout: 5000 })
|
||||
if (pages.length === 0) {
|
||||
throw new Error(NO_TABS_ERROR)
|
||||
}
|
||||
```
|
||||
|
||||
### Step 3: Update `reconnect()` function (line ~448)
|
||||
|
||||
Same change as above.
|
||||
|
||||
### Step 4: Consider the "createInitialTab" flow
|
||||
|
||||
The relay server has a `createInitialTab` message for when no tabs exist. We should:
|
||||
1. First try `waitForPages()` with a short timeout (1-2 seconds)
|
||||
2. If still no pages, request `createInitialTab`
|
||||
3. Then `waitForPages()` again for the new tab
|
||||
|
||||
```typescript
|
||||
let pages = await waitForPages(context, { timeout: 2000 })
|
||||
if (pages.length === 0) {
|
||||
// No tabs - request extension to create one
|
||||
await requestCreateInitialTab()
|
||||
pages = await waitForPages(context, { timeout: 5000 })
|
||||
}
|
||||
if (pages.length === 0) {
|
||||
throw new Error(NO_TABS_ERROR)
|
||||
}
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
1. Run existing tests to ensure no regression
|
||||
2. Add a new test that:
|
||||
- Connects immediately after extension toggle
|
||||
- Verifies pages are available without sleep
|
||||
|
||||
## Files to Modify
|
||||
|
||||
1. `playwriter/src/mcp.ts`:
|
||||
- Add `waitForPages()` helper
|
||||
- Update `connect()` function
|
||||
- Update `reconnect()` function
|
||||
|
||||
## Edge Cases
|
||||
|
||||
1. **All pages are `about:blank`**: May need to filter for real pages
|
||||
2. **Extension not connected**: Existing timeout handling should work
|
||||
3. **Very slow page load**: 5 second timeout should be sufficient for CDP events (actual page content load is not required)
|
||||
@@ -1,102 +0,0 @@
|
||||
# Page Ready Investigation Summary
|
||||
|
||||
## Goal
|
||||
Convert undeterministic `sleep()` calls into deterministic event waits when toggling the extension.
|
||||
|
||||
## Problem
|
||||
After `toggleExtensionForActiveTab()` returns, the page is not immediately available in Playwright's `context.pages()`. Tests currently use arbitrary sleeps (100-400ms) to work around this.
|
||||
|
||||
## Key Events Required
|
||||
|
||||
For a page to appear in `context.pages()`, Playwright needs:
|
||||
|
||||
1. **`Target.attachedToTarget`** - Tells Playwright a new target exists with its URL
|
||||
2. **`Runtime.executionContextCreated`** - Tells Playwright the JS context is ready
|
||||
|
||||
The page appears in `context.pages()` only after BOTH events are processed.
|
||||
|
||||
## Current Flow (Timeline from test)
|
||||
|
||||
```
|
||||
63ms ← EVT Target.attachedToTarget # Extension sends to relay
|
||||
431ms ← EVT Runtime.executionContextCreated # Relay's proactive Runtime.enable
|
||||
437ms → CMD Runtime.enable # Playwright sends its own
|
||||
438ms Toggle completes (pageReady received)
|
||||
467ms ← EVT Runtime.executionContextCreated # Response to Playwright's Runtime.enable
|
||||
728ms Page appears in context.pages()
|
||||
```
|
||||
|
||||
## Root Cause
|
||||
|
||||
The ~300ms delay between toggle completing and page appearing is caused by:
|
||||
|
||||
1. **Extension's `Runtime.enable` handler** (background.ts:256-272):
|
||||
- Always calls `Runtime.disable` then `Runtime.enable`
|
||||
- Has a 50ms `sleep()` between them
|
||||
- This is needed to force Chrome to re-send `executionContextCreated` events for multiple clients
|
||||
|
||||
2. **Double Runtime.enable cycle**:
|
||||
- Relay proactively enables Runtime → events at ~430ms
|
||||
- Playwright receives `Target.attachedToTarget`, sends its own `Runtime.enable`
|
||||
- Extension does disable+enable again → events at ~470ms
|
||||
- Playwright waits for ITS events before adding page to pages()
|
||||
|
||||
3. **Playwright's internal processing**:
|
||||
- Even after receiving events, Playwright takes time to create Page objects
|
||||
- This is async and cannot be controlled from our side
|
||||
|
||||
## What We Tried
|
||||
|
||||
1. **Proactive Runtime.enable in relay** - Enable Runtime before forwarding `Target.attachedToTarget`
|
||||
- Helps get events faster, but Playwright still calls Runtime.enable itself
|
||||
|
||||
2. **Skip disable cycle if recently enabled** - Track recent enables in extension
|
||||
- Breaks because relay's Runtime.enable handler waits for events that won't come
|
||||
|
||||
3. **pageReady handshake** - Extension waits for relay confirmation before returning from attachTab
|
||||
- Toggle now waits for executionContextCreated
|
||||
- But Playwright STILL calls Runtime.enable after, causing another cycle
|
||||
|
||||
## The Core Issue
|
||||
|
||||
**Playwright always calls `Runtime.enable` after receiving `Target.attachedToTarget`**, regardless of whether we pre-enabled it. The extension's disable+enable cycle adds ~200ms, and we cannot skip it without breaking the multi-client case.
|
||||
|
||||
## Possible Solutions
|
||||
|
||||
### Option 1: Accept the delay, use proper waiting
|
||||
Instead of sleep, use `context.waitForEvent('page')` with a predicate:
|
||||
```typescript
|
||||
await serviceWorker.evaluate(() => globalThis.toggleExtensionForActiveTab())
|
||||
const page = await context.waitForEvent('page', {
|
||||
predicate: p => p.url().includes('target-url')
|
||||
})
|
||||
```
|
||||
|
||||
### Option 2: Expose a "page ready" promise from the relay
|
||||
Add an endpoint or event that resolves when the page is fully ready in Playwright:
|
||||
```typescript
|
||||
await serviceWorker.evaluate(() => globalThis.toggleExtensionForActiveTab())
|
||||
await relay.waitForPageReady(sessionId) // Waits for Playwright to process everything
|
||||
```
|
||||
|
||||
### Option 3: Have extension track Runtime state per session
|
||||
Skip the disable+enable if:
|
||||
- This is the SAME Playwright client that just received `Target.attachedToTarget`
|
||||
- The session was JUST created (within last 100ms)
|
||||
|
||||
This requires tracking which client enabled Runtime and when.
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Option 1 is the simplest and most reliable.** The delay is inherent to how Playwright processes CDP events. We cannot eliminate it, but we can wait for it properly using `context.waitForEvent('page')` instead of arbitrary sleeps.
|
||||
|
||||
The test should be:
|
||||
```typescript
|
||||
const pagePromise = context.waitForEvent('page', {
|
||||
predicate: p => p.url().includes('discord.com'),
|
||||
timeout: 5000
|
||||
})
|
||||
await serviceWorker.evaluate(() => globalThis.toggleExtensionForActiveTab())
|
||||
const page = await pagePromise
|
||||
// Page is now guaranteed to be ready
|
||||
```
|
||||
@@ -0,0 +1,332 @@
|
||||
---
|
||||
title: Playwright Accessibility Implementation Findings
|
||||
description: Analysis of how Playwright implements accessibility snapshots and handles iframes
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
This document contains findings from exploring the Playwright source code to understand:
|
||||
1. How Playwright implements accessibility snapshots (ariaSnapshot)
|
||||
2. Whether Playwright supports getting accessibility tree for iframes/child frames
|
||||
3. How Playwright handles frame locators for accessibility
|
||||
4. What CDP commands Playwright uses for accessibility
|
||||
|
||||
## Key Findings
|
||||
|
||||
### 1. Playwright's Accessibility Implementation Strategy
|
||||
|
||||
**Playwright does NOT use CDP Accessibility commands for ariaSnapshot**. Instead, it uses:
|
||||
|
||||
- **Browser-side JavaScript injection** via `InjectedScript` class
|
||||
- The injected script traverses the DOM directly using browser APIs
|
||||
- No CDP `Accessibility.getFullAXTree` or `Accessibility.getPartialAXTree` calls
|
||||
|
||||
#### Code Flow:
|
||||
|
||||
```typescript
|
||||
// Server-side (packages/playwright-core/src/server/frames.ts:1371)
|
||||
async ariaSnapshot(progress: Progress, selector: string): Promise<string> {
|
||||
return await this._retryWithProgressIfNotConnected(
|
||||
progress,
|
||||
selector,
|
||||
true,
|
||||
true,
|
||||
handle => progress.race(handle.ariaSnapshot())
|
||||
);
|
||||
}
|
||||
|
||||
// Element handle (packages/playwright-core/src/server/dom.ts:757)
|
||||
async ariaSnapshot(): Promise<string> {
|
||||
return await this.evaluateInUtility(
|
||||
([injected, element]) => injected.ariaSnapshot(element, { mode: 'expect' }),
|
||||
{}
|
||||
);
|
||||
}
|
||||
|
||||
// Injected script (packages/injected/src/injectedScript.ts:305)
|
||||
ariaSnapshot(node: Node, options: AriaTreeOptions): string {
|
||||
return this.incrementalAriaSnapshot(node, options).full;
|
||||
}
|
||||
|
||||
incrementalAriaSnapshot(node: Node, options: AriaTreeOptions & { track?: string }):
|
||||
{ full: string, incremental?: string, iframeRefs: string[] } {
|
||||
if (node.nodeType !== Node.ELEMENT_NODE)
|
||||
throw this.createStacklessError('Can only capture aria snapshot of Element nodes.');
|
||||
const ariaSnapshot = generateAriaTree(node as Element, options);
|
||||
const full = renderAriaTree(ariaSnapshot, options);
|
||||
// ...
|
||||
return { full, incremental, iframeRefs: ariaSnapshot.iframeRefs };
|
||||
}
|
||||
```
|
||||
|
||||
### 2. How Playwright Handles IFrames in Accessibility Snapshots
|
||||
|
||||
**IFrames are recognized but their content is NOT included** in the accessibility tree.
|
||||
|
||||
#### Evidence:
|
||||
|
||||
From `packages/injected/src/ariaSnapshot.ts:217-232`:
|
||||
|
||||
```typescript
|
||||
function toAriaNode(element: Element, options: InternalOptions): aria.AriaNode | null {
|
||||
const active = element.ownerDocument.activeElement === element;
|
||||
if (element.nodeName === 'IFRAME') {
|
||||
const ariaNode: aria.AriaNode = {
|
||||
role: 'iframe',
|
||||
name: '',
|
||||
children: [], // ⚠️ Empty children - no content from iframe
|
||||
props: {},
|
||||
box: computeBox(element),
|
||||
receivesPointerEvents: true,
|
||||
active
|
||||
};
|
||||
setAriaNodeElement(ariaNode, element);
|
||||
computeAriaRef(ariaNode, options);
|
||||
return ariaNode;
|
||||
}
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
**Key observations:**
|
||||
- IFrame elements are detected and added to the tree with `role: 'iframe'`
|
||||
- The `children` array is ALWAYS empty for iframes
|
||||
- IFrame refs are tracked separately in `snapshot.iframeRefs` array
|
||||
- The content document of the iframe is NEVER traversed
|
||||
- No attempt is made to access `iframe.contentDocument` or `iframe.contentWindow`
|
||||
|
||||
### 3. AriaSnapshot Return Value Structure
|
||||
|
||||
```typescript
|
||||
export type AriaSnapshot = {
|
||||
root: aria.AriaNode;
|
||||
elements: Map<string, Element>; // ref -> Element mapping
|
||||
refs: Map<Element, string>; // Element -> ref mapping
|
||||
iframeRefs: string[]; // List of iframe ref IDs
|
||||
};
|
||||
```
|
||||
|
||||
The `iframeRefs` array contains references to iframe elements but NOT their content.
|
||||
|
||||
### 4. CDP Accessibility Commands (Available but NOT Used)
|
||||
|
||||
The CDP protocol DOES support frame-specific accessibility queries:
|
||||
|
||||
```typescript
|
||||
// From protocol.d.ts
|
||||
export type getFullAXTreeParameters = {
|
||||
depth?: number;
|
||||
frameId?: Page.FrameId; // ⚠️ Supports frame-specific queries!
|
||||
}
|
||||
|
||||
export type getPartialAXTreeParameters = {
|
||||
nodeId?: DOM.NodeId;
|
||||
backendNodeId?: DOM.BackendNodeId;
|
||||
objectId?: Runtime.RemoteObjectId;
|
||||
fetchRelatives?: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
**However, Playwright does NOT use these CDP commands anywhere in the codebase.**
|
||||
|
||||
Search results show:
|
||||
- CDP types defined in `protocol.d.ts`
|
||||
- No actual usage in Chromium implementation files
|
||||
- Firefox uses a custom `Accessibility.getFullAXTree` via Juggler protocol
|
||||
|
||||
### 5. Why Playwright Uses DOM Traversal Instead of CDP
|
||||
|
||||
**Advantages of DOM traversal approach:**
|
||||
1. **Cross-browser compatibility** - Works in Firefox, WebKit, Chromium
|
||||
2. **Full control** - Can customize what gets included/excluded
|
||||
3. **Performance** - No serialization overhead for large trees
|
||||
4. **Flexibility** - Can implement custom filtering (visibility, aria roles, etc.)
|
||||
|
||||
**Disadvantages:**
|
||||
1. **Cannot access iframe content** - Browser security prevents cross-origin access
|
||||
2. **Must execute in each frame separately** - No single command for entire page tree
|
||||
3. **Slower for deep trees** - Must traverse DOM node-by-node
|
||||
|
||||
### 6. Current Iframe Handling Limitations
|
||||
|
||||
Based on code analysis:
|
||||
|
||||
**Playwright CANNOT get accessibility tree for iframe content because:**
|
||||
|
||||
1. **Security restrictions**: JavaScript cannot access `iframe.contentDocument` for cross-origin iframes
|
||||
2. **No CDP fallback**: Playwright doesn't use CDP Accessibility commands that could bypass this
|
||||
3. **By design**: The `generateAriaTree` function explicitly skips iframe children
|
||||
|
||||
**To get iframe content accessibility tree, you would need to:**
|
||||
1. Switch to the iframe's frame context
|
||||
2. Call `ariaSnapshot()` again on that frame
|
||||
3. Manually combine the results
|
||||
|
||||
Example:
|
||||
```typescript
|
||||
// Get main frame snapshot
|
||||
const mainSnapshot = await page.locator('body').ariaSnapshot();
|
||||
|
||||
// Get iframe content
|
||||
const frameElement = await page.frameLocator('iframe');
|
||||
const frame = await frameElement.owner();
|
||||
const frameSnapshot = await frame.contentFrame().locator('body').ariaSnapshot();
|
||||
|
||||
// Results are separate - no automatic merging
|
||||
```
|
||||
|
||||
### 7. Alternative: Using CDP Accessibility Commands
|
||||
|
||||
**IF Playwriter wanted to support iframe content in accessibility snapshots**, it could:
|
||||
|
||||
1. Use `Accessibility.getFullAXTree({ frameId })` for each frame
|
||||
2. Recursively call it for all child frames
|
||||
3. Merge the results into a single tree
|
||||
|
||||
**CDP Command:**
|
||||
```typescript
|
||||
await session.send('Accessibility.getFullAXTree', {
|
||||
frameId: 'frame-id-here',
|
||||
depth: -1 // unlimited depth
|
||||
});
|
||||
```
|
||||
|
||||
This would return the full accessibility tree including iframe content, but:
|
||||
- Only works in Chromium (not Firefox/WebKit)
|
||||
- Returns CDP's AXNode format, not Playwright's AriaNode format
|
||||
- Would need conversion logic
|
||||
|
||||
## Recommendations for Playwriter
|
||||
|
||||
### Option 1: Multi-Frame Snapshot Approach (Current Playwright Pattern)
|
||||
|
||||
Get accessibility tree for each frame separately:
|
||||
|
||||
```typescript
|
||||
// Pseudo-code for MCP implementation
|
||||
async function getAccessibilitySnapshot({ sessionId, includeFrames = false }) {
|
||||
const page = getPage(sessionId);
|
||||
|
||||
// Get main frame snapshot
|
||||
const mainSnapshot = await page.evaluate(() => {
|
||||
return injected.ariaSnapshot(document.body, { mode: 'ai' });
|
||||
});
|
||||
|
||||
if (!includeFrames) {
|
||||
return mainSnapshot;
|
||||
}
|
||||
|
||||
// Get all iframe snapshots
|
||||
const frames = page.frames();
|
||||
const frameSnapshots = await Promise.all(
|
||||
frames.slice(1).map(async (frame) => {
|
||||
return {
|
||||
frameId: frame.name() || frame.url(),
|
||||
snapshot: await frame.evaluate(() => {
|
||||
return injected.ariaSnapshot(document.body, { mode: 'ai' });
|
||||
})
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
return {
|
||||
main: mainSnapshot,
|
||||
frames: frameSnapshots
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
### Option 2: CDP-Based Approach (Chromium Only)
|
||||
|
||||
Use CDP `Accessibility.getFullAXTree` for each frame:
|
||||
|
||||
```typescript
|
||||
async function getFullAccessibilityTree({ sessionId }) {
|
||||
const page = getPage(sessionId);
|
||||
const frames = page.frames();
|
||||
|
||||
const trees = await Promise.all(
|
||||
frames.map(async (frame) => {
|
||||
const session = await frame._client; // Get CDP session
|
||||
const { nodes } = await session.send('Accessibility.getFullAXTree', {
|
||||
frameId: frame._id
|
||||
});
|
||||
return {
|
||||
frameId: frame.url(),
|
||||
nodes
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
// Convert CDP AXNode[] to Playwright AriaNode format
|
||||
return convertCDPtoAria(trees);
|
||||
}
|
||||
```
|
||||
|
||||
**Note:** This would only work for Chromium, not Firefox/WebKit.
|
||||
|
||||
### Option 3: Hybrid Approach (Recommended)
|
||||
|
||||
1. Use Playwright's existing `ariaSnapshot()` for the current frame
|
||||
2. Detect iframes via `snapshot.iframeRefs`
|
||||
3. Recursively get snapshots for each iframe's content frame
|
||||
4. Mark iframe boundaries in the output
|
||||
|
||||
```typescript
|
||||
async function getRecursiveSnapshot({ sessionId, maxDepth = 3 }) {
|
||||
const page = getPage(sessionId);
|
||||
|
||||
async function getFrameSnapshot(frame, depth = 0) {
|
||||
if (depth >= maxDepth) return null;
|
||||
|
||||
// Get snapshot for this frame
|
||||
const result = await frame.evaluate(() => {
|
||||
return injected.incrementalAriaSnapshot(document.body, {
|
||||
mode: 'ai',
|
||||
refPrefix: `f${depth}_`
|
||||
});
|
||||
});
|
||||
|
||||
// Find iframe elements
|
||||
const iframeElements = await frame.$$('iframe');
|
||||
|
||||
// Get snapshots for child frames
|
||||
const childSnapshots = await Promise.all(
|
||||
iframeElements.map(async (iframeEl) => {
|
||||
const childFrame = await iframeEl.contentFrame();
|
||||
if (!childFrame) return null;
|
||||
return {
|
||||
iframeSrc: await iframeEl.getAttribute('src'),
|
||||
content: await getFrameSnapshot(childFrame, depth + 1)
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
return {
|
||||
snapshot: result.full,
|
||||
iframes: childSnapshots.filter(Boolean)
|
||||
};
|
||||
}
|
||||
|
||||
return await getFrameSnapshot(page.mainFrame());
|
||||
}
|
||||
```
|
||||
|
||||
## Summary
|
||||
|
||||
| Feature | Playwright Support | Notes |
|
||||
|---------|-------------------|-------|
|
||||
| Accessibility snapshot for current frame | ✅ Yes | Via injected script DOM traversal |
|
||||
| Accessibility snapshot for iframe content | ❌ No | Iframes detected but content not included |
|
||||
| CDP Accessibility commands | ❌ Not used | Available but Playwright doesn't use them |
|
||||
| Cross-browser support | ✅ Yes | Works in all browsers via DOM traversal |
|
||||
| Frame-specific queries via CDP | ⚠️ Available | `frameId` parameter exists but unused |
|
||||
| Multi-frame snapshot | ⚠️ Manual | Must query each frame separately |
|
||||
|
||||
**Bottom line:** Playwright's `ariaSnapshot()` works on a single frame at a time. To get iframe content, you must:
|
||||
1. Get the iframe element
|
||||
2. Access its `contentFrame()`
|
||||
3. Call `ariaSnapshot()` on that frame
|
||||
4. Manually combine results
|
||||
|
||||
There is no built-in way to get a recursive accessibility tree that includes iframe content in a single call.
|
||||
@@ -1,398 +0,0 @@
|
||||
# Playwright CDP Connection Flow
|
||||
|
||||
This document describes exactly what CDP commands, responses, and events Playwright
|
||||
sends and expects when connecting to a browser via `connectOverCDP`. Understanding
|
||||
this flow is critical for implementing a CDP relay that works correctly with Playwright.
|
||||
|
||||
## Connection Sequence Overview
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 1. ENDPOINT RESOLUTION │
|
||||
├─────────────────────────────────────────────────────────────────────────────┤
|
||||
│ If HTTP URL → fetch /json/version/ to get webSocketDebuggerUrl │
|
||||
│ If WS URL → use directly │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 2. WEBSOCKET CONNECT │
|
||||
├─────────────────────────────────────────────────────────────────────────────┤
|
||||
│ Connect to ws://host:port/devtools/browser/<id> │
|
||||
│ Creates root CDP session with sessionId = '' (empty string) │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 3. BROWSER INITIALIZATION (root session) │
|
||||
├─────────────────────────────────────────────────────────────────────────────┤
|
||||
│ Command: Browser.getVersion │
|
||||
│ Response: { protocolVersion, product, userAgent, ... } │
|
||||
│ │
|
||||
│ Command: Target.setAutoAttach │
|
||||
│ params: { autoAttach: true, waitForDebuggerOnStart: true, flatten: true } │
|
||||
│ Response: {} │
|
||||
│ │
|
||||
│ (Chrome bug workaround) │
|
||||
│ Command: Target.getTargetInfo │
|
||||
│ Response: { targetInfo: { type: 'browser', ... } } │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 4. TARGET DISCOVERY (events from browser) │
|
||||
├─────────────────────────────────────────────────────────────────────────────┤
|
||||
│ For each existing page/target, Chrome sends: │
|
||||
│ │
|
||||
│ Event: Target.attachedToTarget │
|
||||
│ params: { │
|
||||
│ targetInfo: { │
|
||||
│ type: 'page' | 'service_worker' | 'browser' | 'other', │
|
||||
│ targetId: string, │
|
||||
│ browserContextId: string, │
|
||||
│ url: string, ← page URL is here │
|
||||
│ openerId?: string ← for popups │
|
||||
│ }, │
|
||||
│ sessionId: string, ← CDP session ID for this target │
|
||||
│ waitingForDebugger: boolean │
|
||||
│ } │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 5. PAGE INITIALIZATION (per-page session) │
|
||||
├─────────────────────────────────────────────────────────────────────────────┤
|
||||
│ For each page, Playwright sends these commands on the page's sessionId: │
|
||||
│ │
|
||||
│ Parallel batch 1: │
|
||||
│ - Page.enable │
|
||||
│ - Page.getFrameTree │
|
||||
│ - Network.enable │
|
||||
│ - Runtime.enable ← triggers executionContextCreated │
|
||||
│ - Page.setLifecycleEventsEnabled { enabled: true } │
|
||||
│ - Log.enable │
|
||||
│ - Page.addScriptToEvaluateOnNewDocument { ... } │
|
||||
│ - Target.setAutoAttach { autoAttach: true, ... } ← for iframes │
|
||||
│ │
|
||||
│ Then: │
|
||||
│ - Runtime.runIfWaitingForDebugger ← unpauses the page │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 6. EXECUTION CONTEXT CREATION (events from browser) │
|
||||
├─────────────────────────────────────────────────────────────────────────────┤
|
||||
│ After Runtime.enable, Chrome sends: │
|
||||
│ │
|
||||
│ Event: Runtime.executionContextCreated │
|
||||
│ params: { │
|
||||
│ context: { │
|
||||
│ id: number, ← contextId for Runtime.evaluate │
|
||||
│ auxData: { │
|
||||
│ frameId: string, │
|
||||
│ isDefault: boolean ← true = main world │
|
||||
│ }, │
|
||||
│ name: string ← matches utility world name if isolated │
|
||||
│ } │
|
||||
│ } │
|
||||
│ │
|
||||
│ Playwright creates a ManualPromise for each world ('main', 'utility'). │
|
||||
│ The promise resolves when executionContextCreated fires for that world. │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## CDP Commands Sent by Playwright
|
||||
|
||||
### On Browser Connection (root session, sessionId = '')
|
||||
|
||||
| Order | Command | Parameters | Purpose |
|
||||
|-------|---------|------------|---------|
|
||||
| 1 | `Browser.getVersion` | none | Get browser version, user agent, detect headless |
|
||||
| 2 | `Target.setAutoAttach` | `autoAttach: true, waitForDebuggerOnStart: true, flatten: true` | Auto-attach to all targets |
|
||||
| 3 | `Target.getTargetInfo` | none | Chrome bug workaround - ensures targets attached |
|
||||
|
||||
### On Page Attachment (page session)
|
||||
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `Page.enable` | Enable Page domain events |
|
||||
| `Page.getFrameTree` | Get frame hierarchy |
|
||||
| `Network.enable` | Enable network interception |
|
||||
| `Runtime.enable` | Enable JS execution contexts |
|
||||
| `Page.setLifecycleEventsEnabled` | Enable load/DOMContentLoaded events |
|
||||
| `Log.enable` | Enable console message capture |
|
||||
| `Page.addScriptToEvaluateOnNewDocument` | Inject Playwright's utility scripts |
|
||||
| `Target.setAutoAttach` | Auto-attach to iframes |
|
||||
| `Runtime.runIfWaitingForDebugger` | Unpause the page |
|
||||
|
||||
## CDP Events Playwright Listens For
|
||||
|
||||
### Browser-Level Events (root session)
|
||||
|
||||
| Event | Handler | Purpose |
|
||||
|-------|---------|---------|
|
||||
| `Target.attachedToTarget` | `_onAttachedToTarget` | New page/target discovered |
|
||||
| `Target.detachedFromTarget` | `_onDetachedFromTarget` | Page/target closed |
|
||||
| `Browser.downloadWillBegin` | `_onDownloadWillBegin` | Download started |
|
||||
| `Browser.downloadProgress` | `_onDownloadProgress` | Download progress |
|
||||
|
||||
### Page-Level Events (page session)
|
||||
|
||||
| Event | Purpose |
|
||||
|-------|---------|
|
||||
| `Runtime.executionContextCreated` | JavaScript context ready |
|
||||
| `Runtime.executionContextDestroyed` | Context destroyed (navigation) |
|
||||
| `Page.frameAttached` | New iframe |
|
||||
| `Page.frameDetached` | Iframe removed |
|
||||
| `Page.navigatedWithinDocument` | SPA navigation |
|
||||
| `Page.lifecycleEvent` | load, DOMContentLoaded, etc. |
|
||||
| `Page.frameStoppedLoading` | Frame finished loading |
|
||||
| `Network.requestWillBeSent` | Request started |
|
||||
| `Network.responseReceived` | Response received |
|
||||
| `Log.entryAdded` | Console message |
|
||||
|
||||
## The Context Promise Mechanism
|
||||
|
||||
Playwright uses a `ManualPromise` pattern for execution contexts:
|
||||
|
||||
```
|
||||
Frame created
|
||||
↓
|
||||
ManualPromise created for 'main' and 'utility' worlds
|
||||
↓
|
||||
Runtime.enable sent
|
||||
↓
|
||||
Chrome sends Runtime.executionContextCreated
|
||||
↓
|
||||
_contextCreated() resolves the ManualPromise
|
||||
↓
|
||||
page.evaluate() can now execute
|
||||
```
|
||||
|
||||
### How User APIs Wait for Context
|
||||
|
||||
Every DOM operation in Playwright awaits `_context()` before running:
|
||||
|
||||
```typescript
|
||||
// frames.ts
|
||||
async evaluateExpression(expression, options, arg) {
|
||||
const context = await this._context(options.world ?? 'main'); // waits here
|
||||
return context.evaluateExpression(expression, options, arg);
|
||||
}
|
||||
|
||||
_context(world: 'main' | 'utility'): Promise<FrameExecutionContext> {
|
||||
return this._contextData.get(world)!.contextPromise.then(...);
|
||||
}
|
||||
```
|
||||
|
||||
### Context Lifecycle on Navigation
|
||||
|
||||
```
|
||||
Navigation starts
|
||||
↓
|
||||
Runtime.executionContextDestroyed fires
|
||||
↓
|
||||
_setContext(world, null) → creates NEW ManualPromise
|
||||
↓
|
||||
Any evaluate() call now waits on new promise
|
||||
↓
|
||||
New page loads, Runtime.executionContextCreated fires
|
||||
↓
|
||||
_contextCreated() resolves the new promise
|
||||
↓
|
||||
Waiting evaluate() calls proceed
|
||||
```
|
||||
|
||||
## World Types
|
||||
|
||||
Playwright uses two JavaScript execution worlds:
|
||||
|
||||
| World | Purpose | APIs That Use It |
|
||||
|-------|---------|------------------|
|
||||
| **main** | Page's own JS context. User's `evaluate()` runs here. | `evaluate()`, `evaluateHandle()`, `addScriptTag()`, `addStyleTag()` |
|
||||
| **utility** | Isolated world. Playwright's internal code runs here. | `$()`, `$$()`, `click()`, `fill()`, `type()`, `content()`, `waitForSelector()` |
|
||||
|
||||
### Why Two Worlds?
|
||||
|
||||
- **main**: User code needs access to page variables and functions
|
||||
- **utility**: Playwright's selectors/actions need isolation from page code that might override `document.querySelector`, etc.
|
||||
|
||||
### Element Handle Adoption
|
||||
|
||||
When `page.$()` or `waitForSelector()` returns an element:
|
||||
1. Query runs in **utility** world
|
||||
2. Result is adopted to **main** world
|
||||
3. User receives handle bound to main context
|
||||
|
||||
## What Must Happen Before User Code Can Run
|
||||
|
||||
For `page.evaluate()` or any DOM operation to work:
|
||||
|
||||
1. **WebSocket connected** to CDP endpoint
|
||||
2. **Browser.getVersion** returned successfully
|
||||
3. **Target.setAutoAttach** sent and acknowledged
|
||||
4. **Target.attachedToTarget** received for the page
|
||||
5. **Runtime.enable** sent on page session
|
||||
6. **Runtime.executionContextCreated** received for the frame
|
||||
7. **ManualPromise resolved** for the appropriate world
|
||||
|
||||
If any step is missing or delayed, `evaluate()` will hang waiting for the context promise.
|
||||
|
||||
## Key Parameters Explained
|
||||
|
||||
### Target.setAutoAttach Parameters
|
||||
|
||||
| Parameter | Value | Purpose |
|
||||
|-----------|-------|---------|
|
||||
| `autoAttach` | `true` | Automatically attach to new targets |
|
||||
| `waitForDebuggerOnStart` | `true` | Pause new targets until `Runtime.runIfWaitingForDebugger` |
|
||||
| `flatten` | `true` | Use flat session IDs (not nested) |
|
||||
|
||||
### Why waitForDebuggerOnStart?
|
||||
|
||||
Without this, a page could start executing before Playwright injects its utility scripts. With it:
|
||||
1. Chrome pauses the page immediately after creation
|
||||
2. Playwright sends initialization commands
|
||||
3. Playwright sends `Runtime.runIfWaitingForDebugger`
|
||||
4. Page starts executing with Playwright ready
|
||||
|
||||
## Implications for CDP Relay Implementation
|
||||
|
||||
A CDP relay (like playwriter) must:
|
||||
|
||||
1. **Forward Target.setAutoAttach** and return response before any targets attach
|
||||
2. **Send Target.attachedToTarget** with correct `targetInfo` including `url`
|
||||
3. **Forward Runtime.enable** and ensure `Runtime.executionContextCreated` follows
|
||||
4. **Use correct sessionId** for page-specific commands/events
|
||||
5. **Handle session lifecycle** - detach events when tabs close
|
||||
|
||||
### Critical Timing
|
||||
|
||||
The relay should ensure:
|
||||
- `Target.attachedToTarget` is sent before any page commands
|
||||
- `Runtime.executionContextCreated` is sent after `Runtime.enable` returns
|
||||
- Events use the correct `sessionId` matching the target
|
||||
|
||||
See `cdp-timing.md` for details on our relay's event synchronization.
|
||||
|
||||
## When `context.on('page')` / `context.waitForEvent('page')` Fires
|
||||
|
||||
Both `context.on('page')` and `context.waitForEvent('page')` listen to the **same
|
||||
underlying event**. The difference is only in consumption pattern:
|
||||
|
||||
| Aspect | `context.on('page', cb)` | `context.waitForEvent('page')` |
|
||||
|--------|--------------------------|-------------------------------|
|
||||
| Return | EventEmitter | Promise\<Page\> |
|
||||
| Fires | Every time | First match only |
|
||||
| Timeout | None | Built-in timeout |
|
||||
| Predicate | Manual | Native `{ predicate: fn }` |
|
||||
| Cleanup | Manual `off()` | Auto-removes listener |
|
||||
| Close handling | Manual | Auto-rejects if context closes |
|
||||
|
||||
The 'page' event on BrowserContext is NOT immediate after `Target.attachedToTarget`.
|
||||
It requires page initialization to complete first.
|
||||
|
||||
### Event Chain
|
||||
|
||||
```
|
||||
Target.attachedToTarget (CDP event)
|
||||
↓
|
||||
CRPage created (crBrowser.ts)
|
||||
↓
|
||||
FrameSession._initialize() starts
|
||||
↓
|
||||
Parallel CDP commands sent:
|
||||
- Page.enable
|
||||
- Page.getFrameTree ← must complete
|
||||
- Runtime.enable
|
||||
- Log.enable
|
||||
- Page.setLifecycleEventsEnabled
|
||||
- Runtime.runIfWaitingForDebugger
|
||||
↓
|
||||
WAIT for _firstNonInitialNavigationCommittedPromise
|
||||
↓
|
||||
_initialize() resolves
|
||||
↓
|
||||
page.reportAsNew() called
|
||||
↓
|
||||
page._markInitialized()
|
||||
↓
|
||||
this.emitOnContext(BrowserContext.Events.Page, this) ← 'page' EVENT FIRES
|
||||
```
|
||||
|
||||
### The Navigation Wait Condition
|
||||
|
||||
The critical gate is `_firstNonInitialNavigationCommittedPromise`. The 'page' event
|
||||
does NOT fire until:
|
||||
|
||||
1. `Page.getFrameTree` returns the frame structure
|
||||
2. **AND** one of:
|
||||
- Page URL is not `:` (not an initial empty page)
|
||||
- `Page.frameNavigated` event arrives for a non-initial navigation
|
||||
|
||||
This ensures the page has a **valid URL and frame structure** before user code sees it.
|
||||
|
||||
### Key Insights
|
||||
|
||||
- `context.on('page')` and `context.waitForEvent('page')` fire on the **same event**
|
||||
- **`waitForEvent('page')` only waits for NEW pages** - it will NOT resolve with existing pages
|
||||
- To handle existing pages, check `context.pages()` first before waiting
|
||||
- The 'page' event fires AFTER page initialization, not immediately on attach
|
||||
- `Runtime.executionContextCreated` is NOT required for the 'page' event
|
||||
- The 'page' event guarantees the page has a valid URL and frame tree
|
||||
- `page.evaluate()` may still need to wait for execution context after 'page' fires
|
||||
|
||||
### Handling Existing vs New Pages
|
||||
|
||||
```typescript
|
||||
// waitForEvent only catches NEW pages, not existing ones
|
||||
const existingPages = context.pages();
|
||||
if (existingPages.length > 0) {
|
||||
return existingPages[0]; // use existing
|
||||
}
|
||||
// Only wait if no pages exist yet
|
||||
const newPage = await context.waitForEvent('page');
|
||||
```
|
||||
|
||||
### Timing Implications for CDP Relay
|
||||
|
||||
For `context.on('page')` to fire, the relay must:
|
||||
|
||||
1. Send `Target.attachedToTarget` with valid `targetInfo.url`
|
||||
2. Respond to `Page.getFrameTree` with frame structure
|
||||
3. Either:
|
||||
- Provide a non-empty URL (not `:`) in the frame tree
|
||||
- Or send `Page.frameNavigated` event after navigation
|
||||
|
||||
## Empty URL Detection
|
||||
|
||||
Empty URLs in `Target.attachedToTarget` cause Playwright to create broken pages that
|
||||
never recover. Both the extension and relay log errors when this happens:
|
||||
|
||||
### Extension logging
|
||||
|
||||
```typescript
|
||||
// In attachTab(), after Target.getTargetInfo
|
||||
if (!targetInfo.url || targetInfo.url === '' || targetInfo.url === ':') {
|
||||
logger.error('WARNING: Target.attachedToTarget will be sent with empty URL!')
|
||||
}
|
||||
```
|
||||
|
||||
### Relay logging
|
||||
|
||||
The relay logs `logger.error` warnings when:
|
||||
- `Target.attachedToTarget` is sent/received with empty URL
|
||||
- `Target.targetCreated` is sent with empty URL
|
||||
|
||||
### Why empty URLs break Playwright
|
||||
|
||||
If `Target.attachedToTarget` is sent with empty URL:
|
||||
- Playwright creates a broken page
|
||||
- `_firstNonInitialNavigationCommittedPromise` may never resolve
|
||||
- `page.evaluate()` hangs forever
|
||||
- **No recovery possible**
|
||||
|
||||
### Debugging empty URL issues
|
||||
|
||||
Check the relay server logs (run `playwriter logfile` to get the path) for:
|
||||
```
|
||||
WARNING: Target.attachedToTarget sent with empty URL
|
||||
WARNING: Target.attachedToTarget received with empty URL
|
||||
```
|
||||
|
||||
These indicate timing issues where the page hasn't fully loaded when attached.
|
||||
Reference in New Issue
Block a user