From 73c9723840b827ee07e993c070be58372bf57634 Mon Sep 17 00:00:00 2001 From: "Tommy D. Rossi" Date: Wed, 4 Feb 2026 23:21:33 +0100 Subject: [PATCH] slopping around --- slop/aria-ref-labels-plan.md | 352 ------------------- slop/aria-snapshot-analysis.md | 330 ++++++++++++++++++ slop/mcp-page-ready-plan.md | 115 ------- slop/page-ready-investigation.md | 102 ------ slop/playwright-accessibility-findings.md | 332 ++++++++++++++++++ slop/playwright-cdp-connection.md | 398 ---------------------- 6 files changed, 662 insertions(+), 967 deletions(-) delete mode 100644 slop/aria-ref-labels-plan.md create mode 100644 slop/aria-snapshot-analysis.md delete mode 100644 slop/mcp-page-ready-plan.md delete mode 100644 slop/page-ready-investigation.md create mode 100644 slop/playwright-accessibility-findings.md delete mode 100644 slop/playwright-cdp-connection.md diff --git a/slop/aria-ref-labels-plan.md b/slop/aria-ref-labels-plan.md deleted file mode 100644 index 1ecf166..0000000 --- a/slop/aria-ref-labels-plan.md +++ /dev/null @@ -1,352 +0,0 @@ -# Plan: Visual Aria Ref Labels (Vimium-style) - -## Goal - -Add functions to overlay visual ref labels on interactive elements, allowing agents to: -1. Take a screenshot with visible ref labels (e.g., `[e1]`, `[e2]`, `[e3]`) -2. See which elements are interactive and their ref IDs -3. Use `page.locator('aria-ref=e5')` to interact with labeled elements - -## Reference Implementation - -Based on `/Users/morse/Documents/GitHub/unframer-private/autofill-extension/src/content/`: - -### gui.js - Container setup -```js -const guiRoot = document.createElement('div') -guiRoot.style = ` - position: absolute; - left: 0; - top: 0; - width: 100%; - height: 100%; - z-index: 2147483647; // Max z-index to be on top of everything - pointer-events: none; // Don't block clicks on underlying elements -` -document.documentElement.appendChild(guiRoot) -``` - -### HintRenderer.tsx - Label positioning -```tsx -// Position label at element's top-left corner, accounting for scroll -left: `${window.scrollX + rect.left - 16}px` -top: `${window.scrollY + rect.top - 16}px` // 16px above element -``` - -### Vimium-style CSS -```css -font-family: Helvetica, Arial, sans-serif; -font-weight: bold; -font-size: 12px; -padding: 0px 2px; -background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#FFF785), color-stop(100%,#FFC542)); -border: 1px solid #E3BE23; -border-radius: 4px; -color: black; -text-shadow: rgba(255, 255, 255, 0.6) 0px 1px 0px; -``` - -## Existing Code Context - -### `getAriaSnapshot()` in `playwriter/src/aria-snapshot.ts` - -Already implemented - discovers refs and provides lookup: - -```ts -export async function getAriaSnapshot({ page }: { page: Page }): Promise { - // 1. Calls page._snapshotForAI() to populate Playwright's internal ref tracking - // 2. Probes aria-ref=e1, e2, e3... to discover all refs - // 3. Caches ElementHandles in refHandles array - // 4. Returns { snapshot, refToElement, refHandles, getRefsForLocators, ... } -} -``` - -The `refHandles` array contains `{ ref: string, handle: ElementHandle }` for each interactive element. We'll use these handles to get bounding boxes and create labels. - -### Which elements get refs? - -Playwright's `_snapshotForAI()` uses `mode: 'ai'` which sets `refs: 'interactable'`: - -```ts -// From playwright/packages/injected/src/ariaSnapshot.ts -function computeAriaRef(ariaNode, options) { - if (options.refs === 'interactable' && (!ariaNode.box.visible || !ariaNode.receivesPointerEvents)) - return; // Skip non-interactive elements - // ... assign ref like 'e1', 'e2', etc. -} -``` - -**Included:** buttons, links, inputs, textareas, selects, checkboxes, radios, sliders, any clickable element -**Excluded:** hidden elements, static text, decorative images, elements behind overlays - -## API Design - -```ts -// Show Vimium-style labels on all interactive elements -export async function showAriaRefLabels({ page }: { page: Page }): Promise<{ - snapshot: string // The accessibility snapshot text - labelCount: number // Number of labels shown -}> - -// Remove all labels from the page -export async function hideAriaRefLabels({ page }: { page: Page }): Promise -``` - -## Implementation - -Add to `playwriter/src/aria-snapshot.ts`: - -```ts -const LABELS_CONTAINER_ID = '__playwriter_labels__' - -// Use css`` tagged template for IDE syntax highlighting (identity function) -const css = String.raw - -const LABEL_STYLES = css` - .__pw_label__ { - position: absolute; - font: bold 11px Helvetica, Arial, sans-serif; - padding: 1px 4px; - background: linear-gradient(to bottom, #FFF785 0%, #FFC542 100%); - border: 1px solid #E3BE23; - border-radius: 3px; - color: black; - text-shadow: 0 1px 0 rgba(255, 255, 255, 0.6); - white-space: nowrap; - } -` - -const CONTAINER_STYLES = css` - position: absolute; - left: 0; - top: 0; - z-index: 2147483647; - pointer-events: none; -` - -export async function showAriaRefLabels({ page }: { page: Page }) { - // Get snapshot and cached element handles - const { snapshot, refHandles } = await getAriaSnapshot({ page }) - - // Single evaluate call: create container, styles, and all labels - // Pass ElementHandles which get unwrapped to DOM elements in browser context - const labelCount = await page.evaluate( - ({ refs, containerId, containerStyles, labelStyles }) => { - // Remove existing labels if present (idempotent) - document.getElementById(containerId)?.remove() - - // Create container - absolute positioned, max z-index, no pointer events - const container = document.createElement('div') - container.id = containerId - container.style.cssText = containerStyles - - // Inject Vimium-style CSS - const style = document.createElement('style') - style.textContent = labelStyles - container.appendChild(style) - - // Create label for each interactive element - let count = 0 - for (const { ref, element } of refs) { - const rect = element.getBoundingClientRect() - - // Skip elements with no size (hidden) - if (rect.width === 0 || rect.height === 0) continue - - const label = document.createElement('div') - label.className = '__pw_label__' - label.textContent = ref - - // Position above element, accounting for scroll - // Use scrollX/scrollY so labels scroll with the page - label.style.left = `${scrollX + rect.left}px` - label.style.top = `${scrollY + Math.max(0, rect.top - 16)}px` - - container.appendChild(label) - count++ - } - - document.documentElement.appendChild(container) - return count - }, - { - refs: refHandles.map(({ ref, handle }) => ({ ref, element: handle })), - containerId: LABELS_CONTAINER_ID, - containerStyles: CONTAINER_STYLES, - labelStyles: LABEL_STYLES, - } - ) - - return { snapshot, labelCount } -} - -export async function hideAriaRefLabels({ page }: { page: Page }) { - await page.evaluate( - (id) => document.getElementById(id)?.remove(), - LABELS_CONTAINER_ID - ) -} -``` - -## Scroll Behavior - -Labels use `position: absolute` with `scrollX/scrollY` offsets: -- Labels stay positioned on their elements when user scrolls -- Labels move with the document (not fixed to viewport) -- This matches Vimium's behavior - -## Export from index.ts - -```ts -export { getAriaSnapshot, showAriaRefLabels, hideAriaRefLabels } from './aria-snapshot.js' -``` - -## Test Implementation - -Add to `playwriter/src/mcp.test.ts`: - -```ts -it('should show aria ref labels and capture in screenshot', async () => { - const browserContext = getBrowserContext() - const serviceWorker = await getExtensionServiceWorker(browserContext) - - // Create page with interactive elements - const page = await browserContext.newPage() - await page.setContent(` - - -

Test Page

- - About Us - - - - - `) - await page.bringToFront() - - // Enable extension for this tab - await serviceWorker.evaluate(async () => { - await globalThis.toggleExtensionForActiveTab() - }) - await new Promise(r => setTimeout(r, 400)) - - // Connect via CDP - const browser = await chromium.connectOverCDP(getCdpUrl({ port: TEST_PORT })) - const cdpPage = browser.contexts()[0].pages().find(p => p.url().startsWith('about:')) - expect(cdpPage).toBeDefined() - - // Import and show labels - const { showAriaRefLabels, hideAriaRefLabels } = await import('./aria-snapshot.js') - const { snapshot, labelCount } = await showAriaRefLabels({ page: cdpPage! }) - - console.log('Snapshot:', snapshot.slice(0, 200)) - console.log('Label count:', labelCount) - - expect(labelCount).toBeGreaterThan(0) - expect(snapshot).toContain('button') - expect(snapshot).toContain('link') - - // Verify labels are in DOM - const labelElements = await cdpPage!.evaluate(() => - document.querySelectorAll('.__pw_label__').length - ) - expect(labelElements).toBe(labelCount) - - // Verify label text matches refs - const labelTexts = await cdpPage!.evaluate(() => - Array.from(document.querySelectorAll('.__pw_label__')).map(el => el.textContent) - ) - expect(labelTexts.every(t => /^e\d+$/.test(t!))).toBe(true) - console.log('Label texts:', labelTexts) - - // Take screenshot with labels visible - const screenshot = await cdpPage!.screenshot({ type: 'png' }) - expect(screenshot.length).toBeGreaterThan(5000) // Not empty - - // Save screenshot for visual verification - const fs = await import('node:fs') - const path = await import('node:path') - const snapshotsDir = path.join(path.dirname(new URL(import.meta.url).pathname), 'snapshots') - if (!fs.existsSync(snapshotsDir)) fs.mkdirSync(snapshotsDir, { recursive: true }) - fs.writeFileSync(path.join(snapshotsDir, 'aria-labels-screenshot.png'), screenshot) - console.log('Screenshot saved to snapshots/aria-labels-screenshot.png') - - // Test that labels can be used to find elements - const buttonLabel = labelTexts.find(t => t === 'e2') // Typically button is e2 - if (buttonLabel) { - const buttonViaRef = cdpPage!.locator(`aria-ref=${buttonLabel}`) - const buttonText = await buttonViaRef.textContent() - console.log(`Button via ${buttonLabel}:`, buttonText) - } - - // Cleanup - await hideAriaRefLabels({ page: cdpPage! }) - - // Verify labels removed - const labelsAfterHide = await cdpPage!.evaluate(() => - document.getElementById('__playwriter_labels__') - ) - expect(labelsAfterHide).toBeNull() - - await browser.close() - await page.close() -}, 60000) -``` - -## Visual Result - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ │ -│ Test Page │ -│ │ -│ [e2] [e3] [e4] [e5] │ -│ ┌──────────────┐ ┌──────────┐ ┌───────────────┐ ┌────────┐ │ -│ │ Submit Form │ │ About Us │ │ Enter name... │ │Option 1│ │ -│ └──────────────┘ └──────────┘ └───────────────┘ └────────┘ │ -│ button link textbox combobox │ -│ │ -└─────────────────────────────────────────────────────────────────┘ -``` - -Yellow Vimium-style badges positioned 16px above each interactive element. -Agent sees screenshot → identifies `[e3]` is the "About Us" link → uses `page.locator('aria-ref=e3').click()` - -## Files to Modify - -| File | Change | -|------|--------| -| `playwriter/src/aria-snapshot.ts` | Add `showAriaRefLabels()`, `hideAriaRefLabels()`, `LABELS_CONTAINER_ID` constant | -| `playwriter/src/index.ts` | Add exports: `showAriaRefLabels`, `hideAriaRefLabels` | -| `playwriter/src/mcp.test.ts` | Add screenshot test | -| `playwriter/src/snapshots/` | Directory for test screenshots (gitignored) | - -## Usage Example (Agent Workflow) - -```ts -// 1. Show labels on page -const { snapshot, labelCount } = await showAriaRefLabels({ page }) -console.log(`Found ${labelCount} interactive elements`) - -// 2. Take screenshot - labels are visible -const screenshot = await page.screenshot() -// Agent analyzes screenshot, sees [e5] on "Submit" button - -// 3. Interact using ref -await page.locator('aria-ref=e5').click() - -// 4. Hide labels when done -await hideAriaRefLabels({ page }) -``` - -## Edge Cases to Handle - -1. **Elements near top edge** - Label would go off-screen, position below element instead -2. **Overlapping labels** - Could add offset, but keep simple for now -3. **Very long pages** - Labels use absolute positioning, works with scroll -4. **Dynamic content** - Call `showAriaRefLabels` again to refresh after DOM changes -5. **Iframes** - Current implementation only handles main frame (could extend later) diff --git a/slop/aria-snapshot-analysis.md b/slop/aria-snapshot-analysis.md new file mode 100644 index 0000000..eea2ea4 --- /dev/null +++ b/slop/aria-snapshot-analysis.md @@ -0,0 +1,330 @@ +# Aria Snapshot Implementation Analysis + +## Overview + +The aria snapshot feature in playwriter extracts an accessibility tree from the browser using Chrome DevTools Protocol (CDP). This document details how it works, its CDP interactions, and frame/iframe handling capabilities. + +## Main Implementation File + +**File:** `playwriter/src/aria-snapshot.ts` (1358 lines) + +## How It Works + +### High-Level Pipeline + +``` +1. Get CDP Session for Page +2. Enable DOM and Accessibility domains +3. Fetch DOM tree (with pierce: true for iframes) +4. Fetch Accessibility tree +5. Build raw snapshot tree +6. Filter tree (interactive-only or full) +7. Generate locators and refs +8. Return formatted snapshot + utilities +``` + +### Key Functions + +#### `getAriaSnapshot()` +**Location:** Line 749-1033 in `aria-snapshot.ts` + +Main entry point that returns `AriaSnapshotResult` containing: +- `snapshot`: String representation of the accessibility tree +- `tree`: Structured tree with nodes +- `refs`: Array of references to interactive elements +- `getSelectorForRef()`: Get CSS selector for a ref +- `getRefsForLocators()`: Get refs for Playwright locators + +**Signature:** +```typescript +export async function getAriaSnapshot({ + page, + locator, + refFilter, + wsUrl, + interactiveOnly = false, + cdp +}: { + page: Page + locator?: Locator + refFilter?: (info: { role: string; name: string }) => boolean + wsUrl?: string + interactiveOnly?: boolean + cdp?: ICDPSession +}): Promise +``` + +## CDP Commands Used + +### 1. DOM Domain + +**Command:** `DOM.getFlattenedDocument` +**Location:** Line 772 +```typescript +const { nodes: domNodes } = await session.send('DOM.getFlattenedDocument', { + depth: -1, + pierce: true +}) as Protocol.DOM.GetFlattenedDocumentResponse +``` + +**Parameters:** +- `depth: -1` - Get entire subtree +- `pierce: true` - **Traverses iframes and shadow roots** + +**Purpose:** Get all DOM nodes to map accessibility nodes to their attributes (test IDs, etc.) + +### 2. Accessibility Domain + +**Command:** `Accessibility.getFullAXTree` +**Location:** Line 791 +```typescript +const { nodes: axNodes } = await session.send('Accessibility.getFullAXTree') + as Protocol.Accessibility.GetFullAXTreeResponse +``` + +**Parameters:** None specified (uses defaults) +- Default: Returns AX tree for root frame +- **Has optional `frameId` parameter** (not currently used) + +**Purpose:** Get the complete accessibility tree with roles, names, and relationships + +## Frame/Iframe Support + +### Current Implementation + +**DOM Level:** ✅ **FULL SUPPORT** +- `DOM.getFlattenedDocument` with `pierce: true` traverses all iframes and shadow roots +- All DOM nodes from all frames are included in the flattened document + +**Accessibility Level:** ⚠️ **LIMITED** +- `Accessibility.getFullAXTree` is called **without `frameId` parameter** +- According to CDP spec, when `frameId` is omitted, **only the root frame is used** +- Cross-origin iframes may have additional restrictions + +### CDP Spec Details + +From `Accessibility.pdl`: +``` +experimental command getFullAXTree + parameters + # The maximum depth at which descendants of the root node should be retrieved. + # If omitted, the full tree is returned. + optional integer depth + # The frame for whose document the AX tree should be retrieved. + # If omitted, the root frame is used. + optional Page.FrameId frameId + returns + array of AXNode nodes +``` + +### Implications + +1. **Same-origin iframes:** May be included in the root frame's AX tree +2. **Cross-origin iframes:** Likely **NOT included** due to security restrictions +3. **Shadow DOM:** Likely included (as `pierce: true` affects DOM and shadow roots are part of the same document) + +### Evidence from Code + +**Scope handling (Line 760-789):** +- Uses `data-pw-scope` attribute to scope snapshots to a locator +- Builds `allowedBackendIds` set from DOM tree traversal +- Filters AX nodes based on `backendDOMNodeId` membership in this set + +**Node mapping:** +- Each AX node has `backendDOMNodeId` property linking to DOM node +- DOM nodes fetched with `pierce: true` include iframe contents +- But AX tree without `frameId` may not cover all frames + +## Key Data Structures + +### AriaSnapshotNode +```typescript +type AriaSnapshotNode = { + role: string + name: string + locator?: string + ref?: string + shortRef?: string + backendNodeId?: Protocol.DOM.BackendNodeId + children: AriaSnapshotNode[] +} +``` + +### AriaRef +```typescript +interface AriaRef { + role: string + name: string + ref: string // Full ref (testid or e1, e2, e3...) + shortRef: string // Short ref (e1, e2, e3...) + backendNodeId?: Protocol.DOM.BackendNodeId +} +``` + +## Locator Generation + +**File:** Lines 209-242 + +Generates Playwright-compatible locators: + +1. **Stable refs** (preferred): + - `[data-testid="submit-btn"]` + - `[id="login"]` + - Test ID attributes: `data-testid`, `data-test-id`, `data-test`, `data-cy`, `data-pw`, `data-qa`, `data-e2e`, `data-automation-id` + +2. **Role-based fallback**: + - `role=button[name="Submit"]` + - `role=link[name="Learn More"]` + +3. **Nth handling**: + - Duplicates get `>> nth=0`, `>> nth=1`, etc. + +## Interactive Roles + +**Location:** Lines 123-143 + +Only these roles get refs in interactive mode: +```typescript +const INTERACTIVE_ROLES = new Set([ + 'button', 'link', 'textbox', 'combobox', 'searchbox', + 'checkbox', 'radio', 'slider', 'spinbutton', 'switch', + 'menuitem', 'menuitemcheckbox', 'menuitemradio', + 'option', 'tab', 'treeitem', 'img', 'video', 'audio', +]) +``` + +## Executor Integration + +**File:** `playwriter/src/executor.ts` (Lines 533-619) + +The `accessibilitySnapshot()` function exposed to agents: + +```typescript +const accessibilitySnapshot = async (options: { + page: Page + locator?: Locator + search?: string | RegExp + showDiffSinceLastCall?: boolean + format?: SnapshotFormat + interactiveOnly?: boolean +}) => { + const { snapshot, refs, getSelectorForRef } = await getAriaSnapshot({ + page: targetPage, + locator, + wsUrl: getCdpUrl(this.cdpConfig), + interactiveOnly, + }) + + // Store refs for refToLocator() function + // Handle search filtering + // Handle diff mode + // Return formatted snapshot +} +``` + +## Screenshot with Labels + +**Function:** `screenshotWithAccessibilityLabels()` +**Location:** Lines 1258-1354 + +Takes a screenshot with Vimium-style labels overlaid on interactive elements: + +1. Calls `showAriaRefLabels()` to render labels +2. Takes screenshot with viewport clipping +3. Resizes to max 1568px (Claude token optimization) +4. Hides labels +5. Returns screenshot with snapshot + +## Tests + +**File:** `playwriter/src/aria-snapshot.test.ts` + +Tests against real websites: +- Hacker News +- GitHub + +**Coverage:** +- Snapshot generation +- Interactive-only mode +- Locator format validation +- No wrapper roles in output + +**Note:** No explicit iframe tests found + +## Limitations & Findings + +### Frame Handling Limitations + +1. **Root frame only:** `Accessibility.getFullAXTree()` without `frameId` only returns root frame AX tree +2. **No iframe iteration:** Code doesn't enumerate frames and call `getFullAXTree()` per frame +3. **Cross-origin restrictions:** Even if frames were enumerated, cross-origin iframes would be blocked + +### Potential Issues + +1. **Elements in iframes may not appear** in accessibility snapshot +2. **DOM includes iframe content** (via `pierce: true`) but AX tree may not +3. **Inconsistency:** `backendNodeId` from DOM may not have matching AX node for iframe elements + +### Evidence of Limitation + +- No `frameId` parameter passed to `Accessibility.getFullAXTree()` +- No frame enumeration logic +- No tests for iframe scenarios +- CDP spec clearly states "If omitted, the root frame is used" + +## Possible Enhancements + +To support iframes properly: + +1. **Enumerate frames:** + ```typescript + const { frameTree } = await session.send('Page.getFrameTree') + // Recursively collect all frame IDs + ``` + +2. **Get AX tree per frame:** + ```typescript + for (const frameId of frameIds) { + const { nodes } = await session.send('Accessibility.getFullAXTree', { frameId }) + // Merge nodes + } + ``` + +3. **Handle cross-origin frames:** + - Detect cross-origin frames + - Skip or show placeholder for restricted frames + - Document limitation in output + +4. **Test with iframes:** + - Add test cases with same-origin iframes + - Add test cases with cross-origin iframes + - Validate behavior + +## Summary + +**File Paths:** +- Main implementation: `playwriter/src/aria-snapshot.ts` +- Executor integration: `playwriter/src/executor.ts` (lines 533-619) +- CDP session: `playwriter/src/cdp-session.ts` +- Tests: `playwriter/src/aria-snapshot.test.ts` + +**CDP Commands:** +- `DOM.enable` - Enable DOM domain +- `DOM.getFlattenedDocument({ depth: -1, pierce: true })` - Get all DOM nodes including iframes +- `Accessibility.enable` - Enable accessibility domain +- `Accessibility.getFullAXTree()` - Get AX tree (root frame only by default) +- `DOM.getBoxModel({ backendNodeId })` - Get element positions for labels + +**Frame Handling:** +- ✅ DOM tree includes iframe content (`pierce: true`) +- ⚠️ Accessibility tree likely **only root frame** (no `frameId` parameter) +- ❌ No frame enumeration or per-frame AX tree fetching +- ❌ No explicit iframe tests + +**Key Insight:** +The current implementation may miss interactive elements inside iframes because: +1. `Accessibility.getFullAXTree()` without `frameId` only returns root frame +2. No iteration over child frames to collect their AX trees +3. Cross-origin iframes would be blocked anyway for security + +For multi-browser support with Firefox/WebKit, this limitation would need to be addressed as those browsers may handle iframes differently in their accessibility trees. diff --git a/slop/mcp-page-ready-plan.md b/slop/mcp-page-ready-plan.md deleted file mode 100644 index ff56543..0000000 --- a/slop/mcp-page-ready-plan.md +++ /dev/null @@ -1,115 +0,0 @@ -# Plan: Add Deterministic Page Waiting to mcp.ts - -## Current Problem - -In `mcp.ts`, both `connect()` and `reconnect()` functions: - -```typescript -const browser = await chromium.connectOverCDP(cdpEndpoint) -const context = contexts[0] -const pages = context.pages() // May be empty! -if (pages.length === 0) { - throw new Error(NO_TABS_ERROR) -} -``` - -This fails when: -1. Extension has tabs connected, but... -2. Playwright hasn't finished processing `Target.attachedToTarget` + `Runtime.executionContextCreated` events - -## Solution - -Use `context.waitForEvent('page')` with a timeout, similar to our test fix. - -## Implementation Plan - -### Step 1: Create a helper function - -```typescript -async function waitForPages(context: BrowserContext, options?: { - timeout?: number - minPages?: number -}): Promise { - const { timeout = 5000, minPages = 1 } = options || {} - - // Check if pages already exist - let pages = context.pages() - if (pages.length >= minPages) { - return pages - } - - // Wait for page events - const startTime = Date.now() - while (pages.length < minPages && Date.now() - startTime < timeout) { - try { - await context.waitForEvent('page', { timeout: timeout - (Date.now() - startTime) }) - pages = context.pages() - } catch { - // Timeout - check one more time - pages = context.pages() - break - } - } - - return pages -} -``` - -### Step 2: Update `connect()` function (line ~305) - -```typescript -// Before: -const pages = context.pages() -if (pages.length === 0) { - throw new Error(NO_TABS_ERROR) -} - -// After: -const pages = await waitForPages(context, { timeout: 5000 }) -if (pages.length === 0) { - throw new Error(NO_TABS_ERROR) -} -``` - -### Step 3: Update `reconnect()` function (line ~448) - -Same change as above. - -### Step 4: Consider the "createInitialTab" flow - -The relay server has a `createInitialTab` message for when no tabs exist. We should: -1. First try `waitForPages()` with a short timeout (1-2 seconds) -2. If still no pages, request `createInitialTab` -3. Then `waitForPages()` again for the new tab - -```typescript -let pages = await waitForPages(context, { timeout: 2000 }) -if (pages.length === 0) { - // No tabs - request extension to create one - await requestCreateInitialTab() - pages = await waitForPages(context, { timeout: 5000 }) -} -if (pages.length === 0) { - throw new Error(NO_TABS_ERROR) -} -``` - -## Testing - -1. Run existing tests to ensure no regression -2. Add a new test that: - - Connects immediately after extension toggle - - Verifies pages are available without sleep - -## Files to Modify - -1. `playwriter/src/mcp.ts`: - - Add `waitForPages()` helper - - Update `connect()` function - - Update `reconnect()` function - -## Edge Cases - -1. **All pages are `about:blank`**: May need to filter for real pages -2. **Extension not connected**: Existing timeout handling should work -3. **Very slow page load**: 5 second timeout should be sufficient for CDP events (actual page content load is not required) diff --git a/slop/page-ready-investigation.md b/slop/page-ready-investigation.md deleted file mode 100644 index f89cc82..0000000 --- a/slop/page-ready-investigation.md +++ /dev/null @@ -1,102 +0,0 @@ -# Page Ready Investigation Summary - -## Goal -Convert undeterministic `sleep()` calls into deterministic event waits when toggling the extension. - -## Problem -After `toggleExtensionForActiveTab()` returns, the page is not immediately available in Playwright's `context.pages()`. Tests currently use arbitrary sleeps (100-400ms) to work around this. - -## Key Events Required - -For a page to appear in `context.pages()`, Playwright needs: - -1. **`Target.attachedToTarget`** - Tells Playwright a new target exists with its URL -2. **`Runtime.executionContextCreated`** - Tells Playwright the JS context is ready - -The page appears in `context.pages()` only after BOTH events are processed. - -## Current Flow (Timeline from test) - -``` - 63ms ← EVT Target.attachedToTarget # Extension sends to relay - 431ms ← EVT Runtime.executionContextCreated # Relay's proactive Runtime.enable - 437ms → CMD Runtime.enable # Playwright sends its own - 438ms Toggle completes (pageReady received) - 467ms ← EVT Runtime.executionContextCreated # Response to Playwright's Runtime.enable - 728ms Page appears in context.pages() -``` - -## Root Cause - -The ~300ms delay between toggle completing and page appearing is caused by: - -1. **Extension's `Runtime.enable` handler** (background.ts:256-272): - - Always calls `Runtime.disable` then `Runtime.enable` - - Has a 50ms `sleep()` between them - - This is needed to force Chrome to re-send `executionContextCreated` events for multiple clients - -2. **Double Runtime.enable cycle**: - - Relay proactively enables Runtime → events at ~430ms - - Playwright receives `Target.attachedToTarget`, sends its own `Runtime.enable` - - Extension does disable+enable again → events at ~470ms - - Playwright waits for ITS events before adding page to pages() - -3. **Playwright's internal processing**: - - Even after receiving events, Playwright takes time to create Page objects - - This is async and cannot be controlled from our side - -## What We Tried - -1. **Proactive Runtime.enable in relay** - Enable Runtime before forwarding `Target.attachedToTarget` - - Helps get events faster, but Playwright still calls Runtime.enable itself - -2. **Skip disable cycle if recently enabled** - Track recent enables in extension - - Breaks because relay's Runtime.enable handler waits for events that won't come - -3. **pageReady handshake** - Extension waits for relay confirmation before returning from attachTab - - Toggle now waits for executionContextCreated - - But Playwright STILL calls Runtime.enable after, causing another cycle - -## The Core Issue - -**Playwright always calls `Runtime.enable` after receiving `Target.attachedToTarget`**, regardless of whether we pre-enabled it. The extension's disable+enable cycle adds ~200ms, and we cannot skip it without breaking the multi-client case. - -## Possible Solutions - -### Option 1: Accept the delay, use proper waiting -Instead of sleep, use `context.waitForEvent('page')` with a predicate: -```typescript -await serviceWorker.evaluate(() => globalThis.toggleExtensionForActiveTab()) -const page = await context.waitForEvent('page', { - predicate: p => p.url().includes('target-url') -}) -``` - -### Option 2: Expose a "page ready" promise from the relay -Add an endpoint or event that resolves when the page is fully ready in Playwright: -```typescript -await serviceWorker.evaluate(() => globalThis.toggleExtensionForActiveTab()) -await relay.waitForPageReady(sessionId) // Waits for Playwright to process everything -``` - -### Option 3: Have extension track Runtime state per session -Skip the disable+enable if: -- This is the SAME Playwright client that just received `Target.attachedToTarget` -- The session was JUST created (within last 100ms) - -This requires tracking which client enabled Runtime and when. - -## Recommendation - -**Option 1 is the simplest and most reliable.** The delay is inherent to how Playwright processes CDP events. We cannot eliminate it, but we can wait for it properly using `context.waitForEvent('page')` instead of arbitrary sleeps. - -The test should be: -```typescript -const pagePromise = context.waitForEvent('page', { - predicate: p => p.url().includes('discord.com'), - timeout: 5000 -}) -await serviceWorker.evaluate(() => globalThis.toggleExtensionForActiveTab()) -const page = await pagePromise -// Page is now guaranteed to be ready -``` diff --git a/slop/playwright-accessibility-findings.md b/slop/playwright-accessibility-findings.md new file mode 100644 index 0000000..a694896 --- /dev/null +++ b/slop/playwright-accessibility-findings.md @@ -0,0 +1,332 @@ +--- +title: Playwright Accessibility Implementation Findings +description: Analysis of how Playwright implements accessibility snapshots and handles iframes +--- + +## Overview + +This document contains findings from exploring the Playwright source code to understand: +1. How Playwright implements accessibility snapshots (ariaSnapshot) +2. Whether Playwright supports getting accessibility tree for iframes/child frames +3. How Playwright handles frame locators for accessibility +4. What CDP commands Playwright uses for accessibility + +## Key Findings + +### 1. Playwright's Accessibility Implementation Strategy + +**Playwright does NOT use CDP Accessibility commands for ariaSnapshot**. Instead, it uses: + +- **Browser-side JavaScript injection** via `InjectedScript` class +- The injected script traverses the DOM directly using browser APIs +- No CDP `Accessibility.getFullAXTree` or `Accessibility.getPartialAXTree` calls + +#### Code Flow: + +```typescript +// Server-side (packages/playwright-core/src/server/frames.ts:1371) +async ariaSnapshot(progress: Progress, selector: string): Promise { + return await this._retryWithProgressIfNotConnected( + progress, + selector, + true, + true, + handle => progress.race(handle.ariaSnapshot()) + ); +} + +// Element handle (packages/playwright-core/src/server/dom.ts:757) +async ariaSnapshot(): Promise { + return await this.evaluateInUtility( + ([injected, element]) => injected.ariaSnapshot(element, { mode: 'expect' }), + {} + ); +} + +// Injected script (packages/injected/src/injectedScript.ts:305) +ariaSnapshot(node: Node, options: AriaTreeOptions): string { + return this.incrementalAriaSnapshot(node, options).full; +} + +incrementalAriaSnapshot(node: Node, options: AriaTreeOptions & { track?: string }): + { full: string, incremental?: string, iframeRefs: string[] } { + if (node.nodeType !== Node.ELEMENT_NODE) + throw this.createStacklessError('Can only capture aria snapshot of Element nodes.'); + const ariaSnapshot = generateAriaTree(node as Element, options); + const full = renderAriaTree(ariaSnapshot, options); + // ... + return { full, incremental, iframeRefs: ariaSnapshot.iframeRefs }; +} +``` + +### 2. How Playwright Handles IFrames in Accessibility Snapshots + +**IFrames are recognized but their content is NOT included** in the accessibility tree. + +#### Evidence: + +From `packages/injected/src/ariaSnapshot.ts:217-232`: + +```typescript +function toAriaNode(element: Element, options: InternalOptions): aria.AriaNode | null { + const active = element.ownerDocument.activeElement === element; + if (element.nodeName === 'IFRAME') { + const ariaNode: aria.AriaNode = { + role: 'iframe', + name: '', + children: [], // ⚠️ Empty children - no content from iframe + props: {}, + box: computeBox(element), + receivesPointerEvents: true, + active + }; + setAriaNodeElement(ariaNode, element); + computeAriaRef(ariaNode, options); + return ariaNode; + } + // ... +} +``` + +**Key observations:** +- IFrame elements are detected and added to the tree with `role: 'iframe'` +- The `children` array is ALWAYS empty for iframes +- IFrame refs are tracked separately in `snapshot.iframeRefs` array +- The content document of the iframe is NEVER traversed +- No attempt is made to access `iframe.contentDocument` or `iframe.contentWindow` + +### 3. AriaSnapshot Return Value Structure + +```typescript +export type AriaSnapshot = { + root: aria.AriaNode; + elements: Map; // ref -> Element mapping + refs: Map; // Element -> ref mapping + iframeRefs: string[]; // List of iframe ref IDs +}; +``` + +The `iframeRefs` array contains references to iframe elements but NOT their content. + +### 4. CDP Accessibility Commands (Available but NOT Used) + +The CDP protocol DOES support frame-specific accessibility queries: + +```typescript +// From protocol.d.ts +export type getFullAXTreeParameters = { + depth?: number; + frameId?: Page.FrameId; // ⚠️ Supports frame-specific queries! +} + +export type getPartialAXTreeParameters = { + nodeId?: DOM.NodeId; + backendNodeId?: DOM.BackendNodeId; + objectId?: Runtime.RemoteObjectId; + fetchRelatives?: boolean; +} +``` + +**However, Playwright does NOT use these CDP commands anywhere in the codebase.** + +Search results show: +- CDP types defined in `protocol.d.ts` +- No actual usage in Chromium implementation files +- Firefox uses a custom `Accessibility.getFullAXTree` via Juggler protocol + +### 5. Why Playwright Uses DOM Traversal Instead of CDP + +**Advantages of DOM traversal approach:** +1. **Cross-browser compatibility** - Works in Firefox, WebKit, Chromium +2. **Full control** - Can customize what gets included/excluded +3. **Performance** - No serialization overhead for large trees +4. **Flexibility** - Can implement custom filtering (visibility, aria roles, etc.) + +**Disadvantages:** +1. **Cannot access iframe content** - Browser security prevents cross-origin access +2. **Must execute in each frame separately** - No single command for entire page tree +3. **Slower for deep trees** - Must traverse DOM node-by-node + +### 6. Current Iframe Handling Limitations + +Based on code analysis: + +**Playwright CANNOT get accessibility tree for iframe content because:** + +1. **Security restrictions**: JavaScript cannot access `iframe.contentDocument` for cross-origin iframes +2. **No CDP fallback**: Playwright doesn't use CDP Accessibility commands that could bypass this +3. **By design**: The `generateAriaTree` function explicitly skips iframe children + +**To get iframe content accessibility tree, you would need to:** +1. Switch to the iframe's frame context +2. Call `ariaSnapshot()` again on that frame +3. Manually combine the results + +Example: +```typescript +// Get main frame snapshot +const mainSnapshot = await page.locator('body').ariaSnapshot(); + +// Get iframe content +const frameElement = await page.frameLocator('iframe'); +const frame = await frameElement.owner(); +const frameSnapshot = await frame.contentFrame().locator('body').ariaSnapshot(); + +// Results are separate - no automatic merging +``` + +### 7. Alternative: Using CDP Accessibility Commands + +**IF Playwriter wanted to support iframe content in accessibility snapshots**, it could: + +1. Use `Accessibility.getFullAXTree({ frameId })` for each frame +2. Recursively call it for all child frames +3. Merge the results into a single tree + +**CDP Command:** +```typescript +await session.send('Accessibility.getFullAXTree', { + frameId: 'frame-id-here', + depth: -1 // unlimited depth +}); +``` + +This would return the full accessibility tree including iframe content, but: +- Only works in Chromium (not Firefox/WebKit) +- Returns CDP's AXNode format, not Playwright's AriaNode format +- Would need conversion logic + +## Recommendations for Playwriter + +### Option 1: Multi-Frame Snapshot Approach (Current Playwright Pattern) + +Get accessibility tree for each frame separately: + +```typescript +// Pseudo-code for MCP implementation +async function getAccessibilitySnapshot({ sessionId, includeFrames = false }) { + const page = getPage(sessionId); + + // Get main frame snapshot + const mainSnapshot = await page.evaluate(() => { + return injected.ariaSnapshot(document.body, { mode: 'ai' }); + }); + + if (!includeFrames) { + return mainSnapshot; + } + + // Get all iframe snapshots + const frames = page.frames(); + const frameSnapshots = await Promise.all( + frames.slice(1).map(async (frame) => { + return { + frameId: frame.name() || frame.url(), + snapshot: await frame.evaluate(() => { + return injected.ariaSnapshot(document.body, { mode: 'ai' }); + }) + }; + }) + ); + + return { + main: mainSnapshot, + frames: frameSnapshots + }; +} +``` + +### Option 2: CDP-Based Approach (Chromium Only) + +Use CDP `Accessibility.getFullAXTree` for each frame: + +```typescript +async function getFullAccessibilityTree({ sessionId }) { + const page = getPage(sessionId); + const frames = page.frames(); + + const trees = await Promise.all( + frames.map(async (frame) => { + const session = await frame._client; // Get CDP session + const { nodes } = await session.send('Accessibility.getFullAXTree', { + frameId: frame._id + }); + return { + frameId: frame.url(), + nodes + }; + }) + ); + + // Convert CDP AXNode[] to Playwright AriaNode format + return convertCDPtoAria(trees); +} +``` + +**Note:** This would only work for Chromium, not Firefox/WebKit. + +### Option 3: Hybrid Approach (Recommended) + +1. Use Playwright's existing `ariaSnapshot()` for the current frame +2. Detect iframes via `snapshot.iframeRefs` +3. Recursively get snapshots for each iframe's content frame +4. Mark iframe boundaries in the output + +```typescript +async function getRecursiveSnapshot({ sessionId, maxDepth = 3 }) { + const page = getPage(sessionId); + + async function getFrameSnapshot(frame, depth = 0) { + if (depth >= maxDepth) return null; + + // Get snapshot for this frame + const result = await frame.evaluate(() => { + return injected.incrementalAriaSnapshot(document.body, { + mode: 'ai', + refPrefix: `f${depth}_` + }); + }); + + // Find iframe elements + const iframeElements = await frame.$$('iframe'); + + // Get snapshots for child frames + const childSnapshots = await Promise.all( + iframeElements.map(async (iframeEl) => { + const childFrame = await iframeEl.contentFrame(); + if (!childFrame) return null; + return { + iframeSrc: await iframeEl.getAttribute('src'), + content: await getFrameSnapshot(childFrame, depth + 1) + }; + }) + ); + + return { + snapshot: result.full, + iframes: childSnapshots.filter(Boolean) + }; + } + + return await getFrameSnapshot(page.mainFrame()); +} +``` + +## Summary + +| Feature | Playwright Support | Notes | +|---------|-------------------|-------| +| Accessibility snapshot for current frame | ✅ Yes | Via injected script DOM traversal | +| Accessibility snapshot for iframe content | ❌ No | Iframes detected but content not included | +| CDP Accessibility commands | ❌ Not used | Available but Playwright doesn't use them | +| Cross-browser support | ✅ Yes | Works in all browsers via DOM traversal | +| Frame-specific queries via CDP | ⚠️ Available | `frameId` parameter exists but unused | +| Multi-frame snapshot | ⚠️ Manual | Must query each frame separately | + +**Bottom line:** Playwright's `ariaSnapshot()` works on a single frame at a time. To get iframe content, you must: +1. Get the iframe element +2. Access its `contentFrame()` +3. Call `ariaSnapshot()` on that frame +4. Manually combine results + +There is no built-in way to get a recursive accessibility tree that includes iframe content in a single call. diff --git a/slop/playwright-cdp-connection.md b/slop/playwright-cdp-connection.md deleted file mode 100644 index 3f9323e..0000000 --- a/slop/playwright-cdp-connection.md +++ /dev/null @@ -1,398 +0,0 @@ -# Playwright CDP Connection Flow - -This document describes exactly what CDP commands, responses, and events Playwright -sends and expects when connecting to a browser via `connectOverCDP`. Understanding -this flow is critical for implementing a CDP relay that works correctly with Playwright. - -## Connection Sequence Overview - -``` -┌─────────────────────────────────────────────────────────────────────────────┐ -│ 1. ENDPOINT RESOLUTION │ -├─────────────────────────────────────────────────────────────────────────────┤ -│ If HTTP URL → fetch /json/version/ to get webSocketDebuggerUrl │ -│ If WS URL → use directly │ -└─────────────────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ 2. WEBSOCKET CONNECT │ -├─────────────────────────────────────────────────────────────────────────────┤ -│ Connect to ws://host:port/devtools/browser/ │ -│ Creates root CDP session with sessionId = '' (empty string) │ -└─────────────────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ 3. BROWSER INITIALIZATION (root session) │ -├─────────────────────────────────────────────────────────────────────────────┤ -│ Command: Browser.getVersion │ -│ Response: { protocolVersion, product, userAgent, ... } │ -│ │ -│ Command: Target.setAutoAttach │ -│ params: { autoAttach: true, waitForDebuggerOnStart: true, flatten: true } │ -│ Response: {} │ -│ │ -│ (Chrome bug workaround) │ -│ Command: Target.getTargetInfo │ -│ Response: { targetInfo: { type: 'browser', ... } } │ -└─────────────────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ 4. TARGET DISCOVERY (events from browser) │ -├─────────────────────────────────────────────────────────────────────────────┤ -│ For each existing page/target, Chrome sends: │ -│ │ -│ Event: Target.attachedToTarget │ -│ params: { │ -│ targetInfo: { │ -│ type: 'page' | 'service_worker' | 'browser' | 'other', │ -│ targetId: string, │ -│ browserContextId: string, │ -│ url: string, ← page URL is here │ -│ openerId?: string ← for popups │ -│ }, │ -│ sessionId: string, ← CDP session ID for this target │ -│ waitingForDebugger: boolean │ -│ } │ -└─────────────────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ 5. PAGE INITIALIZATION (per-page session) │ -├─────────────────────────────────────────────────────────────────────────────┤ -│ For each page, Playwright sends these commands on the page's sessionId: │ -│ │ -│ Parallel batch 1: │ -│ - Page.enable │ -│ - Page.getFrameTree │ -│ - Network.enable │ -│ - Runtime.enable ← triggers executionContextCreated │ -│ - Page.setLifecycleEventsEnabled { enabled: true } │ -│ - Log.enable │ -│ - Page.addScriptToEvaluateOnNewDocument { ... } │ -│ - Target.setAutoAttach { autoAttach: true, ... } ← for iframes │ -│ │ -│ Then: │ -│ - Runtime.runIfWaitingForDebugger ← unpauses the page │ -└─────────────────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ 6. EXECUTION CONTEXT CREATION (events from browser) │ -├─────────────────────────────────────────────────────────────────────────────┤ -│ After Runtime.enable, Chrome sends: │ -│ │ -│ Event: Runtime.executionContextCreated │ -│ params: { │ -│ context: { │ -│ id: number, ← contextId for Runtime.evaluate │ -│ auxData: { │ -│ frameId: string, │ -│ isDefault: boolean ← true = main world │ -│ }, │ -│ name: string ← matches utility world name if isolated │ -│ } │ -│ } │ -│ │ -│ Playwright creates a ManualPromise for each world ('main', 'utility'). │ -│ The promise resolves when executionContextCreated fires for that world. │ -└─────────────────────────────────────────────────────────────────────────────┘ -``` - -## CDP Commands Sent by Playwright - -### On Browser Connection (root session, sessionId = '') - -| Order | Command | Parameters | Purpose | -|-------|---------|------------|---------| -| 1 | `Browser.getVersion` | none | Get browser version, user agent, detect headless | -| 2 | `Target.setAutoAttach` | `autoAttach: true, waitForDebuggerOnStart: true, flatten: true` | Auto-attach to all targets | -| 3 | `Target.getTargetInfo` | none | Chrome bug workaround - ensures targets attached | - -### On Page Attachment (page session) - -| Command | Purpose | -|---------|---------| -| `Page.enable` | Enable Page domain events | -| `Page.getFrameTree` | Get frame hierarchy | -| `Network.enable` | Enable network interception | -| `Runtime.enable` | Enable JS execution contexts | -| `Page.setLifecycleEventsEnabled` | Enable load/DOMContentLoaded events | -| `Log.enable` | Enable console message capture | -| `Page.addScriptToEvaluateOnNewDocument` | Inject Playwright's utility scripts | -| `Target.setAutoAttach` | Auto-attach to iframes | -| `Runtime.runIfWaitingForDebugger` | Unpause the page | - -## CDP Events Playwright Listens For - -### Browser-Level Events (root session) - -| Event | Handler | Purpose | -|-------|---------|---------| -| `Target.attachedToTarget` | `_onAttachedToTarget` | New page/target discovered | -| `Target.detachedFromTarget` | `_onDetachedFromTarget` | Page/target closed | -| `Browser.downloadWillBegin` | `_onDownloadWillBegin` | Download started | -| `Browser.downloadProgress` | `_onDownloadProgress` | Download progress | - -### Page-Level Events (page session) - -| Event | Purpose | -|-------|---------| -| `Runtime.executionContextCreated` | JavaScript context ready | -| `Runtime.executionContextDestroyed` | Context destroyed (navigation) | -| `Page.frameAttached` | New iframe | -| `Page.frameDetached` | Iframe removed | -| `Page.navigatedWithinDocument` | SPA navigation | -| `Page.lifecycleEvent` | load, DOMContentLoaded, etc. | -| `Page.frameStoppedLoading` | Frame finished loading | -| `Network.requestWillBeSent` | Request started | -| `Network.responseReceived` | Response received | -| `Log.entryAdded` | Console message | - -## The Context Promise Mechanism - -Playwright uses a `ManualPromise` pattern for execution contexts: - -``` -Frame created - ↓ -ManualPromise created for 'main' and 'utility' worlds - ↓ -Runtime.enable sent - ↓ -Chrome sends Runtime.executionContextCreated - ↓ -_contextCreated() resolves the ManualPromise - ↓ -page.evaluate() can now execute -``` - -### How User APIs Wait for Context - -Every DOM operation in Playwright awaits `_context()` before running: - -```typescript -// frames.ts -async evaluateExpression(expression, options, arg) { - const context = await this._context(options.world ?? 'main'); // waits here - return context.evaluateExpression(expression, options, arg); -} - -_context(world: 'main' | 'utility'): Promise { - return this._contextData.get(world)!.contextPromise.then(...); -} -``` - -### Context Lifecycle on Navigation - -``` -Navigation starts - ↓ -Runtime.executionContextDestroyed fires - ↓ -_setContext(world, null) → creates NEW ManualPromise - ↓ -Any evaluate() call now waits on new promise - ↓ -New page loads, Runtime.executionContextCreated fires - ↓ -_contextCreated() resolves the new promise - ↓ -Waiting evaluate() calls proceed -``` - -## World Types - -Playwright uses two JavaScript execution worlds: - -| World | Purpose | APIs That Use It | -|-------|---------|------------------| -| **main** | Page's own JS context. User's `evaluate()` runs here. | `evaluate()`, `evaluateHandle()`, `addScriptTag()`, `addStyleTag()` | -| **utility** | Isolated world. Playwright's internal code runs here. | `$()`, `$$()`, `click()`, `fill()`, `type()`, `content()`, `waitForSelector()` | - -### Why Two Worlds? - -- **main**: User code needs access to page variables and functions -- **utility**: Playwright's selectors/actions need isolation from page code that might override `document.querySelector`, etc. - -### Element Handle Adoption - -When `page.$()` or `waitForSelector()` returns an element: -1. Query runs in **utility** world -2. Result is adopted to **main** world -3. User receives handle bound to main context - -## What Must Happen Before User Code Can Run - -For `page.evaluate()` or any DOM operation to work: - -1. **WebSocket connected** to CDP endpoint -2. **Browser.getVersion** returned successfully -3. **Target.setAutoAttach** sent and acknowledged -4. **Target.attachedToTarget** received for the page -5. **Runtime.enable** sent on page session -6. **Runtime.executionContextCreated** received for the frame -7. **ManualPromise resolved** for the appropriate world - -If any step is missing or delayed, `evaluate()` will hang waiting for the context promise. - -## Key Parameters Explained - -### Target.setAutoAttach Parameters - -| Parameter | Value | Purpose | -|-----------|-------|---------| -| `autoAttach` | `true` | Automatically attach to new targets | -| `waitForDebuggerOnStart` | `true` | Pause new targets until `Runtime.runIfWaitingForDebugger` | -| `flatten` | `true` | Use flat session IDs (not nested) | - -### Why waitForDebuggerOnStart? - -Without this, a page could start executing before Playwright injects its utility scripts. With it: -1. Chrome pauses the page immediately after creation -2. Playwright sends initialization commands -3. Playwright sends `Runtime.runIfWaitingForDebugger` -4. Page starts executing with Playwright ready - -## Implications for CDP Relay Implementation - -A CDP relay (like playwriter) must: - -1. **Forward Target.setAutoAttach** and return response before any targets attach -2. **Send Target.attachedToTarget** with correct `targetInfo` including `url` -3. **Forward Runtime.enable** and ensure `Runtime.executionContextCreated` follows -4. **Use correct sessionId** for page-specific commands/events -5. **Handle session lifecycle** - detach events when tabs close - -### Critical Timing - -The relay should ensure: -- `Target.attachedToTarget` is sent before any page commands -- `Runtime.executionContextCreated` is sent after `Runtime.enable` returns -- Events use the correct `sessionId` matching the target - -See `cdp-timing.md` for details on our relay's event synchronization. - -## When `context.on('page')` / `context.waitForEvent('page')` Fires - -Both `context.on('page')` and `context.waitForEvent('page')` listen to the **same -underlying event**. The difference is only in consumption pattern: - -| Aspect | `context.on('page', cb)` | `context.waitForEvent('page')` | -|--------|--------------------------|-------------------------------| -| Return | EventEmitter | Promise\ | -| Fires | Every time | First match only | -| Timeout | None | Built-in timeout | -| Predicate | Manual | Native `{ predicate: fn }` | -| Cleanup | Manual `off()` | Auto-removes listener | -| Close handling | Manual | Auto-rejects if context closes | - -The 'page' event on BrowserContext is NOT immediate after `Target.attachedToTarget`. -It requires page initialization to complete first. - -### Event Chain - -``` -Target.attachedToTarget (CDP event) - ↓ -CRPage created (crBrowser.ts) - ↓ -FrameSession._initialize() starts - ↓ -Parallel CDP commands sent: - - Page.enable - - Page.getFrameTree ← must complete - - Runtime.enable - - Log.enable - - Page.setLifecycleEventsEnabled - - Runtime.runIfWaitingForDebugger - ↓ -WAIT for _firstNonInitialNavigationCommittedPromise - ↓ -_initialize() resolves - ↓ -page.reportAsNew() called - ↓ -page._markInitialized() - ↓ -this.emitOnContext(BrowserContext.Events.Page, this) ← 'page' EVENT FIRES -``` - -### The Navigation Wait Condition - -The critical gate is `_firstNonInitialNavigationCommittedPromise`. The 'page' event -does NOT fire until: - -1. `Page.getFrameTree` returns the frame structure -2. **AND** one of: - - Page URL is not `:` (not an initial empty page) - - `Page.frameNavigated` event arrives for a non-initial navigation - -This ensures the page has a **valid URL and frame structure** before user code sees it. - -### Key Insights - -- `context.on('page')` and `context.waitForEvent('page')` fire on the **same event** -- **`waitForEvent('page')` only waits for NEW pages** - it will NOT resolve with existing pages -- To handle existing pages, check `context.pages()` first before waiting -- The 'page' event fires AFTER page initialization, not immediately on attach -- `Runtime.executionContextCreated` is NOT required for the 'page' event -- The 'page' event guarantees the page has a valid URL and frame tree -- `page.evaluate()` may still need to wait for execution context after 'page' fires - -### Handling Existing vs New Pages - -```typescript -// waitForEvent only catches NEW pages, not existing ones -const existingPages = context.pages(); -if (existingPages.length > 0) { - return existingPages[0]; // use existing -} -// Only wait if no pages exist yet -const newPage = await context.waitForEvent('page'); -``` - -### Timing Implications for CDP Relay - -For `context.on('page')` to fire, the relay must: - -1. Send `Target.attachedToTarget` with valid `targetInfo.url` -2. Respond to `Page.getFrameTree` with frame structure -3. Either: - - Provide a non-empty URL (not `:`) in the frame tree - - Or send `Page.frameNavigated` event after navigation - -## Empty URL Detection - -Empty URLs in `Target.attachedToTarget` cause Playwright to create broken pages that -never recover. Both the extension and relay log errors when this happens: - -### Extension logging - -```typescript -// In attachTab(), after Target.getTargetInfo -if (!targetInfo.url || targetInfo.url === '' || targetInfo.url === ':') { - logger.error('WARNING: Target.attachedToTarget will be sent with empty URL!') -} -``` - -### Relay logging - -The relay logs `logger.error` warnings when: -- `Target.attachedToTarget` is sent/received with empty URL -- `Target.targetCreated` is sent with empty URL - -### Why empty URLs break Playwright - -If `Target.attachedToTarget` is sent with empty URL: -- Playwright creates a broken page -- `_firstNonInitialNavigationCommittedPromise` may never resolve -- `page.evaluate()` hangs forever -- **No recovery possible** - -### Debugging empty URL issues - -Check the relay server logs (run `playwriter logfile` to get the path) for: -``` -WARNING: Target.attachedToTarget sent with empty URL -WARNING: Target.attachedToTarget received with empty URL -``` - -These indicate timing issues where the page hasn't fully loaded when attached.