playwriter
Code Issues

add showAriaRefLabels/hideAriaRefLabels for Vimium-style visual labels

Browse Source 
- getAriaSnapshot() discovers aria refs and caches ElementHandles
- showAriaRefLabels() overlays yellow badges on interactive elements
- hideAriaRefLabels() removes labels from page
- test captures screenshots from HN, Google, GitHub
This commit is contained in:
Tommy D. Rossi
2026-01-03 15:17:47 +01:00
parent 22a255e1dd
commit 802e6768e2
10 changed files with 2520 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/aria-ref-labels-plan.md
+352
View File
@@ -0,0 +1,352 @@
# Plan: Visual Aria Ref Labels (Vimium-style)
## Goal
Add functions to overlay visual ref labels on interactive elements, allowing agents to:
1. Take a screenshot with visible ref labels (e.g., `[e1]`, `[e2]`, `[e3]`)
2. See which elements are interactive and their ref IDs
3. Use `page.locator('aria-ref=e5')` to interact with labeled elements
## Reference Implementation
Based on `/Users/morse/Documents/GitHub/unframer-private/autofill-extension/src/content/`:
### gui.js - Container setup
```js
const guiRoot = document.createElement('div')
guiRoot.style = `
position: absolute;
left: 0;
top: 0;
width: 100%;
height: 100%;
z-index: 2147483647; // Max z-index to be on top of everything
pointer-events: none; // Don't block clicks on underlying elements
`
document.documentElement.appendChild(guiRoot)
```
### HintRenderer.tsx - Label positioning
```tsx
// Position label at element's top-left corner, accounting for scroll
left: `${window.scrollX + rect.left - 16}px`
top: `${window.scrollY + rect.top - 16}px` // 16px above element
```
### Vimium-style CSS
```css
font-family: Helvetica, Arial, sans-serif;
font-weight: bold;
font-size: 12px;
padding: 0px 2px;
background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#FFF785), color-stop(100%,#FFC542));
border: 1px solid #E3BE23;
border-radius: 4px;
color: black;
text-shadow: rgba(255, 255, 255, 0.6) 0px 1px 0px;
```
## Existing Code Context
### `getAriaSnapshot()` in `playwriter/src/aria-snapshot.ts`
Already implemented - discovers refs and provides lookup:
```ts
export async function getAriaSnapshot({ page }: { page: Page }): Promise<AriaSnapshotResult> {
// 1. Calls page._snapshotForAI() to populate Playwright's internal ref tracking
// 2. Probes aria-ref=e1, e2, e3... to discover all refs
// 3. Caches ElementHandles in refHandles array
// 4. Returns { snapshot, refToElement, refHandles, getRefsForLocators, ... }
}
```
The `refHandles` array contains `{ ref: string, handle: ElementHandle }` for each interactive element. We'll use these handles to get bounding boxes and create labels.
### Which elements get refs?
Playwright's `_snapshotForAI()` uses `mode: 'ai'` which sets `refs: 'interactable'`:
```ts
// From playwright/packages/injected/src/ariaSnapshot.ts
function computeAriaRef(ariaNode, options) {
if (options.refs === 'interactable' && (!ariaNode.box.visible || !ariaNode.receivesPointerEvents))
return; // Skip non-interactive elements
// ... assign ref like 'e1', 'e2', etc.
}
```
**Included:** buttons, links, inputs, textareas, selects, checkboxes, radios, sliders, any clickable element
**Excluded:** hidden elements, static text, decorative images, elements behind overlays
## API Design
```ts
// Show Vimium-style labels on all interactive elements
export async function showAriaRefLabels({ page }: { page: Page }): Promise<{
snapshot: string // The accessibility snapshot text
labelCount: number // Number of labels shown
}>
// Remove all labels from the page
export async function hideAriaRefLabels({ page }: { page: Page }): Promise<void>
```
## Implementation
Add to `playwriter/src/aria-snapshot.ts`:
```ts
const LABELS_CONTAINER_ID = '__playwriter_labels__'
// Use css`` tagged template for IDE syntax highlighting (identity function)
const css = String.raw
const LABEL_STYLES = css`
.__pw_label__ {
position: absolute;
font: bold 11px Helvetica, Arial, sans-serif;
padding: 1px 4px;
background: linear-gradient(to bottom, #FFF785 0%, #FFC542 100%);
border: 1px solid #E3BE23;
border-radius: 3px;
color: black;
text-shadow: 0 1px 0 rgba(255, 255, 255, 0.6);
white-space: nowrap;
}
`
const CONTAINER_STYLES = css`
position: absolute;
left: 0;
top: 0;
z-index: 2147483647;
pointer-events: none;
`
export async function showAriaRefLabels({ page }: { page: Page }) {
// Get snapshot and cached element handles
const { snapshot, refHandles } = await getAriaSnapshot({ page })
// Single evaluate call: create container, styles, and all labels
// Pass ElementHandles which get unwrapped to DOM elements in browser context
const labelCount = await page.evaluate(
({ refs, containerId, containerStyles, labelStyles }) => {
// Remove existing labels if present (idempotent)
document.getElementById(containerId)?.remove()
// Create container - absolute positioned, max z-index, no pointer events
const container = document.createElement('div')
container.id = containerId
container.style.cssText = containerStyles
// Inject Vimium-style CSS
const style = document.createElement('style')
style.textContent = labelStyles
container.appendChild(style)
// Create label for each interactive element
let count = 0
for (const { ref, element } of refs) {
const rect = element.getBoundingClientRect()
// Skip elements with no size (hidden)
if (rect.width === 0 || rect.height === 0) continue
const label = document.createElement('div')
label.className = '__pw_label__'
label.textContent = ref
// Position above element, accounting for scroll
// Use scrollX/scrollY so labels scroll with the page
label.style.left = `${scrollX + rect.left}px`
label.style.top = `${scrollY + Math.max(0, rect.top - 16)}px`
container.appendChild(label)
count++
}
document.documentElement.appendChild(container)
return count
},
{
refs: refHandles.map(({ ref, handle }) => ({ ref, element: handle })),
containerId: LABELS_CONTAINER_ID,
containerStyles: CONTAINER_STYLES,
labelStyles: LABEL_STYLES,
}
)
return { snapshot, labelCount }
}
export async function hideAriaRefLabels({ page }: { page: Page }) {
await page.evaluate(
(id) => document.getElementById(id)?.remove(),
LABELS_CONTAINER_ID
)
}
```
## Scroll Behavior
Labels use `position: absolute` with `scrollX/scrollY` offsets:
- Labels stay positioned on their elements when user scrolls
- Labels move with the document (not fixed to viewport)
- This matches Vimium's behavior
## Export from index.ts
```ts
export { getAriaSnapshot, showAriaRefLabels, hideAriaRefLabels } from './aria-snapshot.js'
```
## Test Implementation
Add to `playwriter/src/mcp.test.ts`:
```ts
it('should show aria ref labels and capture in screenshot', async () => {
const browserContext = getBrowserContext()
const serviceWorker = await getExtensionServiceWorker(browserContext)
// Create page with interactive elements
const page = await browserContext.newPage()
await page.setContent(`
<html>
<body style="padding: 50px; font-family: sans-serif;">
<h1>Test Page</h1>
<button id="submit-btn">Submit Form</button>
<a href="/about" style="margin-left: 20px;">About Us</a>
<input type="text" placeholder="Enter your name" style="margin-left: 20px;" />
<select style="margin-left: 20px;">
<option>Option 1</option>
<option>Option 2</option>
</select>
</body>
</html>
`)
await page.bringToFront()
// Enable extension for this tab
await serviceWorker.evaluate(async () => {
await globalThis.toggleExtensionForActiveTab()
})
await new Promise(r => setTimeout(r, 400))
// Connect via CDP
const browser = await chromium.connectOverCDP(getCdpUrl({ port: TEST_PORT }))
const cdpPage = browser.contexts()[0].pages().find(p => p.url().startsWith('about:'))
expect(cdpPage).toBeDefined()
// Import and show labels
const { showAriaRefLabels, hideAriaRefLabels } = await import('./aria-snapshot.js')
const { snapshot, labelCount } = await showAriaRefLabels({ page: cdpPage! })
console.log('Snapshot:', snapshot.slice(0, 200))
console.log('Label count:', labelCount)
expect(labelCount).toBeGreaterThan(0)
expect(snapshot).toContain('button')
expect(snapshot).toContain('link')
// Verify labels are in DOM
const labelElements = await cdpPage!.evaluate(() =>
document.querySelectorAll('.__pw_label__').length
)
expect(labelElements).toBe(labelCount)
// Verify label text matches refs
const labelTexts = await cdpPage!.evaluate(() =>
Array.from(document.querySelectorAll('.__pw_label__')).map(el => el.textContent)
)
expect(labelTexts.every(t => /^e\d+$/.test(t!))).toBe(true)
console.log('Label texts:', labelTexts)
// Take screenshot with labels visible
const screenshot = await cdpPage!.screenshot({ type: 'png' })
expect(screenshot.length).toBeGreaterThan(5000) // Not empty
// Save screenshot for visual verification
const fs = await import('node:fs')
const path = await import('node:path')
const snapshotsDir = path.join(path.dirname(new URL(import.meta.url).pathname), 'snapshots')
if (!fs.existsSync(snapshotsDir)) fs.mkdirSync(snapshotsDir, { recursive: true })
fs.writeFileSync(path.join(snapshotsDir, 'aria-labels-screenshot.png'), screenshot)
console.log('Screenshot saved to snapshots/aria-labels-screenshot.png')
// Test that labels can be used to find elements
const buttonLabel = labelTexts.find(t => t === 'e2') // Typically button is e2
if (buttonLabel) {
const buttonViaRef = cdpPage!.locator(`aria-ref=${buttonLabel}`)
const buttonText = await buttonViaRef.textContent()
console.log(`Button via ${buttonLabel}:`, buttonText)
}
// Cleanup
await hideAriaRefLabels({ page: cdpPage! })
// Verify labels removed
const labelsAfterHide = await cdpPage!.evaluate(() =>
document.getElementById('__playwriter_labels__')
)
expect(labelsAfterHide).toBeNull()
await browser.close()
await page.close()
}, 60000)
```
## Visual Result
```
┌─────────────────────────────────────────────────────────────────┐
│ │
│ Test Page │
│ │
│ [e2] [e3] [e4] [e5] │
│ ┌──────────────┐ ┌──────────┐ ┌───────────────┐ ┌────────┐ │
│ │ Submit Form │ │ About Us │ │ Enter name... │ │Option 1│ │
│ └──────────────┘ └──────────┘ └───────────────┘ └────────┘ │
│ button link textbox combobox │
│ │
└─────────────────────────────────────────────────────────────────┘
```
Yellow Vimium-style badges positioned 16px above each interactive element.
Agent sees screenshot → identifies `[e3]` is the "About Us" link → uses `page.locator('aria-ref=e3').click()`
## Files to Modify
| File | Change |
|------|--------|
| `playwriter/src/aria-snapshot.ts` | Add `showAriaRefLabels()`, `hideAriaRefLabels()`, `LABELS_CONTAINER_ID` constant |
| `playwriter/src/index.ts` | Add exports: `showAriaRefLabels`, `hideAriaRefLabels` |
| `playwriter/src/mcp.test.ts` | Add screenshot test |
| `playwriter/src/snapshots/` | Directory for test screenshots (gitignored) |
## Usage Example (Agent Workflow)
```ts
// 1. Show labels on page
const { snapshot, labelCount } = await showAriaRefLabels({ page })
console.log(`Found ${labelCount} interactive elements`)
// 2. Take screenshot - labels are visible
const screenshot = await page.screenshot()
// Agent analyzes screenshot, sees [e5] on "Submit" button
// 3. Interact using ref
await page.locator('aria-ref=e5').click()
// 4. Hide labels when done
await hideAriaRefLabels({ page })
```
## Edge Cases to Handle
1. **Elements near top edge** - Label would go off-screen, position below element instead
2. **Overlapping labels** - Could add offset, but keep simple for now
3. **Very long pages** - Labels use absolute positioning, works with scroll
4. **Dynamic content** - Call `showAriaRefLabels` again to refresh after DOM changes
5. **Iframes** - Current implementation only handles main frame (could extend later)
playwriter/src/aria-snapshot.ts
+237
View File
@@ -0,0 +1,237 @@
import type { Page, Locator, ElementHandle } from 'playwright-core'
export interface AriaRef {
role: string
name: string
ref: string
}
export interface AriaSnapshotResult {
snapshot: string
refToElement: Map<string, { role: string; name: string }>
refHandles: Array<{ ref: string; handle: ElementHandle }>
getRefsForLocators: (locators: Array<Locator | ElementHandle>) => Promise<Array<AriaRef | null>>
getRefForLocator: (locator: Locator | ElementHandle) => Promise<AriaRef | null>
getRefStringForLocator: (locator: Locator | ElementHandle) => Promise<string | null>
}
const LABELS_CONTAINER_ID = '__playwriter_labels__'
// Use String.raw for CSS syntax highlighting in editors
const css = String.raw
const LABEL_STYLES = css`
.__pw_label__ {
position: absolute;
font: bold 11px Helvetica, Arial, sans-serif;
padding: 1px 4px;
background: linear-gradient(to bottom, #FFF785 0%, #FFC542 100%);
border: 1px solid #E3BE23;
border-radius: 3px;
color: black;
text-shadow: 0 1px 0 rgba(255, 255, 255, 0.6);
white-space: nowrap;
}
`
const CONTAINER_STYLES = css`
position: absolute;
left: 0;
top: 0;
z-index: 2147483647;
pointer-events: none;
`
/**
* Get an accessibility snapshot with utilities to look up aria refs for elements.
* Uses Playwright's internal aria-ref selector engine.
*
* @example
* ```ts
* const { snapshot, getRefsForLocators } = await getAriaSnapshot({ page })
* const refs = await getRefsForLocators([page.locator('button'), page.locator('a')])
* // refs[0].ref is e.g. "e5" - use page.locator('aria-ref=e5') to select
* ```
*/
export async function getAriaSnapshot({ page }: { page: Page }): Promise<AriaSnapshotResult> {
const snapshotMethod = (page as any)._snapshotForAI
if (!snapshotMethod) {
throw new Error('_snapshotForAI not available. Ensure you are using Playwright.')
}
const snapshot = await snapshotMethod.call(page)
const snapshotStr = typeof snapshot === 'string' ? snapshot : (snapshot.full || JSON.stringify(snapshot, null, 2))
// Discover refs by probing aria-ref=e1, e2, e3... until 10 consecutive misses
const refToElement = new Map<string, { role: string; name: string }>()
const refHandles: Array<{ ref: string; handle: ElementHandle }> = []
let consecutiveMisses = 0
let refNum = 1
while (consecutiveMisses < 10) {
const ref = `e${refNum++}`
try {
const locator = page.locator(`aria-ref=${ref}`)
if (await locator.count() === 1) {
consecutiveMisses = 0
const [info, handle] = await Promise.all([
locator.evaluate((el: any) => ({
role: el.getAttribute('role') || {
a: el.hasAttribute('href') ? 'link' : 'generic',
button: 'button', input: { button: 'button', checkbox: 'checkbox', radio: 'radio',
text: 'textbox', search: 'searchbox', number: 'spinbutton', range: 'slider',
}[el.type] || 'textbox', select: 'combobox', textarea: 'textbox', img: 'img',
nav: 'navigation', main: 'main', header: 'banner', footer: 'contentinfo',
}[el.tagName.toLowerCase()] || 'generic',
name: el.getAttribute('aria-label') || el.textContent?.trim() || el.placeholder || '',
})),
locator.elementHandle({ timeout: 1000 }),
])
refToElement.set(ref, info)
if (handle) {
refHandles.push({ ref, handle })
}
} else {
consecutiveMisses++
}
} catch {
consecutiveMisses++
}
}
// Find refs for multiple locators in a single evaluate call
const getRefsForLocators = async (locators: Array<Locator | ElementHandle>): Promise<Array<AriaRef | null>> => {
if (locators.length === 0 || refHandles.length === 0) {
return locators.map(() => null)
}
const targetHandles = await Promise.all(
locators.map(async (loc) => {
try {
return 'elementHandle' in loc
? await (loc as Locator).elementHandle({ timeout: 1000 })
: (loc as ElementHandle)
} catch {
return null
}
})
)
const matchingRefs = await page.evaluate(
({ targets, candidates }) => targets.map((target) => {
if (!target) return null
return candidates.find(({ element }) => element === target)?.ref ?? null
}),
{ targets: targetHandles, candidates: refHandles.map(({ ref, handle }) => ({ ref, element: handle })) }
)
return matchingRefs.map((ref) => {
if (!ref) return null
const info = refToElement.get(ref)
return info ? { ...info, ref } : null
})
}
return {
snapshot: snapshotStr,
refToElement,
refHandles,
getRefsForLocators,
getRefForLocator: async (loc) => (await getRefsForLocators([loc]))[0],
getRefStringForLocator: async (loc) => (await getRefsForLocators([loc]))[0]?.ref ?? null,
}
}
/**
* Show Vimium-style labels on all interactive elements.
* Labels are yellow badges positioned above each element showing the aria ref (e.g., "e1", "e2").
* Use with screenshots so agents can see which elements are interactive.
*
* @example
* ```ts
* const { snapshot, labelCount } = await showAriaRefLabels({ page })
* const screenshot = await page.screenshot()
* // Agent sees [e5] label on "Submit" button
* await page.locator('aria-ref=e5').click()
* await hideAriaRefLabels({ page })
* ```
*/
export async function showAriaRefLabels({ page }: { page: Page }): Promise<{
snapshot: string
labelCount: number
}> {
const { snapshot, refHandles } = await getAriaSnapshot({ page })
// Single evaluate call: create container, styles, and all labels
// ElementHandles get unwrapped to DOM elements in browser context
// Using 'any' types here since this code runs in browser context
const labelCount = await page.evaluate(
({ refs, containerId, containerStyles, labelStyles }: {
refs: Array<{ ref: string; element: { getBoundingClientRect(): { width: number; height: number; left: number; top: number } } }>
containerId: string
containerStyles: string
labelStyles: string
}) => {
const doc = (globalThis as any).document
const win = globalThis as any
// Remove existing labels if present (idempotent)
doc.getElementById(containerId)?.remove()
// Create container - absolute positioned, max z-index, no pointer events
const container = doc.createElement('div')
container.id = containerId
container.style.cssText = containerStyles
// Inject Vimium-style CSS
const style = doc.createElement('style')
style.textContent = labelStyles
container.appendChild(style)
// Create label for each interactive element
let count = 0
for (const { ref, element } of refs) {
const rect = element.getBoundingClientRect()
// Skip elements with no size (hidden)
if (rect.width === 0 || rect.height === 0) {
continue
}
const label = doc.createElement('div')
label.className = '__pw_label__'
label.textContent = ref
// Position above element, accounting for scroll
// Use scrollX/scrollY so labels scroll with the page
label.style.left = `${win.scrollX + rect.left}px`
label.style.top = `${win.scrollY + Math.max(0, rect.top - 16)}px`
container.appendChild(label)
count++
}
doc.documentElement.appendChild(container)
return count
},
{
refs: refHandles.map(({ ref, handle }) => ({ ref, element: handle })),
containerId: LABELS_CONTAINER_ID,
containerStyles: CONTAINER_STYLES,
labelStyles: LABEL_STYLES,
}
)
return { snapshot, labelCount }
}
/**
* Remove all aria ref labels from the page.
*/
export async function hideAriaRefLabels({ page }: { page: Page }): Promise<void> {
await page.evaluate((id) => {
const doc = (globalThis as any).document
doc.getElementById(id)?.remove()
}, LABELS_CONTAINER_ID)
}
playwriter/src/assets/aria-labels-github-snapshot.txt
+605
View File
@@ -0,0 +1,605 @@
- generic [ref=e2]:
- region
- generic:
- link "Skip to content" [ref=e3] [cursor=pointer]:
- /url: "#start-of-content"
- banner [ref=e5]:
- heading "Navigation Menu" [level=2] [ref=e6]
- generic [ref=e7]:
- link "Homepage" [ref=e9] [cursor=pointer]:
- /url: /
- img [ref=e10]
- generic [ref=e12]:
- navigation "Global" [ref=e15]:
- list [ref=e16]:
- listitem [ref=e17]:
- button "Platform" [ref=e19] [cursor=pointer]:
- text: Platform
- img [ref=e20]
- listitem [ref=e22]:
- button "Solutions" [ref=e24] [cursor=pointer]:
- text: Solutions
- img [ref=e25]
- listitem [ref=e27]:
- button "Resources" [ref=e29] [cursor=pointer]:
- text: Resources
- img [ref=e30]
- listitem [ref=e32]:
- button "Open Source" [ref=e34] [cursor=pointer]:
- text: Open Source
- img [ref=e35]
- listitem [ref=e37]:
- button "Enterprise" [ref=e39] [cursor=pointer]:
- text: Enterprise
- img [ref=e40]
- listitem [ref=e42]:
- link "Pricing" [ref=e43] [cursor=pointer]:
- /url: https://github.com/pricing
- generic [ref=e44]: Pricing
- generic [ref=e45]:
- button "Search or jump to…" [ref=e48] [cursor=pointer]:
- img [ref=e50]
- link "Sign in" [ref=e53] [cursor=pointer]:
- /url: /login
- link "Sign up" [ref=e54] [cursor=pointer]:
- /url: /signup?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2F&source=header-home
- main [ref=e57]:
- generic [ref=e61]:
- generic [ref=e62]:
- generic [ref=e63]:
- generic [ref=e66]:
- img [ref=e68]
- img [ref=e79]
- generic [ref=e98]: Mona the Octocat rises upward from behind the GitHub product demo accompanied by a purple glow and some clouds. A white butterfly gently flutters in from the right and lands on Mona's nose.
- generic [ref=e99]:
- link "Explore the latest tools from Universe '25" [ref=e100] [cursor=pointer]:
- /url: /events/universe/recap
- generic [ref=e101]:
- img [ref=e104]
- generic [ref=e108]: Explore the latest tools from Universe '25
- img [ref=e111]
- region "The future of building happens together" [ref=e113]:
- generic [ref=e116]:
- heading "The future of building happens together" [level=1] [ref=e117]
- paragraph [ref=e118]: Tools and trends evolve, but collaboration endures. With GitHub, developers, agents, and code come together on one platform.
- generic [ref=e119]:
- form "Sign up for GitHub" [ref=e120]:
- generic [ref=e122]:
- generic [ref=e123]:
- generic [ref=e124]: Enter your email
- textbox "Enter your email" [ref=e126]:
- /placeholder: you@domain.com
- button "Sign up for GitHub" [ref=e127] [cursor=pointer]:
- generic [ref=e128]: Sign up for GitHub
- link "Try GitHub Copilot free" [ref=e129] [cursor=pointer]:
- /url: /github-copilot/pro
- generic [ref=e130]: Try GitHub Copilot free
- generic [ref=e132]:
- heading "GitHub features" [level=2] [ref=e133]
- button "Pause" [ref=e134] [cursor=pointer]
- generic [ref=e139]: A demonstration animation of a code editor using GitHub Copilot Chat, where the user requests GitHub Copilot to refactor duplicated logic and extract it into a reusable function for a given code snippet.
- generic [ref=e141]:
- tablist [ref=e143]:
- generic [ref=e144]:
- tab "Code" [ref=e146] [cursor=pointer]
- tab "Plan" [ref=e147] [cursor=pointer]
- tab "Collaborate" [ref=e148] [cursor=pointer]
- tab "Automate" [ref=e149] [cursor=pointer]
- tab "Secure" [ref=e150] [cursor=pointer]
- region [ref=e151]: Write, test, and fix code quickly with GitHub Copilot, from simple boilerplate to complex features.
- generic [ref=e153]:
- heading "GitHub customers" [level=2] [ref=e154]
- generic [ref=e155]:
- generic [ref=e156]:
- generic [ref=e157]:
- img "American Airlines" [ref=e158]
- img "Duolingo" [ref=e161]
- img "Ernst and Young" [ref=e163]
- img "Ford" [ref=e167]
- img "InfoSys" [ref=e170]
- img "Mercado Libre" [ref=e173]
- img "Mercedes-Benz" [ref=e189]
- img "Shopify" [ref=e192]
- img "Philips" [ref=e204]
- img "Société Générale" [ref=e207]
- img "Spotify" [ref=e225]
- img "Vodafone" [ref=e228]
- generic [ref=e239]:
- img [ref=e240]
- img [ref=e243]
- img [ref=e245]
- img [ref=e249]
- img [ref=e252]
- img [ref=e255]
- img [ref=e271]
- img [ref=e274]
- img [ref=e286]
- img [ref=e289]
- img [ref=e307]
- img [ref=e310]
- button "Pause animation" [ref=e321] [cursor=pointer]:
- img [ref=e322]
- generic [ref=e325]:
- generic [ref=e327]:
- generic [ref=e333]:
- heading "Accelerate your entire workflow" [level=2] [ref=e334]:
- generic [ref=e335]: Accelerate your entire workflow
- paragraph [ref=e336]: From your first line of code to final deployment, GitHub provides AI and automation tools to help you build and ship better software faster.
- generic [ref=e338]:
- button "Pause video" [ref=e343] [cursor=pointer]
- generic [ref=e349]: A Copilot chat window with the 'Ask' mode enabled. The user switches from 'Ask' mode to 'Agent' mode from a dropdown menu, then sends the prompt 'Update the website to allow searching for running races by name.' Copilot analyzes the codebase, then explains the required edits for three files before generating them. Copilot then confirms completion and summarizes the implemented changes for the new functionality allowing users to search races by name and view paginated, filtered results.
- generic [ref=e353]:
- generic [ref=e355]:
- heading "Your AI partner everywhere. Copilot is ready to work with you at each step of the software development lifecycle." [level=3] [ref=e356]:
- generic [ref=e357]: Your AI partner everywhere. Copilot is ready to work with you at each step of the software development lifecycle.
- link "Explore GitHub Copilot" [ref=e359] [cursor=pointer]:
- /url: /features/copilot
- generic [ref=e360]: Explore GitHub Copilot
- img [ref=e361]
- generic [ref=e363]:
- generic [ref=e365]:
- paragraph [ref=e366]: Duolingo boosts developer speed by 25% with GitHub Copilot
- link "Read customer story" [ref=e367] [cursor=pointer]:
- /url: /customer-stories/duolingo
- generic [ref=e368]: Read customer story
- img [ref=e369]
- generic [ref=e372]:
- paragraph [ref=e373]: 2025 Gartner® Magic Quadrant™ for AI Code Assistants
- link "Read industry report" [ref=e374] [cursor=pointer]:
- /url: https://www.gartner.com/reprints/?id=1-2LVTG7RP&ct=250915&st=sb
- generic [ref=e375]: Read industry report
- img [ref=e376]
- generic [ref=e383]:
- generic [ref=e384]:
- term [ref=e385]:
- button "Automate your path to production" [expanded] [ref=e386]:
- generic [ref=e387]:
- heading "Automate your path to production" [level=3] [ref=e388]
- img [ref=e389]
- definition [ref=e391]:
- generic [ref=e393]:
- paragraph [ref=e394]: Ship faster with secure, reliable CI/CD.
- link "Explore GitHub Actions" [ref=e396] [cursor=pointer]:
- /url: /features/actions
- generic [ref=e397]: Explore GitHub Actions
- img [ref=e398]
- generic [ref=e400]:
- term [ref=e401]:
- button "Code instantly from anywhere" [ref=e402] [cursor=pointer]:
- generic [ref=e403]:
- heading "Code instantly from anywhere" [level=3] [ref=e404]
- img [ref=e405]
- paragraph [ref=e407]: Launch a full, cloud-based development environment in seconds.
- link [ref=e409] [cursor=pointer]:
- /url: /features/codespaces
- generic [ref=e410]: Explore GitHub Codespaces
- img [ref=e411]
- generic [ref=e413]:
- term [ref=e414]:
- button "Keep momentum on the go" [ref=e415] [cursor=pointer]:
- generic [ref=e416]:
- heading "Keep momentum on the go" [level=3] [ref=e417]
- img [ref=e418]
- paragraph [ref=e420]: Manage projects and assign tasks to Copilot, all from your mobile device.
- link [ref=e422] [cursor=pointer]:
- /url: /mobile
- generic [ref=e423]: Explore GitHub Mobile
- img [ref=e424]
- generic [ref=e426]:
- term [ref=e427]:
- button "Shape your toolchain" [ref=e428] [cursor=pointer]:
- generic [ref=e429]:
- heading "Shape your toolchain" [level=3] [ref=e430]
- img [ref=e431]
- paragraph [ref=e433]: Extend your stack with apps, actions, and AI models.
- link [ref=e435] [cursor=pointer]:
- /url: /marketplace
- generic [ref=e436]: Explore GitHub Marketplace
- img [ref=e437]
- generic [ref=e439]:
- generic [ref=e441]:
- generic [ref=e447]:
- heading "Built-in application security where found means fixed" [level=2] [ref=e448]:
- generic [ref=e449]: Built-in application security where found means fixed
- paragraph [ref=e450]: Use AI to find and fix vulnerabilities so your team can ship more secure software faster.
- generic [ref=e456]:
- generic [ref=e457]:
- heading "Apply fixes in seconds. Spend less time debugging and more time building features with Copilot Autofix." [level=3] [ref=e458]
- link "Explore GitHub Advanced Security" [ref=e460] [cursor=pointer]:
- /url: /security/advanced-security
- generic [ref=e461]: Explore GitHub Advanced Security
- img [ref=e462]
- img "Copilot Autofix identifies vulnerable code and provides an explanation, together with a secure code suggestion to remediate the vulnerability." [ref=e467]
- generic [ref=e469]:
- generic [ref=e470]:
- generic [ref=e472]:
- generic [ref=e473]:
- paragraph [ref=e474]:
- generic [ref=e475]: Security debt, solved. Leverage security campaigns and Copilot Autofix to reduce application vulnerabilities.
- link "Learn about GitHub Code Security" [ref=e476] [cursor=pointer]:
- /url: /security/advanced-security/code-security
- generic [ref=e477]: Learn about GitHub Code Security
- img [ref=e478]
- generic [ref=e480]:
- img "A security campaign screen displays the campaign’s progress bar with 97% completed of 701 alerts. A total of 23 alerts are left with 13 in progress, and the campaign started 20 days ago. The status below shows that there are 7 days left in the campaign with a due date of November 15, 2024."
- generic [ref=e482]:
- generic [ref=e483]:
- paragraph [ref=e484]:
- generic [ref=e485]: Dependencies you can depend on. Update vulnerable dependencies with supported fixes for breaking changes.
- link "Learn about Dependabot" [ref=e486] [cursor=pointer]:
- /url: /security/advanced-security/software-supply-chain
- generic [ref=e487]: Learn about Dependabot
- img [ref=e488]
- generic [ref=e490]:
- img "List of dependencies defined in a requirements .txt file."
- generic [ref=e492]:
- generic [ref=e493]:
- paragraph [ref=e494]:
- generic [ref=e495]: Your secrets, your business. Detect, prevent, and remediate leaked secrets across your organization.
- link "Learn about GitHub Secret Protection" [ref=e496] [cursor=pointer]:
- /url: /security/advanced-security/secret-protection
- generic [ref=e497]: Learn about GitHub Secret Protection
- img [ref=e498]
- generic [ref=e500]:
- img "GitHub push protection confirms and displays an active secret, and blocks the push."
- generic [ref=e501]:
- paragraph [ref=e505]:
- text: 70% MTTR reduction
- generic [ref=e507]:
- text: with Copilot Autofix
- superscript [ref=e508]:
- link "Footnote 1" [ref=e509] [cursor=pointer]:
- /url: "#footnote-1"
- text: "1"
- paragraph [ref=e513]:
- text: 8.3M secret leaks stopped
- generic [ref=e515]:
- text: in the past 12 months with push protection
- superscript [ref=e516]:
- link "Footnote 1" [ref=e517] [cursor=pointer]:
- /url: "#footnote-1"
- text: "1"
- generic [ref=e519]:
- generic [ref=e521]:
- generic [ref=e527]:
- heading "Work together, achieve more" [level=2] [ref=e528]:
- generic [ref=e529]: Work together, achieve more
- paragraph [ref=e530]: From planning and discussion to code review, GitHub keeps your team’s conversation and context next to your code.
- img "A project management dashboard showing tasks for the ‘OctoArcade Invaders’ project, with tasks grouped under project phase categories like ‘Prototype,’ ‘Beta,’ and ‘Launch’ in a table layout. One of the columns displays sub-issue progress bars with percentages for each issue." [ref=e541]
- generic [ref=e544]:
- generic [ref=e546]:
- heading "Plan with clarity. Organize everything from high-level roadmaps to everyday tasks." [level=3] [ref=e547]:
- generic [ref=e548]: Plan with clarity. Organize everything from high-level roadmaps to everyday tasks.
- link "Explore GitHub Projects" [ref=e550] [cursor=pointer]:
- /url: /features/issues
- generic [ref=e551]: Explore GitHub Projects
- img [ref=e552]
- figure [ref=e556]:
- generic [ref=e557]: “
- blockquote [ref=e558]:
- generic [ref=e559]: It helps us onboard new software engineers and get them productive right away. We have all our source code, issues, and pull requests in one place... GitHub is a complete platform that frees us from menial tasks and enables us to do our best work.
- generic [ref=e561]:
- generic [ref=e562]: Fabian Faulhaber
- generic [ref=e563]: Application manager at Mercedes-Benz
- generic [ref=e569]:
- generic [ref=e570]:
- term [ref=e571]:
- button "Keep track of your tasks" [expanded] [ref=e572]:
- generic [ref=e573]:
- heading "Keep track of your tasks" [level=3] [ref=e574]
- img [ref=e575]
- definition [ref=e577]:
- generic [ref=e579]:
- paragraph [ref=e580]: Create issues and manage projects with tools that adapt to your code.
- link "Explore GitHub Issues" [ref=e582] [cursor=pointer]:
- /url: /features/issues
- generic [ref=e583]: Explore GitHub Issues
- img [ref=e584]
- generic [ref=e586]:
- term [ref=e587]:
- button "Share ideas and ask questions" [ref=e588] [cursor=pointer]:
- generic [ref=e589]:
- heading "Share ideas and ask questions" [level=3] [ref=e590]
- img [ref=e591]
- paragraph [ref=e593]: Create space for open-ended conversations alongside your project.
- link [ref=e595] [cursor=pointer]:
- /url: /features/discussions
- generic [ref=e596]: Explore GitHub Discussions
- img [ref=e597]
- generic [ref=e599]:
- term [ref=e600]:
- button "Review code changes together" [ref=e601] [cursor=pointer]:
- generic [ref=e602]:
- heading "Review code changes together" [level=3] [ref=e603]
- img [ref=e604]
- paragraph [ref=e606]: Assign initial reviews to Copilot for greater speed and quality.
- link [ref=e608] [cursor=pointer]:
- /url: /features/code-review
- generic [ref=e609]: Explore code review
- img [ref=e610]
- generic [ref=e612]:
- term [ref=e613]:
- button "Fund open source projects" [ref=e614] [cursor=pointer]:
- generic [ref=e615]:
- heading "Fund open source projects" [level=3] [ref=e616]
- img [ref=e617]
- paragraph [ref=e619]: Become an open source partner and support the tools and libraries that power your work.
- link [ref=e621] [cursor=pointer]:
- /url: /sponsors
- generic [ref=e622]: Explore GitHub Sponsors
- img [ref=e623]
- generic [ref=e625]:
- generic [ref=e626]:
- heading "From startups to enterprises, GitHub scales with teams of any size in any industry." [level=2] [ref=e635]:
- generic [ref=e636]: From startups to enterprises, GitHub scales with teams of any size in any industry.
- tablist [ref=e640]:
- generic [ref=e641]:
- tab "By industry" [ref=e643] [cursor=pointer]
- tab "By size" [ref=e644] [cursor=pointer]
- tab "By use case" [ref=e645] [cursor=pointer]
- separator [ref=e647]
- generic [ref=e650]:
- link "Technology Figma streamlines development and strengthens security Read customer story" [ref=e652] [cursor=pointer]:
- /url: /customer-stories/figma
- generic [ref=e657]:
- generic [ref=e658]: Technology
- paragraph [ref=e659]: Figma streamlines development and strengthens security
- generic [ref=e661]:
- text: Read customer story
- img [ref=e662]
- link "Automotive Mercedes-Benz standardizes source code and automates onboarding Read customer story" [ref=e665] [cursor=pointer]:
- /url: /customer-stories/mercedes-benz
- generic [ref=e670]:
- generic [ref=e671]: Automotive
- paragraph [ref=e672]: Mercedes-Benz standardizes source code and automates onboarding
- generic [ref=e674]:
- text: Read customer story
- img [ref=e675]
- link "Financial services Mercado Libre cuts coding time by 50% Read customer story" [ref=e678] [cursor=pointer]:
- /url: /customer-stories/mercado-libre
- generic [ref=e683]:
- generic [ref=e684]: Financial services
- paragraph [ref=e685]: Mercado Libre cuts coding time by 50%
- generic [ref=e687]:
- text: Read customer story
- img [ref=e688]
- generic [ref=e693]:
- link "Explore customer stories" [ref=e695] [cursor=pointer]:
- /url: /customer-stories
- generic [ref=e696]: Explore customer stories
- img [ref=e697]
- separator [ref=e699]
- link "View all solutions" [ref=e701] [cursor=pointer]:
- /url: /solutions
- generic [ref=e702]: View all solutions
- img [ref=e703]
- generic [ref=e706]:
- generic [ref=e709]:
- generic [ref=e710]: A subtle purple glow fades in as Mona the Octocat, Copilot, and Ducky dramatically fall into place next to one another while gazing optimistically into the distance. To celebrate Universe 2025, a butterfly flies in from the right to gently land on Mona's nose, Ducky has a flower on their head, and Copilot has a bright star reflecting in their goggles.
- img [ref=e717]
- img [ref=e719]
- img [ref=e721]
- generic [ref=e743]:
- heading "Millions of developers and businesses call GitHub home" [level=2] [ref=e744]
- paragraph [ref=e745]: Whether you’re scaling your development process or just learning how to code, GitHub is where you belong. Join the world’s most widely adopted developer platform to build the technologies that shape what’s next.
- generic [ref=e746]:
- form "Sign up for GitHub" [ref=e747]:
- generic [ref=e749]:
- generic [ref=e750]:
- generic [ref=e751]: Enter your email
- textbox "Enter your email" [ref=e753]:
- /placeholder: you@domain.com
- button "Sign up for GitHub" [ref=e754] [cursor=pointer]:
- generic [ref=e755]: Sign up for GitHub
- link "Try GitHub Copilot free" [ref=e756] [cursor=pointer]:
- /url: /github-copilot/pro
- generic [ref=e757]: Try GitHub Copilot free
- generic [ref=e759]:
- heading "Footnotes" [level=2] [ref=e760]
- list [ref=e761]:
- listitem [ref=e762]:
- paragraph [ref=e763]:
- text: GitHub internal customer data, 2025.
- link "Back to content" [ref=e764] [cursor=pointer]:
- /url: "#footnote-1-ref-0"
- img [ref=e765]
- link "Back to top" [ref=e767] [cursor=pointer]:
- /url: "#hero"
- img [ref=e768]
- contentinfo [ref=e770]:
- heading "Site-wide Links" [level=2] [ref=e771]
- generic [ref=e773]:
- generic [ref=e774]:
- link "Go to GitHub homepage" [ref=e775] [cursor=pointer]:
- /url: /
- img [ref=e776]
- heading "Subscribe to our developer newsletter" [level=3] [ref=e780]
- paragraph [ref=e781]: Get tips, technical guides, and best practices. Twice a month.
- link "Subscribe" [ref=e782] [cursor=pointer]:
- /url: https://github.com/newsletter
- navigation "Platform" [ref=e783]:
- heading "Platform" [level=3] [ref=e784]
- list [ref=e785]:
- listitem [ref=e786]:
- link "Features" [ref=e787] [cursor=pointer]:
- /url: /features
- listitem [ref=e788]:
- link "Enterprise" [ref=e789] [cursor=pointer]:
- /url: /enterprise
- listitem [ref=e790]:
- link "Copilot" [ref=e791] [cursor=pointer]:
- /url: /features/copilot
- listitem [ref=e792]:
- link "AI" [ref=e793] [cursor=pointer]:
- /url: /features/ai
- listitem [ref=e794]:
- link "Security" [ref=e795] [cursor=pointer]:
- /url: /security
- listitem [ref=e796]:
- link "Pricing" [ref=e797] [cursor=pointer]:
- /url: /pricing
- listitem [ref=e798]:
- link "Team" [ref=e799] [cursor=pointer]:
- /url: /team
- listitem [ref=e800]:
- link "Resources" [ref=e801] [cursor=pointer]:
- /url: https://resources.github.com
- listitem [ref=e802]:
- link "Roadmap" [ref=e803] [cursor=pointer]:
- /url: https://github.com/github/roadmap
- listitem [ref=e804]:
- link "Compare GitHub" [ref=e805] [cursor=pointer]:
- /url: https://resources.github.com/devops/tools/compare
- navigation "Ecosystem" [ref=e806]:
- heading "Ecosystem" [level=3] [ref=e807]
- list [ref=e808]:
- listitem [ref=e809]:
- link "Developer API" [ref=e810] [cursor=pointer]:
- /url: https://docs.github.com/get-started/exploring-integrations/about-building-integrations
- listitem [ref=e811]:
- link "Partners" [ref=e812] [cursor=pointer]:
- /url: https://partner.github.com
- listitem [ref=e813]:
- link "Education" [ref=e814] [cursor=pointer]:
- /url: https://github.com/edu
- listitem [ref=e815]:
- link "GitHub CLI" [ref=e816] [cursor=pointer]:
- /url: https://cli.github.com
- listitem [ref=e817]:
- link "GitHub Desktop" [ref=e818] [cursor=pointer]:
- /url: https://desktop.github.com
- listitem [ref=e819]:
- link "GitHub Mobile" [ref=e820] [cursor=pointer]:
- /url: https://github.com/mobile
- listitem [ref=e821]:
- link "GitHub Marketplace" [ref=e822] [cursor=pointer]:
- /url: https://github.com/marketplace
- listitem [ref=e823]:
- link "MCP Registry" [ref=e824] [cursor=pointer]:
- /url: https://github.com/mcp
- navigation "Support" [ref=e825]:
- heading "Support" [level=3] [ref=e826]
- list [ref=e827]:
- listitem [ref=e828]:
- link "Docs" [ref=e829] [cursor=pointer]:
- /url: https://docs.github.com
- listitem [ref=e830]:
- link "Community Forum" [ref=e831] [cursor=pointer]:
- /url: https://github.community
- listitem [ref=e832]:
- link "Professional Services" [ref=e833] [cursor=pointer]:
- /url: https://services.github.com
- listitem [ref=e834]:
- link "Premium Support" [ref=e835] [cursor=pointer]:
- /url: /enterprise/premium-support
- listitem [ref=e836]:
- link "Skills" [ref=e837] [cursor=pointer]:
- /url: https://skills.github.com
- listitem [ref=e838]:
- link "Status" [ref=e839] [cursor=pointer]:
- /url: https://www.githubstatus.com
- listitem [ref=e840]:
- link "Contact GitHub" [ref=e841] [cursor=pointer]:
- /url: https://support.github.com?tags=dotcom-footer
- navigation "Company" [ref=e842]:
- heading "Company" [level=3] [ref=e843]
- list [ref=e844]:
- listitem [ref=e845]:
- link "About" [ref=e846] [cursor=pointer]:
- /url: https://github.com/about
- listitem [ref=e847]:
- link "Why GitHub" [ref=e848] [cursor=pointer]:
- /url: https://github.com/why-github
- listitem [ref=e849]:
- link "Customer stories" [ref=e850] [cursor=pointer]:
- /url: /customer-stories?type=enterprise
- listitem [ref=e851]:
- link "Blog" [ref=e852] [cursor=pointer]:
- /url: https://github.blog
- listitem [ref=e853]:
- link "The ReadME Project" [ref=e854] [cursor=pointer]:
- /url: /readme
- listitem [ref=e855]:
- link "Careers" [ref=e856] [cursor=pointer]:
- /url: https://github.careers
- listitem [ref=e857]:
- link "Newsroom" [ref=e858] [cursor=pointer]:
- /url: /newsroom
- listitem [ref=e859]:
- link "Inclusion" [ref=e860] [cursor=pointer]:
- /url: /about/diversity
- listitem [ref=e861]:
- link "Social Impact" [ref=e862] [cursor=pointer]:
- /url: https://socialimpact.github.com
- listitem [ref=e863]:
- link "Shop" [ref=e864] [cursor=pointer]:
- /url: https://shop.github.com
- generic [ref=e866]:
- navigation "Legal and Resource Links" [ref=e867]:
- list [ref=e868]:
- listitem [ref=e869]:
- text: ©
- time [ref=e870]: "2026"
- text: GitHub, Inc.
- listitem [ref=e871]:
- link "Terms" [ref=e872] [cursor=pointer]:
- /url: https://docs.github.com/site-policy/github-terms/github-terms-of-service
- listitem [ref=e873]:
- link "Privacy" [ref=e874] [cursor=pointer]:
- /url: https://docs.github.com/site-policy/privacy-policies/github-privacy-statement
- link "(Updated 02/2024) 02/2024" [ref=e875] [cursor=pointer]:
- /url: https://github.com/github/site-policy/pull/582
- text: (Updated 02/2024)
- time [ref=e876]: 02/2024
- listitem [ref=e877]:
- link "Sitemap" [ref=e878] [cursor=pointer]:
- /url: /sitemap
- listitem [ref=e879]:
- link "What is Git?" [ref=e880] [cursor=pointer]:
- /url: /git-guides
- listitem [ref=e881]:
- button "Manage cookies" [ref=e883] [cursor=pointer]
- listitem [ref=e884]:
- button "Do not share my personal information" [ref=e886] [cursor=pointer]
- navigation "GitHub's Social Media Links" [ref=e887]:
- list [ref=e888]:
- listitem [ref=e889]:
- link "GitHub on LinkedIn" [ref=e890] [cursor=pointer]:
- /url: https://www.linkedin.com/company/github
- img [ref=e891]
- generic [ref=e893]: GitHub on LinkedIn
- listitem [ref=e894]:
- link "GitHub on Instagram" [ref=e895] [cursor=pointer]:
- /url: https://www.instagram.com/github
- img [ref=e896]
- generic [ref=e898]: GitHub on Instagram
- listitem [ref=e899]:
- link "GitHub on YouTube" [ref=e900] [cursor=pointer]:
- /url: https://www.youtube.com/github
- img [ref=e901]
- generic [ref=e903]: GitHub on YouTube
- listitem [ref=e904]:
- link "GitHub on X" [ref=e905] [cursor=pointer]:
- /url: https://x.com/github
- img [ref=e906]
- generic [ref=e908]: GitHub on X
- listitem [ref=e909]:
- link "GitHub on TikTok" [ref=e910] [cursor=pointer]:
- /url: https://www.tiktok.com/@github
- img [ref=e911]
- generic [ref=e913]: GitHub on TikTok
- listitem [ref=e914]:
- link "GitHub on Twitch" [ref=e915] [cursor=pointer]:
- /url: https://www.twitch.tv/github
- img [ref=e916]
- generic [ref=e918]: GitHub on Twitch
- listitem [ref=e919]:
- link "GitHub’s organization on GitHub" [ref=e920] [cursor=pointer]:
- /url: https://github.com/github
- img [ref=e921]
- generic [ref=e923]: GitHub’s organization on GitHub
- button "English - Select language" [ref=e926] [cursor=pointer]:
- generic [ref=e929]:
- img [ref=e930]
- text: English
- img [ref=e932]
playwriter/src/assets/aria-labels-github.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 256 KiB

playwriter/src/assets/aria-labels-google-snapshot.txt
+110
View File
@@ -0,0 +1,110 @@
- generic [ref=e1]:
- generic [ref=e2]:
- navigation [ref=e3]:
- generic [ref=e5]:
- generic [ref=e6]:
- link [ref=e8] [cursor=pointer]:
- /url: https://mail.google.com/mail/&ogbl
- text: Gmail
- link [ref=e10] [cursor=pointer]:
- /url: https://www.google.com/imghp?hl=en&ogbl
- text: Images
- button [ref=e13] [cursor=pointer]:
- img [ref=e14]
- link [ref=e18] [cursor=pointer]:
- /url: https://accounts.google.com/ServiceLogin?hl=en&passive=true&continue=https://www.google.com/&ec=futura_exp_og_so_72776762_e
- text: Sign in
- img [ref=e21]
- search [ref=e29]:
- generic [ref=e31]:
- generic [ref=e33]:
- img [ref=e37]
- combobox [ref=e40]
- generic [ref=e42]:
- button [ref=e43] [cursor=pointer]:
- img [ref=e44]
- button [ref=e46] [cursor=pointer]:
- img [ref=e47]
- generic [ref=e50]:
- button [ref=e51] [cursor=pointer]: Google Search
- button [ref=e52] [cursor=pointer]: I'm Feeling Lucky
- generic [ref=e55]:
- text: "Google offered in:"
- link [ref=e56] [cursor=pointer]:
- /url: https://www.google.com/setprefs?sig=0_AUHfGjXqCW2k22WacFgno6C3bdw%3D&hl=it&source=homepage&sa=X&ved=0ahUKEwjJxPW8xu-RAxWF6ckDHZ8wN1QQ2ZgBCBU
- text: Italiano
- contentinfo [ref=e58]:
- generic [ref=e59]: Italy
- generic [ref=e60]:
- generic [ref=e61]:
- link [ref=e62] [cursor=pointer]:
- /url: https://about.google/?utm_source=google-IT&utm_medium=referral&utm_campaign=hp-footer&fg=1
- text: About
- link [ref=e63] [cursor=pointer]:
- /url: https://www.google.com/intl/en_it/ads/?subid=ww-ww-et-g-awa-a-g_hpafoot1_1!o2&utm_source=google.com&utm_medium=referral&utm_campaign=google_hpafooter&fg=1
- text: Advertising
- link [ref=e64] [cursor=pointer]:
- /url: https://www.google.com/services/?subid=ww-ww-et-g-awa-a-g_hpbfoot1_1!o2&utm_source=google.com&utm_medium=referral&utm_campaign=google_hpbfooter&fg=1
- text: Business
- link [ref=e65] [cursor=pointer]:
- /url: https://google.com/search/howsearchworks/?fg=1
- text: How Search works
- generic [ref=e66]:
- link [ref=e67] [cursor=pointer]:
- /url: https://policies.google.com/privacy?hl=en-IT&fg=1
- text: Privacy
- link [ref=e68] [cursor=pointer]:
- /url: https://policies.google.com/terms?hl=en-IT&fg=1
- text: Terms
- button [ref=e72] [cursor=pointer]:
- generic [ref=e73]: Settings
- generic:
- dialog "Before you continue to Google Search":
- generic [ref=e80]:
- generic [ref=e82]:
- img "Google" [ref=e83]
- generic [ref=e84]:
- 'button "Language: ‪English‬" [active] [ref=e86] [cursor=pointer]':
- generic [ref=e87]:
- img
- generic [ref=e88]: en
- link "Sign in" [ref=e89] [cursor=pointer]
- generic [ref=e90]:
- heading "Before you continue to Google" [level=1] [ref=e91]
- generic [ref=e92]:
- text: A reminder that showing ads is the primary way that we fund our services. Learn more about our use of
- link "data for advertising" [ref=e93] [cursor=pointer]:
- /url: https://howwemakemoney.withgoogle.com/en?utm_source=ucbs
- text: .
- generic [ref=e94]:
- generic [ref=e95]:
- text: We use
- link "cookies" [ref=e96] [cursor=pointer]:
- /url: https://policies.google.com/technologies/cookies?utm_source=ucbs&hl=en-IT
- text: and data to
- list [ref=e97]:
- listitem [ref=e98]: Deliver and maintain Google services
- listitem [ref=e99]: Track outages and protect against spam, fraud and abuse
- listitem [ref=e100]: Measure audience engagement and site statistics to understand how our services are used and enhance the quality of those services
- generic [ref=e101]:
- text: If you choose to 'Accept all', we will also use cookies and data to
- list [ref=e102]:
- listitem [ref=e103]: Develop and improve new services
- listitem [ref=e104]: Deliver and measure the effectiveness of ads
- listitem [ref=e105]: Show personalised content, depending on your settings
- listitem [ref=e106]: Show personalised ads, depending on your settings
- generic [ref=e107]: If you choose to 'Reject all', we will not use cookies for these additional purposes.
- generic [ref=e108]: Non-personalised content is influenced by things like the content that you’re currently viewing, activity in your active Search session, and your location. Non-personalised ads are influenced by the content that you’re currently viewing and your general location. Personalised content and ads can also include more relevant results, recommendations and tailored ads based on past activity from this browser, like previous Google searches. We also use cookies and data to tailor the experience to be age-appropriate, if relevant.
- generic [ref=e109]: Select 'More options' to see additional information, including details about managing your privacy settings. You can also visit g.co/privacytools at any time.
- generic [ref=e110]:
- generic [ref=e111]:
- button "Reject all" [ref=e112] [cursor=pointer]
- button "Accept all" [ref=e113] [cursor=pointer]
- link "More options for personalisation settings and cookies" [ref=e115] [cursor=pointer]:
- generic "More options for personalisation settings and cookies" [ref=e116]: More options
- generic [ref=e117]:
- link "Privacy" [ref=e118] [cursor=pointer]:
- /url: https://policies.google.com/privacy?hl=en-IT&fg=1&utm_source=ucbs
- generic [ref=e119]: ·
- link "Terms" [ref=e120] [cursor=pointer]:
- /url: https://policies.google.com/terms?hl=en-IT&fg=1&utm_source=ucbs
playwriter/src/assets/aria-labels-google.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 108 KiB

playwriter/src/assets/aria-labels-hacker-news-snapshot.txt
+1023
View File
File diff suppressed because it is too large Load Diff
playwriter/src/assets/aria-labels-hacker-news.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 208 KiB

playwriter/src/index.ts
+2
View File
@@ -6,3 +6,5 @@ export { Editor } from './editor.js'
export type { ReadResult, SearchMatch, EditResult } from './editor.js'
export { Debugger } from './debugger.js'
export type { BreakpointInfo, LocationInfo, EvaluateResult, ScriptInfo } from './debugger.js'
export { getAriaSnapshot, showAriaRefLabels, hideAriaRefLabels } from './aria-snapshot.js'
export type { AriaRef, AriaSnapshotResult } from './aria-snapshot.js'
playwriter/src/mcp.test.ts
+191 -3
View File
@@ -686,7 +686,7 @@ describe('MCP Server Tests', () => {
await browser.close()
await page.close()
})
}, 60000)
it('should be able to reconnect after disconnecting everything', async () => {
const browserContext = getBrowserContext()
@@ -2106,8 +2106,8 @@ describe('MCP Server Tests', () => {
{
"text": "Return value:
{
"matchesDark": false,
"matchesLight": true
\"matchesDark\": false,
\"matchesLight\": true
}",
"type": "text",
},
@@ -2117,7 +2117,195 @@ describe('MCP Server Tests', () => {
await page.close()
}, 60000)
it('should get aria ref for locator using getAriaSnapshot', async () => {
const browserContext = getBrowserContext()
const serviceWorker = await getExtensionServiceWorker(browserContext)
const page = await browserContext.newPage()
await page.setContent(`
<html>
<body>
<button id="submit-btn">Submit Form</button>
<a href="/about">About Us</a>
<input type="text" placeholder="Enter your name" />
</body>
</html>
`)
await page.bringToFront()
await serviceWorker.evaluate(async () => {
await globalThis.toggleExtensionForActiveTab()
})
await new Promise(r => setTimeout(r, 400))
const browser = await chromium.connectOverCDP(getCdpUrl({ port: TEST_PORT }))
let cdpPage
for (const p of browser.contexts()[0].pages()) {
const html = await p.content()
if (html.includes('submit-btn')) {
cdpPage = p
break
}
}
expect(cdpPage).toBeDefined()
const { getAriaSnapshot } = await import('./aria-snapshot.js')
// Get aria snapshot and verify we can get refs
const ariaResult = await getAriaSnapshot({ page: cdpPage! })
expect(ariaResult.snapshot).toBeDefined()
expect(ariaResult.snapshot.length).toBeGreaterThan(0)
expect(ariaResult.snapshot).toContain('Submit Form')
// Verify refToElement map is populated
expect(ariaResult.refToElement.size).toBeGreaterThan(0)
console.log('RefToElement map size:', ariaResult.refToElement.size)
console.log('RefToElement entries:', [...ariaResult.refToElement.entries()])
// Verify we can select elements using aria-ref selectors
const btnViaAriaRef = cdpPage!.locator('aria-ref=e2')
const btnTextViaRef = await btnViaAriaRef.textContent()
console.log('Button text via aria-ref=e2:', btnTextViaRef)
expect(btnTextViaRef).toBe('Submit Form')
// Get ref for the submit button using getRefForLocator
const submitBtn = cdpPage!.locator('#submit-btn')
const btnAriaRef = await ariaResult.getRefForLocator(submitBtn)
console.log('Button ariaRef:', btnAriaRef)
expect(btnAriaRef).toBeDefined()
expect(btnAriaRef?.role).toBe('button')
expect(btnAriaRef?.name).toBe('Submit Form')
expect(btnAriaRef?.ref).toMatch(/^e\d+$/)
// Verify the ref matches what we can use to select
const btnFromRef = cdpPage!.locator(`aria-ref=${btnAriaRef?.ref}`)
const btnText = await btnFromRef.textContent()
expect(btnText).toBe('Submit Form')
// Test getRefStringForLocator
const btnRefStr = await ariaResult.getRefStringForLocator(submitBtn)
console.log('Button ref string:', btnRefStr)
expect(btnRefStr).toBe(btnAriaRef?.ref)
// Test link
const aboutLink = cdpPage!.locator('a')
const linkAriaRef = await ariaResult.getRefForLocator(aboutLink)
console.log('Link ariaRef:', linkAriaRef)
expect(linkAriaRef).toBeDefined()
expect(linkAriaRef?.role).toBe('link')
expect(linkAriaRef?.name).toBe('About Us')
// Verify the link ref works
const linkFromRef = cdpPage!.locator(`aria-ref=${linkAriaRef?.ref}`)
const linkText = await linkFromRef.textContent()
expect(linkText).toBe('About Us')
// Test input field
const inputField = cdpPage!.locator('input')
const inputAriaRef = await ariaResult.getRefForLocator(inputField)
console.log('Input ariaRef:', inputAriaRef)
expect(inputAriaRef).toBeDefined()
expect(inputAriaRef?.role).toBe('textbox')
// Test batch getRefsForLocators - single evaluate call for multiple elements
const batchRefs = await ariaResult.getRefsForLocators([submitBtn, aboutLink, inputField])
console.log('Batch refs:', batchRefs)
expect(batchRefs).toHaveLength(3)
expect(batchRefs[0]?.ref).toBe(btnAriaRef?.ref)
expect(batchRefs[1]?.ref).toBe(linkAriaRef?.ref)
expect(batchRefs[2]?.ref).toBe(inputAriaRef?.ref)
await browser.close()
await page.close()
}, 60000)
it('should show aria ref labels on real pages and save screenshots', async () => {
const browserContext = getBrowserContext()
const serviceWorker = await getExtensionServiceWorker(browserContext)
const { showAriaRefLabels, hideAriaRefLabels } = await import('./aria-snapshot.js')
const fs = await import('node:fs')
const path = await import('node:path')
// Create assets folder for screenshots
const assetsDir = path.join(path.dirname(new URL(import.meta.url).pathname), 'assets')
if (!fs.existsSync(assetsDir)) {
fs.mkdirSync(assetsDir, { recursive: true })
}
const testPages = [
{ name: 'hacker-news', url: 'https://news.ycombinator.com/' },
{ name: 'google', url: 'https://www.google.com/' },
{ name: 'github', url: 'https://github.com/' },
]
// Create all pages and enable extension for each
const pages = await Promise.all(
testPages.map(async ({ name, url }) => {
const page = await browserContext.newPage()
await page.goto(url, { waitUntil: 'domcontentloaded' })
return { name, url, page }
})
)
// Enable extension for each tab (must be done sequentially as it uses active tab)
for (const { page } of pages) {
await page.bringToFront()
await serviceWorker.evaluate(async () => {
await globalThis.toggleExtensionForActiveTab()
})
}
// Connect CDP and process all pages concurrently
const browser = await chromium.connectOverCDP(getCdpUrl({ port: TEST_PORT }))
await Promise.all(
pages.map(async ({ name, url, page }) => {
const cdpPage = browser.contexts()[0].pages().find(p => p.url().includes(new URL(url).hostname))
if (!cdpPage) {
console.log(`Could not find CDP page for ${name}, skipping...`)
return
}
// Show aria ref labels
const { snapshot, labelCount } = await showAriaRefLabels({ page: cdpPage })
console.log(`${name}: ${labelCount} labels shown`)
expect(labelCount).toBeGreaterThan(0)
// Take screenshot with labels visible
const screenshot = await cdpPage.screenshot({ type: 'png', fullPage: false })
const screenshotPath = path.join(assetsDir, `aria-labels-${name}.png`)
fs.writeFileSync(screenshotPath, screenshot)
console.log(`Screenshot saved: ${screenshotPath}`)
// Save snapshot text for reference
const snapshotPath = path.join(assetsDir, `aria-labels-${name}-snapshot.txt`)
fs.writeFileSync(snapshotPath, snapshot)
// Verify labels are in DOM
const labelElements = await cdpPage.evaluate(() =>
document.querySelectorAll('.__pw_label__').length
)
expect(labelElements).toBe(labelCount)
// Cleanup
await hideAriaRefLabels({ page: cdpPage })
// Verify labels removed
const labelsAfterHide = await cdpPage.evaluate(() =>
document.getElementById('__playwriter_labels__')
)
expect(labelsAfterHide).toBeNull()
await page.close()
})
)
await browser.close()
console.log(`Screenshots saved to: ${assetsDir}`)
}, 120000)
})