playwriter
Code Issues

filter labels for elements covered by opaque overlays

Browse Source 
Uses elementsFromPoint() to check if element is blocked by opaque
overlays (modals, cookie banners, etc.). Skips elements where an
opaque element with pointer-events appears above the target in
the stacking order.

Correctly shows only visible, interactable elements.
This commit is contained in:
Tommy D. Rossi
2026-01-03 15:45:48 +01:00
parent d9783b15be
commit 3cd59600e9
5 changed files with 492 additions and 410 deletions
Download Patch File Download Diff File Expand all files Collapse all files
playwriter/package.json
+1
View File
@@ -44,6 +44,7 @@
"@modelcontextprotocol/sdk": "^1.21.1",
"cac": "^6.7.14",
"chalk": "^5.6.2",
"colord": "^2.9.3",
"devtools-protocol": "^0.0.1543509",
"diff": "^8.0.2",
"hono": "^4.10.6",
playwriter/src/aria-snapshot.ts
+66 -1
View File
@@ -237,11 +237,12 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
// ElementHandles get unwrapped to DOM elements in browser context
// Using 'any' types here since this code runs in browser context
const labelCount = await page.evaluate(
// Using 'any' for browser types since this runs in browser context
({ refs, containerId, containerStyles, labelStyles, roleColors, defaultColors }: {
refs: Array<{
ref: string
role: string
element: { getBoundingClientRect(): { width: number; height: number; left: number; top: number; right: number; bottom: number } }
element: any // Element in browser context
}>
containerId: string
containerStyles: string
@@ -273,6 +274,65 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
const LABEL_HEIGHT = 16
const LABEL_CHAR_WIDTH = 7 // approximate width per character
// Parse alpha from rgb/rgba color string (getComputedStyle always returns these formats)
const getColorAlpha = (color: string): number => {
if (color === 'transparent') return 0
// Match rgba(r, g, b, a) or rgb(r, g, b)
const match = color.match(/rgba?\(\s*[\d.]+\s*,\s*[\d.]+\s*,\s*[\d.]+\s*(?:,\s*([\d.]+)\s*)?\)/)
if (match) {
return match[1] !== undefined ? parseFloat(match[1]) : 1
}
return 1 // Default to opaque for unrecognized formats
}
// Check if an element has an opaque background that would block elements behind it
const isOpaqueElement = (el: any): boolean => {
const style = win.getComputedStyle(el)
// Check element opacity
const opacity = parseFloat(style.opacity)
if (opacity < 0.1) return false
// Check background-color alpha
const bgAlpha = getColorAlpha(style.backgroundColor)
if (bgAlpha > 0.1) return true
// Check if has background-image (usually opaque)
if (style.backgroundImage !== 'none') return true
return false
}
// Check if element is visible (not covered by opaque overlay)
const isElementVisible = (element: any, rect: any): boolean => {
const centerX = rect.left + rect.width / 2
const centerY = rect.top + rect.height / 2
// Get all elements at this point, from top to bottom
const stack = doc.elementsFromPoint(centerX, centerY) as any[]
// Find our target element in the stack
const targetIndex = stack.findIndex((el: any) =>
element.contains(el) || el.contains(element)
)
// Element not in stack at all - not visible
if (targetIndex === -1) return false
// Check if any opaque element is above our target
for (let i = 0; i < targetIndex; i++) {
const el = stack[i]
// Skip our own overlay container
if (el.id === containerId) continue
// Skip pointer-events: none elements (decorative overlays)
if (win.getComputedStyle(el).pointerEvents === 'none') continue
// If this element is opaque, our target is blocked
if (isOpaqueElement(el)) return false
}
return true
}
// Check if two rectangles overlap
const rectsOverlap = (
a: { left: number; top: number; right: number; bottom: number },
@@ -291,6 +351,11 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
continue
}
// Skip elements that are covered by opaque overlays
if (!isElementVisible(element, rect)) {
continue
}
// Calculate label position and dimensions
const labelWidth = ref.length * LABEL_CHAR_WIDTH + 8 // +8 for padding
const labelLeft = rect.left
playwriter/src/assets/aria-labels-google-snapshot.txt
+1 -1
View File
@@ -31,7 +31,7 @@
- generic [ref=e55]:
- text: "Google offered in:"
- link [ref=e56] [cursor=pointer]:
- /url: https://www.google.com/setprefs?sig=0_9hKalRtXzD5AWtzEh3KSYnYsv7w%3D&hl=it&source=homepage&sa=X&ved=0ahUKEwjUxd_Qyu-RAxUqwskDHd_ZFnQQ2ZgBCBU
- /url: https://www.google.com/setprefs?sig=0_GcFvy0kvBkaZ22ghfnLBdpCO7v0%3D&hl=it&source=homepage&sa=X&ved=0ahUKEwiqtPX4ze-RAxWEkyYFHdD-Ma8Q2ZgBCBU
- text: Italiano
- contentinfo [ref=e58]:
- generic [ref=e59]: Italy
playwriter/src/assets/aria-labels-hacker-news-snapshot.txt
+404 -404
View File
File diff suppressed because it is too large Load Diff
pnpm-lock.yaml
Generated
+20 -4
View File
@@ -73,6 +73,9 @@ importers:
chalk:
specifier: ^5.6.2
version: 5.6.2
colord:
specifier: ^2.9.3
version: 2.9.3
devtools-protocol:
specifier: ^0.0.1543509
version: 0.0.1543509
@@ -227,7 +230,7 @@ importers:
version: 6.0.3(typescript@5.9.3)(vite@7.3.0(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))
vitest:
specifier: ^4.0.16
version: 4.0.16(@opentelemetry/api@1.9.0)(@types/node@25.0.3)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6)
version: 4.0.16(@opentelemetry/api@1.9.0)(@types/node@25.0.3)(@vitest/ui@4.0.8(vitest@4.0.8))(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6)
packages:
@@ -2605,6 +2608,9 @@ packages:
color-name@1.1.4:
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
colord@2.9.3:
resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==}
colorette@2.0.20:
resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==}
@@ -7221,6 +7227,14 @@ snapshots:
optionalDependencies:
vite: 7.3.0(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)
'@vitest/mocker@4.0.8(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))':
dependencies:
'@vitest/spy': 4.0.8
estree-walker: 3.0.3
magic-string: 0.30.21
optionalDependencies:
vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)
'@vitest/mocker@4.0.8(vite@7.2.2(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))':
dependencies:
'@vitest/spy': 4.0.8
@@ -7272,7 +7286,7 @@ snapshots:
sirv: 3.0.2
tinyglobby: 0.2.15
tinyrainbow: 3.0.3
vitest: 4.0.8(@types/node@25.0.3)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6)
vitest: 4.0.8(@types/node@24.10.1)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6)
'@vitest/utils@4.0.16':
dependencies:
@@ -7650,6 +7664,8 @@ snapshots:
color-name@1.1.4: {}
colord@2.9.3: {}
colorette@2.0.20: {}
combined-stream@1.0.8:
@@ -9890,7 +9906,7 @@ snapshots:
lightningcss: 1.30.2
tsx: 4.20.6
vitest@4.0.16(@opentelemetry/api@1.9.0)(@types/node@25.0.3)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6):
vitest@4.0.16(@opentelemetry/api@1.9.0)(@types/node@25.0.3)(@vitest/ui@4.0.8(vitest@4.0.8))(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6):
dependencies:
'@vitest/expect': 4.0.16
'@vitest/mocker': 4.0.16(vite@7.3.0(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))
@@ -9933,7 +9949,7 @@ snapshots:
vitest@4.0.8(@types/node@24.10.1)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6):
dependencies:
'@vitest/expect': 4.0.8
'@vitest/mocker': 4.0.8(vite@7.2.2(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))
'@vitest/mocker': 4.0.8(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))
'@vitest/pretty-format': 4.0.8
'@vitest/runner': 4.0.8
'@vitest/snapshot': 4.0.8