playwriter
Code Issues

bigger labels. fix screenshots sizes

Browse Source
This commit is contained in:
Tommy D. Rossi
2026-01-05 11:11:00 +01:00
parent e87f801700
commit 8f2285a63c
5 changed files with 710 additions and 614 deletions
Download Patch File Download Diff File Expand all files Collapse all files
playwriter/src/aria-snapshot.ts
+16 -8
View File
@@ -93,7 +93,7 @@ const css = String.raw
const LABEL_STYLES = css`
.__pw_label__ {
position: absolute;
font: bold 11px Helvetica, Arial, sans-serif;
font: bold 12px Helvetica, Arial, sans-serif;
padding: 1px 4px;
border-radius: 3px;
color: black;
@@ -300,8 +300,8 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
// Each rect is { left, top, right, bottom } in viewport coordinates
const placedLabels: Array<{ left: number; top: number; right: number; bottom: number }> = []
// Estimate label dimensions (11px font + padding)
const LABEL_HEIGHT = 16
// Estimate label dimensions (12px font + padding)
const LABEL_HEIGHT = 17
const LABEL_CHAR_WIDTH = 7 // approximate width per character
// Parse alpha from rgb/rgba color string (getComputedStyle always returns these formats)
@@ -518,7 +518,7 @@ export async function screenshotWithAccessibilityLabels({ page, interactiveOnly
const timestamp = Date.now()
const random = Math.random().toString(36).slice(2, 6)
const filename = `playwriter-screenshot-${timestamp}-${random}.jpg`
// Use ./tmp folder (gitignored) instead of system temp
const tmpDir = path.join(process.cwd(), 'tmp')
if (!fs.existsSync(tmpDir)) {
@@ -526,12 +526,20 @@ export async function screenshotWithAccessibilityLabels({ page, interactiveOnly
}
const screenshotPath = path.join(tmpDir, filename)
// Take screenshot
const buffer = await page.screenshot({ type: 'jpeg', quality: 80 })
// Get actual viewport size (innerWidth/innerHeight, not outer window size)
const viewport = await page.evaluate('(() => ({ width: window.innerWidth, height: window.innerHeight }))()') as { width: number; height: number }
// Take screenshot clipped to actual viewport (excludes browser chrome)
const buffer = await page.screenshot({
type: 'jpeg',
quality: 80,
scale: 'css',
clip: { x: 0, y: 0, width: viewport.width, height: viewport.height },
})
// Save to file
fs.writeFileSync(screenshotPath, buffer)
// Convert to base64
const base64 = buffer.toString('base64')
playwriter/src/assets/aria-labels-google-snapshot.txt
+1 -1
View File
@@ -31,7 +31,7 @@
- generic [ref=e55]:
- text: "Google offered in:"
- link [ref=e56] [cursor=pointer]:
- /url: https://www.google.com/setprefs?sig=0_U_LREl4THfXa2sQ-b49g0Jd45jk%3D&hl=it&source=homepage&sa=X&ved=0ahUKEwiYgJn4hfSRAxU00xoGHUTZBMEQ2ZgBCBU
- /url: https://www.google.com/setprefs?sig=0_gi5Q9KT3akeon6vbuxB8Wv_u0vw%3D&hl=it&source=homepage&sa=X&ved=0ahUKEwiG0LuSlPSRAxWiLtAFHc53GkwQ2ZgBCBU
- text: Italiano
- contentinfo [ref=e58]:
- generic [ref=e59]: Italy
playwriter/src/assets/aria-labels-hacker-news-snapshot.txt
+607 -601
View File
File diff suppressed because it is too large Load Diff
playwriter/src/mcp.test.ts
+62 -3
View File
@@ -2314,10 +2314,58 @@ describe('MCP Server Tests', () => {
const page = await browserContext.newPage()
await page.setContent(`
<html>
<head>
<style>
body {
margin: 0;
background: #e8f4f8;
position: relative;
min-height: 100vh;
}
.controls {
padding: 20px;
position: relative;
z-index: 10;
}
.grid-marker {
position: absolute;
background: rgba(255, 100, 100, 0.3);
border: 1px solid #ff6464;
font-size: 10px;
color: #333;
display: flex;
align-items: center;
justify-content: center;
}
.h-marker {
left: 0;
width: 100%;
height: 20px;
}
.v-marker {
top: 0;
height: 100%;
width: 20px;
}
</style>
</head>
<body>
<button id="submit-btn">Submit Form</button>
<a href="/about">About Us</a>
<input type="text" placeholder="Enter your name" />
<div class="controls">
<button id="submit-btn">Submit Form</button>
<a href="/about">About Us</a>
<input type="text" placeholder="Enter your name" />
</div>
<!-- Horizontal markers every 200px -->
<div class="grid-marker h-marker" style="top: 200px;">200px</div>
<div class="grid-marker h-marker" style="top: 400px;">400px</div>
<div class="grid-marker h-marker" style="top: 600px;">600px</div>
<!-- Vertical markers every 200px -->
<div class="grid-marker v-marker" style="left: 200px;">200</div>
<div class="grid-marker v-marker" style="left: 400px;">400</div>
<div class="grid-marker v-marker" style="left: 600px;">600</div>
<div class="grid-marker v-marker" style="left: 800px;">800</div>
<div class="grid-marker v-marker" style="left: 1000px;">1000</div>
<div class="grid-marker v-marker" style="left: 1200px;">1200</div>
</body>
</html>
`)
@@ -2370,6 +2418,17 @@ describe('MCP Server Tests', () => {
// Verify the image is valid JPEG by checking base64
const buffer = Buffer.from(imageContent.data, 'base64')
const dimensions = imageSize(buffer)
// Get actual viewport size from page
const viewport = await page.evaluate(() => ({
innerWidth: window.innerWidth,
innerHeight: window.innerHeight,
outerWidth: window.outerWidth,
outerHeight: window.outerHeight,
}))
console.log('Screenshot dimensions:', dimensions.width, 'x', dimensions.height)
console.log('Window viewport:', viewport)
expect(dimensions.type).toBe('jpg')
expect(dimensions.width).toBeGreaterThan(0)
expect(dimensions.height).toBeGreaterThan(0)
playwriter/src/prompt.md
+24 -1
View File
@@ -31,6 +31,8 @@ After any action (click, submit, navigate), verify what happened:
console.log('url:', page.url()); console.log(await accessibilitySnapshot({ page }).then(x => x.split('\n').slice(0, 30).join('\n')));
```
For visually complex pages (grids, galleries, dashboards), use `screenshotWithAccessibilityLabels({ page })` instead to understand spatial layout.
If nothing changed, try `await page.waitForLoadState('networkidle', {timeout: 3000})` or you may have clicked the wrong element.
## accessibility snapshots
@@ -66,6 +68,24 @@ Search for specific elements:
const snapshot = await accessibilitySnapshot({ page, search: /button|submit/i })
```
## choosing between snapshot methods
Both `accessibilitySnapshot` and `screenshotWithAccessibilityLabels` use the same `aria-ref` system, so you can combine them effectively.
**Use `accessibilitySnapshot` when:**
- Page has simple, semantic structure (articles, forms, lists)
- You need to search for specific text or patterns
- Token usage matters (text is smaller than images)
- You need to process the output programmatically
**Use `screenshotWithAccessibilityLabels` when:**
- Page has complex visual layout (grids, galleries, dashboards, maps)
- Spatial position matters (e.g., "first image", "top-left button")
- DOM order doesn't match visual order
- You need to understand the visual hierarchy
**Combining both:** Use screenshot first to understand layout and identify target elements visually, then use `accessibilitySnapshot({ search: /pattern/ })` for efficient searching in subsequent calls.
## selector best practices
**For unknown websites**: use `accessibilitySnapshot()` with `aria-ref` - it shows what's actually interactive.
@@ -206,7 +226,9 @@ const matches = await editor.grep({ regex: /console\.log/ });
await editor.edit({ url: matches[0].url, oldString: 'DEBUG = false', newString: 'DEBUG = true' });
```
**screenshotWithAccessibilityLabels** - take a screenshot with Vimium-style visual labels overlaid on interactive elements. Shows labels, captures screenshot, then removes labels. The image and accessibility snapshot are automatically included in the response. Can be called multiple times to capture multiple screenshots. Use a timeout of 10 seconds at least.
**screenshotWithAccessibilityLabels** - take a screenshot with Vimium-style visual labels overlaid on interactive elements. Shows labels, captures screenshot, then removes labels. The image and accessibility snapshot are automatically included in the response. Can be called multiple times to capture multiple screenshots. Use a timeout of **20 seconds** for complex pages.
Prefer this for pages with grids, image galleries, maps, or complex visual layouts where spatial position matters. For simple text-heavy pages, `accessibilitySnapshot` with search is faster and uses fewer tokens.
```js
await screenshotWithAccessibilityLabels({ page });
@@ -296,5 +318,6 @@ Examples of what playwriter can do:
- Intercept network requests to reverse-engineer APIs and build SDKs
- Scrape data by replaying paginated API calls instead of scrolling DOM
- Get accessibility snapshot to find elements, then automate interactions
- Use visual screenshots to understand complex layouts like image grids, dashboards, or maps
- Debug issues by collecting logs and controlling the page simultaneously
- Handle popups, downloads, iframes, and dialog boxes