diff --git a/playwriter/package.json b/playwriter/package.json index 8b4ad63..fd2200c 100644 --- a/playwriter/package.json +++ b/playwriter/package.json @@ -7,7 +7,7 @@ "types": "dist/index.d.ts", "repository": "https://github.com/remorses/playwriter", "scripts": { - "build": "rm -rf dist *.tsbuildinfo && mkdir dist && bun scripts/build-selector-generator.ts && bun scripts/build-bippy.ts && bun scripts/build-a11y-client.ts && tsc && bun scripts/build-resources.ts", + "build": "rm -rf dist *.tsbuildinfo && mkdir dist && bun scripts/build-client-bundles.ts && tsc && bun scripts/build-resources.ts", "prepublishOnly": "pnpm build", "watch": "tsc -w", "cli": "vite-node src/cli.ts", @@ -32,6 +32,7 @@ "@types/node": "^24.10.1", "@types/ws": "^8.18.1", "@vitest/ui": "^4.0.8", + "@mozilla/readability": "^0.6.0", "bippy": "^0.5.27", "devtools-protocol": "^0.0.1568893", "image-size": "^2.0.2", diff --git a/playwriter/scripts/build-a11y-client.ts b/playwriter/scripts/build-a11y-client.ts deleted file mode 100644 index b3764b5..0000000 --- a/playwriter/scripts/build-a11y-client.ts +++ /dev/null @@ -1,39 +0,0 @@ -import fs from 'node:fs' -import path from 'node:path' -import { fileURLToPath } from 'node:url' - -const __dirname = path.dirname(fileURLToPath(import.meta.url)) -const distDir = path.join(__dirname, '..', 'dist') -const srcDir = path.join(__dirname, '..', 'src') - -async function main() { - if (!fs.existsSync(distDir)) { - fs.mkdirSync(distDir, { recursive: true }) - } - - console.log('Bundling a11y-client...') - - const result = await Bun.build({ - entrypoints: [path.join(srcDir, 'a11y-client.ts')], - target: 'browser', - format: 'iife', - define: { - 'process.env.NODE_ENV': '"development"', - }, - }) - - if (!result.success) { - console.error('Bundle errors:', result.logs) - process.exit(1) - } - - const bundledCode = await result.outputs[0].text() - const outputPath = path.join(distDir, 'a11y-client.js') - fs.writeFileSync(outputPath, bundledCode) - console.log(`Saved to ${outputPath} (${Math.round(bundledCode.length / 1024)}kb)`) -} - -main().catch((err) => { - console.error(err) - process.exit(1) -}) diff --git a/playwriter/scripts/build-bippy.ts b/playwriter/scripts/build-bippy.ts deleted file mode 100644 index fa35239..0000000 --- a/playwriter/scripts/build-bippy.ts +++ /dev/null @@ -1,60 +0,0 @@ -import fs from 'node:fs' -import path from 'node:path' -import { fileURLToPath } from 'node:url' - -const __dirname = path.dirname(fileURLToPath(import.meta.url)) -const distDir = path.join(__dirname, '..', 'dist') - -const ENTRY_CODE = ` -import { getFiberFromHostInstance, getDisplayName, traverseFiber, isCompositeFiber, isHostFiber } from 'bippy' -import { getSource, getOwnerStack, normalizeFileName, isSourceFile } from 'bippy/source' - -globalThis.__bippy = { - getFiberFromHostInstance, - getDisplayName, - traverseFiber, - isCompositeFiber, - isHostFiber, - getSource, - getOwnerStack, - normalizeFileName, - isSourceFile, -} -` - -async function main() { - if (!fs.existsSync(distDir)) { - fs.mkdirSync(distDir, { recursive: true }) - } - - const entryPath = path.join(distDir, '_bippy-entry.js') - fs.writeFileSync(entryPath, ENTRY_CODE) - - console.log('Bundling bippy...') - - const result = await Bun.build({ - entrypoints: [entryPath], - target: 'browser', - format: 'iife', - define: { - 'process.env.NODE_ENV': '"development"', - }, - }) - - fs.unlinkSync(entryPath) - - if (!result.success) { - console.error('Bundle errors:', result.logs) - process.exit(1) - } - - const bundledCode = await result.outputs[0].text() - const outputPath = path.join(distDir, 'bippy.js') - fs.writeFileSync(outputPath, bundledCode) - console.log(`Saved to ${outputPath} (${Math.round(bundledCode.length / 1024)}kb)`) -} - -main().catch((err) => { - console.error(err) - process.exit(1) -}) diff --git a/playwriter/scripts/build-client-bundles.ts b/playwriter/scripts/build-client-bundles.ts new file mode 100644 index 0000000..79ef335 --- /dev/null +++ b/playwriter/scripts/build-client-bundles.ts @@ -0,0 +1,151 @@ +/** + * Consolidated build script for all client-side JavaScript bundles. + * + * These bundles are injected into pages via CDP Runtime.evaluate or page.evaluate(). + * All bundles are built as browser-targeted IIFEs that expose their APIs on globalThis. + * + * Injection flow (see react-source.ts, page-markdown.ts for examples): + * 1. Bundle is read from dist/*.js via fs.readFileSync (cached after first read) + * 2. Check if already injected: `await page.evaluate(() => !!globalThis.__name)` + * 3. If not present, inject: `await page.evaluate(code)` or `cdp.send('Runtime.evaluate', { expression: code })` + * 4. Use the exposed global in subsequent evaluate calls: `globalThis.__readability`, `globalThis.__bippy`, etc. + * + * Each bundle uses a separate Bun.build() call (not multiple entrypoints in one call) + * to ensure fully self-contained output with no shared chunks. + * + * Two types of bundles: + * 1. Source file bundles - directly bundle a TypeScript source file + * 2. Wrapper bundles - create entry code that imports from npm packages and exposes on globalThis + */ + +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const distDir = path.join(__dirname, '..', 'dist') +const srcDir = path.join(__dirname, '..', 'src') + +interface SourceBundle { + name: string + type: 'source' + entry: string // relative to src/ +} + +interface WrapperBundle { + name: string + type: 'wrapper' + code: string +} + +type BundleConfig = SourceBundle | WrapperBundle + +const BUNDLES: BundleConfig[] = [ + // Source file bundles + { + name: 'a11y-client', + type: 'source', + entry: 'a11y-client.ts', + }, + + // Wrapper bundles (npm packages → globalThis) + { + name: 'selector-generator', + type: 'wrapper', + code: ` +import { createSelectorGenerator, toLocator } from '@mizchi/selector-generator' + +globalThis.__selectorGenerator = { createSelectorGenerator, toLocator } +`, + }, + { + name: 'bippy', + type: 'wrapper', + code: ` +import { getFiberFromHostInstance, getDisplayName, traverseFiber, isCompositeFiber, isHostFiber } from 'bippy' +import { getSource, getOwnerStack, normalizeFileName, isSourceFile } from 'bippy/source' + +globalThis.__bippy = { + getFiberFromHostInstance, + getDisplayName, + traverseFiber, + isCompositeFiber, + isHostFiber, + getSource, + getOwnerStack, + normalizeFileName, + isSourceFile, +} +`, + }, + { + name: 'readability', + type: 'wrapper', + code: ` +import { Readability, isProbablyReaderable } from '@mozilla/readability' + +globalThis.__readability = { Readability, isProbablyReaderable } +`, + }, +] + +async function buildBundle(config: BundleConfig): Promise { + const startTime = Date.now() + let entryPath: string + let cleanupEntry = false + + if (config.type === 'source') { + entryPath = path.join(srcDir, config.entry) + } else { + // Create temporary entry file for wrapper bundles + entryPath = path.join(distDir, `_${config.name}-entry.js`) + fs.writeFileSync(entryPath, config.code) + cleanupEntry = true + } + + const result = await Bun.build({ + entrypoints: [entryPath], + target: 'browser', + format: 'iife', + define: { + 'process.env.NODE_ENV': '"development"', + }, + }) + + // Cleanup temporary entry file + if (cleanupEntry) { + fs.unlinkSync(entryPath) + } + + if (!result.success) { + console.error(`Bundle errors for ${config.name}:`, result.logs) + throw new Error(`Failed to bundle ${config.name}`) + } + + const bundledCode = await result.outputs[0].text() + const outputPath = path.join(distDir, `${config.name}.js`) + fs.writeFileSync(outputPath, bundledCode) + + const sizeKb = Math.round(bundledCode.length / 1024) + const timeMs = Date.now() - startTime + console.log(` ${config.name}.js (${sizeKb}kb) [${timeMs}ms]`) +} + +async function main() { + if (!fs.existsSync(distDir)) { + fs.mkdirSync(distDir, { recursive: true }) + } + + console.log('Building client bundles...') + + // Each buildBundle() call runs its own Bun.build() with a single entrypoint, + // so parallel execution is safe - no shared chunks between bundles + await Promise.all(BUNDLES.map(buildBundle)) + + console.log(`Done! Built ${BUNDLES.length} bundles.`) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/playwriter/scripts/build-selector-generator.ts b/playwriter/scripts/build-selector-generator.ts deleted file mode 100644 index 135f381..0000000 --- a/playwriter/scripts/build-selector-generator.ts +++ /dev/null @@ -1,46 +0,0 @@ -import fs from 'node:fs' -import path from 'node:path' -import { fileURLToPath } from 'node:url' - -const __dirname = path.dirname(fileURLToPath(import.meta.url)) -const distDir = path.join(__dirname, '..', 'dist') - -const ENTRY_CODE = ` -import { createSelectorGenerator, toLocator } from '@mizchi/selector-generator' - -globalThis.__selectorGenerator = { createSelectorGenerator, toLocator } -` - -async function main() { - if (!fs.existsSync(distDir)) { - fs.mkdirSync(distDir, { recursive: true }) - } - - const entryPath = path.join(distDir, '_selector-generator-entry.js') - fs.writeFileSync(entryPath, ENTRY_CODE) - - console.log('Bundling selector-generator...') - - const result = await Bun.build({ - entrypoints: [entryPath], - target: 'browser', - format: 'iife', - }) - - fs.unlinkSync(entryPath) - - if (!result.success) { - console.error('Bundle errors:', result.logs) - process.exit(1) - } - - const bundledCode = await result.outputs[0].text() - const outputPath = path.join(distDir, 'selector-generator.js') - fs.writeFileSync(outputPath, bundledCode) - console.log(`Saved to ${outputPath} (${Math.round(bundledCode.length / 1024)}kb)`) -} - -main().catch((err) => { - console.error(err) - process.exit(1) -}) diff --git a/playwriter/src/executor.ts b/playwriter/src/executor.ts index 63dd500..5b8622c 100644 --- a/playwriter/src/executor.ts +++ b/playwriter/src/executor.ts @@ -24,6 +24,7 @@ import { ScopedFS } from './scoped-fs.js' import { screenshotWithAccessibilityLabels, formatSnapshot, DEFAULT_SNAPSHOT_FORMAT, getAriaSnapshot, type ScreenshotResult, type SnapshotFormat } from './aria-snapshot.js' export type { SnapshotFormat } import { getCleanHTML, type GetCleanHTMLOptions } from './clean-html.js' +import { getPageMarkdown, type GetPageMarkdownOptions } from './page-markdown.js' import { startRecording, stopRecording, isRecording, cancelRecording } from './screen-recording.js' const __filename = fileURLToPath(import.meta.url) @@ -617,6 +618,7 @@ export class PlaywrightExecutor { console: customConsole, accessibilitySnapshot, getCleanHTML, + getPageMarkdown, getLocatorStringForElement, getLatestLogs, clearAllLogs, diff --git a/playwriter/src/page-markdown.ts b/playwriter/src/page-markdown.ts new file mode 100644 index 0000000..68e6d7c --- /dev/null +++ b/playwriter/src/page-markdown.ts @@ -0,0 +1,247 @@ +/** + * Extract page content as markdown using Mozilla Readability. + * + * This utility injects the Readability library into the page and extracts + * the main content, similar to Firefox Reader View. + */ + +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import type { Page } from 'playwright-core' +import { createPatch } from 'diff' + +export interface PageMarkdownResult { + /** Extracted content as plain text (HTML tags stripped) */ + content: string + /** Article title */ + title: string | null + /** Article author/byline */ + author: string | null + /** Article excerpt/description */ + excerpt: string | null + /** Site name */ + siteName: string | null + /** Content language */ + lang: string | null + /** Published time */ + publishedTime: string | null + /** Word count */ + wordCount: number +} + +export interface GetPageMarkdownOptions { + page: Page + /** String or regex to filter content (returns matching lines with context) */ + search?: string | RegExp + /** Return diff since last call for this page */ + showDiffSinceLastCall?: boolean +} + +// Cache for the bundled readability code +let readabilityCode: string | null = null + +function getReadabilityCode(): string { + if (readabilityCode) { + return readabilityCode + } + const currentDir = path.dirname(fileURLToPath(import.meta.url)) + const readabilityPath = path.join(currentDir, '..', 'dist', 'readability.js') + readabilityCode = fs.readFileSync(readabilityPath, 'utf-8') + return readabilityCode +} + +// Store last snapshots per page for diffing +const lastMarkdownSnapshots: WeakMap = new WeakMap() + +function isRegExp(value: unknown): value is RegExp { + return ( + typeof value === 'object' && + value !== null && + typeof (value as RegExp).test === 'function' && + typeof (value as RegExp).exec === 'function' + ) +} + +/** + * Extract page content as markdown using Mozilla Readability. + * + * Injects Readability into the page if not already present, then extracts + * the main content. Returns plain text content (no HTML). + */ +export async function getPageMarkdown(options: GetPageMarkdownOptions): Promise { + const { page, search, showDiffSinceLastCall = false } = options + + // Check if readability is already injected + const hasReadability = await page.evaluate(() => !!(globalThis as any).__readability) + + if (!hasReadability) { + const code = getReadabilityCode() + await page.evaluate(code) + } + + // Extract content using Readability + const result = await page.evaluate(() => { + const readability = (globalThis as any).__readability + if (!readability) { + throw new Error('Readability not loaded') + } + + // Clone document to avoid modifying the original + const documentClone = document.cloneNode(true) + + // Check if page is probably readable + if (!readability.isProbablyReaderable(documentClone)) { + return { + content: document.body?.innerText || '', + title: document.title || null, + author: null, + excerpt: null, + siteName: null, + lang: document.documentElement?.lang || null, + publishedTime: null, + wordCount: (document.body?.innerText || '').split(/\s+/).filter(Boolean).length, + _notReadable: true, + } + } + + const article = new readability.Readability(documentClone).parse() + + if (!article) { + return { + content: document.body?.innerText || '', + title: document.title || null, + author: null, + excerpt: null, + siteName: null, + lang: document.documentElement?.lang || null, + publishedTime: null, + wordCount: (document.body?.innerText || '').split(/\s+/).filter(Boolean).length, + _notReadable: true, + } + } + + return { + content: article.textContent || '', + title: article.title || null, + author: article.byline || null, + excerpt: article.excerpt || null, + siteName: article.siteName || null, + lang: article.lang || null, + publishedTime: article.publishedTime || null, + wordCount: (article.textContent || '').split(/\s+/).filter(Boolean).length, + } + }) as PageMarkdownResult & { _notReadable?: boolean } + + // Format output + const lines: string[] = [] + + if (result.title) { + lines.push(`# ${result.title}`) + lines.push('') + } + + const metadata: string[] = [] + if (result.author) { + metadata.push(`Author: ${result.author}`) + } + if (result.siteName) { + metadata.push(`Site: ${result.siteName}`) + } + if (result.publishedTime) { + metadata.push(`Published: ${result.publishedTime}`) + } + if (metadata.length > 0) { + lines.push(metadata.join(' | ')) + lines.push('') + } + + if (result.excerpt && result.excerpt !== result.content.slice(0, result.excerpt.length)) { + lines.push(`> ${result.excerpt}`) + lines.push('') + } + + lines.push(result.content) + + let markdown = lines.join('\n').trim() + + // Sanitize to remove unpaired surrogates that break JSON encoding + markdown = markdown.toWellFormed?.() ?? markdown + + // Handle diffing + if (showDiffSinceLastCall) { + const previousSnapshot = lastMarkdownSnapshots.get(page) + + if (!previousSnapshot) { + lastMarkdownSnapshots.set(page, markdown) + return 'No previous snapshot available. This is the first call. Full snapshot stored for next diff.' + } + + const patch = createPatch('content', previousSnapshot, markdown, 'previous', 'current', { + context: 3, + }) + + lastMarkdownSnapshots.set(page, markdown) + + if (patch.split('\n').length <= 4) { + return 'No changes detected since last snapshot' + } + return patch + } + + // Store snapshot for future diffs + lastMarkdownSnapshots.set(page, markdown) + + // Handle search + if (search) { + const contentLines = markdown.split('\n') + const matchIndices: number[] = [] + + for (let i = 0; i < contentLines.length; i++) { + const line = contentLines[i] + let isMatch = false + if (isRegExp(search)) { + isMatch = search.test(line) + } else { + isMatch = line.toLowerCase().includes(search.toLowerCase()) + } + + if (isMatch) { + matchIndices.push(i) + if (matchIndices.length >= 10) { + break + } + } + } + + if (matchIndices.length === 0) { + return 'No matches found' + } + + // Collect lines with 5 lines of context above and below each match + const CONTEXT_LINES = 5 + const includedLines = new Set() + for (const idx of matchIndices) { + const start = Math.max(0, idx - CONTEXT_LINES) + const end = Math.min(contentLines.length - 1, idx + CONTEXT_LINES) + for (let i = start; i <= end; i++) { + includedLines.add(i) + } + } + + // Build result with separators between non-contiguous sections + const sortedIndices = [...includedLines].sort((a, b) => a - b) + const resultLines: string[] = [] + for (let i = 0; i < sortedIndices.length; i++) { + const lineIdx = sortedIndices[i] + if (i > 0 && sortedIndices[i - 1] !== lineIdx - 1) { + resultLines.push('---') + } + resultLines.push(contentLines[lineIdx]) + } + + return resultLines.join('\n') + } + + return markdown +} diff --git a/playwriter/src/relay-core.test.ts b/playwriter/src/relay-core.test.ts index 724612a..5071fc0 100644 --- a/playwriter/src/relay-core.test.ts +++ b/playwriter/src/relay-core.test.ts @@ -721,6 +721,104 @@ describe('Relay Core Tests', () => { await page.close() }, 60000) + it('should extract page content as markdown with getPageMarkdown', async () => { + const browserContext = getBrowserContext() + const serviceWorker = await getExtensionServiceWorker(browserContext) + + const page = await browserContext.newPage() + // Create a realistic article-like page structure + await page.setContent(` + + + Test Article Title + + + + + + +
+

Test Article Title

+

This is the first paragraph of the article content.

+

This is the second paragraph with more details about the topic.

+

The article continues with important information here.

+
+ + + + + `) + await page.bringToFront() + + await serviceWorker.evaluate(async () => { + await globalThis.toggleExtensionForActiveTab() + }) + await new Promise(r => setTimeout(r, 400)) + + // Test basic getPageMarkdown + const result = await client.callTool({ + name: 'execute', + arguments: { + code: js` + let testPage; + for (const p of context.pages()) { + const html = await p.content(); + if (html.includes('Test Article Title')) { testPage = p; break; } + } + if (!testPage) throw new Error('Test page not found'); + const content = await getPageMarkdown({ page: testPage }); + console.log(content); + `, + timeout: 15000, + }, + }) + + expect(result.isError).toBeFalsy() + const text = (result.content as any)[0]?.text || '' + + // Snapshot the full output + await expect(text).toMatchFileSnapshot('./snapshots/page-markdown-output.txt') + + // Should contain article content + expect(text).toContain('Test Article Title') + expect(text).toContain('first paragraph') + expect(text).toContain('second paragraph') + + // Should NOT contain script/style content + expect(text).not.toContain('analytics') + expect(text).not.toContain('background: blue') + + // Test search functionality + const searchResult = await client.callTool({ + name: 'execute', + arguments: { + code: js` + let testPage; + for (const p of context.pages()) { + const html = await p.content(); + if (html.includes('Test Article Title')) { testPage = p; break; } + } + if (!testPage) throw new Error('Test page not found'); + const content = await getPageMarkdown({ page: testPage, search: /important/i }); + return content; + `, + timeout: 15000, + }, + }) + + expect(searchResult.isError).toBeFalsy() + const searchText = (searchResult.content as any)[0]?.text || '' + expect(searchText).toContain('important') + + await page.close() + }, 60000) + it('should handle default page being closed and switch to another available page', async () => { // This test verifies that when the default `page` in MCP scope is closed, // the MCP automatically switches to another available page instead of failing diff --git a/playwriter/src/skill.md b/playwriter/src/skill.md index 7ce08cd..b4da563 100644 --- a/playwriter/src/skill.md +++ b/playwriter/src/skill.md @@ -493,6 +493,37 @@ console.log((await getCleanHTML({ locator: page })).split('\n').slice(0, 50).joi console.log((await getCleanHTML({ locator: page })).split('\n').slice(50, 100).join('\n')); // next 50 lines ``` +**getPageMarkdown** - extract main page content as plain text using Mozilla Readability (same algorithm as Firefox Reader View). Strips navigation, ads, sidebars, and other clutter. Returns formatted text with title, author, and content: + +```js +await getPageMarkdown({ page, search?, showDiffSinceLastCall? }) +// Examples: +const content = await getPageMarkdown({ page }) // full article as plain text +const matches = await getPageMarkdown({ page, search: /API/i }) // search within content +const diff = await getPageMarkdown({ page, showDiffSinceLastCall: true }) // track content changes +``` + +**Output format:** +``` +# Article Title + +Author: John Doe | Site: example.com | Published: 2024-01-15 + +> Article excerpt or description + +The main article content as plain text, with paragraphs preserved... +``` + +**Parameters:** +- `page` - Playwright Page to extract content from +- `search` - string/regex to filter content (returns first 10 matching lines with 5 lines context) +- `showDiffSinceLastCall` - returns unified diff since last call. Useful for tracking content changes. + +**Use cases:** +- Extract article text for LLM processing without HTML noise +- Get readable content from news sites, blogs, documentation +- Compare content changes after interactions + **waitForPageLoad** - smart load detection that ignores analytics/ads: ```js diff --git a/playwriter/src/snapshots/page-markdown-output.txt b/playwriter/src/snapshots/page-markdown-output.txt new file mode 100644 index 0000000..9e11a82 --- /dev/null +++ b/playwriter/src/snapshots/page-markdown-output.txt @@ -0,0 +1,16 @@ +Console output: +[log] '# Test Article Title\n' + + '\n' + + 'Home About\n' + + 'Test Article Title\n' + + '\n' + + 'This is the first paragraph of the article content.\n' + + '\n' + + 'This is the second paragraph with more details about the topic.\n' + + '\n' + + 'The article continues with important information here.\n' + + '\n' + + 'Related Posts\n' + + 'Post 1\n' + + 'Post 2\n' + + 'Copyright 2024' \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2b7f827..addc63c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,7 +14,7 @@ importers: devDependencies: '@changesets/cli': specifier: ^2.29.7 - version: 2.29.7(@types/node@25.0.3) + version: 2.29.7(@types/node@25.2.0) prettier: specifier: ^3.6.2 version: 3.6.2 @@ -26,10 +26,10 @@ importers: version: 5.9.3 vite: specifier: ^7.2.2 - version: 7.2.2(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) + version: 7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) vitest: specifier: ^4.0.8 - version: 4.0.8(@types/node@25.0.3)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6) + version: 4.0.8(@types/node@25.2.0)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6) extension: dependencies: @@ -51,10 +51,26 @@ importers: version: 5.9.3 vite: specifier: ^5.4.21 - version: 5.4.21(@types/node@25.0.3)(lightningcss@1.30.2) + version: 5.4.21(@types/node@25.2.0)(lightningcss@1.30.2) vite-plugin-static-copy: specifier: ^3.1.1 - version: 3.1.4(vite@5.4.21(@types/node@25.0.3)(lightningcss@1.30.2)) + version: 3.1.4(vite@5.4.21(@types/node@25.2.0)(lightningcss@1.30.2)) + + gmail-oauth: + dependencies: + google-auth-library: + specifier: ^10.5.0 + version: 10.5.0 + googleapis: + specifier: ^171.0.0 + version: 171.0.0 + devDependencies: + '@types/node': + specifier: ^25.2.0 + version: 25.2.0 + typescript: + specifier: ^5.9.3 + version: 5.9.3 playwriter: dependencies: @@ -113,6 +129,9 @@ importers: '@mizchi/selector-generator': specifier: 1.50.0-next version: 1.50.0-next + '@mozilla/readability': + specifier: ^0.6.0 + version: 0.6.0 '@types/chrome': specifier: ^0.0.315 version: 0.0.315 @@ -1366,6 +1385,10 @@ packages: '@cfworker/json-schema': optional: true + '@mozilla/readability@0.6.0': + resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==} + engines: {node: '>=14.0.0'} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -2317,6 +2340,9 @@ packages: '@types/node@25.0.3': resolution: {integrity: sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==} + '@types/node@25.2.0': + resolution: {integrity: sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w==} + '@types/pg-pool@2.0.6': resolution: {integrity: sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==} @@ -3390,6 +3416,14 @@ packages: resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} engines: {node: '>=14'} + googleapis-common@8.0.1: + resolution: {integrity: sha512-eCzNACUXPb1PW5l0ULTzMHaL/ltPRADoPgjBlT8jWsTbxkCp6siv+qKJ/1ldaybCthGwsYFYallF7u9AkU4L+A==} + engines: {node: '>=18.0.0'} + + googleapis@171.0.0: + resolution: {integrity: sha512-z+wpYZ9wfO/v58b/7fM0JqwDR6dx6yE3UdQZ9vWrzmzNVHFd85Uud8QewFfm8ht/3Hx2ISLbCs8RfnbrqR8X4A==} + engines: {node: '>=18'} + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -4827,6 +4861,9 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + url-template@2.0.8: + resolution: {integrity: sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==} + urlpattern-polyfill@10.0.0: resolution: {integrity: sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==} @@ -5845,7 +5882,7 @@ snapshots: dependencies: '@changesets/types': 6.1.0 - '@changesets/cli@2.29.7(@types/node@25.0.3)': + '@changesets/cli@2.29.7(@types/node@25.2.0)': dependencies: '@changesets/apply-release-plan': 7.0.13 '@changesets/assemble-release-plan': 6.0.9 @@ -5861,7 +5898,7 @@ snapshots: '@changesets/should-skip-package': 0.1.2 '@changesets/types': 6.1.0 '@changesets/write': 0.4.0 - '@inquirer/external-editor': 1.0.3(@types/node@25.0.3) + '@inquirer/external-editor': 1.0.3(@types/node@25.2.0) '@manypkg/get-packages': 1.1.3 ansi-colors: 4.1.3 ci-info: 3.9.0 @@ -6347,12 +6384,12 @@ snapshots: '@img/sharp-win32-x64@0.34.5': optional: true - '@inquirer/external-editor@1.0.3(@types/node@25.0.3)': + '@inquirer/external-editor@1.0.3(@types/node@25.2.0)': dependencies: chardet: 2.1.1 iconv-lite: 0.7.0 optionalDependencies: - '@types/node': 25.0.3 + '@types/node': 25.2.0 '@isaacs/cliui@8.0.2': dependencies: @@ -6459,6 +6496,8 @@ snapshots: - hono - supports-color + '@mozilla/readability@0.6.0': {} + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -7502,7 +7541,7 @@ snapshots: '@types/node-fetch@2.6.13': dependencies: - '@types/node': 24.10.1 + '@types/node': 25.0.3 form-data: 4.0.5 '@types/node@12.20.55': {} @@ -7519,6 +7558,10 @@ snapshots: dependencies: undici-types: 7.16.0 + '@types/node@25.2.0': + dependencies: + undici-types: 7.16.0 + '@types/pg-pool@2.0.6': dependencies: '@types/pg': 8.15.6 @@ -7559,11 +7602,11 @@ snapshots: '@types/ws@8.18.1': dependencies: - '@types/node': 24.10.1 + '@types/node': 25.0.3 '@types/yauzl@2.10.3': dependencies: - '@types/node': 24.10.1 + '@types/node': 25.0.3 optional: true '@typespec/ts-http-runtime@0.3.2': @@ -7630,21 +7673,13 @@ snapshots: optionalDependencies: vite: 7.3.0(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) - '@vitest/mocker@4.0.8(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))': + '@vitest/mocker@4.0.8(vite@7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))': dependencies: '@vitest/spy': 4.0.8 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) - - '@vitest/mocker@4.0.8(vite@7.2.2(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))': - dependencies: - '@vitest/spy': 4.0.8 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 7.2.2(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) + vite: 7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) '@vitest/pretty-format@4.0.16': dependencies: @@ -7689,7 +7724,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vitest: 4.0.8(@types/node@24.10.1)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6) + vitest: 4.0.8(@types/node@25.2.0)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6) '@vitest/utils@4.0.16': dependencies: @@ -8029,7 +8064,7 @@ snapshots: chrome-launcher@1.2.1: dependencies: - '@types/node': 24.10.1 + '@types/node': 25.0.3 escape-string-regexp: 4.0.0 is-wsl: 2.2.0 lighthouse-logger: 2.0.2 @@ -8830,6 +8865,23 @@ snapshots: google-logging-utils@1.1.3: {} + googleapis-common@8.0.1: + dependencies: + extend: 3.0.2 + gaxios: 7.1.3 + google-auth-library: 10.5.0 + qs: 6.14.1 + url-template: 2.0.8 + transitivePeerDependencies: + - supports-color + + googleapis@171.0.0: + dependencies: + google-auth-library: 10.5.0 + googleapis-common: 8.0.1 + transitivePeerDependencies: + - supports-color + gopd@1.2.0: {} graceful-fs@4.2.11: {} @@ -10328,6 +10380,8 @@ snapshots: dependencies: punycode: 2.3.1 + url-template@2.0.8: {} + urlpattern-polyfill@10.0.0: optional: true @@ -10391,13 +10445,13 @@ snapshots: dependencies: vite: 7.3.0(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) - vite-plugin-static-copy@3.1.4(vite@5.4.21(@types/node@25.0.3)(lightningcss@1.30.2)): + vite-plugin-static-copy@3.1.4(vite@5.4.21(@types/node@25.2.0)(lightningcss@1.30.2)): dependencies: chokidar: 3.6.0 p-map: 7.0.4 picocolors: 1.1.1 tinyglobby: 0.2.15 - vite: 5.4.21(@types/node@25.0.3)(lightningcss@1.30.2) + vite: 5.4.21(@types/node@25.2.0)(lightningcss@1.30.2) vite-tsconfig-paths@6.0.3(typescript@5.9.3)(vite@7.3.0(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)): dependencies: @@ -10410,13 +10464,13 @@ snapshots: - supports-color - typescript - vite@5.4.21(@types/node@25.0.3)(lightningcss@1.30.2): + vite@5.4.21(@types/node@25.2.0)(lightningcss@1.30.2): dependencies: esbuild: 0.21.5 postcss: 8.5.6 rollup: 4.53.2 optionalDependencies: - '@types/node': 25.0.3 + '@types/node': 25.2.0 fsevents: 2.3.3 lightningcss: 1.30.2 @@ -10435,7 +10489,7 @@ snapshots: lightningcss: 1.30.2 tsx: 4.20.6 - vite@7.2.2(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6): + vite@7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6): dependencies: esbuild: 0.25.12 fdir: 6.5.0(picomatch@4.0.3) @@ -10444,7 +10498,7 @@ snapshots: rollup: 4.53.2 tinyglobby: 0.2.15 optionalDependencies: - '@types/node': 25.0.3 + '@types/node': 25.2.0 fsevents: 2.3.3 jiti: 2.6.1 lightningcss: 1.30.2 @@ -10523,7 +10577,7 @@ snapshots: vitest@4.0.8(@types/node@24.10.1)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6): dependencies: '@vitest/expect': 4.0.8 - '@vitest/mocker': 4.0.8(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)) + '@vitest/mocker': 4.0.8(vite@7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)) '@vitest/pretty-format': 4.0.8 '@vitest/runner': 4.0.8 '@vitest/snapshot': 4.0.8 @@ -10560,10 +10614,10 @@ snapshots: - tsx - yaml - vitest@4.0.8(@types/node@25.0.3)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6): + vitest@4.0.8(@types/node@25.2.0)(@vitest/ui@4.0.8)(jiti@2.6.1)(jsdom@27.2.0(bufferutil@4.0.9))(lightningcss@1.30.2)(tsx@4.20.6): dependencies: '@vitest/expect': 4.0.8 - '@vitest/mocker': 4.0.8(vite@7.2.2(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)) + '@vitest/mocker': 4.0.8(vite@7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)) '@vitest/pretty-format': 4.0.8 '@vitest/runner': 4.0.8 '@vitest/snapshot': 4.0.8 @@ -10580,10 +10634,10 @@ snapshots: tinyexec: 0.3.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vite: 7.2.2(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) + vite: 7.2.2(@types/node@25.2.0)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 25.0.3 + '@types/node': 25.2.0 '@vitest/ui': 4.0.8(vitest@4.0.8) jsdom: 27.2.0(bufferutil@4.0.9) transitivePeerDependencies: diff --git a/website/public/SKILL.md b/website/public/SKILL.md index 7ce08cd..b4da563 100644 --- a/website/public/SKILL.md +++ b/website/public/SKILL.md @@ -493,6 +493,37 @@ console.log((await getCleanHTML({ locator: page })).split('\n').slice(0, 50).joi console.log((await getCleanHTML({ locator: page })).split('\n').slice(50, 100).join('\n')); // next 50 lines ``` +**getPageMarkdown** - extract main page content as plain text using Mozilla Readability (same algorithm as Firefox Reader View). Strips navigation, ads, sidebars, and other clutter. Returns formatted text with title, author, and content: + +```js +await getPageMarkdown({ page, search?, showDiffSinceLastCall? }) +// Examples: +const content = await getPageMarkdown({ page }) // full article as plain text +const matches = await getPageMarkdown({ page, search: /API/i }) // search within content +const diff = await getPageMarkdown({ page, showDiffSinceLastCall: true }) // track content changes +``` + +**Output format:** +``` +# Article Title + +Author: John Doe | Site: example.com | Published: 2024-01-15 + +> Article excerpt or description + +The main article content as plain text, with paragraphs preserved... +``` + +**Parameters:** +- `page` - Playwright Page to extract content from +- `search` - string/regex to filter content (returns first 10 matching lines with 5 lines context) +- `showDiffSinceLastCall` - returns unified diff since last call. Useful for tracking content changes. + +**Use cases:** +- Extract article text for LLM processing without HTML noise +- Get readable content from news sites, blogs, documentation +- Compare content changes after interactions + **waitForPageLoad** - smart load detection that ignores analytics/ads: ```js