Create wait-for-page-load.ts

playwriter/src/mcp.ts

@@ -12,6 +12,7 @@ import vm from 'node:vm'
 import dedent from 'string-dedent'
 import { createPatch } from 'diff'
 import { getCdpUrl } from './utils.js'
+import { waitForPageLoad, WaitForPageLoadOptions, WaitForPageLoadResult } from './wait-for-page-load.js'
 
 const require = createRequire(import.meta.url)
 
@@ -60,6 +61,7 @@ interface VMContext {
   resetPlaywright: () => Promise<{ page: Page; context: BrowserContext }>
   getLatestLogs: (options?: { page?: Page; count?: number; search?: string | RegExp }) => Promise<string[]>
   clearAllLogs: () => void
+  waitForPageLoad: (options: WaitForPageLoadOptions) => Promise<WaitForPageLoadResult>
   require: NodeRequire
   import: (specifier: string) => Promise<any>
 }
@@ -527,6 +529,7 @@ server.tool(
         getLocatorStringForElement,
         getLatestLogs,
         clearAllLogs,
+        waitForPageLoad,
         resetPlaywright: async () => {
           const { page: newPage, context: newContext } = await resetConnection()
 
@@ -539,6 +542,7 @@ server.tool(
             getLocatorStringForElement,
             getLatestLogs,
             clearAllLogs,
+            waitForPageLoad,
             resetPlaywright: vmContextObj.resetPlaywright,
             require,
             // TODO --experimental-vm-modules is needed to make import work in vm

playwriter/src/prompt.md

@@ -81,6 +81,13 @@ you have access to some functions in addition to playwright methods:
     - `page`: (optional) filter logs by a specific page instance. Only returns logs from that page
     - `count`: (optional) limit number of logs to return. If not specified, returns all available logs
     - `search`: (optional) string or regex to filter logs. Only returns logs that match
+- `waitForPageLoad({ page, timeout, pollInterval, minWait })`: smart network-aware page load detection. Playwright's `networkidle` waits for ALL requests to finish, which often times out on sites with analytics/ads. This function ignores those and returns when meaningful content is loaded.
+    - `page`: the page object to wait on
+    - `timeout`: (optional) max wait time in ms (default: 30000)
+    - `pollInterval`: (optional) how often to check in ms (default: 100)
+    - `minWait`: (optional) minimum wait before checking in ms (default: 500)
+    - Returns: `{ success, readyState, pendingRequests, waitTimeMs, timedOut }`
+    - Filters out: ad networks (doubleclick, googlesyndication), analytics (google-analytics, mixpanel, segment), social (facebook.net, twitter), support widgets (intercom, zendesk), and slow fonts/images
 
 To bring a tab to front and focus it, use the standard Playwright method `await page.bringToFront()`
 

playwriter/src/wait-for-page-load.ts

@@ -0,0 +1,162 @@
+import type { Page } from 'playwright-core'
+
+const FILTERED_DOMAINS = [
+  'doubleclick',
+  'googlesyndication',
+  'googleadservices',
+  'google-analytics',
+  'googletagmanager',
+  'facebook.net',
+  'fbcdn.net',
+  'twitter.com',
+  'linkedin.com',
+  'hotjar',
+  'mixpanel',
+  'segment.io',
+  'segment.com',
+  'newrelic',
+  'datadoghq',
+  'sentry.io',
+  'fullstory',
+  'amplitude',
+  'intercom',
+  'crisp.chat',
+  'zdassets.com',
+  'zendesk',
+  'tawk.to',
+  'hubspot',
+  'marketo',
+  'pardot',
+  'optimizely',
+  'crazyegg',
+  'mouseflow',
+  'clarity.ms',
+  'bing.com/bat',
+  'ads.',
+  'analytics.',
+  'tracking.',
+  'pixel.',
+]
+
+const FILTERED_EXTENSIONS = ['.gif', '.ico', '.cur', '.woff', '.woff2', '.ttf', '.otf', '.eot']
+
+export interface WaitForPageLoadOptions {
+  page: Page
+  timeout?: number
+  pollInterval?: number
+  minWait?: number
+}
+
+export interface WaitForPageLoadResult {
+  success: boolean
+  readyState: string
+  pendingRequests: string[]
+  waitTimeMs: number
+  timedOut: boolean
+}
+
+export async function waitForPageLoad(options: WaitForPageLoadOptions): Promise<WaitForPageLoadResult> {
+  const { page, timeout = 30000, pollInterval = 100, minWait = 500 } = options
+
+  const startTime = Date.now()
+  let timedOut = false
+  let lastReadyState = ''
+  let lastPendingRequests: string[] = []
+
+  const checkPageReady = async (): Promise<{ ready: boolean; readyState: string; pendingRequests: string[] }> => {
+    const result = await page.evaluate(
+      ({ filteredDomains, filteredExtensions, stuckThreshold, slowResourceThreshold }): {
+        ready: boolean
+        readyState: string
+        pendingRequests: string[]
+      } => {
+        const doc = globalThis.document as { readyState: string }
+        const readyState = doc.readyState
+
+        if (readyState !== 'complete') {
+          return { ready: false, readyState, pendingRequests: [`document.readyState: ${readyState}`] }
+        }
+
+        const resources = (performance as any).getEntriesByType('resource') as Array<{
+          name: string
+          startTime: number
+          responseEnd: number
+        }>
+        const now = (performance as any).now() as number
+
+        const pendingRequests = resources
+          .filter((r) => {
+            if (r.responseEnd > 0) {
+              return false
+            }
+
+            const elapsed = now - r.startTime
+            const url = r.name.toLowerCase()
+
+            if (url.startsWith('data:')) {
+              return false
+            }
+
+            if (filteredDomains.some((domain: string) => url.includes(domain))) {
+              return false
+            }
+
+            if (elapsed > stuckThreshold) {
+              return false
+            }
+
+            if (elapsed > slowResourceThreshold && filteredExtensions.some((ext: string) => url.includes(ext))) {
+              return false
+            }
+
+            return true
+          })
+          .map((r) => r.name)
+
+        return {
+          ready: pendingRequests.length === 0,
+          readyState,
+          pendingRequests,
+        }
+      },
+      {
+        filteredDomains: FILTERED_DOMAINS,
+        filteredExtensions: FILTERED_EXTENSIONS,
+        stuckThreshold: 10000,
+        slowResourceThreshold: 3000,
+      },
+    )
+
+    return result
+  }
+
+  await new Promise((resolve) => setTimeout(resolve, minWait))
+
+  while (Date.now() - startTime < timeout) {
+    const { ready, readyState, pendingRequests } = await checkPageReady()
+    lastReadyState = readyState
+    lastPendingRequests = pendingRequests
+
+    if (ready) {
+      return {
+        success: true,
+        readyState,
+        pendingRequests: [],
+        waitTimeMs: Date.now() - startTime,
+        timedOut: false,
+      }
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, pollInterval))
+  }
+
+  timedOut = true
+
+  return {
+    success: false,
+    readyState: lastReadyState,
+    pendingRequests: lastPendingRequests.slice(0, 10),
+    waitTimeMs: Date.now() - startTime,
+    timedOut,
+  }
+}