removed relayConnection.ts. added diff for ai snapshot. use search param

This commit is contained in:
Tommy D. Rossi
2025-11-25 10:35:39 +01:00
parent 4e3be9b497
commit 0df52a08c0
10 changed files with 664 additions and 903 deletions
+1 -1
View File
@@ -55,7 +55,7 @@ remember that every time the extension is activated in a tab that tab gets added
to debug server or extension issues you can also inspect the file @playwriter/relay-server.log to see both extension and server logs. with all cdp events sent. to see if there are events missing or something broken. this file is recreated every time the server is started and appended in real time. use rg to only read relevant lines and parts because it can get quite long
tests will take about 30 seconds, so set a timeout of at least 60 seconds when running the test bash command
IMPORTANT: `pnpm test` will take about 30 seconds so set a timeout of at least 3600ms when running the pnpm test bash command
## changelogs
+1 -1
View File
@@ -55,7 +55,7 @@ remember that every time the extension is activated in a tab that tab gets added
to debug server or extension issues you can also inspect the file @playwriter/relay-server.log to see both extension and server logs. with all cdp events sent. to see if there are events missing or something broken. this file is recreated every time the server is started and appended in real time. use rg to only read relevant lines and parts because it can get quite long
tests will take about 30 seconds, so set a timeout of at least 60 seconds when running the test bash command
IMPORTANT: `pnpm test` will take about 30 seconds so set a timeout of at least 3600ms when running the pnpm test bash command
## changelogs
File diff suppressed because it is too large Load Diff
-494
View File
@@ -1,494 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { CDPEvent, Protocol } from 'playwriter/src/cdp-types'
import type { ExtensionCommandMessage, ExtensionResponseMessage } from 'playwriter/src/extension/protocol'
let activeConnection: RelayConnection | undefined
export const logger = {
log: (...args: any[]) => logToRemote('log', args),
debug: (...args: any[]) => logToRemote('debug', args),
info: (...args: any[]) => logToRemote('info', args),
warn: (...args: any[]) => logToRemote('warn', args),
error: (...args: any[]) => logToRemote('error', args),
}
function logToRemote(level: 'log' | 'debug' | 'info' | 'warn' | 'error', args: any[]) {
// Always log to local console
console[level](...args)
if (activeConnection) {
activeConnection.sendLog(level, args)
}
}
function safeSerialize(arg: any): string {
if (arg === undefined) return 'undefined'
if (arg === null) return 'null'
if (typeof arg === 'function') return `[Function: ${arg.name || 'anonymous'}]`
if (typeof arg === 'symbol') return String(arg)
if (arg instanceof Error) {
return arg.stack || arg.message || String(arg)
}
if (typeof arg === 'object') {
try {
const seen = new WeakSet()
return JSON.stringify(arg, (key, value) => {
if (typeof value === 'object' && value !== null) {
if (seen.has(value)) return '[Circular]'
seen.add(value)
if (value instanceof Map) {
return {
dataType: 'Map',
value: Array.from(value.entries()),
}
}
if (value instanceof Set) {
return {
dataType: 'Set',
value: Array.from(value.values()),
}
}
}
return value
})
} catch (e) {
return String(arg)
}
}
return String(arg)
}
interface AttachedTab {
debuggee: chrome.debugger.Debuggee
targetId: string
sessionId: string
targetInfo: Protocol.Target.TargetInfo
}
export class RelayConnection {
private _attachedTabs: Map<number, AttachedTab> = new Map()
private _childSessions: Map<string, number> = new Map()
private _nextSessionId: number = 1
private _ws: WebSocket
private _closed = false
private _onCloseCallback?: (reason: string, code: number) => void
private _onTabDetachedCallback?: (tabId: number, reason: `${chrome.debugger.DetachReason}`) => void
private _onTabAttachedCallback?: (tabId: number, targetId: string) => void
constructor({
ws,
onClose,
onTabDetached,
onTabAttached,
}: {
ws: WebSocket
onClose?: (reason: string, code: number) => void
onTabDetached?: (tabId: number, reason: `${chrome.debugger.DetachReason}`) => void
onTabAttached?: (tabId: number, targetId: string) => void
}) {
this._ws = ws
this._onCloseCallback = onClose
this._onTabDetachedCallback = onTabDetached
this._onTabAttachedCallback = onTabAttached
this._ws.onmessage = async (event: MessageEvent) => {
let message: ExtensionCommandMessage
try {
message = JSON.parse(event.data)
} catch (error: any) {
logger.debug('Error parsing message:', error)
this._sendMessage({
error: {
code: -32700,
message: `Error parsing message: ${error.message}`,
},
})
return
}
// logger.debug('Received message:', message)
const response: ExtensionResponseMessage = {
id: message.id,
}
try {
response.result = await this._handleCommand(message)
} catch (error: any) {
logger.debug('Error handling command:', error)
response.error = error.message
}
logger.debug('Sending response:', response)
this._sendMessage(response)
}
this._ws.onclose = (event: CloseEvent) => {
logger.debug('WebSocket onclose event:', {
code: event.code,
reason: event.reason,
wasClean: event.wasClean,
})
this._onClose(event.reason, event.code)
}
this._ws.onerror = (event: Event) => {
logger.debug('WebSocket onerror event:', event)
}
chrome.debugger.onEvent.addListener(this._onDebuggerEvent)
chrome.debugger.onDetach.addListener(this._onDebuggerDetach)
logger.debug('RelayConnection created, WebSocket readyState:', this._ws.readyState)
activeConnection = this
}
sendLog(level: string, args: any[]) {
this._sendMessage({
method: 'log',
params: {
level,
args: args.map((arg) => safeSerialize(arg)),
},
})
}
async attachTab(tabId: number): Promise<Protocol.Target.TargetInfo> {
const debuggee = { tabId }
logger.debug('Attaching debugger to tab:', tabId, 'WebSocket state:', this._ws.readyState)
try {
await chrome.debugger.attach(debuggee, '1.3')
logger.debug('Debugger attached successfully to tab:', tabId)
} catch (error: any) {
logger.debug('ERROR attaching debugger to tab:', tabId, error)
throw error
}
logger.debug('Sending Target.getTargetInfo command for tab:', tabId)
const result = (await chrome.debugger.sendCommand(
debuggee,
'Target.getTargetInfo',
)) as Protocol.Target.GetTargetInfoResponse
logger.debug('Received targetInfo for tab:', tabId, result.targetInfo)
const targetInfo = result.targetInfo
const sessionId = `pw-tab-${this._nextSessionId++}`
this._attachedTabs.set(tabId, {
debuggee,
targetId: targetInfo.targetId,
sessionId,
targetInfo,
})
logger.debug('Sending Target.attachedToTarget event, WebSocket state:', this._ws.readyState)
this._sendMessage({
method: 'forwardCDPEvent',
params: {
method: 'Target.attachedToTarget',
params: {
sessionId,
targetInfo: {
...targetInfo,
attached: true,
},
waitingForDebugger: false,
},
},
})
logger.debug('Tab attached successfully:', tabId, 'sessionId:', sessionId, 'targetId:', targetInfo.targetId)
this._onTabAttachedCallback?.(tabId, targetInfo.targetId)
return targetInfo
}
detachTab(tabId: number): void {
this._cleanupTab(tabId, true)
}
private _cleanupTab(tabId: number, shouldDetachDebugger: boolean): void {
const tab = this._attachedTabs.get(tabId)
if (!tab) {
logger.debug('cleanupTab: tab not found in map:', tabId)
return
}
logger.debug('Cleaning up tab:', tabId, 'sessionId:', tab.sessionId, 'shouldDetach:', shouldDetachDebugger)
this._sendMessage({
method: 'forwardCDPEvent',
params: {
method: 'Target.detachedFromTarget',
params: {
sessionId: tab.sessionId,
targetId: tab.targetId,
},
},
})
this._attachedTabs.delete(tabId)
for (const [childSessionId, parentTabId] of this._childSessions.entries()) {
if (parentTabId === tabId) {
logger.debug('Cleaning up child session:', childSessionId, 'for tab:', tabId)
this._childSessions.delete(childSessionId)
}
}
logger.debug('Removed tab from _attachedTabs map. Remaining tabs:', this._attachedTabs.size)
if (shouldDetachDebugger) {
chrome.debugger
.detach(tab.debuggee)
.then(() => {
logger.debug('Successfully detached debugger from tab:', tabId)
})
.catch((err) => {
logger.debug('Error detaching debugger from tab:', tabId, err.message)
})
}
}
close(message: string): void {
logger.debug('Closing RelayConnection, reason:', message, 'current state:', this._ws.readyState)
this._ws.close(1000, message)
this._onClose(message, 1000)
}
private _onClose(reason: string = 'Unknown', code: number = 1000) {
if (this._closed) {
logger.debug('_onClose called but already closed')
return
}
logger.debug('Connection closing, attached tabs count:', this._attachedTabs.size)
this._closed = true
chrome.debugger.onEvent.removeListener(this._onDebuggerEvent)
chrome.debugger.onDetach.removeListener(this._onDebuggerDetach)
const tabIds = Array.from(this._attachedTabs.keys())
logger.debug('Detaching all tabs:', tabIds)
for (const [tabId, tab] of this._attachedTabs) {
logger.debug('Detaching debugger from tab:', tabId)
chrome.debugger
.detach(tab.debuggee)
.then(() => {
logger.debug('Successfully detached from tab:', tabId)
})
.catch((err) => {
logger.debug('Error detaching from tab:', tabId, err.message)
})
}
this._attachedTabs.clear()
logger.debug('All tabs cleared from map. Chrome automation bar should disappear in a few seconds.')
logger.debug('Connection closed, calling onClose callback')
if (activeConnection === this) {
activeConnection = undefined
}
this._onCloseCallback?.(reason, code)
}
private _onDebuggerEvent = (source: chrome.debugger.DebuggerSession, method: string, params: any): void => {
const tab = this._attachedTabs.get(source.tabId!)
if (!tab) return
// Track execution contexts so we can replay them when Playwright reconnects.
// Chrome's debugger only sends Runtime.executionContextCreated events once per context,
// not on every Runtime.enable call. We cache them here and replay on reconnection.
logger.debug('Forwarding CDP event:', method, 'from tab:', source.tabId)
if (method === 'Target.attachedToTarget' && params?.sessionId) {
logger.debug('Child target attached:', params.sessionId, 'for tab:', source.tabId)
this._childSessions.set(params.sessionId, source.tabId!)
}
if (method === 'Target.detachedFromTarget' && params?.sessionId) {
logger.debug('Child target detached:', params.sessionId)
this._childSessions.delete(params.sessionId)
}
this._sendMessage({
method: 'forwardCDPEvent',
params: {
sessionId: source.sessionId || tab.sessionId,
method,
params,
},
})
}
private _onDebuggerDetach = (source: chrome.debugger.Debuggee, reason: `${chrome.debugger.DetachReason}`): void => {
const tabId = source.tabId
logger.debug(
'_onDebuggerDetach called for tab:',
tabId,
'reason:',
reason,
'isAttached:',
tabId ? this._attachedTabs.has(tabId) : false,
)
if (!tabId || !this._attachedTabs.has(tabId)) {
logger.debug('Ignoring debugger detach event for untracked tab:', tabId)
return
}
logger.debug(`Manual debugger detachment detected for tab ${tabId}: ${reason}`)
logger.debug('User closed debugger via Chrome automation bar, calling onTabDetached callback')
this._onTabDetachedCallback?.(tabId, reason)
this._cleanupTab(tabId, false)
}
private async _handleCommand(msg: ExtensionCommandMessage): Promise<any> {
if (msg.method === 'forwardCDPCommand') {
let targetTab: AttachedTab | undefined
// find tab by sessionId
if (msg.params.sessionId) {
for (const [tabId, tab] of this._attachedTabs) {
if (tab.sessionId === msg.params.sessionId) {
targetTab = tab
break
}
}
}
// find parent tab if this is a child target
if (!targetTab && msg.params.sessionId) {
const parentTabId = this._childSessions.get(msg.params.sessionId)
if (parentTabId) {
targetTab = this._attachedTabs.get(parentTabId)
logger.debug('Found parent tab for child session:', msg.params.sessionId, 'tabId:', parentTabId)
}
}
// find tab by targetId in params
if (!targetTab && msg.params.params && 'targetId' in msg.params.params && msg.params.params.targetId) {
const paramTargetId = msg.params.params.targetId as string
for (const [tabId, tab] of this._attachedTabs) {
if (tab.targetId === paramTargetId) {
targetTab = tab
logger.debug('Found tab for targetId:', paramTargetId, 'tabId:', tabId)
break
}
}
}
switch (msg.params.method) {
// CRITICAL FIX FOR PLAYWRIGHT HANGS:
// When Playwright connects to an existing page, it sends 'Runtime.enable' and waits for
// 'Runtime.executionContextCreated' events to map out available execution contexts (frames, workers).
//
// However, if the Runtime domain is already enabled or if Chrome doesn't re-emit these events
// for existing contexts upon a standard 'Runtime.enable' call, Playwright never gets the
// context IDs it needs. This causes commands like `page.evaluate()` to hang indefinitely
// because Playwright is waiting for a context that it believes hasn't been created yet.
//
// To fix this, we intercept 'Runtime.enable' and force a reset:
// 1. We send 'Runtime.disable'. This clears Chrome's internal Runtime state for this session.
// 2. We wait a brief moment (100ms) to ensure the disable command propagates.
// 3. Then we allow the original 'Runtime.enable' command to proceed.
//
// This sequence forces Chrome to treat the enablement as a fresh start, causing it to
// re-scan and emit 'Runtime.executionContextCreated' events for ALL existing contexts.
// Playwright receives these events, populates its context map, and the hang is resolved.
case 'Runtime.enable': {
if (!targetTab?.debuggee) throw new Error(`No debuggee found for tab (sessionId: ${msg.params.sessionId}, tabId: ${targetTab?.debuggee?.tabId ?? 'unknown'})`)
try {
await chrome.debugger.sendCommand(targetTab?.debuggee!, 'Runtime.disable')
await new Promise((resolve) => setTimeout(resolve, 200))
} catch (e) {
logger.debug('Error disabling Runtime (ignoring):', e)
}
const result = await chrome.debugger.sendCommand(targetTab?.debuggee!, 'Runtime.enable', msg.params.params)
return result
}
case 'Target.createTarget': {
const url = msg.params.params?.url || 'about:blank'
logger.debug('Creating new tab with URL:', url)
const tab = await chrome.tabs.create({ url, active: false })
if (!tab.id) {
throw new Error('Failed to create tab')
}
logger.debug('Created tab:', tab.id, 'waiting for it to load...')
await new Promise((resolve) => setTimeout(resolve, 100))
const targetInfo = await this.attachTab(tab.id)
return { targetId: targetInfo.targetId } satisfies Protocol.Target.CreateTargetResponse
}
case 'Target.closeTarget': {
if (!targetTab || !targetTab.debuggee.tabId) {
logger.log(`Target not found: ${msg.params.params?.targetId}`)
return { success: false } satisfies Protocol.Target.CloseTargetResponse
}
await chrome.tabs.remove(targetTab.debuggee.tabId!)
return { success: true } satisfies Protocol.Target.CloseTargetResponse
}
}
if (!targetTab) {
throw new Error(
`No tab found for method ${msg.params.method} sessionId: ${msg.params.sessionId} params: ${JSON.stringify(msg.params.params || null)}`,
)
}
logger.debug('CDP command:', msg.params.method, 'for tab:', targetTab.debuggee.tabId)
const debuggerSession: chrome.debugger.DebuggerSession = {
...targetTab.debuggee,
sessionId: msg.params.sessionId !== targetTab.sessionId ? msg.params.sessionId : undefined,
}
const result = await chrome.debugger.sendCommand(debuggerSession, msg.params.method, msg.params.params)
return result
}
}
private _sendMessage(message: any): void {
if (this._ws.readyState === WebSocket.OPEN) {
try {
this._ws.send(JSON.stringify(message))
// logger.debug('Message sent successfully, type:', message.method || 'response');
} catch (error: any) {
// Use console directly to avoid infinite recursion if logger tries to send log over this same connection
console.debug('ERROR sending message:', error, 'message type:', message.method || 'response')
}
} else {
// Use console directly to avoid infinite recursion
console.debug(
'Cannot send message, WebSocket not open. State:',
this._ws.readyState,
'message type:',
message.method || 'response',
)
}
}
}
+3 -5
View File
@@ -1,17 +1,15 @@
import type { RelayConnection } from './relayConnection'
export type ConnectionState = 'disconnected' | 'reconnecting' | 'connected' | 'error'
export type TabState = 'connecting' | 'connected' | 'error'
export interface TabInfo {
targetId: string
sessionId?: string
targetId?: string
state: TabState
errorText?: string
}
export interface ExtensionState {
connection: RelayConnection | undefined
connectedTabs: Map<number, TabInfo>
tabs: Map<number, TabInfo>
connectionState: ConnectionState
currentTabId: number | undefined
errorText: string | undefined
+1
View File
@@ -40,6 +40,7 @@
"@modelcontextprotocol/sdk": "^1.21.1",
"chalk": "^5.6.2",
"devtools-protocol": "^0.0.1543509",
"diff": "^8.0.2",
"hono": "^4.10.6",
"playwright-core": "^1.56.1",
"string-dedent": "^3.0.2",
+7 -7
View File
@@ -226,9 +226,9 @@ describe('MCP Server Tests', () => {
const tabs = await chrome.tabs.query({})
const testTab = tabs.find((t: any) => t.url?.includes('mcp-test'))
return {
connected: !!testTab && !!testTab.id && state.connectedTabs.has(testTab.id),
connected: !!testTab && !!testTab.id && state.tabs.has(testTab.id),
tabId: testTab?.id,
tabInfo: testTab?.id ? state.connectedTabs.get(testTab.id) : null,
tabInfo: testTab?.id ? state.tabs.get(testTab.id) : null,
connectionState: state.connectionState
}
})
@@ -581,8 +581,8 @@ describe('MCP Server Tests', () => {
const tabA = tabs.find((t: any) => t.url?.includes('tab-a'))
const tabB = tabs.find((t: any) => t.url?.includes('tab-b'))
return {
idA: state.connectedTabs.get(tabA?.id ?? -1)?.targetId,
idB: state.connectedTabs.get(tabB?.id ?? -1)?.targetId
idA: state.tabs.get(tabA?.id ?? -1)?.targetId,
idB: state.tabs.get(tabB?.id ?? -1)?.targetId
}
})
@@ -667,8 +667,8 @@ describe('MCP Server Tests', () => {
const tabA = tabs.find((t: any) => t.url?.includes('tab-a'))
const tabB = tabs.find((t: any) => t.url?.includes('tab-b'))
return {
idA: state.connectedTabs.get(tabA?.id ?? -1)?.targetId,
idB: state.connectedTabs.get(tabB?.id ?? -1)?.targetId
idA: state.tabs.get(tabA?.id ?? -1)?.targetId,
idB: state.tabs.get(tabB?.id ?? -1)?.targetId
}
})
@@ -961,7 +961,7 @@ describe('MCP Server Tests', () => {
name: 'execute',
arguments: {
code: js`
const logs = await getLatestLogs({ searchFilter: 'error' });
const logs = await getLatestLogs({ search: 'error' });
logs.forEach(log => console.log(log));
`,
},
+24 -1
View File
@@ -8,6 +8,7 @@ import { spawn } from 'node:child_process'
import { createRequire } from 'node:module'
import vm from 'node:vm'
import dedent from 'string-dedent'
import { createPatch } from 'diff'
import { getCdpUrl } from './utils.js'
const require = createRequire(import.meta.url)
@@ -51,6 +52,7 @@ interface VMContext {
page: Page
search?: string | RegExp
contextLines?: number
showDiffSinceLastCall?: boolean
}) => Promise<string>
getLocatorStringForElement: (element: any) => Promise<string>
resetPlaywright: () => Promise<{ page: Page; context: BrowserContext }>
@@ -77,6 +79,9 @@ const userState: Record<string, any> = {}
const browserLogs: Map<string, string[]> = new Map()
const MAX_LOGS_PER_PAGE = 5000
// Store last accessibility snapshot per page for diff feature
const lastSnapshots: WeakMap<Page, string> = new WeakMap()
const RELAY_PORT = 19988
const NO_TABS_ERROR = `No browser tabs are connected. Please install and enable the Playwriter extension on at least one tab: https://chromewebstore.google.com/detail/playwriter-mcp/jfeammnjpkecdekppnclgkkffahnhfhe`
@@ -390,12 +395,30 @@ server.tool(
page: Page
search?: string | RegExp
contextLines?: number
showDiffSinceLastCall?: boolean
}) => {
const { page: targetPage, search, contextLines = 10 } = options
const { page: targetPage, search, contextLines = 10, showDiffSinceLastCall = false } = options
if ((targetPage as any)._snapshotForAI) {
const snapshot = await (targetPage as any)._snapshotForAI()
const snapshotStr = typeof snapshot === 'string' ? snapshot : JSON.stringify(snapshot, null, 2)
if (showDiffSinceLastCall) {
const previousSnapshot = lastSnapshots.get(targetPage)
lastSnapshots.set(targetPage, snapshotStr)
if (!previousSnapshot) {
return 'No previous snapshot available. This is the first call for this page. Full snapshot stored for next diff.'
}
const patch = createPatch('snapshot', previousSnapshot, snapshotStr, 'previous', 'current')
if (patch.split('\n').length <= 4) {
return 'No changes detected since last snapshot'
}
return patch
}
lastSnapshots.set(targetPage, snapshotStr)
if (!search) {
return snapshotStr
}
+2 -1
View File
@@ -72,10 +72,11 @@ always detach event listener you create at the end of a message using `page.remo
you have access to some functions in addition to playwright methods:
- `async accessibilitySnapshot({ page, search, contextLines })`: gets a human readable snapshot of clickable elements on the page. useful to see the overall structure of the page and what elements you can interact with.
- `async accessibilitySnapshot({ page, search, contextLines, showDiffSinceLastCall })`: gets a human readable snapshot of clickable elements on the page. useful to see the overall structure of the page and what elements you can interact with.
- `page`: the page object to snapshot
- `search`: (optional) a string or regex to filter the snapshot. If provided, returns the first 10 matches with surrounding context
- `contextLines`: (optional) number of lines of context to show around each match (default: 10)
- `showDiffSinceLastCall`: (optional) if true, returns a unified diff patch showing only changes since the last snapshot call for this page. Disables search when enabled. Useful to see what changed after an action.
- `getLatestLogs({ page, count, search })`: retrieves browser console logs. The system automatically captures and stores up to 5000 logs per page. Logs are cleared when a page reloads or navigates.
- `page`: (optional) filter logs by a specific page instance. Only returns logs from that page
- `count`: (optional) limit number of logs to return. If not specified, returns all available logs
+9
View File
@@ -75,6 +75,9 @@ importers:
devtools-protocol:
specifier: ^0.0.1543509
version: 0.0.1543509
diff:
specifier: ^8.0.2
version: 8.0.2
hono:
specifier: ^4.10.6
version: 4.10.6
@@ -931,6 +934,10 @@ packages:
devtools-protocol@0.0.1543509:
resolution: {integrity: sha512-JlHmIPtgDeqcL2uwPA7xryDNSb1ug9h11IexYQUG98vcyV6/L3taLRECgUk/B/7yQhXC/sgBzKj9kaB+bxmdWg==}
diff@8.0.2:
resolution: {integrity: sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg==}
engines: {node: '>=0.3.1'}
dir-glob@3.0.1:
resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==}
engines: {node: '>=8'}
@@ -2653,6 +2660,8 @@ snapshots:
devtools-protocol@0.0.1543509: {}
diff@8.0.2: {}
dir-glob@3.0.1:
dependencies:
path-type: 4.0.0