Files
playwriter/extension-kapture/page-helpers.js
Tommy D. Rossi ec02a90812 adding examples
2025-11-13 19:42:44 +01:00

684 lines
22 KiB
JavaScript

// page-helpers.js - Content script that provides helper functions
let kaptureIdCounter = 0;
function getUniqueSelector(element) {
if (!element || !(element instanceof Element)) return null;
// Special handling for html, head, and body elements - their tagName is unique
const tagName = element.tagName.toLowerCase();
if (tagName === 'html' || tagName === 'head' || tagName === 'body') {
return tagName;
}
// If element has an ID, use it (unless it's empty or contains special chars)
if (element.id && /^[a-zA-Z][\w-]*$/.test(element.id)) {
// Check if ID is truly unique
if (document.querySelectorAll('#' + CSS.escape(element.id)).length === 1) {
return '#' + CSS.escape(element.id);
}
}
const uniqueId = 'kapture-' + (++kaptureIdCounter)
if (!element.id) {
element.id = uniqueId;
return '#' + uniqueId;
}
element.classList.add(uniqueId);
return '.' + uniqueId
}
function findScrollableParent(element) {
function isScrollable(element) {
const hasScrollableContent = element.scrollHeight > element.clientHeight ||
element.scrollWidth > element.clientWidth;
if (!hasScrollableContent) return false;
const style = getComputedStyle(element);
return /(auto|scroll)/.test(style.overflow + style.overflowY + style.overflowX);
}
let parent = element.parentElement;
while (parent && parent !== document.body) {
if (isScrollable(parent)) return parent;
parent = parent.parentElement;
}
return document.documentElement;
}
function serializeValue(value, depth = 0, maxDepth = 3, seen = new WeakSet()) {
// Handle primitive types
if (value === null || value === undefined) return value;
if (typeof value === 'boolean' || typeof value === 'number' || typeof value === 'string') {
return value;
}
if (typeof value === 'function') return '[Function: ' + (value.name || 'anonymous') + ']';
if (typeof value === 'symbol') return value.toString();
if (value instanceof Date) return value.toISOString();
if (value instanceof RegExp) return value.toString();
// Handle errors
if (value instanceof Error) {
return {
name: value.name,
message: value.message,
stack: value.stack
};
}
// Handle DOM elements
if (value instanceof Element) {
const selector = getUniqueSelector(value);
return {
nodeType: 'ELEMENT_NODE',
selector: selector,
tagName: value.tagName,
id: value.id || undefined,
className: value.className || undefined,
attributes: Array.from(value.attributes).reduce((acc, attr) => {
acc[attr.name] = attr.value;
return acc;
}, {})
};
}
// Prevent infinite recursion
if (depth >= maxDepth) {
return '[Max depth reached]';
}
// Handle circular references
if (typeof value === 'object' && seen.has(value)) {
return '[Circular reference]';
}
// Mark object as seen
if (typeof value === 'object') {
seen.add(value);
}
// Handle arrays
if (Array.isArray(value)) {
return value.map(item => serializeValue(item, depth + 1, maxDepth, seen));
}
// Handle NodeList and HTMLCollection
if (value instanceof NodeList || value instanceof HTMLCollection) {
return {
nodeType: value instanceof NodeList ? 'NodeList' : 'HTMLCollection',
length: value.length,
items: Array.from(value).map(item => serializeValue(item, depth + 1, maxDepth, seen))
};
}
// Handle typed arrays
if (ArrayBuffer.isView(value)) {
return {
type: value.constructor.name,
length: value.length,
data: '[Binary data]'
};
}
// Handle other objects
if (typeof value === 'object') {
const result = {};
const keys = Object.keys(value);
// Limit number of keys to prevent huge objects
const maxKeys = 100;
const limitedKeys = keys.slice(0, maxKeys);
for (const key of limitedKeys) {
try {
const serialized = this.serializeValue(value[key], depth + 1, maxDepth, seen);
if (serialized === undefined || serialized === null) continue; // Skip undefined values
result[key] = serialized;
} catch (e) {
result[key] = '[Error accessing property]';
}
}
if (keys.length > maxKeys) {
result['...'] = `${keys.length - maxKeys} more properties`;
}
return result;
}
// Fallback for unknown types
return String(value);
}
function getTabInfo() {
const de = document.documentElement;
return {
url: window.location.href,
title: document.title,
domSize: de.outerHTML.length,
fullPageDimensions: { width: de.scrollWidth, height: de.scrollHeight },
viewportDimensions: { width: window.innerWidth, height: window.innerHeight },
scrollPosition: { x: window.pageXOffset || de.scrollLeft, y: window.pageYOffset || de.scrollTop },
pageVisibility: { visible: !document.hidden, visibilityState: document.visibilityState }
};
}
function findAllElements(selector, xpath) {
if (selector) {
try {
return Array.from(document.querySelectorAll(selector));
} catch (e) {
throw new Error(`Invalid selector: ${e.message}`);
}
}
try {
const result = document.evaluate(xpath, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
return Array.from({length: result.snapshotLength}, (_, i) => result.snapshotItem(i));
}
catch (e) {
throw new Error(`Invalid XPath: ${e.message}`);
}
}
function getElementData(element) {
const rect = element.getBoundingClientRect();
const computedStyle = window.getComputedStyle(element);
// Get the selector (which may add an ID to the element)
const selector = getUniqueSelector(element);
// Comprehensive visibility check
const visible = isElementVisible(element, rect, computedStyle);
const data = {
tagName: element.tagName.toLowerCase(),
id: element.id || undefined,
className: element.className || undefined,
selector: selector,
bounds: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
visible: visible,
focused: element === document.activeElement,
position: computedStyle.position
};
// Conditionally add attributes
["href", "src", "value", "name"].forEach(attr => element[attr] && (data[attr] = element[attr]));
// If it's a select element, add the options
if (data.tagName === 'select') {
data.options = Array.from(element.options).map((option, optionIndex) => ({
index: optionIndex,
value: option.value,
text: option.text,
selected: option.selected,
disabled: option.disabled
}));
}
// Add scrollable parent if exists
const scrollParent = findScrollableParent(element);
data.scrollParent = getUniqueSelector(scrollParent);
return data;
}
function isElementVisible(element, rect, computedStyle) {
// If rect and computedStyle not provided, calculate them
if (!rect) rect = element.getBoundingClientRect();
if (!computedStyle) computedStyle = window.getComputedStyle(element);
// Check if element has dimensions
if (rect.width <= 0 || rect.height <= 0) return false;
// Check CSS visibility properties
if (computedStyle.display === 'none' || computedStyle.visibility === 'hidden' || computedStyle.opacity === '0') {
return false;
}
// Check if element is in viewport
const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
// Check if any part of the element is within the viewport
const inViewport = rect.bottom > 0 &&
rect.right > 0 &&
rect.top < viewportHeight &&
rect.left < viewportWidth;
if (!inViewport) return false;
// Get element's center point (used multiple times below)
const centerX = rect.left + rect.width / 2;
const centerY = rect.top + rect.height / 2;
// Helper function to check if element is visible at a point
const isElementAtPoint = (x, y) => {
const elementAtPoint = document.elementFromPoint(x, y);
if (!elementAtPoint) return false;
return elementAtPoint === element || element.contains(elementAtPoint) || elementAtPoint.contains(element);
};
// Check if element is visible at center point - this is the most reliable check
if (isElementAtPoint(centerX, centerY)) return true;
// Check if element is hidden by ancestor's properties
let parent = element.parentElement;
while (parent && parent !== document.body) {
const parentStyle = window.getComputedStyle(parent);
if (parentStyle.display === 'none' || parentStyle.visibility === 'hidden' || parentStyle.opacity === '0') {
return false;
}
// Check for overflow hidden that might hide the element
if (parentStyle.overflow === 'hidden' || parentStyle.overflowX === 'hidden' || parentStyle.overflowY === 'hidden') {
const parentRect = parent.getBoundingClientRect();
// Check if element is outside parent's visible area
if (rect.bottom < parentRect.top || rect.top > parentRect.bottom || rect.right < parentRect.left || rect.left > parentRect.right) {
// Before returning false, check if the element is actually visible using elementsFromPoint
const elementsAtPoint = document.elementsFromPoint(centerX, centerY);
// If the element is in the elements chain at its center point, it's visible
if (elementsAtPoint.includes(element)) {
continue; // Skip this parent check and continue checking other parents
}
return false;
}
}
parent = parent.parentElement;
}
// Element might be partially covered, check multiple points
const points = [
{ x: rect.left + rect.width * 0.1, y: rect.top + rect.height * 0.1 },
{ x: rect.right - rect.width * 0.1, y: rect.top + rect.height * 0.1 },
{ x: rect.left + rect.width * 0.1, y: rect.bottom - rect.height * 0.1 },
{ x: rect.right - rect.width * 0.1, y: rect.bottom - rect.height * 0.1 }
];
// Check if any of the points hit our element
return points.some(point => isElementAtPoint(point.x, point.y));
}
function respondWith(obj, selector, xpath) {
return {
success: !obj.error,
selector,
xpath: !selector ? xpath : undefined,
...getTabInfo(),
...obj
};
}
function respondWithError(code, message, selector, xpath) {
return respondWith({ error: { code, message } }, selector, xpath);
}
function elementNotFound(selector, xpath) {
return respondWithError('ELEMENT_NOT_FOUND', 'Element not found', selector, xpath);
}
function requireSelectorOrXpath(selector, xpath) {
return respondWithError('SELECTOR_OR_XPATH_REQUIRED', 'Selector or XPath parameter required', selector, xpath);
}
const helpers = {
//called by the background script
_navigate: ({ url }) => {
window.location.href = url;
},
_elementPosition: ({ id }) => {
const element = document.getElementById(id);
return element.getBoundingClientRect();
},
_connectionStateChanged: ({ status, connected }) => {
// Remove existing connection classes
document.body.classList.remove('kapture-connected', 'kapture-connecting');
// Add appropriate class based on status
if (status === 'connected') {
document.body.classList.add('kapture-connected');
} else if (status === 'retrying') {
document.body.classList.add('kapture-connecting');
}
// No class for disconnected state
return { success: true };
},
// tool calls
getTabInfo,
/**
* Limits the nesting depth of HTML by replacing deeply nested elements with comments
* indicating how many more layers exist.
*/
limitDOMNesting: (htmlString, maxNesting) => {
if (typeof maxNesting !== 'number' || maxNesting < 0) {
return htmlString;
}
// Parse the HTML string
const parser = new DOMParser();
const doc = parser.parseFromString(htmlString, 'text/html');
// Get the root element (could be body or a specific element)
const root = doc.body.firstElementChild || doc.body;
/**
* Counts the maximum nesting depth from a given element
*/
function countDepth(element) {
if (!element || !element.children || element.children.length === 0) {
return 0;
}
let maxChildDepth = 0;
for (const child of element.children) {
maxChildDepth = Math.max(maxChildDepth, countDepth(child));
}
return 1 + maxChildDepth;
}
/**
* Recursively processes elements and limits nesting depth
*/
function processElement(element, currentDepth) {
if (!element || !element.children) return;
// If we've reached the max nesting depth
if (currentDepth >= maxNesting) {
// Count how many more layers exist
const remainingDepth = countDepth(element);
if (remainingDepth > 0) {
// Replace all children with a comment
const comment = doc.createComment(` ... ${remainingDepth} more layer${remainingDepth === 1 ? '' : 's'} of nesting `);
// Remove all children
while (element.firstChild) {
element.removeChild(element.firstChild);
}
// Add the comment
element.appendChild(comment);
}
return;
}
// Process children recursively
const children = Array.from(element.children);
for (const child of children) {
processElement(child, currentDepth + 1);
}
}
// Start processing from depth 0
processElement(root, 0);
// Return the serialized HTML
return root.outerHTML;
},
dom: ({selector, xpath, maxNesting}) => {
let html;
if (!selector && !xpath) {
html = document.body.outerHTML;
} else {
const element = findAllElements(selector, xpath)[0];
if (!element) return elementNotFound(selector, xpath);
html = element.outerHTML;
}
// Apply nesting limit (default is 5)
const nestingLimit = typeof maxNesting === 'number' && maxNesting >= 0 ? maxNesting : 5;
html = window.pageHelpers.limitDOMNesting(html, nestingLimit);
return respondWith({ html }, selector, xpath);
},
elementsFromPoint: ({x, y}) => {
if (typeof x !== 'number' || typeof y !== 'number') {
return respondWithError('XY_REQUIRED', 'Both x and y coordinates are required');
}
const elements = document.elementsFromPoint(x, y);
return respondWith({ x, y, elements: elements.map(getElementData) });
},
elements: ({selector, xpath, visible = 'all'}) => {
if (!selector && !xpath) return requireSelectorOrXpath();
let elements;
try {
elements = findAllElements(selector, xpath).map(getElementData);
} catch (e) {
const errorCode = selector ? 'INVALID_SELECTOR' : 'INVALID_XPATH';
return respondWithError(errorCode, e.message, selector, xpath);
}
// Apply visibility filter
if (visible !== 'all') {
const filterVisible = String(visible) === 'true';
elements = elements.filter(el => el.visible === filterVisible);
}
return respondWith({elements: elements, visible: visible !== 'all' ? visible : undefined}, selector, xpath);
},
element: ({selector, xpath, visible = 'all'}) => {
const result = helpers.elements({selector, xpath, visible});
if (result.error) return result;
if (!result.elements.length) return elementNotFound(selector, xpath);
result.element = result.elements[0];
delete result.elements;
return result;
},
focus: ({selector, xpath}) => {
if (!selector && !xpath) return requireSelectorOrXpath();
let element;
try {
element = findAllElements(selector, xpath)[0];
} catch (e) {
const errorCode = selector ? 'INVALID_SELECTOR' : 'INVALID_XPATH';
return respondWithError(errorCode, e.message, selector, xpath);
}
if (!element) return elementNotFound(selector, xpath);
// Focus the element
element.focus();
// Check if element is actually focusable
const focusableElements = ['input', 'textarea', 'select', 'button', 'a'];
const tagName = element.tagName.toLowerCase();
const isFocusable = focusableElements.includes(tagName) ||
element.hasAttribute('tabindex') ||
element.isContentEditable;
if (!isFocusable) {
// Still return success but with a warning
return respondWith({
focused: true,
warning: 'Element may not be focusable'
}, selector, xpath);
}
return respondWith({ focused: true }, selector, xpath);
},
fill: ({selector, xpath, value}) => {
if (!selector && !xpath) return requireSelectorOrXpath();
const element = findAllElements(selector, xpath)[0];
if (!element) return elementNotFound(selector, xpath);
// Check if it's an input element
const tagName = element.tagName.toLowerCase();
const inputTypes = ['input', 'textarea'];
if (!inputTypes.includes(tagName) && !element.isContentEditable) {
return respondWithError('INVALID_ELEMENT', 'Element is not fillable: ' + tagName, selector, xpath);
}
// Focus the element
element.focus();
// Clear existing value
if (element.value !== undefined) {
element.value = '';
} else if (element.isContentEditable) {
element.textContent = '';
}
// Set new value
if (element.value !== undefined) {
element.value = value;
} else if (element.isContentEditable) {
element.textContent = value;
}
// Trigger input and change events
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
// Blur to trigger any blur handlers
element.blur();
return respondWith({ filled: true }, selector, xpath);
},
select: ({selector, xpath, value}) => {
if (!selector && !xpath) return requireSelectorOrXpath();
const element = findAllElements(selector, xpath)[0];
if (!element) return elementNotFound(selector, xpath);
if (element.tagName !== 'SELECT') {
return respondWithError('INVALID_ELEMENT', 'Element is not fillable: ' + element.name, selector, xpath);
}
// Find option by value
const option = Array.from(element.options).find(opt => opt.value === value);
if (!option) {
return respondWithError('OPTION_NOT_FOUND', 'Option not found with value: ' + value, selector, xpath);
}
// Select the option
element.value = value;
option.selected = true;
// Trigger change event
element.dispatchEvent(new Event('change', { bubbles: true }));
return respondWith({ selected: true }, selector, xpath);
},
blur: ({selector, xpath}) => {
if (!selector && !xpath) return requireSelectorOrXpath();
let element;
try {
element = findAllElements(selector, xpath)[0];
} catch (e) {
const errorCode = selector ? 'INVALID_SELECTOR' : 'INVALID_XPATH';
return respondWithError(errorCode, e.message, selector, xpath);
}
if (!element) return elementNotFound(selector, xpath);
// Blur the element
element.blur();
// Also remove focus from document.activeElement if it's different
if (document.activeElement && document.activeElement !== element) {
document.activeElement.blur();
}
return respondWith({ blurred: true }, selector, xpath);
},
_cursor: ({show}) => {
const cursorId = 'kapture-cursor';
let cursor = document.getElementById(cursorId);
try {
if (show === false) {
// Hide cursor
if (cursor) {
cursor.style.display = 'none';
}
return respondWith({ visible: false });
}
// Show cursor - create if doesn't exist
if (!cursor) {
cursor = document.createElement('div');
cursor.id = cursorId;
// Create cursor SVG
cursor.innerHTML = `
<svg width="20" height="20" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
<path d="M0 0 L0 16 L4.5 12.5 L7.5 20 L10 19 L7 11.5 L12 11 Z"
fill="white"
stroke="black"
stroke-width="1"/>
</svg>
`;
// Style the cursor container
cursor.style.cssText = `
position: fixed;
top: 0;
left: 0;
width: 20px;
height: 20px;
z-index: 2147483647;
pointer-events: none;
transform: translate(-2px, -2px);
transition: none;
will-change: transform;
`;
document.body.appendChild(cursor);
}
cursor.style.display = 'block';
return respondWith({ visible: true });
} catch (e) {
return respondWithError('CURSOR_ERROR', e.message);
}
},
_moveMouseSVG: ({x, y}) => {
if (typeof x !== 'number' || typeof y !== 'number') {
return respondWithError('XY_REQUIRED', 'Both x and y coordinates are required');
}
try {
const cursor = document.getElementById('kapture-cursor');
if (!cursor) {
return respondWithError('CURSOR_NOT_FOUND', 'Cursor element not found. Call _cursor with show=true first');
}
cursor.style.transform = `translate(${x - 2}px, ${y - 2}px)`;
return respondWith({ moved: true, x, y });
} catch (e) {
return respondWithError('MOVE_MOUSE_SVG_ERROR', e.message);
}
}
};
// Mouse position tracking with throttling
let lastMouseSendTime = 0;
const MOUSE_THROTTLE_MS = 50; // Throttle to 20 updates per second
document.addEventListener('mousemove', (event) => {
const now = Date.now();
if (now - lastMouseSendTime < MOUSE_THROTTLE_MS) return;
lastMouseSendTime = now;
chrome.runtime.sendMessage({
type: 'mousePosition',
x: event.clientX,
y: event.clientY
});
});
// Listen for requests from the extension
chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
if (!request.command) return;
if (helpers[request.command]) {
const result = helpers[request.command](request.params);
Promise.resolve(result).then(sendResponse);
return true; // Keep channel open for async response
}
else {
sendResponse(respondWith({
error: { code: 'UNKNOWN_COMMAND', message: `Command '${request.command}' not found` }
}));
}
});