more compact htmlrewrite

playwriter/src/htmlrewrite.test.ts

@@ -1,4 +1,5 @@
 import { expect, test } from 'vitest'
+import { readFileSync } from 'fs'
 import { formatHtmlForPrompt } from './htmlrewrite.js'
 
 test('formatHtmlForPrompt', async () => {
@@ -16351,48 +16352,39 @@ test('formatHtmlForPrompt', async () => {
   )
 })
 
-test('collapses 3+ consecutive empty elements of same type', async () => {
+test('removes empty elements', async () => {
   const html = `
     <div>
-      <span></span>
-      <span></span>
       <span></span>
       <span></span>
       <p>content</p>
       <div></div>
-      <div></div>
     </div>
   `
   const result = await formatHtmlForPrompt({ html })
 
-  // Should collapse 4 spans into 1, but keep 2 divs (not >= 3)
+  // All empty elements should be removed
   expect(result).toMatchInlineSnapshot(`
-    "<div>
+    "<p>content</p>
-     <span></span>
-     <p>content</p>
-     <div></div>
-     <div></div>
-    </div>
     "
   `)
 })
 
-test('does not collapse empty elements with attributes', async () => {
+test('keeps empty elements that have attributes', async () => {
   const html = `
     <div>
       <span data-framer-name="a"></span>
       <span data-framer-name="b"></span>
-      <span data-framer-name="c"></span>
+      <span></span>
     </div>
   `
   const result = await formatHtmlForPrompt({ html })
 
-  // All 3 should remain because they have attributes
+  // Empty elements with attributes should remain, empty without attrs removed
   expect(result).toMatchInlineSnapshot(`
     "<div>
      <span data-framer-name="a"></span>
      <span data-framer-name="b"></span>
-     <span data-framer-name="c"></span>
     </div>
     "
   `)
@@ -16483,3 +16475,50 @@ test('always keeps test ID attributes', async () => {
     "
   `)
 })
+
+test('processes x.com.html with size savings', async () => {
+  const html = readFileSync(new URL('./assets/x.com.html', import.meta.url), 'utf-8')
+
+  const result = await formatHtmlForPrompt({ html })
+  const resultWithStyles = await formatHtmlForPrompt({ html, keepStyles: true })
+
+  const originalSize = html.length
+  const originalTokens = Math.ceil(originalSize / 4)
+
+  const processedSize = result.length
+  const processedTokens = Math.ceil(processedSize / 4)
+  const savings = originalSize - processedSize
+  const savingsPercent = ((savings / originalSize) * 100).toFixed(1)
+
+  const withStylesSize = resultWithStyles.length
+  const withStylesTokens = Math.ceil(withStylesSize / 4)
+  const withStylesSavings = originalSize - withStylesSize
+  const withStylesPercent = ((withStylesSavings / originalSize) * 100).toFixed(1)
+
+  console.log(`\n📊 x.com.html processing stats:`)
+  console.log(`   Original:     ${originalSize.toLocaleString()} chars (${originalTokens.toLocaleString()} tokens)`)
+  console.log(`   Without styles: ${processedSize.toLocaleString()} chars (${processedTokens.toLocaleString()} tokens) - ${savingsPercent}% savings`)
+  console.log(`   With styles:    ${withStylesSize.toLocaleString()} chars (${withStylesTokens.toLocaleString()} tokens) - ${withStylesPercent}% savings`)
+
+  await expect(result).toMatchFileSnapshot('./__snapshots__/x.com.processed.html')
+  await expect(resultWithStyles).toMatchFileSnapshot('./__snapshots__/x.com.processed.withStyles.html')
+})
+
+test('unwraps unnecessary nested wrapper divs', async () => {
+  const html = `
+    <div>
+      <div>
+        <div>
+          <p>content</p>
+        </div>
+      </div>
+    </div>
+  `
+  const result = await formatHtmlForPrompt({ html })
+
+  // Should unwrap to a single div containing the p
+  expect(result).toMatchInlineSnapshot(`
+    "<p>content</p>
+    "
+  `)
+})

playwriter/src/htmlrewrite.ts

@@ -26,6 +26,7 @@ export async function formatHtmlForPrompt({
     ]
 
     const attributesToKeep = [
+        // Standard descriptive attributes
         'label',
         'title',
         'alt',
@@ -37,10 +38,24 @@ export async function formatHtmlForPrompt({
         'type',
         'role',
         'target',
-        'vimium-label',
+        // Descriptive aria attributes (text content)
+        'aria-label',
+        'aria-placeholder',
+        'aria-valuetext',
+        'aria-roledescription',
+        // Useful aria state attributes
+        'aria-hidden',
+        'aria-expanded',
+        'aria-checked',
+        'aria-selected',
+        'aria-disabled',
+        'aria-pressed',
+        'aria-required',
+        'aria-current',
         // Test IDs (data-testid, data-test, data-cy are covered by data-* prefix)
         'testid',
         'test-id',
+        'vimium-label',
         // Conditionally added: 'style', 'class'
     ]
 
@@ -85,7 +100,6 @@ export async function formatHtmlForPrompt({
                     const newAttrs: typeof node.attrs = {}
                     for (const [attr, value] of Object.entries(node.attrs)) {
                         const shouldKeep =
-                            attr.startsWith('aria-') ||
                             attr.startsWith('data-') ||
                             attributesToKeep.includes(attr)
 
@@ -121,8 +135,63 @@ export async function formatHtmlForPrompt({
         }
     }
 
-    // Plugin to collapse 3+ consecutive empty elements of the same type
+    // Plugin to unwrap unnecessary nested wrapper elements
-    const collapseEmptyElementsPlugin = () => {
+    // e.g., <div><div><div><p>text</p></div></div></div> -> <div><p>text</p></div>
+    const unwrapNestedWrappersPlugin = () => {
+        return (tree) => {
+            const isWhitespaceOnly = (node) => {
+                return typeof node === 'string' && node.trim().length === 0
+            }
+
+            const hasNoAttrs = (node) => {
+                return !node.attrs || Object.keys(node.attrs).length === 0
+            }
+
+            const unwrapNode = (node) => {
+                if (typeof node === 'string') return node
+                if (!node.tag) return node
+
+                // First, recursively process children
+                if (node.content && Array.isArray(node.content)) {
+                    node.content = node.content.map(unwrapNode)
+                }
+
+                // Check if this node is an unnecessary wrapper:
+                // - has no attributes
+                // - has exactly one non-whitespace child that is an element
+                if (hasNoAttrs(node) && node.content && Array.isArray(node.content)) {
+                    const nonWhitespaceChildren = node.content.filter(c => !isWhitespaceOnly(c))
+
+                    if (nonWhitespaceChildren.length === 1) {
+                        const onlyChild = nonWhitespaceChildren[0]
+                        // If the only child is also an element (not text), unwrap
+                        if (typeof onlyChild !== 'string' && onlyChild.tag) {
+                            // Replace this node with its child
+                            return onlyChild
+                        }
+                    }
+                }
+
+                return node
+            }
+
+            // Apply multiple passes until stable (handles deeply nested wrappers)
+            let result = tree.map(unwrapNode)
+            let prevJson = ''
+            let currJson = JSON.stringify(result)
+            while (prevJson !== currJson) {
+                prevJson = currJson
+                result = result.map(unwrapNode)
+                currJson = JSON.stringify(result)
+            }
+
+            return result
+        }
+    }
+
+    // Plugin to remove empty elements (no attrs, no content)
+    // Runs repeatedly until no more empty elements exist
+    const removeEmptyElementsPlugin = () => {
         return (tree) => {
             const isEmptyElement = (node) => {
                 if (typeof node === 'string') return false
@@ -134,63 +203,39 @@ export async function formatHtmlForPrompt({
                 return !hasAttrs && !hasContent
             }
 
-            const isWhitespaceOnly = (node) => {
+            const removeEmpty = (content) => {
-                return typeof node === 'string' && node.trim().length === 0
-            }
-
-            const collapseConsecutive = (content) => {
                 if (!content || !Array.isArray(content)) return content
 
-                const result: typeof content = []
+                return content
-                let i = 0
+                    .map(node => {
-                while (i < content.length) {
+                        if (typeof node === 'string') return node
-                    const current = content[i]
+                        if (node.content) {
-
+                            node.content = removeEmpty(node.content)
-                    // Process children first (recursive)
-                    if (typeof current !== 'string' && current.content) {
-                        current.content = collapseConsecutive(current.content)
-                    }
-
-                    // Check for consecutive empty elements (skipping whitespace-only strings)
-                    if (isEmptyElement(current)) {
-                        const emptyElements = [current]
-                        let j = i + 1
-
-                        while (j < content.length) {
-                            if (isWhitespaceOnly(content[j])) {
-                                j++
-                                continue
-                            }
-                            if (isEmptyElement(content[j]) && content[j].tag === current.tag) {
-                                emptyElements.push(content[j])
-                                j++
-                                continue
-                            }
-                            break
                         }
-
+                        return node
-                        if (emptyElements.length >= 3) {
+                    })
-                            // Collapse: keep only one element
+                    .filter(node => !isEmptyElement(node))
-                            result.push(current)
-                            i = j
-                            continue
-                        }
-                    }
-
-                    result.push(current)
-                    i++
-                }
-                return result
             }
 
-            return collapseConsecutive(tree)
+            // Apply multiple passes until stable
+            let result = removeEmpty(tree)
+            let prevJson = ''
+            let currJson = JSON.stringify(result)
+            while (prevJson !== currJson) {
+                prevJson = currJson
+                result = removeEmpty(result)
+                currJson = JSON.stringify(result)
+            }
+
+            return result
         }
     }
 
     // Process HTML
     const processor = posthtml()
         .use(removeTagsAndAttrsPlugin())
-        .use(collapseEmptyElementsPlugin())
+        .use(removeEmptyElementsPlugin())
+        .use(unwrapNestedWrappersPlugin())
         .use(beautify({
             rules: {
                 indent: 1,          // 1-space indent