Add visual cursor to MCP browser tools

silentpilot/mcp_server/src/tools/browser.ts

@@ -13,9 +13,11 @@
  */
 
 import { chromium, Browser, Page } from "playwright";
+import { PageCursor } from "./cursor.js";
 
 let browser: Browser | null = null;
 let page: Page | null = null;
+let cursor: PageCursor | null = null;
 
 const VIEWPORT = { width: 1280, height: 800 };
 
@@ -40,6 +42,10 @@ export async function ensureBrowser(): Promise<Page> {
       viewport: VIEWPORT,
     });
     page = await context.newPage();
+
+    // Initialize cursor
+    cursor = new PageCursor();
+    await cursor.attach(page);
   }
   return page;
 }
@@ -49,6 +55,7 @@ export async function closeBrowser(): Promise<void> {
     await browser.close();
     browser = null;
     page = null;
+    cursor = null;
   }
 }
 
@@ -57,12 +64,27 @@ export async function closeBrowser(): Promise<void> {
 export async function browserGoto(url: string): Promise<string> {
   const p = await ensureBrowser();
   await p.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
+
+  // Re-attach cursor after navigation
+  if (cursor) {
+    await cursor.attach(p);
+  }
+
   return `Navigated to ${p.url()}`;
 }
 
 export async function browserClick(selector: string): Promise<string> {
   const p = await ensureBrowser();
   try {
+    const locator = p.locator(selector).first();
+    const box = await locator.boundingBox();
+    if (box && cursor) {
+      const cx = box.x + box.width / 2;
+      const cy = box.y + box.height / 2;
+      await cursor.moveTo(cx, cy);
+      await cursor.clickEffect();
+    }
+
     await p.click(selector, { timeout: 5000 });
     return `Clicked: ${selector}`;
   } catch (e) {
@@ -76,6 +98,15 @@ export async function browserType(
 ): Promise<string> {
   const p = await ensureBrowser();
   try {
+    const locator = p.locator(selector).first();
+    const box = await locator.boundingBox();
+    if (box && cursor) {
+      const cx = box.x + box.width / 2;
+      const cy = box.y + box.height / 2;
+      await cursor.moveTo(cx, cy);
+      await cursor.clickEffect();
+    }
+
     await p.fill(selector, text, { timeout: 5000 });
     return `Typed into ${selector}: "${text}"`;
   } catch (e) {
@@ -85,6 +116,9 @@ export async function browserType(
 
 export async function browserPress(key: string): Promise<string> {
   const p = await ensureBrowser();
+  if (cursor) {
+    await cursor.ensureAlive();
+  }
   await p.keyboard.press(key);
   return `Pressed key: ${key}`;
 }
@@ -94,6 +128,14 @@ export async function browserScroll(
   amount: number = 300
 ): Promise<string> {
   const p = await ensureBrowser();
+
+  if (cursor) {
+    const viewport = p.viewportSize();
+    if (viewport) {
+      await cursor.moveTo(viewport.width / 2, viewport.height / 2);
+    }
+  }
+
   const delta = direction === "down" ? amount : -amount;
   await p.mouse.wheel(0, delta);
   return `Scrolled ${direction} by ${amount}px`;

silentpilot/mcp_server/src/tools/cursor.ts

@@ -0,0 +1,204 @@
+/**
+ * Smooth animated cursor overlay injected into the browser page.
+ *
+ * Uses a visible arrow cursor (div + inline SVG) that starts at viewport center
+ * and physically animates to each target using requestAnimationFrame.
+ * Ported from silentpilot/actions/cursor.py
+ */
+
+import { Page } from "playwright";
+
+const CURSOR_MOVE_DURATION_MS = 400;
+const CURSOR_CLICK_DELAY_MS = 200;
+
+// Inline SVG arrow — red with white outline, classic pointer shape
+const SVG_ARROW = `
+<svg xmlns="http://www.w3.org/2000/svg" width="38" height="38" viewBox="0 0 24 24">
+<path d="M2 2 L2 20 L7.5 14.5 L12 22 L15 20.5 L10.5 13 L18 13 Z"
+fill="#dc2626" stroke="white" stroke-width="1.5" stroke-linejoin="round"/>
+</svg>
+`.replace(/\n/g, "");
+
+// CSS for cursor — includes subtle blink (gentle opacity pulse)
+const CURSOR_CSS = `
+#__sp_cursor { position:fixed; z-index:2147483647; pointer-events:none;
+ width:38px; height:38px; filter:drop-shadow(1px 2px 4px rgba(0,0,0,0.45));
+ left:50vw; top:50vh; animation: __sp_blink 1.8s ease-in-out infinite; }
+ @keyframes __sp_blink { 0%,100%{opacity:1;} 50%{opacity:0.55;} }
+ #__sp_ripple { position:fixed; z-index:2147483646; pointer-events:none;
+ width:40px; height:40px; border-radius:50%;
+ border:2.5px solid rgba(220,38,38,0.85);
+ transform:translate(-50%,-50%) scale(0); opacity:0; }
+`.replace(/\n/g, "");
+
+// JS that creates the cursor element — idempotent (safe to call multiple times)
+const INJECT_JS = `(() => {
+    if (document.getElementById('__sp_cursor')) return;
+    const s = document.createElement('style');
+    s.id = '__sp_cursor_style';
+    s.textContent = \`${CURSOR_CSS.replace(/`/g, '\\`')}\`;
+    (document.head || document.documentElement).appendChild(s);
+    const d = document.createElement('div');
+    d.id = '__sp_cursor';
+    d.innerHTML = '${SVG_ARROW}';
+    document.documentElement.appendChild(d);
+    const r = document.createElement('div');
+    r.id = '__sp_ripple';
+    document.documentElement.appendChild(r);
+})()`;
+
+// Template for animate JS
+const ANIMATE_TPL = (tx: number, ty: number, dur: number) => `(() => {
+    const c = document.getElementById('__sp_cursor');
+    if (!c) return;
+    const startX = parseFloat(c.style.left) || (window.innerWidth / 2);
+    const startY = parseFloat(c.style.top) || (window.innerHeight / 2);
+    const tx = ${tx}; const ty = ${ty}; const dur = ${dur};
+    const dx = tx - startX; const dy = ty - startY;
+    if (Math.abs(dx) < 1 && Math.abs(dy) < 1) {
+        c.style.left = tx + 'px'; c.style.top = ty + 'px'; return;
+    }
+    const startTime = performance.now();
+    function step(now) {
+        let t = Math.min((now - startTime) / dur, 1);
+        t = 1 - Math.pow(1 - t, 3);
+        c.style.left = (startX + dx * t) + 'px';
+        c.style.top = (startY + dy * t) + 'px';
+        if (t < 1) requestAnimationFrame(step);
+    }
+    requestAnimationFrame(step);
+})()`;
+
+// Template for ripple JS
+const RIPPLE_TPL = (x: number, y: number) => `(() => {
+    const r = document.getElementById('__sp_ripple');
+    if (!r) return;
+    r.style.left = '${x}px'; r.style.top = '${y}px';
+    r.style.opacity = '1';
+    r.style.transform = 'translate(-50%,-50%) scale(0)';
+    void r.offsetWidth;
+    r.style.transition = 'transform 0.35s ease-out, opacity 0.35s ease-out';
+    r.style.transform = 'translate(-50%,-50%) scale(2)';
+    r.style.opacity = '0';
+    setTimeout(() => { r.style.transition = 'none'; }, 350);
+})()`;
+
+export class PageCursor {
+    private page: Page | null = null;
+    private x: number = 0;
+    private y: number = 0;
+
+    async attach(page: Page): Promise<void> {
+        this.page = page;
+        await this.injectCursor();
+        await this.center();
+    }
+
+    private async injectCursor(): Promise<void> {
+        if (!this.page) return;
+        try {
+            await this.page.evaluate(INJECT_JS);
+        } catch (e) {
+            // Ignore injection errors
+        }
+    }
+
+    private async center(): Promise<void> {
+        if (!this.page) return;
+        try {
+            const pos = await this.page.evaluate(() => ({
+                x: Math.round(window.innerWidth / 2),
+                y: Math.round(window.innerHeight / 2)
+            }));
+            this.x = pos.x;
+            this.y = pos.y;
+            await this.page.evaluate(`(() => {
+                const c = document.getElementById('__sp_cursor');
+                if (c) { c.style.left = '${this.x}px'; c.style.top = '${this.y}px'; }
+            })()`);
+        } catch (e) {
+            // Ignore errors
+        }
+    }
+
+    async ensureAlive(): Promise<void> {
+        if (!this.page) return;
+        try {
+            const exists = await this.page.evaluate(() => !!document.getElementById('__sp_cursor'));
+            if (!exists) {
+                await this.injectCursor();
+                // Snap to last known position
+                await this.page.evaluate(`(() => {
+                    const c = document.getElementById('__sp_cursor');
+                    if (c) { c.style.left = '${this.x}px'; c.style.top = '${this.y}px'; }
+                })()`);
+            }
+        } catch (e) {
+            await this.injectCursor();
+        }
+    }
+
+    async moveTo(x: number, y: number, durationMs: number = CURSOR_MOVE_DURATION_MS): Promise<void> {
+        if (!this.page) return;
+        await this.ensureAlive();
+        const js = ANIMATE_TPL(x, y, durationMs);
+        try {
+            await this.page.evaluate(js);
+        } catch (e) {
+            await this.injectCursor();
+            try {
+                // Try simpler move if animation failed
+                await this.page.evaluate(`(() => {
+                    const c = document.getElementById('__sp_cursor');
+                    if (c) { c.style.left = '${x}px'; c.style.top = '${y}px'; }
+                })()`);
+            } catch (inner) {
+                // Ignore
+            }
+        }
+
+        // Wait for animation (+ buffer)
+        await new Promise(resolve => setTimeout(resolve, durationMs + 50));
+        this.x = x;
+        this.y = y;
+    }
+
+    async clickEffect(): Promise<void> {
+        if (!this.page) return;
+        const js = RIPPLE_TPL(this.x, this.y);
+        try {
+            await this.page.evaluate(js);
+        } catch (e) {
+            // Ignore
+        }
+        await new Promise(resolve => setTimeout(resolve, CURSOR_CLICK_DELAY_MS));
+    }
+
+    async hide(): Promise<void> {
+        if (!this.page) return;
+        try {
+            await this.page.evaluate(() => {
+                const c = document.getElementById('__sp_cursor');
+                if (c) c.style.display = 'none';
+                const r = document.getElementById('__sp_ripple');
+                if (r) r.style.display = 'none';
+            });
+        } catch (e) {
+            // Ignore
+        }
+    }
+
+    async show(): Promise<void> {
+        if (!this.page) return;
+        try {
+            await this.page.evaluate(() => {
+                const c = document.getElementById('__sp_cursor');
+                if (c) c.style.display = '';
+                const r = document.getElementById('__sp_ripple');
+                if (r) r.style.display = '';
+            });
+        } catch (e) {
+            // Ignore
+        }
+    }
+}

silentpilot/mcp_server/verify_cursor.ts

@@ -0,0 +1,30 @@
+
+import { browserGoto, browserType, browserScreenshot, closeBrowser } from "./src/tools/index";
+import * as fs from "fs";
+
+async function main() {
+    try {
+        console.log("Navigating to google.com...");
+        await browserGoto("https://www.google.com");
+
+        console.log("Typing 'hello world'...");
+        // Wait a bit to ensure page is loaded
+        await new Promise(r => setTimeout(r, 2000));
+
+        // This will trigger cursor movement and type. Selector for Google search box.
+        // It's usually a textarea with name='q' or title='Search'.
+        // To be safe, try textarea[name='q']
+        await browserType("textarea[name='q']", "hello world");
+
+        console.log("Taking screenshot...");
+        const b64 = await browserScreenshot();
+        fs.writeFileSync("cursor_verify.png", Buffer.from(b64, 'base64'));
+        console.log("Screenshot saved to cursor_verify.png");
+    } catch (e) {
+        console.error("Error:", e);
+    } finally {
+        await closeBrowser();
+    }
+}
+
+main();