diff --git a/silentpilot/mcp_server/src/tools/browser.ts b/silentpilot/mcp_server/src/tools/browser.ts index 98905e35..b94ac598 100644 --- a/silentpilot/mcp_server/src/tools/browser.ts +++ b/silentpilot/mcp_server/src/tools/browser.ts @@ -13,9 +13,11 @@ */ import { chromium, Browser, Page } from "playwright"; +import { PageCursor } from "./cursor.js"; let browser: Browser | null = null; let page: Page | null = null; +let cursor: PageCursor | null = null; const VIEWPORT = { width: 1280, height: 800 }; @@ -40,6 +42,10 @@ export async function ensureBrowser(): Promise { viewport: VIEWPORT, }); page = await context.newPage(); + + // Initialize cursor + cursor = new PageCursor(); + await cursor.attach(page); } return page; } @@ -49,6 +55,7 @@ export async function closeBrowser(): Promise { await browser.close(); browser = null; page = null; + cursor = null; } } @@ -57,12 +64,27 @@ export async function closeBrowser(): Promise { export async function browserGoto(url: string): Promise { const p = await ensureBrowser(); await p.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 }); + + // Re-attach cursor after navigation + if (cursor) { + await cursor.attach(p); + } + return `Navigated to ${p.url()}`; } export async function browserClick(selector: string): Promise { const p = await ensureBrowser(); try { + const locator = p.locator(selector).first(); + const box = await locator.boundingBox(); + if (box && cursor) { + const cx = box.x + box.width / 2; + const cy = box.y + box.height / 2; + await cursor.moveTo(cx, cy); + await cursor.clickEffect(); + } + await p.click(selector, { timeout: 5000 }); return `Clicked: ${selector}`; } catch (e) { @@ -76,6 +98,15 @@ export async function browserType( ): Promise { const p = await ensureBrowser(); try { + const locator = p.locator(selector).first(); + const box = await locator.boundingBox(); + if (box && cursor) { + const cx = box.x + box.width / 2; + const cy = box.y + box.height / 2; + await cursor.moveTo(cx, cy); + await cursor.clickEffect(); + } + await p.fill(selector, text, { timeout: 5000 }); return `Typed into ${selector}: "${text}"`; } catch (e) { @@ -85,6 +116,9 @@ export async function browserType( export async function browserPress(key: string): Promise { const p = await ensureBrowser(); + if (cursor) { + await cursor.ensureAlive(); + } await p.keyboard.press(key); return `Pressed key: ${key}`; } @@ -94,6 +128,14 @@ export async function browserScroll( amount: number = 300 ): Promise { const p = await ensureBrowser(); + + if (cursor) { + const viewport = p.viewportSize(); + if (viewport) { + await cursor.moveTo(viewport.width / 2, viewport.height / 2); + } + } + const delta = direction === "down" ? amount : -amount; await p.mouse.wheel(0, delta); return `Scrolled ${direction} by ${amount}px`; diff --git a/silentpilot/mcp_server/src/tools/cursor.ts b/silentpilot/mcp_server/src/tools/cursor.ts new file mode 100644 index 00000000..114f754f --- /dev/null +++ b/silentpilot/mcp_server/src/tools/cursor.ts @@ -0,0 +1,204 @@ +/** + * Smooth animated cursor overlay injected into the browser page. + * + * Uses a visible arrow cursor (div + inline SVG) that starts at viewport center + * and physically animates to each target using requestAnimationFrame. + * Ported from silentpilot/actions/cursor.py + */ + +import { Page } from "playwright"; + +const CURSOR_MOVE_DURATION_MS = 400; +const CURSOR_CLICK_DELAY_MS = 200; + +// Inline SVG arrow — red with white outline, classic pointer shape +const SVG_ARROW = ` + + + +`.replace(/\n/g, ""); + +// CSS for cursor — includes subtle blink (gentle opacity pulse) +const CURSOR_CSS = ` +#__sp_cursor { position:fixed; z-index:2147483647; pointer-events:none; + width:38px; height:38px; filter:drop-shadow(1px 2px 4px rgba(0,0,0,0.45)); + left:50vw; top:50vh; animation: __sp_blink 1.8s ease-in-out infinite; } + @keyframes __sp_blink { 0%,100%{opacity:1;} 50%{opacity:0.55;} } + #__sp_ripple { position:fixed; z-index:2147483646; pointer-events:none; + width:40px; height:40px; border-radius:50%; + border:2.5px solid rgba(220,38,38,0.85); + transform:translate(-50%,-50%) scale(0); opacity:0; } +`.replace(/\n/g, ""); + +// JS that creates the cursor element — idempotent (safe to call multiple times) +const INJECT_JS = `(() => { + if (document.getElementById('__sp_cursor')) return; + const s = document.createElement('style'); + s.id = '__sp_cursor_style'; + s.textContent = \`${CURSOR_CSS.replace(/`/g, '\\`')}\`; + (document.head || document.documentElement).appendChild(s); + const d = document.createElement('div'); + d.id = '__sp_cursor'; + d.innerHTML = '${SVG_ARROW}'; + document.documentElement.appendChild(d); + const r = document.createElement('div'); + r.id = '__sp_ripple'; + document.documentElement.appendChild(r); +})()`; + +// Template for animate JS +const ANIMATE_TPL = (tx: number, ty: number, dur: number) => `(() => { + const c = document.getElementById('__sp_cursor'); + if (!c) return; + const startX = parseFloat(c.style.left) || (window.innerWidth / 2); + const startY = parseFloat(c.style.top) || (window.innerHeight / 2); + const tx = ${tx}; const ty = ${ty}; const dur = ${dur}; + const dx = tx - startX; const dy = ty - startY; + if (Math.abs(dx) < 1 && Math.abs(dy) < 1) { + c.style.left = tx + 'px'; c.style.top = ty + 'px'; return; + } + const startTime = performance.now(); + function step(now) { + let t = Math.min((now - startTime) / dur, 1); + t = 1 - Math.pow(1 - t, 3); + c.style.left = (startX + dx * t) + 'px'; + c.style.top = (startY + dy * t) + 'px'; + if (t < 1) requestAnimationFrame(step); + } + requestAnimationFrame(step); +})()`; + +// Template for ripple JS +const RIPPLE_TPL = (x: number, y: number) => `(() => { + const r = document.getElementById('__sp_ripple'); + if (!r) return; + r.style.left = '${x}px'; r.style.top = '${y}px'; + r.style.opacity = '1'; + r.style.transform = 'translate(-50%,-50%) scale(0)'; + void r.offsetWidth; + r.style.transition = 'transform 0.35s ease-out, opacity 0.35s ease-out'; + r.style.transform = 'translate(-50%,-50%) scale(2)'; + r.style.opacity = '0'; + setTimeout(() => { r.style.transition = 'none'; }, 350); +})()`; + +export class PageCursor { + private page: Page | null = null; + private x: number = 0; + private y: number = 0; + + async attach(page: Page): Promise { + this.page = page; + await this.injectCursor(); + await this.center(); + } + + private async injectCursor(): Promise { + if (!this.page) return; + try { + await this.page.evaluate(INJECT_JS); + } catch (e) { + // Ignore injection errors + } + } + + private async center(): Promise { + if (!this.page) return; + try { + const pos = await this.page.evaluate(() => ({ + x: Math.round(window.innerWidth / 2), + y: Math.round(window.innerHeight / 2) + })); + this.x = pos.x; + this.y = pos.y; + await this.page.evaluate(`(() => { + const c = document.getElementById('__sp_cursor'); + if (c) { c.style.left = '${this.x}px'; c.style.top = '${this.y}px'; } + })()`); + } catch (e) { + // Ignore errors + } + } + + async ensureAlive(): Promise { + if (!this.page) return; + try { + const exists = await this.page.evaluate(() => !!document.getElementById('__sp_cursor')); + if (!exists) { + await this.injectCursor(); + // Snap to last known position + await this.page.evaluate(`(() => { + const c = document.getElementById('__sp_cursor'); + if (c) { c.style.left = '${this.x}px'; c.style.top = '${this.y}px'; } + })()`); + } + } catch (e) { + await this.injectCursor(); + } + } + + async moveTo(x: number, y: number, durationMs: number = CURSOR_MOVE_DURATION_MS): Promise { + if (!this.page) return; + await this.ensureAlive(); + const js = ANIMATE_TPL(x, y, durationMs); + try { + await this.page.evaluate(js); + } catch (e) { + await this.injectCursor(); + try { + // Try simpler move if animation failed + await this.page.evaluate(`(() => { + const c = document.getElementById('__sp_cursor'); + if (c) { c.style.left = '${x}px'; c.style.top = '${y}px'; } + })()`); + } catch (inner) { + // Ignore + } + } + + // Wait for animation (+ buffer) + await new Promise(resolve => setTimeout(resolve, durationMs + 50)); + this.x = x; + this.y = y; + } + + async clickEffect(): Promise { + if (!this.page) return; + const js = RIPPLE_TPL(this.x, this.y); + try { + await this.page.evaluate(js); + } catch (e) { + // Ignore + } + await new Promise(resolve => setTimeout(resolve, CURSOR_CLICK_DELAY_MS)); + } + + async hide(): Promise { + if (!this.page) return; + try { + await this.page.evaluate(() => { + const c = document.getElementById('__sp_cursor'); + if (c) c.style.display = 'none'; + const r = document.getElementById('__sp_ripple'); + if (r) r.style.display = 'none'; + }); + } catch (e) { + // Ignore + } + } + + async show(): Promise { + if (!this.page) return; + try { + await this.page.evaluate(() => { + const c = document.getElementById('__sp_cursor'); + if (c) c.style.display = ''; + const r = document.getElementById('__sp_ripple'); + if (r) r.style.display = ''; + }); + } catch (e) { + // Ignore + } + } +} diff --git a/silentpilot/mcp_server/verify_cursor.ts b/silentpilot/mcp_server/verify_cursor.ts new file mode 100644 index 00000000..14dc0c74 --- /dev/null +++ b/silentpilot/mcp_server/verify_cursor.ts @@ -0,0 +1,30 @@ + +import { browserGoto, browserType, browserScreenshot, closeBrowser } from "./src/tools/index"; +import * as fs from "fs"; + +async function main() { + try { + console.log("Navigating to google.com..."); + await browserGoto("https://www.google.com"); + + console.log("Typing 'hello world'..."); + // Wait a bit to ensure page is loaded + await new Promise(r => setTimeout(r, 2000)); + + // This will trigger cursor movement and type. Selector for Google search box. + // It's usually a textarea with name='q' or title='Search'. + // To be safe, try textarea[name='q'] + await browserType("textarea[name='q']", "hello world"); + + console.log("Taking screenshot..."); + const b64 = await browserScreenshot(); + fs.writeFileSync("cursor_verify.png", Buffer.from(b64, 'base64')); + console.log("Screenshot saved to cursor_verify.png"); + } catch (e) { + console.error("Error:", e); + } finally { + await closeBrowser(); + } +} + +main();