Skip to content

Commit a24beed

Browse files
author
Harsha240105
committed
feat: full Eyes build — bugs fixed, learning organ, overlay, self-improving loop
Phase 1 — Bug fixes: - Fix planner→tool name mismatch (executor.py maps short names to _tool suffix) - Fix risk checks fail open → fail closed for non-guest tools - Replace google.com with DuckDuckGo across 5 files (zero-API) Phase 2 — Self-improving loop: - Create SelfImprovingLoop (generate→critique→refine) using local llama.cpp - ContextBoundary, ExitCondition with cycle cap + score threshold + deadline Phase 3 — Auto-learning: - Auto-generalize successful plans into ProceduralStore skills - LLM generalization prompt fires on plan.completed = True Phase 4 — Screen attention overlay: - Fix get_active_window() to return position bounds - Add active_window_bounds to DesktopState - Create AttentionOverlay (PySide6, yellow=elements, blue=focused window) Phase 5 — 3-pane UI already wired (Log|Aura|Explainer, activity feed, stats, status bar)
1 parent 7185a95 commit a24beed

11 files changed

Lines changed: 326 additions & 14 deletions

File tree

‎moso_core/agents/executor.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,14 @@
1010

1111
logger = logging.getLogger(__name__)
1212

13+
# ponytail: planner uses short names ("file", "app", "browser"), registry uses "_tool" suffix
14+
_TOOL_NAME_MAP = {
15+
"file": "file_tool",
16+
"app": "app_tool",
17+
"browser": "browser_tool",
18+
"terminal": "terminal_tool",
19+
}
20+
1321

1422
class Executor:
1523
def __init__(self, tool_registry=None, identity=None, memory=None, resources=None, automation_engine=None):
@@ -193,7 +201,7 @@ def _execute_task(self, task: Task, requester: str):
193201
logger.error("Cannot execute task: no tool_registry available")
194202
return None
195203
request = ToolRequest(
196-
tool_name=task.tool_name,
204+
tool_name=_TOOL_NAME_MAP.get(task.tool_name, task.tool_name),
197205
parameters=task.parameters,
198206
requester=requester,
199207
)

‎moso_core/agents/planner.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ def generate(self, goal: Goal) -> list[Task]:
9393
search_match = re.search(r"(?:search|find|look up)\s+(.+?)(?:\.|$)", goal.description, re.IGNORECASE)
9494
query = search_match.group(1).strip() if search_match else goal.description
9595
return [
96-
Task(goal_id=0, title="Search web", description=f"Search for {query}", tool_name="browser", parameters={"action": "search", "query": query}, order=0, verification_method="content_not_empty", verification_target=""),
96+
Task(goal_id=0, title="Search web", description=f"Search for {query}", tool_name="browser", parameters={"action": "search_web", "query": query}, order=0, verification_method="content_not_empty", verification_target=""),
9797
]
9898

9999

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
"""Self-improving loop: generate → critique → refine → execute.
2+
3+
Wraps VisionPlanner in a cycle that scores plans before executing,
4+
refining them until they pass the internal critic or hit exit conditions.
5+
"""
6+
from __future__ import annotations
7+
8+
import json
9+
import logging
10+
import time
11+
from dataclasses import dataclass, field
12+
from typing import Any, Optional, Callable
13+
14+
logger = logging.getLogger(__name__)
15+
16+
CRITIQUE_SYSTEM = """You are a plan critic for MOSO AI. Given a user goal and a proposed plan,
17+
score it 0.0-1.0 and give brief feedback. Return ONLY valid JSON:
18+
{"score": 0.8, "feedback": "one sentence of what to improve"}
19+
Rules: score >= 0.75 means the plan is good enough to execute.
20+
Penalize: missing steps, wrong tool choices, unclear targets, unsafe actions."""
21+
22+
23+
@dataclass
24+
class ContextBoundary:
25+
goal: str
26+
allowed_tools: list[str] = field(default_factory=list)
27+
max_seconds: float = 60.0
28+
29+
30+
@dataclass
31+
class ExitCondition:
32+
max_cycles: int = 3
33+
min_critic_score: float = 0.75
34+
hard_deadline_s: float = 120.0
35+
36+
37+
@dataclass
38+
class LoopResult:
39+
plan: Any = None
40+
cycles: int = 0
41+
final_score: float = 0.0
42+
critiques: list[dict] = field(default_factory=list)
43+
timed_out: bool = False
44+
45+
46+
class SelfImprovingLoop:
47+
def __init__(self, llm: Any = None, exit_condition: Optional[ExitCondition] = None):
48+
self._llm = llm
49+
self._exit = exit_condition or ExitCondition()
50+
51+
def run(
52+
self,
53+
goal: str,
54+
generate_fn: Callable[[ContextBoundary], Any],
55+
execute_fn: Optional[Callable[[Any], Any]] = None,
56+
) -> LoopResult:
57+
boundary = ContextBoundary(goal=goal)
58+
start = time.time()
59+
result = LoopResult()
60+
61+
plan = generate_fn(boundary)
62+
result.plan = plan
63+
64+
for cycle in range(self._exit.max_cycles):
65+
# Check hard deadline
66+
if time.time() - start > self._exit.hard_deadline_s:
67+
result.timed_out = True
68+
logger.warning("Loop timed out after %.0fs", self._exit.hard_deadline_s)
69+
break
70+
71+
score, feedback = self._critique(goal, plan)
72+
result.critiques.append({"cycle": cycle, "score": score, "feedback": feedback})
73+
result.final_score = score
74+
result.cycles = cycle + 1
75+
76+
if score >= self._exit.min_critic_score:
77+
logger.info("Plan passed critique on cycle %d (score %.2f)", cycle + 1, score)
78+
break
79+
80+
# Refine: re-generate with critique feedback appended to goal
81+
refined_goal = f"{goal}\n\n[CRITIC FEEDBACK]: {feedback}"
82+
refined_boundary = ContextBoundary(goal=refined_goal)
83+
plan = generate_fn(refined_boundary)
84+
result.plan = plan
85+
86+
if execute_fn and result.plan:
87+
result.plan = execute_fn(result.plan)
88+
89+
return result
90+
91+
def _critique(self, goal: str, plan: Any) -> tuple[float, str]:
92+
if self._llm is None:
93+
return 0.8, "No LLM available for critique — passing by default"
94+
95+
plan_text = self._plan_to_text(plan)
96+
prompt = f"Goal: {goal}\n\nProposed plan:\n{plan_text}\n\nScore and critique:"
97+
98+
try:
99+
from moso_core.llm.models import LLMRequest
100+
response = self._llm.complete(LLMRequest(
101+
prompt=prompt,
102+
system_prompt=CRITIQUE_SYSTEM,
103+
max_tokens=128,
104+
temperature=0.3,
105+
))
106+
if not response.success:
107+
return 0.8, f"LLM critique unavailable: {response.error}"
108+
109+
parsed = json.loads(response.text.strip())
110+
score = max(0.0, min(1.0, float(parsed.get("score", 0.8))))
111+
feedback = parsed.get("feedback", "No feedback")
112+
return score, feedback
113+
except (json.JSONDecodeError, KeyError, ValueError):
114+
return 0.8, "Critique parse failed — passing by default"
115+
except Exception as e:
116+
logger.debug("Critique error: %s", e)
117+
return 0.8, "Critique unavailable — passing by default"
118+
119+
def _plan_to_text(self, plan: Any) -> str:
120+
if hasattr(plan, "steps"):
121+
lines = []
122+
for i, step in enumerate(plan.steps):
123+
desc = getattr(step, "description", str(step))
124+
action = getattr(step, "action", "")
125+
lines.append(f" {i+1}. [{action}] {desc}")
126+
return "\n".join(lines)
127+
return str(plan)[:500]

‎moso_core/computer_use/agent.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ def plan(self, user_text: str, context: Optional[dict] = None) -> list[Step]:
204204
Step("launch", {"app_name": "chrome"}, "Launch Chrome"),
205205
Step("delay", {"seconds": 2.0}, "Wait for Chrome"),
206206
Step("hotkey", {"keys": ["ctrl", "l"]}, "Focus address bar"),
207-
Step("type_text_action", {"text": f"https://www.google.com/search?q={query}"}, f"Search '{query}'"),
207+
Step("type_text_action", {"text": f"https://duckduckgo.com/?q={query}"}, f"Search '{query}'"),
208208
Step("press_key", {"key": "enter"}, "Press Enter"),
209209
]
210210

‎moso_core/computer_use/windows.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,11 @@ def get_active_window(self) -> ComputerUseResult:
4141
win = self._pygetwindow.getActiveWindow()
4242
if win is None:
4343
return ComputerUseResult(False, "get_active_window", error="No active window found")
44-
return ComputerUseResult(True, "get_active_window", {"title": win.title, "size": (win.width, win.height)})
44+
return ComputerUseResult(True, "get_active_window", {
45+
"title": win.title,
46+
"size": (win.width, win.height),
47+
"bounds": (win.left, win.top, win.width, win.height),
48+
})
4549
except Exception as e:
4650
return ComputerUseResult(False, "get_active_window", error=str(e))
4751

‎moso_core/desktop/perception.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ class DesktopState:
5050
dialogs: list[str] = field(default_factory=list)
5151
notifications: list[str] = field(default_factory=list)
5252
screenshot_path: str = ""
53+
active_window_bounds: tuple[int, int, int, int] = (0, 0, 0, 0) # x, y, w, h
5354

5455
def to_dict(self) -> dict:
5556
return {
@@ -66,6 +67,7 @@ def to_dict(self) -> dict:
6667
"open_windows": self.open_windows,
6768
"dialogs": self.dialogs,
6869
"notifications": self.notifications,
70+
"active_window_bounds": list(self.active_window_bounds),
6971
}
7072

7173
def summary(self) -> str:
@@ -171,7 +173,7 @@ def observe(self) -> DesktopState:
171173
state.resolution = resolution
172174

173175
# 2. Get window info
174-
state.active_app, state.window_title, state.open_windows = self._get_window_info()
176+
state.active_app, state.window_title, state.open_windows, state.active_window_bounds = self._get_window_info()
175177

176178
# 3. Run OCR
177179
if self._ocr and screenshot_path:
@@ -211,23 +213,25 @@ def _capture_screen(self) -> tuple[str, tuple[int, int]]:
211213
res = tuple(r.get("resolution", (0, 0)))
212214
return path, res
213215

214-
def _get_window_info(self) -> tuple[str, str, list[str]]:
216+
def _get_window_info(self) -> tuple[str, str, list[str], tuple[int, int, int, int]]:
215217
active_app = ""
216218
window_title = ""
217219
windows = []
220+
bounds = (0, 0, 0, 0)
218221
if not self._wm or not self._wm.available:
219-
return active_app, window_title, windows
222+
return active_app, window_title, windows, bounds
220223
try:
221224
r = self._wm.get_active_window()
222225
if r.success and r.result:
223226
window_title = r.result.get("title", "")
224227
active_app = self._parse_app_name(window_title)
228+
bounds = tuple(r.result.get("bounds", (0, 0, 0, 0)))
225229
r2 = self._wm.list_windows()
226230
if r2.success and r2.result:
227231
windows = r2.result.get("windows", [])
228232
except Exception as e:
229233
logger.debug("Window info failed: %s", e)
230-
return active_app, window_title, windows
234+
return active_app, window_title, windows, bounds
231235

232236
def _parse_app_name(self, title: str) -> str:
233237
if not title:

‎moso_core/desktop/smart_controllers.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ def _open_url(self, target: str, state: DesktopState) -> str:
238238
def _search(self, target: str, state: DesktopState) -> str:
239239
if not target:
240240
return "What should I search for?"
241-
return self._open_url(f"https://www.google.com/search?q={target}", state)
241+
return self._open_url(f"https://duckduckgo.com/?q={target}", state)
242242

243243
def _click_by_text(self, target: str, state: DesktopState) -> str:
244244
elem = self._find_element(state, target)

‎moso_core/desktop/vision_planner.py‎

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515

1616
logger = logging.getLogger(__name__)
1717

18+
SEARCH_PROVIDER = "https://duckduckgo.com" # zero-API: no Google, no tracking
19+
1820

1921
class StepStatus(str, Enum):
2022
PENDING = "pending"
@@ -407,7 +409,7 @@ def _build_steps(self, template: str, groups: tuple, state: DesktopState) -> lis
407409
steps.append(PlanStep(
408410
action="open_url",
409411
description=f"Navigate to Google",
410-
params={"url": "https://www.google.com"},
412+
params={"url": f"{SEARCH_PROVIDER}"},
411413
))
412414
steps.append(PlanStep(
413415
action="wait",
@@ -541,7 +543,7 @@ def _build_steps(self, template: str, groups: tuple, state: DesktopState) -> lis
541543
steps.append(PlanStep(
542544
action="open_url",
543545
description=f"Search Google for: {query}",
544-
params={"url": f"https://www.google.com/search?q={query}"},
546+
params={"url": f"{SEARCH_PROVIDER}/?q={query}"},
545547
))
546548
steps.append(PlanStep(
547549
action="wait", description="Wait for results",
@@ -746,6 +748,14 @@ def execute_plan(self, plan: Plan) -> Plan:
746748

747749
plan.completed = not plan.failed
748750
self._world.set_task("")
751+
752+
# Auto-learning: generalize successful plans into procedural skills
753+
if plan.completed and self._llm:
754+
try:
755+
self._auto_generalize(plan)
756+
except Exception as e:
757+
logger.debug("Auto-generalize failed: %s", e)
758+
749759
return plan
750760

751761
def plan_and_execute(self, goal: str) -> Plan:
@@ -787,5 +797,52 @@ def _log_perception(self, step: PlanStep, state: DesktopState, verification: Opt
787797
except Exception as e:
788798
logger.debug("Failed to log perception outcome: %s", e)
789799

800+
def _auto_generalize(self, plan: Plan) -> None:
801+
"""Feed a successful plan into the generalization prompt and store as skill."""
802+
from moso_core.llm.models import LLMRequest
803+
804+
steps_text = "\n".join(
805+
f" {i+1}. [{s.action}] {s.description}"
806+
for i, s in enumerate(plan.steps)
807+
if s.status == StepStatus.VERIFIED
808+
)
809+
if not steps_text:
810+
return
811+
812+
prompt = (
813+
f"Goal: {plan.goal.description}\n\n"
814+
f"Steps that worked:\n{steps_text}\n\n"
815+
"Generalize this into a reusable skill. Return ONLY valid JSON:\n"
816+
'{"task_name": "short name", "steps": [{"action":"...", "params":{...}}], '
817+
'"trigger_phrases": ["when user says..."], "app_category": "browser|text_editor|other"}'
818+
)
819+
820+
response = self._llm.complete(LLMRequest(
821+
prompt=prompt,
822+
system_prompt="You are a skill generalizer for MOSO AI. Convert successful plans into reusable skills.",
823+
max_tokens=512,
824+
temperature=0.3,
825+
))
826+
if not response.success:
827+
return
828+
829+
try:
830+
skill = json.loads(response.text.strip())
831+
if not isinstance(skill, dict) or "task_name" not in skill:
832+
return
833+
# Store via memory if available
834+
if self._memory and hasattr(self._memory, "procedural"):
835+
self._memory.procedural.store(
836+
task_name=skill["task_name"],
837+
steps=json.dumps(skill.get("steps", [])),
838+
app_category=skill.get("app_category", "other"),
839+
app_name=plan.goal.description[:100],
840+
trigger_phrases=json.dumps(skill.get("trigger_phrases", [])),
841+
variables="[]",
842+
)
843+
logger.info("Auto-learned skill: %s", skill["task_name"])
844+
except (json.JSONDecodeError, KeyError):
845+
pass
846+
790847
def get_context(self) -> str:
791848
return self._world.get_context_string()

‎moso_core/tools/app_controllers.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,7 @@ def _new_tab(self, target: str = "") -> str:
333333
def _search(self, target: str = "") -> str:
334334
if not target:
335335
return "What should I search for?"
336-
return self._open_url(f"https://www.google.com/search?q={target}")
336+
return self._open_url(f"https://duckduckgo.com/?q={target}")
337337

338338
def _close_tab(self, target: str = "") -> str:
339339
kb = self._kb()

‎moso_core/tools/registry.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,9 +104,22 @@ def execute_tool(
104104
error=f"Risk Engine blocked: {report.max_level.value} - {report.risk.recommendation}",
105105
)
106106
except ImportError:
107-
pass
107+
if tool.permission_level != "guest":
108+
return ToolResult(
109+
success=False,
110+
tool_name=tool.name,
111+
action=action,
112+
error="Risk engine unavailable — blocking non-guest tool",
113+
)
108114
except Exception as e:
109-
logger.warning("Risk check failed (non-blocking): %s", e)
115+
if tool.permission_level != "guest":
116+
logger.warning("Risk check failed (blocking): %s", e)
117+
return ToolResult(
118+
success=False,
119+
tool_name=tool.name,
120+
action=action,
121+
error=f"Risk check failed — blocking: {e}",
122+
)
110123

111124
required_level = tool.get_permission_level(action)
112125
description = tool.describe_action(**params, _action_name=action)

0 commit comments

Comments
 (0)