From b0133139701139ef0f24294f04b2d00bab1384b8 Mon Sep 17 00:00:00 2001 From: singchia Date: Wed, 14 Oct 2026 18:19:30 +0800 Subject: [PATCH 01/24] refactor(aiops): extract shared chat protocol and runtime design (#499) --- docs/adr/ADR-035-unified-agent-runtime.md | 21 +++ docs/rfc/RFC-005-unified-agent-runtime.md | 53 ++++++ internal/manager/biz/aiops/agent/agent.go | 162 +++--------------- .../manager/biz/aiops/chatprotocol/types.go | 149 ++++++++++++++++ internal/manager/biz/imbridge/adapter.go | 22 +-- internal/manager/biz/imbridge/adapter_test.go | 2 +- internal/manager/biz/imbridge/bridge.go | 8 +- .../biz/imbridge/provider/feishu/stream.go | 2 +- .../biz/imbridge/provider/slack/stream.go | 16 +- internal/manager/biz/imbridge/sender.go | 20 +-- internal/manager/biz/imbridge/sender_test.go | 8 +- .../manager/biz/imbridge/stream_supervisor.go | 14 +- internal/manager/biz/imbridge/usecase.go | 4 +- .../biz/imbridge/usecase_allowfrom_test.go | 2 +- internal/manager/server/aiops/http.go | 72 ++++---- internal/manager/server/aiops/http_test.go | 18 +- internal/manager/service/aiops/service.go | 55 +++--- .../service/aiops/service_kernel_test.go | 16 +- .../service/aiops/session_model_test.go | 8 +- internal/manager/service/aiops/user_agent.go | 6 +- 20 files changed, 381 insertions(+), 277 deletions(-) create mode 100644 docs/adr/ADR-035-unified-agent-runtime.md create mode 100644 docs/rfc/RFC-005-unified-agent-runtime.md create mode 100644 internal/manager/biz/aiops/chatprotocol/types.go diff --git a/docs/adr/ADR-035-unified-agent-runtime.md b/docs/adr/ADR-035-unified-agent-runtime.md new file mode 100644 index 000000000..92d8c6259 --- /dev/null +++ b/docs/adr/ADR-035-unified-agent-runtime.md @@ -0,0 +1,21 @@ +# ADR-035：统一 Graph Agent runtime + +- 状态：已接受（用户确认实施方案） +- 日期：2026-10-14 +- 关联：RFC-005、#499、#498 + +## 背景 + +旧循环和 Graph 的工具注册、生命周期和执行约束分叉，造成已注册功能在默认聊天中不可用。 + +## 决策 + +保留 Graph/Eino 为唯一循环，BaseTool 为统一执行接口；Skill 经适配复用原执行服务。共享协议独立于循环；各调用入口对共同工具目录应用自己的权限过滤。发布回滚依赖上一版镜像，不保留隐式旧循环回退。 + +## 备选 + +只补单个工具不能消除结构性分叉；永久维护双循环增加行为兼容成本。两者均不采用。 + +## 后果 + +减少重复执行逻辑和维护成本，但需补齐安全 Skill、联网硬开关、动态目录、历史回放及 IM 回归。初始化失败必须显式报告聊天不可用；本次不变更数据库或外部 API。 diff --git a/docs/rfc/RFC-005-unified-agent-runtime.md b/docs/rfc/RFC-005-unified-agent-runtime.md new file mode 100644 index 000000000..cc946825c --- /dev/null +++ b/docs/rfc/RFC-005-unified-agent-runtime.md @@ -0,0 +1,53 @@ +# RFC-005：统一 Agent runtime 与可执行工具目录 + +- 状态：实施中（用户已确认分阶段方案） +- 日期：2026-10-14 +- 关联：#499、#498、ADR-035 +- 基线：v0.18.1 / a6bc1039 + +## 背景与目标 + +Graph 默认运行路径与旧循环维护独立工具列表，安全 Skill 注册未进入 Graph；工作流目录和执行器也不一致。目标是保留 Graph/Eino 为唯一循环，以同一组可执行工具派生聊天、专家、搜索和工作流视图。HTTP、SSE、IM、会话存储兼容，不更换模型框架，不扩大权限。 + +## 方案 + +1. 抽取 chatprotocol 中的请求、回复、事件类型；旧循环迁移期间仅保留类型别名。 +2. 在 BaseTool 工厂接入安全 Skill 适配器，复用 SkillRunner 的校验、设备路由、审计；过滤由 BaseTool 反向导出的展示 Skill，防止循环桥接和重复注册。 +3. 用完整工具组装入口包含 host-files；运行时动态注册及 MCP 更新同步执行视图和搜索目录。每回合使用一致快照，执行时检查开关，子任务继承。工具来源被移除后不得通过旧引用发起新调用。 +4. 保留只读、写开关、审批和审计控制；联网开关同时限制暴露与执行。工作流的明确发布授权与聊天交互审批不互相替代。 +5. 退役旧循环与内核分支；旧环境变量仅告警，统一 Graph。初始化失败时聊天不可用，不隐式切换执行语义；通过修复配置并重启恢复，其他 Manager 功能不随之停止。 + +接口在消费方定义，不引入跨域依赖。模型接口不变；目录只管理已支持的可执行工具，不把 Markdown 技能元数据当工具。 + +## 能力基线 + +| 能力 | v0.18.1 | 迁移验收 | +| --- | --- | --- | +| web_search、四类 probe、netns、journal、tail | 旧注册表可用，Graph 缺失 | Graph 可发现、可执行；关联网硬拒绝 | +| 外部安全 subprocess Skill | 已加载但 Graph 不可见 | 复用 Skill 服务并接入工具目录 | +| host_find_large_files/du_summary/stat_file | 聊天及工作流执行器存在，画布缺失 | 目录和执行一致 | +| 专家、MCP、审批、页面、通知 | Graph 已接入 | 保持权限、动态更新、停止语义 | +| HTTP/SSE/IM 与历史会话 | 共享旧包类型 | 协议与存储不变，历史回放测试 | +| legacy 显式配置/初始化回退 | 仍存在 | 告警迁移，禁止静默回退 | + +## 备选方案 + +- 仅修 web_search：范围小，但留下外部 Skill 和工作流同类缺口，不采用。 +- 永久维护两个循环：保留切换，但继续产生工具与权限差异，不采用。 +- 统一目录后移除旧循环：分阶段可验证，采用。 + +## 交付与验收 + +2026-10-14 开始；完成时间以验证结果为准，按 1–3 天粒度拆分：共享协议与设计、目录与适配、执行约束与生命周期、综合回归、旧循环清理。 + +- 注册基线覆盖真实内置 Skill 和外部测试 Skill，不只比较名称数量。 +- 默认工具目录、工作流目录/执行、专家可用范围一致；有意排除项显式列出。 +- 覆盖联网开关、动态更新、并发、只读权限、审批、取消、历史回放与 IM；Go 测试带 -race。 +- 不依赖真实 LLM 的确定性回归为主；发布前在有凭据的环境做真实模型、MCP 与升级冒烟。 +- 检查生产是否显式配置 legacy：本地源码不能证明线上使用情况。 + +## 风险、发布与回滚 + +主要风险为漏迁工具、重复桥接、动态目录陈旧、权限弱化、共享类型误删。通过工具执行测试和调用边界硬校验控制。 + +各阶段保持可编译可测试，使用独立提交审查；部署先金丝雀验证聊天与历史会话。无需数据库迁移，回滚恢复上一版 Manager 镜像及对应 Compose 配置，不使用永久双循环。线上盘点、真实环境冒烟未经执行不得宣称完成。 diff --git a/internal/manager/biz/aiops/agent/agent.go b/internal/manager/biz/aiops/agent/agent.go index bd70e4e91..253c9eef9 100644 --- a/internal/manager/biz/aiops/agent/agent.go +++ b/internal/manager/biz/aiops/agent/agent.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" "log/slog" "strings" "time" @@ -59,113 +60,6 @@ type Config struct { HistoryLimit int } -// Reply is the terminal result returned to the caller. -type Reply struct { - Message *model.Message - Usage llm.Usage - Iterations int - ToolCalls []*model.ToolCall -} - -// EventType enumerates streaming events emitted during a run. Each event -// fires exactly once per logical phase, in the same order the agent makes -// progress, so the UI can render incrementally without polling. -type EventType string - -const ( - // EventAssistant fires after every assistant turn is persisted. For - // tool-calling turns Content is empty but PendingToolCalls is non-zero - // — the UI should render placeholders for that many tool calls. - EventAssistant EventType = "assistant" - // EventToolStart fires after a tool_call row is persisted as pending, - // before the tool actually runs. - EventToolStart EventType = "tool_start" - // EventToolEnd fires once the tool finishes (success/error/timeout). - EventToolEnd EventType = "tool_end" - // EventDone fires once at terminal success with the final Reply. - EventDone EventType = "done" - // EventTaskNotification fires when a background sub-agent worker - // reaches a terminal state. The legacy agent itself - // never spawns workers, so this constant exists purely so the SSE - // layer can carry the chatruntime kernel's task_notification frame - // through the same eventName / eventPayload translation as the rest. - EventTaskNotification EventType = "task_notification" - // EventApprovalPending fires when a synchronous-blocking tool - // (HLD-021, e.g. execute_k8s_action) queues a human-approval proposal and is - // about to block on the decision. Carries the live approve/reject card - // payload. Both kernels translate this shape through the same SSE frame. - EventApprovalPending EventType = "approval_pending" -) - -// Event is the streaming envelope. Exactly one of the optional pointer -// fields is set, depending on Type. -type Event struct { - Type EventType - Assistant *AssistantEvent - Tool *ToolEvent - Done *Reply - Notification *TaskNotificationEvent - Approval *ApprovalPendingEvent -} - -// ApprovalPendingEvent carries the approval_pending payload. Mirrors the -// chatruntime.ApprovalPending shape; translation happens in -// service/aiops/service.go::translateRuntimeEvent. Fields stay exported so -// the http SSE renderer can copy them into JSON without reaching into -// chatruntime. -type ApprovalPendingEvent struct { - ApprovalID string - ToolCallID string - Kind string - ToolName string - Command string - Credentials []string -} - -// TaskNotificationEvent carries the task_notification payload. -// Mirrors the chatruntime.TaskNotification shape — translation happens -// in service/aiops/service.go::translateRuntimeEvent. The fields are -// kept exported so the http SSE renderer can copy them into JSON -// without reaching into chatruntime. -type TaskNotificationEvent struct { - TaskID string - Status string - Summary string - Result string - Err string - Usage map[string]any -} - -// AssistantEvent describes one assistant turn's persisted row. -type AssistantEvent struct { - Iteration int - MessageID string - Content string - CreatedAt time.Time - PendingToolCalls int -} - -// ToolEvent describes one tool_call lifecycle event. EndedAt, Status, -// DurationMs and ResultJSON are only meaningful on EventToolEnd; ArgsJSON -// is set on EventToolStart so the UI can show what the agent asked for. -type ToolEvent struct { - ToolCallID string - Name string - DeviceID *uint64 - Status string // pending | success | error | timeout - StartedAt time.Time - EndedAt *time.Time - DurationMs int64 - Error string - ArgsJSON string - ResultJSON string -} - -// Emit is the streaming callback. nil is a no-op (used by Run, which is -// the blocking path); RunStream passes a real fn that fans events out -// over SSE/etc. -type Emit func(Event) - type emitCtxKeyT struct{} var emitCtxKey = emitCtxKeyT{} @@ -194,40 +88,6 @@ type MentionResolver interface { Resolve(ctx context.Context, mentions []Mention) []string } -// Mention is the wire-shape reference the agent receives from the chat -// send endpoint. Mirrors mentions.Mention; declared locally so this -// package doesn't import biz/aiops/mentions (which itself imports -// biz/device + biz/alert and would form a tight coupling chain). -type Mention struct { - Type string - ID string - Label string -} - -// RunOptions bundles per-call overrides — provider/model selection, -// mentions to inline, etc. Empty fields fall back to the agent's -// configured defaults. Pass &RunOptions{} to keep behaviour identical -// to the no-options Run. -// -// WebSearchEnabled toggles whether the agent exposes the manager-scoped -// `web_search` skill to the LLM on this turn. Default false — the -// model will not gratuitously search the public web on every metric -// question. When the user clicks the SPA's globe toggle the SPA -// flips this to true, which makes the tool visible to the model AND -// permits invocation. (Filtering both the schema and the dispatch -// keeps the contract honest: a stale tool_call from history replay -// never silently slips through.) -type RunOptions struct { - Provider string - Model string - Mentions []Mention - Attachments []model.Attachment - WebSearchEnabled bool - // Locale is the UI language the reply should be in ("en-US"/"zh-CN"). - // Threaded to the graph kernel's prompt assembler. Empty = no directive. - Locale string -} - func attachmentIDs(items []model.Attachment) []string { ids := make([]string, 0, len(items)) for _, item := range items { @@ -996,3 +856,23 @@ func classifyToolOutcome(toolCtx context.Context, res tools.ExecuteResult, execE s := string(payload) return model.StatusSuccess, nil, &s } + +type Reply = chatprotocol.Reply +type EventType = chatprotocol.EventType +type Event = chatprotocol.Event +type ApprovalPendingEvent = chatprotocol.ApprovalPendingEvent +type TaskNotificationEvent = chatprotocol.TaskNotificationEvent +type AssistantEvent = chatprotocol.AssistantEvent +type ToolEvent = chatprotocol.ToolEvent +type Emit = chatprotocol.Emit +type Mention = chatprotocol.Mention +type RunOptions = chatprotocol.RunOptions + +const ( + EventAssistant = chatprotocol.EventAssistant + EventToolStart = chatprotocol.EventToolStart + EventToolEnd = chatprotocol.EventToolEnd + EventDone = chatprotocol.EventDone + EventTaskNotification = chatprotocol.EventTaskNotification + EventApprovalPending = chatprotocol.EventApprovalPending +) diff --git a/internal/manager/biz/aiops/chatprotocol/types.go b/internal/manager/biz/aiops/chatprotocol/types.go new file mode 100644 index 000000000..deb02f700 --- /dev/null +++ b/internal/manager/biz/aiops/chatprotocol/types.go @@ -0,0 +1,149 @@ +// Package chatprotocol 定义聊天入口共享的请求、回复和事件，不包含 Agent 执行逻辑。 +package chatprotocol + +import ( + model "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/llm" + "time" +) + +// Reply is the terminal result returned to the caller. +type Reply struct { + Message *model.Message + Usage llm.Usage + Iterations int + ToolCalls []*model.ToolCall +} + +// EventType enumerates streaming events emitted during a run. Each event +// fires exactly once per logical phase, in the same order the agent makes +// progress, so the UI can render incrementally without polling. +type EventType string + +const ( + // EventAssistant fires after every assistant turn is persisted. For + // tool-calling turns Content is empty but PendingToolCalls is non-zero + // — the UI should render placeholders for that many tool calls. + EventAssistant EventType = "assistant" + // EventToolStart fires after a tool_call row is persisted as pending, + // before the tool actually runs. + EventToolStart EventType = "tool_start" + // EventToolEnd fires once the tool finishes (success/error/timeout). + EventToolEnd EventType = "tool_end" + // EventDone fires once at terminal success with the final Reply. + EventDone EventType = "done" + // EventTaskNotification fires when a background sub-agent worker + // reaches a terminal state. The legacy agent itself + // never spawns workers, so this constant exists purely so the SSE + // layer can carry the chatruntime kernel's task_notification frame + // through the same eventName / eventPayload translation as the rest. + EventTaskNotification EventType = "task_notification" + // EventApprovalPending fires when a synchronous-blocking tool + // (HLD-021, e.g. execute_k8s_action) queues a human-approval proposal and is + // about to block on the decision. Carries the live approve/reject card + // payload. Both kernels translate this shape through the same SSE frame. + EventApprovalPending EventType = "approval_pending" +) + +// Event is the streaming envelope. Exactly one of the optional pointer +// fields is set, depending on Type. +type Event struct { + Type EventType + Assistant *AssistantEvent + Tool *ToolEvent + Done *Reply + Notification *TaskNotificationEvent + Approval *ApprovalPendingEvent +} + +// ApprovalPendingEvent carries the approval_pending payload. Mirrors the +// chatruntime.ApprovalPending shape; translation happens in +// service/aiops/service.go::translateRuntimeEvent. Fields stay exported so +// the http SSE renderer can copy them into JSON without reaching into +// chatruntime. +type ApprovalPendingEvent struct { + ApprovalID string + ToolCallID string + Kind string + ToolName string + Command string + Credentials []string +} + +// TaskNotificationEvent carries the task_notification payload. +// Mirrors the chatruntime.TaskNotification shape — translation happens +// in service/aiops/service.go::translateRuntimeEvent. The fields are +// kept exported so the http SSE renderer can copy them into JSON +// without reaching into chatruntime. +type TaskNotificationEvent struct { + TaskID string + Status string + Summary string + Result string + Err string + Usage map[string]any +} + +// AssistantEvent describes one assistant turn's persisted row. +type AssistantEvent struct { + Iteration int + MessageID string + Content string + CreatedAt time.Time + PendingToolCalls int +} + +// ToolEvent describes one tool_call lifecycle event. EndedAt, Status, +// DurationMs and ResultJSON are only meaningful on EventToolEnd; ArgsJSON +// is set on EventToolStart so the UI can show what the agent asked for. +type ToolEvent struct { + ToolCallID string + Name string + DeviceID *uint64 + Status string // pending | success | error | timeout + StartedAt time.Time + EndedAt *time.Time + DurationMs int64 + Error string + ArgsJSON string + ResultJSON string +} + +// Emit is the streaming callback. nil is a no-op (used by Run, which is +// the blocking path); RunStream passes a real fn that fans events out +// over SSE/etc. +type Emit func(Event) + +// Mention is the wire-shape reference the agent receives from the chat +// send endpoint. Mirrors mentions.Mention; declared locally so this +// package doesn't import biz/aiops/mentions (which itself imports +// biz/device + biz/alert and would form a tight coupling chain). +type Mention struct { + Type string + ID string + Label string +} + +// RunOptions bundles per-call overrides — provider/model selection, +// mentions to inline, etc. Empty fields fall back to the agent's +// configured defaults. Pass &RunOptions{} to keep behaviour identical +// to the no-options Run. +// +// WebSearchEnabled toggles whether the agent exposes the manager-scoped +// `web_search` skill to the LLM on this turn. Default false — the +// model will not gratuitously search the public web on every metric +// question. When the user clicks the SPA's globe toggle the SPA +// flips this to true, which makes the tool visible to the model AND +// permits invocation. (Filtering both the schema and the dispatch +// keeps the contract honest: a stale tool_call from history replay +// never silently slips through.) +type RunOptions struct { + Provider string + Model string + Mentions []Mention + Attachments []model.Attachment + WebSearchEnabled bool + // Locale is the UI language the reply should be in ("en-US"/"zh-CN"). + // Threaded to the graph kernel's prompt assembler. Empty = no directive. + Locale string +} diff --git a/internal/manager/biz/imbridge/adapter.go b/internal/manager/biz/imbridge/adapter.go index 68617a4b0..8d2259ee0 100644 --- a/internal/manager/biz/imbridge/adapter.go +++ b/internal/manager/biz/imbridge/adapter.go @@ -5,15 +5,15 @@ import ( "fmt" "log/slog" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" - "github.com/ongridio/ongrid/internal/pkg/llm" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" svcaiops "github.com/ongridio/ongrid/internal/manager/service/aiops" + "github.com/ongridio/ongrid/internal/pkg/llm" ) // LLMDefaultProvider returns the cluster-wide default LLM provider id + // model that web/SPA chats would pick. The IM path uses this so it // stays in sync with whatever the operator selected in the UI picker; -// otherwise agent.New's internal "" → "gpt-5.4" fallback would route IM +// otherwise chatprotocol.New's internal "" → "gpt-5.4" fallback would route IM // traffic to a model that's not in the catalog and the LLM router // rejects it ("this model has beta-limitations…"), which surfaces in // chat apps as "助手执行失败". @@ -38,7 +38,7 @@ type AiopsServiceAdapter struct { } // NewAiopsAdapter wires the adapter. defaults is optional — when nil -// the adapter falls back to agent.RunOptions{} (which puts agent.New's +// the adapter falls back to chatprotocol.RunOptions{} (which puts chatprotocol.New's // hard-coded model default in charge). Pass the LLMSettingsResolver // so the IM path picks up the same default the SPA picker writes. func NewAiopsAdapter(svc *svcaiops.Service, serviceUserID uint64, defaults LLMDefaultProvider, log *slog.Logger) *AiopsServiceAdapter { @@ -78,10 +78,10 @@ func (a *AiopsServiceAdapter) EnsureSession(ctx context.Context, ownerUserID uin } // StreamMessage posts user content to the session and forwards each -// agent.Event to emit. The agent loop runs synchronously on the +// chatprotocol.Event to emit. The agent loop runs synchronously on the // caller's goroutine — the bridge calls this from its own goroutine // (the webhook handler returns 200 immediately). -func (a *AiopsServiceAdapter) StreamMessage(ctx context.Context, sessionID string, userContent string, emit agent.Emit) error { +func (a *AiopsServiceAdapter) StreamMessage(ctx context.Context, sessionID string, userContent string, emit chatprotocol.Emit) error { opts := a.runOptions(ctx) _, err := a.svc.PostMessageStreamWithOpts(ctx, a.caller(), sessionID, userContent, emit, opts) return err @@ -91,10 +91,10 @@ func (a *AiopsServiceAdapter) StreamMessage(ctx context.Context, sessionID strin // uses the same LLM the SPA picker writes. On any resolver error or // missing catalog, returns zero RunOptions and lets the agent layer // apply its own fallback (which currently lands on gpt-5.4 — see -// agent.New). -func (a *AiopsServiceAdapter) runOptions(ctx context.Context) agent.RunOptions { +// chatprotocol.New). +func (a *AiopsServiceAdapter) runOptions(ctx context.Context) chatprotocol.RunOptions { if a.defaults == nil { - return agent.RunOptions{} + return chatprotocol.RunOptions{} } providers, defID, err := a.defaults.ResolveProviders(ctx) if err != nil || len(providers) == 0 { @@ -102,7 +102,7 @@ func (a *AiopsServiceAdapter) runOptions(ctx context.Context) agent.RunOptions { a.log.Warn("imbridge: llm resolver failed; using agent fallback model", slog.Any("err", err)) } - return agent.RunOptions{} + return chatprotocol.RunOptions{} } // If a default provider is set, look it up; otherwise take the // first catalog entry (alphabetical by id — same order the SPA @@ -120,7 +120,7 @@ func (a *AiopsServiceAdapter) runOptions(ctx context.Context) agent.RunOptions { if pick == nil { pick = &providers[0] } - return agent.RunOptions{ + return chatprotocol.RunOptions{ Provider: pick.ID, Model: pick.Model, } diff --git a/internal/manager/biz/imbridge/adapter_test.go b/internal/manager/biz/imbridge/adapter_test.go index 08d0a6eeb..4c0f46c00 100644 --- a/internal/manager/biz/imbridge/adapter_test.go +++ b/internal/manager/biz/imbridge/adapter_test.go @@ -21,7 +21,7 @@ func (f *fakeDefaults) ResolveProviders(_ context.Context) ([]llm.ProviderConfig // TestAdapter_runOptions_picksResolverDefault covers the v0.7.169 fix: // IM bridge must thread the cluster default_provider + _default_model -// into agent.RunOptions instead of leaving them empty (which made agent.New +// into chatprotocol.RunOptions instead of leaving them empty (which made chatprotocol.New // fall back to the hard-coded "gpt-5.4" model and end users saw // "助手执行失败" in Lark/Slack/Telegram). func TestAdapter_runOptions_picksResolverDefault(t *testing.T) { diff --git a/internal/manager/biz/imbridge/bridge.go b/internal/manager/biz/imbridge/bridge.go index 66112765b..f67d86850 100644 --- a/internal/manager/biz/imbridge/bridge.go +++ b/internal/manager/biz/imbridge/bridge.go @@ -14,7 +14,7 @@ import ( "sync" "time" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" model "github.com/ongridio/ongrid/internal/manager/model/imbridge" ) @@ -44,7 +44,7 @@ type AgentSession interface { // EventAssistant (which carries the assistant text chunks) and // EventDone (terminal). This is a thin wrapper over // service.Service.PostMessageStreamWithOpts. - StreamMessage(ctx context.Context, sessionID string, userContent string, emit agent.Emit) error + StreamMessage(ctx context.Context, sessionID string, userContent string, emit chatprotocol.Emit) error } // Bridge is the singleton wired into manager main.go. Exported methods @@ -221,7 +221,7 @@ func (b *Bridge) HandleInbound(ctx context.Context, sender Sender, msg InboundMe // language regardless of persona / model defaults. Empty locale = // auto = LLM mirrors the user. See [[feedback_ai_output_locale]]. editor := newStreamEditor(ctx, sender, msg.ChatID, msg.ReceiveIDType, placeholder, app.DefaultLocale, b.log) - emit := func(e agent.Event) { + emit := func(e chatprotocol.Event) { editor.OnEvent(e) } userContent := msg.Text @@ -236,7 +236,7 @@ func (b *Bridge) HandleInbound(ctx context.Context, sender Sender, msg InboundMe } // localeDirective renders the language hint we append to the user content -// before handing to the agent. Empty locale = "" (no directive, LLM +// before handing to the chatprotocol. Empty locale = "" (no directive, LLM // mirrors). Mirrors the RCA-side helper in alert/investigator but framed // for chat replies, not RCA reports — see [[feedback_ai_output_locale]]. func localeDirective(locale string) string { diff --git a/internal/manager/biz/imbridge/provider/feishu/stream.go b/internal/manager/biz/imbridge/provider/feishu/stream.go index ed43b9fd0..39bf88ba6 100644 --- a/internal/manager/biz/imbridge/provider/feishu/stream.go +++ b/internal/manager/biz/imbridge/provider/feishu/stream.go @@ -14,7 +14,7 @@ import ( ) // StreamClient is the long-connection variant of the Feishu provider -//manager dials out to Feishu's WebSocket-based event +// manager dials out to Feishu's WebSocket-based event // service via the official larksuite/oapi-sdk-go ws.Client; events // land via the dispatcher and are bridged into the agent runtime. // No public webhook URL is required. diff --git a/internal/manager/biz/imbridge/provider/slack/stream.go b/internal/manager/biz/imbridge/provider/slack/stream.go index 8e08930a5..37433fd8a 100644 --- a/internal/manager/biz/imbridge/provider/slack/stream.go +++ b/internal/manager/biz/imbridge/provider/slack/stream.go @@ -39,14 +39,14 @@ type envelope struct { type eventsAPIPayload struct { TeamID string `json:"team_id"` Event struct { - Type string `json:"type"` // "message" | "app_mention" | ... - Subtype string `json:"subtype"` // empty for normal user text; "bot_message", "message_changed" etc. to ignore - Channel string `json:"channel"` // C…/G…/D… channel id - User string `json:"user"` // sender user id - Text string `json:"text"` // raw text (may contain <@U…> mention markup) - TS string `json:"ts"` // message ts (used as event id for dedup) - BotID string `json:"bot_id"` // present on bot-sent messages — used to break echo loops - ThreadTS string `json:"thread_ts"` // present on reply threads — used as ImThread.ImThreadID + Type string `json:"type"` // "message" | "app_mention" | ... + Subtype string `json:"subtype"` // empty for normal user text; "bot_message", "message_changed" etc. to ignore + Channel string `json:"channel"` // C…/G…/D… channel id + User string `json:"user"` // sender user id + Text string `json:"text"` // raw text (may contain <@U…> mention markup) + TS string `json:"ts"` // message ts (used as event id for dedup) + BotID string `json:"bot_id"` // present on bot-sent messages — used to break echo loops + ThreadTS string `json:"thread_ts"` // present on reply threads — used as ImThread.ImThreadID } `json:"event"` } diff --git a/internal/manager/biz/imbridge/sender.go b/internal/manager/biz/imbridge/sender.go index 3449a37a6..300c103f0 100644 --- a/internal/manager/biz/imbridge/sender.go +++ b/internal/manager/biz/imbridge/sender.go @@ -7,7 +7,7 @@ import ( "sync" "time" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" ) // Sender is the minimum platform-agnostic outbound surface. Providers @@ -25,7 +25,7 @@ type MessageEditor interface { EditText(ctx context.Context, messageID, text string) error } -// streamEditor turns the SSE-shape agent.Event stream into throttled +// streamEditor turns the SSE-shape chatprotocol.Event stream into throttled // EditText calls on the platform. It assumes the assistant text grows // monotonically — we always send the full accumulated buffer, never // deltas, because Feishu / DingTalk both replace rather than append on @@ -65,15 +65,15 @@ func newStreamEditor(ctx context.Context, sender Sender, chatID, receiveIDType, } } -// OnEvent is wired as the emit callback for agent.RunStreamWithOpts. +// OnEvent is wired as the emit callback for chatprotocol.RunStreamWithOpts. // We only care about EventAssistant (text chunks) and EventDone // (terminal — force-flush). Tool calls / task notifications are // suppressed in IM for now; they'd be too noisy as inline chat // messages. -func (e *streamEditor) OnEvent(ev agent.Event) { +func (e *streamEditor) OnEvent(ev chatprotocol.Event) { switch ev.Type { - case agent.EventAssistant: - // Pull the text content out of the event. agent.Event for + case chatprotocol.EventAssistant: + // Pull the text content out of the event. chatprotocol.Event for // assistant carries the persisted assistant turn — we // concatenate its Content field. If the runtime later // exposes a per-chunk delta we should switch to that to @@ -89,7 +89,7 @@ func (e *streamEditor) OnEvent(ev agent.Event) { if shouldFlush { e.flush() } - case agent.EventDone: + case chatprotocol.EventDone: e.flush() // terminal: ensure last chunk lands default: // EventToolStart / EventToolEnd / EventTaskNotification — @@ -162,9 +162,9 @@ func (e *streamEditor) flush() error { // EventAssistant payload. The agent runtime stores the full // accumulated assistant turn (not per-token deltas) on the event so we // can just take it. -func assistantText(ev agent.Event) string { - // agent.Event.Assistant is set on assistant events — see - // internal/manager/biz/aiops/agent/agent.go. We avoid a direct +func assistantText(ev chatprotocol.Event) string { + // chatprotocol.Event.Assistant is set on assistant events — see + // internal/manager/biz/aiops/agent/chatprotocol.go. We avoid a direct // type assertion on the field shape to stay forward-compatible: // if the runtime adds richer payloads (citations, attachments) // they live on the same Event struct. diff --git a/internal/manager/biz/imbridge/sender_test.go b/internal/manager/biz/imbridge/sender_test.go index 2f39c1f89..fddb926a5 100644 --- a/internal/manager/biz/imbridge/sender_test.go +++ b/internal/manager/biz/imbridge/sender_test.go @@ -6,7 +6,7 @@ import ( "log/slog" "testing" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" ) type oneShotSender struct { @@ -23,9 +23,9 @@ func TestStreamEditor_WhenSenderCannotEdit_SendsFinalMessageOnce(t *testing.T) { editor := newStreamEditor(context.Background(), sender, "chat", "chat_id", "", "en", slog.New(slog.NewTextHandler(io.Discard, nil))) - editor.OnEvent(agent.Event{Type: agent.EventAssistant, Assistant: &agent.AssistantEvent{Content: "partial"}}) - editor.OnEvent(agent.Event{Type: agent.EventAssistant, Assistant: &agent.AssistantEvent{Content: "final"}}) - editor.OnEvent(agent.Event{Type: agent.EventDone}) + editor.OnEvent(chatprotocol.Event{Type: chatprotocol.EventAssistant, Assistant: &chatprotocol.AssistantEvent{Content: "partial"}}) + editor.OnEvent(chatprotocol.Event{Type: chatprotocol.EventAssistant, Assistant: &chatprotocol.AssistantEvent{Content: "final"}}) + editor.OnEvent(chatprotocol.Event{Type: chatprotocol.EventDone}) if err := editor.Flush(); err != nil { t.Fatalf("Flush() error = %v", err) } diff --git a/internal/manager/biz/imbridge/stream_supervisor.go b/internal/manager/biz/imbridge/stream_supervisor.go index 6c347a279..35713329b 100644 --- a/internal/manager/biz/imbridge/stream_supervisor.go +++ b/internal/manager/biz/imbridge/stream_supervisor.go @@ -42,13 +42,13 @@ type StreamRepo interface { // parent is the supervisor's own context. Cancelling the supervisor // (manager shutdown) propagates a graceful Stop to every client. type StreamSupervisor struct { - repo StreamRepo - bridge *Bridge - factories map[string]StreamClientFactory // provider -> factory - log *slog.Logger - mu sync.Mutex - running map[uint64]*streamSlot // im_app.id -> running slot - reconcile time.Duration + repo StreamRepo + bridge *Bridge + factories map[string]StreamClientFactory // provider -> factory + log *slog.Logger + mu sync.Mutex + running map[uint64]*streamSlot // im_app.id -> running slot + reconcile time.Duration } type streamSlot struct { diff --git a/internal/manager/biz/imbridge/usecase.go b/internal/manager/biz/imbridge/usecase.go index 955a76198..5ce552623 100644 --- a/internal/manager/biz/imbridge/usecase.go +++ b/internal/manager/biz/imbridge/usecase.go @@ -6,8 +6,8 @@ import ( "strings" "time" - "github.com/ongridio/ongrid/internal/pkg/errs" model "github.com/ongridio/ongrid/internal/manager/model/imbridge" + "github.com/ongridio/ongrid/internal/pkg/errs" ) // AdminRepo is the surface UC needs from the data layer. The webhook @@ -154,7 +154,7 @@ func (in *AppInput) validate() error { // Allowlist semantics mirror Telegram: a Slack workspace bot accepts // messages from any workspace member by default, which on a public // or guest-rich workspace would let surprise users command the - // agent. Require at least one Slack user_id (U…) in allow_from so + // chatprotocol. Require at least one Slack user_id (U…) in allow_from so // the operator must consciously open the door. if provider == model.ProviderSlack { if mode != model.ModeStream { diff --git a/internal/manager/biz/imbridge/usecase_allowfrom_test.go b/internal/manager/biz/imbridge/usecase_allowfrom_test.go index 239b243d0..dc59d9fd0 100644 --- a/internal/manager/biz/imbridge/usecase_allowfrom_test.go +++ b/internal/manager/biz/imbridge/usecase_allowfrom_test.go @@ -70,7 +70,7 @@ func TestValidateTelegramRequiresAllowFrom(t *testing.T) { } // Slack mirrors the Telegram safety contract: any workspace member could -// otherwise talk to a tool-equipped agent. Format check rejects obvious +// otherwise talk to a tool-equipped chatprotocol. Format check rejects obvious // typos (numeric or short tokens) so the operator notices, instead of a // silently-empty allowlist that ignores everyone. func TestValidateSlackRequiresAllowFrom(t *testing.T) { diff --git a/internal/manager/server/aiops/http.go b/internal/manager/server/aiops/http.go index 6601426f3..d66e2b1bf 100644 --- a/internal/manager/server/aiops/http.go +++ b/internal/manager/server/aiops/http.go @@ -30,7 +30,7 @@ import ( "github.com/go-chi/chi/v5" biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" "github.com/ongridio/ongrid/internal/manager/biz/aiops/mentions" model "github.com/ongridio/ongrid/internal/manager/model/aiops" @@ -50,10 +50,10 @@ type AIOpsService interface { DeleteSession(ctx context.Context, caller svc.Caller, sessionID string) error RenameSession(ctx context.Context, caller svc.Caller, sessionID string, title string) error UpdateSessionModel(ctx context.Context, caller svc.Caller, sessionID string, provider string, model string) error - PostMessage(ctx context.Context, caller svc.Caller, sessionID string, content string) (*agent.Reply, error) - PostMessageWithOpts(ctx context.Context, caller svc.Caller, sessionID string, content string, opts agent.RunOptions) (*agent.Reply, error) - PostMessageStream(ctx context.Context, caller svc.Caller, sessionID string, content string, emit agent.Emit) (*agent.Reply, error) - PostMessageStreamWithOpts(ctx context.Context, caller svc.Caller, sessionID string, content string, emit agent.Emit, opts agent.RunOptions) (*agent.Reply, error) + PostMessage(ctx context.Context, caller svc.Caller, sessionID string, content string) (*chatprotocol.Reply, error) + PostMessageWithOpts(ctx context.Context, caller svc.Caller, sessionID string, content string, opts chatprotocol.RunOptions) (*chatprotocol.Reply, error) + PostMessageStream(ctx context.Context, caller svc.Caller, sessionID string, content string, emit chatprotocol.Emit) (*chatprotocol.Reply, error) + PostMessageStreamWithOpts(ctx context.Context, caller svc.Caller, sessionID string, content string, emit chatprotocol.Emit, opts chatprotocol.RunOptions) (*chatprotocol.Reply, error) StopSession(ctx context.Context, caller svc.Caller, sessionID string) (bool, error) UsageToday(ctx context.Context) (*biz.DailyUsage, error) ListMutatingProposals(ctx context.Context, caller svc.Caller, f biz.MutatingProposalFilter) ([]*model.MutatingProposal, int64, error) @@ -174,7 +174,7 @@ func (h *Handler) Register(r chi.Router) { r.Delete("/v1/agents/custom/{name}", h.deleteUserAgent) r.Get("/v1/operations/{id}", h.getOperation) r.Post("/v1/operations/{id}/actions/{action}", h.executeOperationAction) - // Generic delete: works on any non-builtin / non-default agent. + // Generic delete: works on any non-builtin / non-default chatprotocol. // Disk-source agents (loaded from agents/*.md) get session-scoped // removal — the in-memory registry drops them, but the .md file // stays and they re-appear on restart. User-source agents go @@ -399,15 +399,15 @@ type mentionInput struct { Label string `json:"label"` } -func (m mentionInput) toAgent() agent.Mention { - return agent.Mention{Type: m.Type, ID: m.ID, Label: m.Label} +func (m mentionInput) toAgent() chatprotocol.Mention { + return chatprotocol.Mention{Type: m.Type, ID: m.ID, Label: m.Label} } -func toAgentMentions(in []mentionInput) []agent.Mention { +func toAgentMentions(in []mentionInput) []chatprotocol.Mention { if len(in) == 0 { return nil } - out := make([]agent.Mention, 0, len(in)) + out := make([]chatprotocol.Mention, 0, len(in)) for _, m := range in { out = append(out, m.toAgent()) } @@ -737,7 +737,7 @@ func (h *Handler) postMessage(w http.ResponseWriter, r *http.Request) { writeErr(w, err) return } - opts := agent.RunOptions{ + opts := chatprotocol.RunOptions{ Provider: req.Provider, Model: req.Model, Mentions: toAgentMentions(req.Mentions), @@ -791,7 +791,7 @@ func (h *Handler) postMessageStream(w http.ResponseWriter, r *http.Request) { writeErr(w, err) return } - opts := agent.RunOptions{ + opts := chatprotocol.RunOptions{ Provider: req.Provider, Model: req.Model, Mentions: toAgentMentions(req.Mentions), @@ -821,7 +821,7 @@ func (h *Handler) postMessageStream(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(": ok\n\n")) flusher.Flush() - emit := func(e agent.Event) { + emit := func(e chatprotocol.Event) { writeSSE(w, flusher, eventName(e.Type), eventPayload(id, e)) } @@ -841,28 +841,28 @@ func (h *Handler) postMessageStream(w http.ResponseWriter, r *http.Request) { } } -func eventName(t agent.EventType) string { +func eventName(t chatprotocol.EventType) string { switch t { - case agent.EventAssistant: + case chatprotocol.EventAssistant: return "assistant" - case agent.EventToolStart: + case chatprotocol.EventToolStart: return "tool_start" - case agent.EventToolEnd: + case chatprotocol.EventToolEnd: return "tool_end" - case agent.EventDone: + case chatprotocol.EventDone: return "done" - case agent.EventTaskNotification: + case chatprotocol.EventTaskNotification: return "task_notification" - case agent.EventApprovalPending: + case chatprotocol.EventApprovalPending: return "approval_pending" default: return string(t) } } -func eventPayload(sessionID string, e agent.Event) any { +func eventPayload(sessionID string, e chatprotocol.Event) any { switch e.Type { - case agent.EventAssistant: + case chatprotocol.EventAssistant: if e.Assistant == nil { return map[string]any{"session_id": sessionID} } @@ -874,7 +874,7 @@ func eventPayload(sessionID string, e agent.Event) any { "created_at": e.Assistant.CreatedAt, "pending_tool_calls": e.Assistant.PendingToolCalls, } - case agent.EventToolStart, agent.EventToolEnd: + case chatprotocol.EventToolStart, chatprotocol.EventToolEnd: if e.Tool == nil { return map[string]any{"session_id": sessionID} } @@ -915,12 +915,12 @@ func eventPayload(sessionID string, e agent.Event) any { } } return out - case agent.EventDone: + case chatprotocol.EventDone: if e.Done == nil { return map[string]any{"session_id": sessionID} } return toPostMessageResp(sessionID, e.Done) - case agent.EventTaskNotification: + case chatprotocol.EventTaskNotification: if e.Notification == nil { return map[string]any{"session_id": sessionID} } @@ -940,7 +940,7 @@ func eventPayload(sessionID string, e agent.Event) any { out["usage"] = e.Notification.Usage } return out - case agent.EventApprovalPending: + case chatprotocol.EventApprovalPending: if e.Approval == nil { return map[string]any{"session_id": sessionID} } @@ -963,7 +963,7 @@ func eventPayload(sessionID string, e agent.Event) any { // writeSSE serialises one event frame. Errors are intentionally swallowed: // if the client closed the connection there is nothing useful to do -// mid-stream, and the next Flush will surface the failure to the agent. +// mid-stream, and the next Flush will surface the failure to the chatprotocol. func writeSSE(w http.ResponseWriter, f http.Flusher, name string, payload any) { body, err := json.Marshal(payload) if err != nil { @@ -1213,10 +1213,10 @@ func (h *Handler) listModels(w http.ResponseWriter, r *http.Request) { return } type providerDTO struct { - ID string `json:"id"` - Label string `json:"label"` - Models []string `json:"models"` - Model string `json:"model,omitempty"` + ID string `json:"id"` + Label string `json:"label"` + Models []string `json:"models"` + Model string `json:"model,omitempty"` } type defaultDTO struct { Provider string `json:"provider"` @@ -1230,10 +1230,10 @@ func (h *Handler) listModels(w http.ResponseWriter, r *http.Request) { if h.catalog != nil { for _, p := range h.catalog.Providers() { out.Providers = append(out.Providers, providerDTO{ - ID: p.ID, - Label: p.Label, - Models: p.Models, - Model: p.Model, + ID: p.ID, + Label: p.Label, + Models: p.Models, + Model: p.Model, }) } defID, defModel := h.catalog.Default() @@ -1289,7 +1289,7 @@ func toMessageDTO(m *model.Message) messageDTO { return out } -func toPostMessageResp(sessionID string, reply *agent.Reply) postMessageResp { +func toPostMessageResp(sessionID string, reply *chatprotocol.Reply) postMessageResp { if reply == nil { return postMessageResp{SessionID: sessionID} } diff --git a/internal/manager/server/aiops/http_test.go b/internal/manager/server/aiops/http_test.go index 268ae2f9e..ffb2c6d6a 100644 --- a/internal/manager/server/aiops/http_test.go +++ b/internal/manager/server/aiops/http_test.go @@ -13,7 +13,7 @@ import ( "github.com/go-chi/chi/v5" biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" model "github.com/ongridio/ongrid/internal/manager/model/aiops" svc "github.com/ongridio/ongrid/internal/manager/service/aiops" "github.com/ongridio/ongrid/internal/pkg/errs" @@ -34,7 +34,7 @@ type fakeService struct { closeErr error - postReply *agent.Reply + postReply *chatprotocol.Reply postErr error usageResp *biz.DailyUsage @@ -129,29 +129,29 @@ func (f *fakeService) UpdateSessionModel(_ context.Context, _ svc.Caller, _, pro f.lastUpdateModel = selectedModel return nil } -func (f *fakeService) PostMessage(_ context.Context, c svc.Caller, _ string, content string) (*agent.Reply, error) { +func (f *fakeService) PostMessage(_ context.Context, c svc.Caller, _ string, content string) (*chatprotocol.Reply, error) { f.lastCaller = c f.lastPostCt = content return f.postReply, f.postErr } -func (f *fakeService) PostMessageWithOpts(_ context.Context, c svc.Caller, _ string, content string, _ agent.RunOptions) (*agent.Reply, error) { +func (f *fakeService) PostMessageWithOpts(_ context.Context, c svc.Caller, _ string, content string, _ chatprotocol.RunOptions) (*chatprotocol.Reply, error) { f.lastCaller = c f.lastPostCt = content return f.postReply, f.postErr } -func (f *fakeService) PostMessageStream(_ context.Context, c svc.Caller, _ string, content string, emit agent.Emit) (*agent.Reply, error) { +func (f *fakeService) PostMessageStream(_ context.Context, c svc.Caller, _ string, content string, emit chatprotocol.Emit) (*chatprotocol.Reply, error) { f.lastCaller = c f.lastPostCt = content if emit != nil && f.postReply != nil { - emit(agent.Event{Type: agent.EventDone, Done: f.postReply}) + emit(chatprotocol.Event{Type: chatprotocol.EventDone, Done: f.postReply}) } return f.postReply, f.postErr } -func (f *fakeService) PostMessageStreamWithOpts(_ context.Context, c svc.Caller, _ string, content string, emit agent.Emit, _ agent.RunOptions) (*agent.Reply, error) { +func (f *fakeService) PostMessageStreamWithOpts(_ context.Context, c svc.Caller, _ string, content string, emit chatprotocol.Emit, _ chatprotocol.RunOptions) (*chatprotocol.Reply, error) { f.lastCaller = c f.lastPostCt = content if emit != nil && f.postReply != nil { - emit(agent.Event{Type: agent.EventDone, Done: f.postReply}) + emit(chatprotocol.Event{Type: chatprotocol.EventDone, Done: f.postReply}) } return f.postReply, f.postErr } @@ -228,7 +228,7 @@ func TestPostMessageHappyPath(t *testing.T) { startedAt := time.Now().UTC() endedAt := startedAt.Add(180 * time.Millisecond) edgeID := uint64(5) - reply := &agent.Reply{ + reply := &chatprotocol.Reply{ Message: &model.Message{ ID: "101", Role: "assistant", diff --git a/internal/manager/service/aiops/service.go b/internal/manager/service/aiops/service.go index 797977916..bb8f2734d 100644 --- a/internal/manager/service/aiops/service.go +++ b/internal/manager/service/aiops/service.go @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" "io" "log/slog" "reflect" @@ -31,7 +32,7 @@ import ( "time" biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" model "github.com/ongridio/ongrid/internal/manager/model/aiops" "github.com/ongridio/ongrid/internal/pkg/errs" @@ -52,7 +53,7 @@ const ( type Kernel string const ( - // KernelLegacy is the pre-PR-9 agent.Agent for-loop (agent.go). + // KernelLegacy is the pre-PR-9 agent.Agent for-loop (chatprotocol.go). KernelLegacy Kernel = "legacy" // KernelGraph is the new eino + chatruntime + graph kernel // (PR-1..PR-7 of). @@ -388,13 +389,13 @@ func (s *Service) UpdateSessionModel(ctx context.Context, caller Caller, session // tunnel dispatches complete before returning. The agent itself re-checks // ownership; to support admin-bypass we resolve the owning user_id here // and pass it in. -func (s *Service) PostMessage(ctx context.Context, caller Caller, sessionID string, content string) (*agent.Reply, error) { - return s.PostMessageWithOpts(ctx, caller, sessionID, content, agent.RunOptions{}) +func (s *Service) PostMessage(ctx context.Context, caller Caller, sessionID string, content string) (*chatprotocol.Reply, error) { + return s.PostMessageWithOpts(ctx, caller, sessionID, content, chatprotocol.RunOptions{}) } // PostMessageWithOpts is the override-aware sibling of PostMessage. Per- // call provider/model + mentions flow into the agent unchanged. -func (s *Service) PostMessageWithOpts(ctx context.Context, caller Caller, sessionID string, content string, opts agent.RunOptions) (*agent.Reply, error) { +func (s *Service) PostMessageWithOpts(ctx context.Context, caller Caller, sessionID string, content string, opts chatprotocol.RunOptions) (*chatprotocol.Reply, error) { return s.runWithKernel(ctx, caller, sessionID, content, nil, opts) } @@ -402,22 +403,22 @@ func (s *Service) PostMessageWithOpts(ctx context.Context, caller Caller, sessio // agent phase (assistant turn / tool start / tool end / done); the final // Reply (or error) is still returned so the handler can decide whether to // emit a trailing SSE error event. -func (s *Service) PostMessageStream(ctx context.Context, caller Caller, sessionID string, content string, emit agent.Emit) (*agent.Reply, error) { - return s.PostMessageStreamWithOpts(ctx, caller, sessionID, content, emit, agent.RunOptions{}) +func (s *Service) PostMessageStream(ctx context.Context, caller Caller, sessionID string, content string, emit chatprotocol.Emit) (*chatprotocol.Reply, error) { + return s.PostMessageStreamWithOpts(ctx, caller, sessionID, content, emit, chatprotocol.RunOptions{}) } // PostMessageStreamWithOpts streams agent events while honouring per-call // provider/model + mention overrides. Empty opts behaves identically to // PostMessageStream. -func (s *Service) PostMessageStreamWithOpts(ctx context.Context, caller Caller, sessionID string, content string, emit agent.Emit, opts agent.RunOptions) (*agent.Reply, error) { +func (s *Service) PostMessageStreamWithOpts(ctx context.Context, caller Caller, sessionID string, content string, emit chatprotocol.Emit, opts chatprotocol.RunOptions) (*chatprotocol.Reply, error) { return s.runWithKernel(ctx, caller, sessionID, content, emit, opts) } // runWithKernel is the single chokepoint that decides which kernel // runs for this request. Both legacy and graph paths return the same -// agent.Reply DTO so the HTTP layer doesn't care which path served +// chatprotocol.Reply DTO so the HTTP layer doesn't care which path served // the response. -func (s *Service) runWithKernel(ctx context.Context, caller Caller, sessionID string, content string, emit agent.Emit, opts agent.RunOptions) (*agent.Reply, error) { +func (s *Service) runWithKernel(ctx context.Context, caller Caller, sessionID string, content string, emit chatprotocol.Emit, opts chatprotocol.RunOptions) (*chatprotocol.Reply, error) { content = strings.TrimSpace(content) if content == "" && len(opts.Attachments) == 0 { return nil, fmt.Errorf("%w: content required", errs.ErrInvalid) @@ -489,7 +490,7 @@ func (s *Service) runWithKernel(ctx context.Context, caller Caller, sessionID st return s.legacyAgent.RunStreamWithOpts(ctx, sessionID, sess.UserID, content, emit, opts) } -func (s *Service) validateImageContext(ctx context.Context, sessionID string, opts *agent.RunOptions) error { +func (s *Service) validateImageContext(ctx context.Context, sessionID string, opts *chatprotocol.RunOptions) error { var total int64 for _, attachment := range opts.Attachments { total += attachment.Size @@ -514,7 +515,7 @@ func (s *Service) validateImageContext(ctx context.Context, sessionID string, op return nil } -func resolveSessionRunOptions(sess *model.Session, opts agent.RunOptions) agent.RunOptions { +func resolveSessionRunOptions(sess *model.Session, opts chatprotocol.RunOptions) chatprotocol.RunOptions { if sess == nil { return opts } @@ -579,11 +580,11 @@ func sameCancel(a, b context.CancelFunc) bool { } // runGraph dispatches the request through chatruntime.Runtime. The -// agent.Reply translation back happens here so the HTTP layer's DTO +// chatprotocol.Reply translation back happens here so the HTTP layer's DTO // stays kernel-agnostic. SSE frames are translated through a closure -// that maps chatruntime events back to agent.Event so the existing +// that maps chatruntime events back to chatprotocol.Event so the existing // http.go writeSSE path reuses unchanged. -func (s *Service) runGraph(ctx context.Context, sess *model.Session, content string, emit agent.Emit, opts agent.RunOptions) (*agent.Reply, error) { +func (s *Service) runGraph(ctx context.Context, sess *model.Session, content string, emit chatprotocol.Emit, opts chatprotocol.RunOptions) (*chatprotocol.Reply, error) { var graphEmit chatruntime.Emit if emit != nil { graphEmit = func(ev chatruntime.Event) { @@ -620,12 +621,12 @@ func (s *Service) runGraph(ctx context.Context, sess *model.Session, content str } // translateRuntimeEvent maps a chatruntime.Event into the legacy -// agent.Event shape so the SSE handler can keep its existing +// chatprotocol.Event shape so the SSE handler can keep its existing // switch-on-Type code path untouched. Frame names stay byte-equal. -func translateRuntimeEvent(ev chatruntime.Event) agent.Event { - out := agent.Event{Type: agent.EventType(ev.Type)} +func translateRuntimeEvent(ev chatruntime.Event) chatprotocol.Event { + out := chatprotocol.Event{Type: chatprotocol.EventType(ev.Type)} if ev.Assistant != nil { - out.Assistant = &agent.AssistantEvent{ + out.Assistant = &chatprotocol.AssistantEvent{ Iteration: ev.Assistant.Iteration, MessageID: ev.Assistant.MessageID, Content: ev.Assistant.Content, @@ -634,7 +635,7 @@ func translateRuntimeEvent(ev chatruntime.Event) agent.Event { } } if ev.Tool != nil { - out.Tool = &agent.ToolEvent{ + out.Tool = &chatprotocol.ToolEvent{ ToolCallID: ev.Tool.ToolCallID, Name: ev.Tool.Name, DeviceID: ev.Tool.DeviceID, @@ -651,7 +652,7 @@ func translateRuntimeEvent(ev chatruntime.Event) agent.Event { out.Done = runtimeReplyToAgentReply(ev.Done) } if ev.Notification != nil { - out.Notification = &agent.TaskNotificationEvent{ + out.Notification = &chatprotocol.TaskNotificationEvent{ TaskID: ev.Notification.TaskID, Status: string(ev.Notification.Status), Summary: ev.Notification.Summary, @@ -661,7 +662,7 @@ func translateRuntimeEvent(ev chatruntime.Event) agent.Event { } } if ev.Approval != nil { - out.Approval = &agent.ApprovalPendingEvent{ + out.Approval = &chatprotocol.ApprovalPendingEvent{ ApprovalID: ev.Approval.ApprovalID, ToolCallID: ev.Approval.ToolCallID, Kind: ev.Approval.Kind, @@ -673,9 +674,9 @@ func translateRuntimeEvent(ev chatruntime.Event) agent.Event { return out } -// translateMentionsToRuntime copies the legacy agent.Mention shape +// translateMentionsToRuntime copies the legacy chatprotocol.Mention shape // into the chatruntime.Mention shape. One alloc per turn — fine. -func translateMentionsToRuntime(in []agent.Mention) []chatruntime.Mention { +func translateMentionsToRuntime(in []chatprotocol.Mention) []chatruntime.Mention { if len(in) == 0 { return nil } @@ -687,14 +688,14 @@ func translateMentionsToRuntime(in []agent.Mention) []chatruntime.Mention { } // runtimeReplyToAgentReply translates the graph kernel's Reply back -// into the legacy agent.Reply shape so the HTTP handler's DTO +// into the legacy chatprotocol.Reply shape so the HTTP handler's DTO // transformer (toPostMessageResp) doesn't care which kernel produced // the answer. -func runtimeReplyToAgentReply(r *chatruntime.Reply) *agent.Reply { +func runtimeReplyToAgentReply(r *chatruntime.Reply) *chatprotocol.Reply { if r == nil { return nil } - return &agent.Reply{ + return &chatprotocol.Reply{ Message: r.Message, Usage: r.Usage, Iterations: r.Iterations, diff --git a/internal/manager/service/aiops/service_kernel_test.go b/internal/manager/service/aiops/service_kernel_test.go index d4a3fe713..00b987d20 100644 --- a/internal/manager/service/aiops/service_kernel_test.go +++ b/internal/manager/service/aiops/service_kernel_test.go @@ -9,7 +9,7 @@ import ( "time" biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" model "github.com/ongridio/ongrid/internal/manager/model/aiops" "github.com/ongridio/ongrid/internal/pkg/errs" @@ -28,7 +28,7 @@ func (f *fakeRuntime) Handle(_ context.Context, req *chatruntime.Request) (*chat f.calls.Add(1) // Drive the SSE adapter through one full sequence so the frame // translation is exercised end-to-end. The call site asserts on - // the resulting agent.Event slice. + // the resulting chatprotocol.Event slice. if req.Emit != nil { now := time.Now().UTC() end := now.Add(20 * time.Millisecond) @@ -188,9 +188,9 @@ func TestService_KernelGraph_DispatchesToRuntime(t *testing.T) { var ( evtMu sync.Mutex - evts []agent.Event + evts []chatprotocol.Event ) - emit := func(ev agent.Event) { + emit := func(ev chatprotocol.Event) { evtMu.Lock() defer evtMu.Unlock() evts = append(evts, ev) @@ -212,10 +212,10 @@ func TestService_KernelGraph_DispatchesToRuntime(t *testing.T) { evtMu.Lock() defer evtMu.Unlock() - wantSequence := []agent.EventType{ - agent.EventAssistant, - agent.EventToolStart, - agent.EventToolEnd, + wantSequence := []chatprotocol.EventType{ + chatprotocol.EventAssistant, + chatprotocol.EventToolStart, + chatprotocol.EventToolEnd, } if len(evts) < len(wantSequence) { t.Fatalf("got %d events, want at least %d (got %#v)", len(evts), len(wantSequence), evts) diff --git a/internal/manager/service/aiops/session_model_test.go b/internal/manager/service/aiops/session_model_test.go index f9358bcac..d5800793d 100644 --- a/internal/manager/service/aiops/session_model_test.go +++ b/internal/manager/service/aiops/session_model_test.go @@ -3,7 +3,7 @@ package aiops import ( "testing" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" model "github.com/ongridio/ongrid/internal/manager/model/aiops" ) @@ -11,17 +11,17 @@ func TestResolveSessionRunOptions(t *testing.T) { provider, selectedModel := "deepseek", "deepseek-chat" sess := &model.Session{Provider: &provider, Model: &selectedModel} - got := resolveSessionRunOptions(sess, agent.RunOptions{}) + got := resolveSessionRunOptions(sess, chatprotocol.RunOptions{}) if got.Provider != provider || got.Model != selectedModel { t.Fatalf("session fallback = %q/%q", got.Provider, got.Model) } - got = resolveSessionRunOptions(sess, agent.RunOptions{Provider: "custom", Model: "qwen"}) + got = resolveSessionRunOptions(sess, chatprotocol.RunOptions{Provider: "custom", Model: "qwen"}) if got.Provider != "custom" || got.Model != "qwen" { t.Fatalf("request override = %q/%q", got.Provider, got.Model) } - got = resolveSessionRunOptions(&model.Session{}, agent.RunOptions{}) + got = resolveSessionRunOptions(&model.Session{}, chatprotocol.RunOptions{}) if got.Provider != "" || got.Model != "" { t.Fatalf("legacy session should fall through: %+v", got) } diff --git a/internal/manager/service/aiops/user_agent.go b/internal/manager/service/aiops/user_agent.go index d4f5c5f99..d01c109b3 100644 --- a/internal/manager/service/aiops/user_agent.go +++ b/internal/manager/service/aiops/user_agent.go @@ -1,4 +1,4 @@ -// Package aiops's user_agent.go is the Phase-3 user-defined persona +// Package aiops's user_chatprotocol.go is the Phase-3 user-defined persona // service. It persists each user-created agent to user_agents table // AND mirrors it into the live chatruntime.AgentRegistry so creates / // updates / deletes take effect immediately (no manager restart). @@ -137,7 +137,7 @@ func (s *UserAgentService) Create(ctx context.Context, in CreateUserAgentInput) return nil, fmt.Errorf("%w: name %q already used by a built-in persona", errs.ErrInvalid, name) } } - // Collision with existing user agent. + // Collision with existing user chatprotocol. if existing, _ := s.repo.GetByName(ctx, name); existing != nil { return nil, fmt.Errorf("%w: name %q already taken", errs.ErrInvalid, name) } @@ -240,7 +240,7 @@ func (s *UserAgentService) Update(ctx context.Context, caller Caller, name strin return updated, nil } -// Delete removes a user agent. Returns ErrNotFound when no user-defined +// Delete removes a user chatprotocol. Returns ErrNotFound when no user-defined // row matches. Disk-loaded personas can't be deleted via this path — // they live as files and survive a restart. // From fe15dbdc73440eb9165b69312ab60a10699967df Mon Sep 17 00:00:00 2001 From: singchia Date: Wed, 14 Oct 2026 19:20:41 +0800 Subject: [PATCH 02/24] fix(aiops): unify safe skill and host-file tool catalogs (#498) --- .../biz/aiops/tools/basetool/web_search.go | 35 +++++ .../biz/aiops/tools/host_files_register.go | 6 + internal/manager/biz/aiops/tools/registry.go | 15 ++- .../biz/aiops/tools/registry_basetool.go | 6 + .../manager/biz/aiops/tools/skill_basetool.go | 74 ++++++++++ .../biz/aiops/tools/skill_basetool_test.go | 126 ++++++++++++++++++ internal/manager/biz/aiops/tools/toolbag.go | 42 ++++++ .../manager/biz/aiops/tools/toolbag_test.go | 19 +++ 8 files changed, 316 insertions(+), 7 deletions(-) create mode 100644 internal/manager/biz/aiops/tools/basetool/web_search.go create mode 100644 internal/manager/biz/aiops/tools/skill_basetool.go create mode 100644 internal/manager/biz/aiops/tools/skill_basetool_test.go diff --git a/internal/manager/biz/aiops/tools/basetool/web_search.go b/internal/manager/biz/aiops/tools/basetool/web_search.go new file mode 100644 index 000000000..777af59e6 --- /dev/null +++ b/internal/manager/biz/aiops/tools/basetool/web_search.go @@ -0,0 +1,35 @@ +package basetool + +import "context" + +// ToolWebSearch 是联网搜索的稳定工具名。 +const ToolWebSearch = "web_search" + +type webSearchKey struct{} + +// WithWebSearchAllowed 设置本轮联网权限，子任务通过 context 继承。 +func WithWebSearchAllowed(ctx context.Context, allowed bool) context.Context { + return context.WithValue(ctx, webSearchKey{}, allowed) +} + +// WebSearchAllowed 未明确授权时默认关闭。 +func WebSearchAllowed(ctx context.Context) bool { + allowed, _ := ctx.Value(webSearchKey{}).(bool) + return allowed +} + +// FilterWebSearch 在模型暴露入口剔除本轮禁止的联网搜索。 +func FilterWebSearch(ctx context.Context, tools []BaseTool) []BaseTool { + if WebSearchAllowed(ctx) { + return tools + } + out := make([]BaseTool, 0, len(tools)) + for _, t := range tools { + info, err := t.Info(ctx) + if err == nil && info != nil && info.Name == ToolWebSearch { + continue + } + out = append(out, t) + } + return out +} diff --git a/internal/manager/biz/aiops/tools/host_files_register.go b/internal/manager/biz/aiops/tools/host_files_register.go index b9ad46a9c..bba2c367b 100644 --- a/internal/manager/biz/aiops/tools/host_files_register.go +++ b/internal/manager/biz/aiops/tools/host_files_register.go @@ -1,6 +1,7 @@ package tools import ( + "context" "log/slog" devicebiz "github.com/ongridio/ongrid/internal/manager/biz/device" @@ -35,6 +36,11 @@ func AppendHostFilesTools(bag *ToolBag, c Caller, e *edgebiz.Usecase, d *deviceb if log == nil { log = slog.Default() } + for _, tool := range bag.AllTools() { + if info, err := tool.Info(context.Background()); err == nil && info != nil && info.Name == ToolNameFindLargeFiles { + return bag + } + } bag.Append(NewFindLargeFilesTool(c, e, d, log)) bag.Append(NewDuSummaryTool(c, e, d, log)) bag.Append(NewStatFileTool(c, e, d, log)) diff --git a/internal/manager/biz/aiops/tools/registry.go b/internal/manager/biz/aiops/tools/registry.go index 15aecc570..602bb5e53 100644 --- a/internal/manager/biz/aiops/tools/registry.go +++ b/internal/manager/biz/aiops/tools/registry.go @@ -70,13 +70,14 @@ type Tool struct { // without that signal). alertUC is a narrow interface (AlertUsecase) so // tests can inject a fake without standing up the full alert biz repo. type Registry struct { - caller Caller - edges *edgebiz.Usecase - devices *devicebiz.Usecase - alertUC AlertUsecase - promQuery PromQuerier - logQuery LogQuerier - traceQuery TraceQuerier + skillRunner SkillRunner + caller Caller + edges *edgebiz.Usecase + devices *devicebiz.Usecase + alertUC AlertUsecase + promQuery PromQuerier + logQuery LogQuerier + traceQuery TraceQuerier // knowledge is the user-curated knowledge base + git repo searcher. // nil-safe — when nil the query_knowledge tool isn't registered. knowledge KnowledgeSearcher diff --git a/internal/manager/biz/aiops/tools/registry_basetool.go b/internal/manager/biz/aiops/tools/registry_basetool.go index 8a5034e0a..f5290cbae 100644 --- a/internal/manager/biz/aiops/tools/registry_basetool.go +++ b/internal/manager/biz/aiops/tools/registry_basetool.go @@ -246,6 +246,12 @@ func (r *Registry) BuildBaseTools() *ToolBag { out = append(out, NewServePageTool(r.pageStore, r.log)) } + // 所有消费方共用完整静态目录，避免工作流画布漏掉 host-files。 + if r.caller != nil && r.edges != nil && r.devices != nil { + out = append(out, NewFindLargeFilesTool(r.caller, r.edges, r.devices, r.log), NewDuSummaryTool(r.caller, r.edges, r.devices, r.log), NewStatFileTool(r.caller, r.edges, r.devices, r.log)) + } + out = r.appendSafeSkillTools(out) + threshold := envIntDefault("ONGRID_TOOLBAG_DEFERRAL_THRESHOLD", defaultDeferralThreshold) bag := NewToolBag(out, threshold) diff --git a/internal/manager/biz/aiops/tools/skill_basetool.go b/internal/manager/biz/aiops/tools/skill_basetool.go new file mode 100644 index 000000000..12c9f4da7 --- /dev/null +++ b/internal/manager/biz/aiops/tools/skill_basetool.go @@ -0,0 +1,74 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + skillcore "github.com/ongridio/ongrid/internal/skill" +) + +// SetSkillRunner 在构建目录前注入 Skill 服务，所有入口使用相同适配器。 +func (r *Registry) SetSkillRunner(runner SkillRunner) { r.skillRunner = runner } + +func (r *Registry) appendSafeSkillTools(out []basetool.BaseTool) []basetool.BaseTool { + if r.skillRunner == nil { + return out + } + seen := make(map[string]bool, len(out)) + for _, t := range out { + if info, err := t.Info(context.Background()); err == nil && info != nil { + seen[info.Name] = true + } + } + for _, executor := range skillcore.AllByClass(skillcore.ClassSafe) { + // 展示目录的反向桥接不是新的工具来源，否则会递归适配。 + if _, exported := executor.(*baseToolSkillExecutor); exported { + continue + } + meta := executor.Metadata() + // 内置 BaseTool 是同名能力的执行实现（如抓包查询），只保留一份。 + if seen[meta.Key] { + if _, external := executor.(*skillcore.SubprocessSkill); external { + r.log.Error("external skill conflicts with built-in tool; registration rejected", "skill", meta.Key) + } + continue + } + schema, err := buildSkillToolSchema(executor) + if err != nil { + r.log.Error("build skill tool schema", "skill", meta.Key, "error", err) + continue + } + origin := basetool.OriginBuiltin + if _, external := executor.(*skillcore.SubprocessSkill); external { + origin = basetool.OriginSkill + } + out = append(out, &skillBaseTool{ + info: basetool.ToolInfo{Name: meta.Key, Description: meta.Description, Parameters: schema, Class: "read", Origin: origin}, + execute: r.newSkillExecutor(r.skillRunner, meta.Key, meta.EffectiveScope()), + }) + seen[meta.Key] = true + } + return out +} + +type skillBaseTool struct { + info basetool.ToolInfo + execute func(context.Context, json.RawMessage) (ExecuteResult, error) +} + +func (t *skillBaseTool) Info(context.Context) (*basetool.ToolInfo, error) { + info := t.info + return &info, nil +} +func (t *skillBaseTool) InvokableRun(ctx context.Context, args string, _ ...basetool.InvokeOption) (string, error) { + if t.info.Name == basetool.ToolWebSearch && !basetool.WebSearchAllowed(ctx) { + return "", fmt.Errorf("web_search disabled for this request") + } + result, err := t.execute(ctx, json.RawMessage(args)) + if err != nil { + return "", err + } + return string(result.ResultJSON), nil +} diff --git a/internal/manager/biz/aiops/tools/skill_basetool_test.go b/internal/manager/biz/aiops/tools/skill_basetool_test.go new file mode 100644 index 000000000..2bf2f82c0 --- /dev/null +++ b/internal/manager/biz/aiops/tools/skill_basetool_test.go @@ -0,0 +1,126 @@ +package tools + +import ( + "context" + "encoding/json" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + skillsvc "github.com/ongridio/ongrid/internal/manager/biz/skill" + skillcore "github.com/ongridio/ongrid/internal/skill" + _ "github.com/ongridio/ongrid/internal/skill/builtin" + "log/slog" + "os" + "path/filepath" + "testing" +) + +func TestGraphCatalogIncludesAndExecutesSafeSkills(t *testing.T) { + runner := &fakeSkillRunner{result: json.RawMessage(`{"ok":true}`)} + r := NewRegistry(nil, nil, nil, nil, nil, nil, nil, slog.Default()) + r.SetSkillRunner(runner) + bag := r.BuildBaseTools() + names := map[string]basetool.BaseTool{} + for _, tool := range bag.AllTools() { + info, err := tool.Info(context.Background()) + if err != nil { + t.Fatal(err) + } + if names[info.Name] != nil { + t.Fatalf("duplicate tool %s", info.Name) + } + names[info.Name] = tool + } + for _, key := range []string{"web_search", "host_probe_dns", "host_probe_tcp", "host_probe_http", "host_probe_ping", "host_netns_inspect", "host_read_journal", "host_tail_file"} { + if names[key] == nil { + t.Errorf("missing executable %s", key) + } + } + tool := names["web_search"] + if tool == nil { + t.Fatal("missing search") + } + if _, err := tool.InvokableRun(context.Background(), `{"query":"test"}`); err == nil { + t.Fatal("search must fail closed") + } + ctx := basetool.WithWebSearchAllowed(context.Background(), true) + if _, err := tool.InvokableRun(ctx, `{"query":"test","device_id":42}`); err != nil { + t.Fatal(err) + } + if runner.gotIn.Key != "web_search" || runner.gotIn.EdgeID != 0 || string(runner.gotIn.Params) != `{"query":"test"}` { + t.Fatalf("wrong dispatch: %+v", runner.gotIn) + } + if names["capture_pcap"] != nil { + t.Fatal("mutating skill exposed as safe") + } +} + +func TestGraphCatalogDoesNotReimportInventoryExports(t *testing.T) { + const key = "test_catalog_export" + skillcore.Register(&baseToolSkillExecutor{meta: skillcore.Metadata{Key: key, Name: key, Description: "inventory", Class: skillcore.ClassSafe, Scope: skillcore.ScopeManager}}) + t.Cleanup(func() { unregisterSkillForTest(key) }) + r := NewRegistry(nil, nil, nil, nil, nil, nil, nil, slog.Default()) + r.SetSkillRunner(&fakeSkillRunner{}) + for _, tool := range r.BuildBaseTools().AllTools() { + info, err := tool.Info(context.Background()) + if err != nil { + t.Fatal(err) + } + if info.Name == key { + t.Fatal("inventory export imported as executable skill") + } + } +} + +func TestGraphCatalogAdaptsExternalSubprocessSkill(t *testing.T) { + const key = "test_catalog_subprocess" + // 使用真实 manifest loader 校验加载到调用之间的链路。 + dir := t.TempDir() + writeCatalogFixture(t, dir) + n, err := skillcore.LoadDirs(skillcore.LoaderConfig{Dirs: []string{dir}}) + if err != nil || n != 1 { + t.Fatalf("load: %d %v", n, err) + } + t.Cleanup(func() { unregisterSkillForTest(key) }) + runner := skillsvc.New(nil, nil, slog.Default()) + r := NewRegistry(nil, nil, nil, nil, nil, nil, nil, slog.Default()) + r.SetSkillRunner(runner) + for _, tool := range r.BuildBaseTools().AllTools() { + info, err := tool.Info(context.Background()) + if err != nil { + t.Fatal(err) + } + if info.Name != key { + continue + } + if info.Origin != basetool.OriginSkill { + t.Fatal("missing dynamic origin") + } + result, err := tool.InvokableRun(context.Background(), `{"query":"external"}`) + if err != nil { + t.Fatal(err) + } + var envelope struct { + Result struct { + OK bool `json:"ok"` + } `json:"result"` + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(result), &envelope); err != nil { + t.Fatal(err) + } + if !envelope.Result.OK || envelope.Error != "" { + t.Fatalf("subprocess execution failed: %s", result) + } + return + } + t.Fatal("external skill missing") +} + +func writeCatalogFixture(t *testing.T, dir string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, "run.sh"), []byte("#!/bin/sh\nprintf '{\"ok\":true}'\n"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "skill.json"), []byte(`{"name":"test_catalog_subprocess","description":"test external tool","entry":"run.sh","class":"safe","schema":{"type":"object","properties":{"query":{"type":"string"}}}}`), 0600); err != nil { + t.Fatal(err) + } +} diff --git a/internal/manager/biz/aiops/tools/toolbag.go b/internal/manager/biz/aiops/tools/toolbag.go index 7677346e1..d61a8d7d2 100644 --- a/internal/manager/biz/aiops/tools/toolbag.go +++ b/internal/manager/biz/aiops/tools/toolbag.go @@ -417,3 +417,45 @@ func (r redactedTool) InvokableRun(ctx context.Context, argsJSON string, opts .. } return r.inner.InvokableRun(ctx, argsJSON, opts...) } + +// UpsertTools 同步后注册工具的完整目录，同名项替换，不生成重复条目。 +func (b *ToolBag) UpsertTools(tools []basetool.BaseTool) { + if b == nil { + return + } + updates := map[string]basetool.BaseTool{} + order := []string{} + for _, tool := range tools { + if tool == nil { + continue + } + info, err := tool.Info(context.Background()) + if err != nil || info == nil || info.Name == "" { + continue + } + if _, ok := updates[info.Name]; !ok { + order = append(order, info.Name) + } + updates[info.Name] = tool + } + b.mu.Lock() + defer b.mu.Unlock() + all := append(append([]basetool.BaseTool{}, b.core...), b.deferred...) + for i, t := range all { + info, err := t.Info(context.Background()) + if err != nil || info == nil { + continue + } + if replacement, ok := updates[info.Name]; ok { + all[i] = replacement + delete(updates, info.Name) + } + } + for _, name := range order { + if t, ok := updates[name]; ok { + all = append(all, t) + } + } + updated := NewToolBag(all, b.threshold) + b.core, b.deferred, b.deferring = updated.core, updated.deferred, updated.deferring +} diff --git a/internal/manager/biz/aiops/tools/toolbag_test.go b/internal/manager/biz/aiops/tools/toolbag_test.go index c47086cfa..811976e94 100644 --- a/internal/manager/biz/aiops/tools/toolbag_test.go +++ b/internal/manager/biz/aiops/tools/toolbag_test.go @@ -380,3 +380,22 @@ func countNameString(values []string, want string) int { } return count } + +func TestToolBagUpsertReplacesWithoutDuplicatesAndSearchUsesNewSchema(t *testing.T) { + old := newStub("host_test", "old") + bag := NewToolBag([]basetool.BaseTool{old}, 0) + replacement := newStub("host_test", "new") + bag.UpsertTools([]basetool.BaseTool{replacement, newStub("host_other", "other")}) + bag.UpsertTools([]basetool.BaseTool{replacement}) + all := bag.AllTools() + if len(all) != 2 { + t.Fatalf("count=%d want=2", len(all)) + } + result, err := NewToolSearchTool(bag, nil).InvokableRun(context.Background(), `{"query":"select:host_test"}`) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(result, `"description":"new"`) { + t.Fatalf("stale search result: %s", result) + } +} From 40665b4141238eb92c457df7d6a68e0fc5ec5af8 Mon Sep 17 00:00:00 2001 From: singchia Date: Wed, 14 Oct 2026 19:20:51 +0800 Subject: [PATCH 03/24] fix(aiops): inherit web permissions and revoke stale tools (#499) --- .../manager/biz/aiops/chatruntime/runtime.go | 75 ++++++---- .../biz/aiops/chatruntime/tool_lifecycle.go | 36 +++++ .../aiops/chatruntime/tool_lifecycle_test.go | 134 ++++++++++++++++++ .../manager/biz/aiops/chatruntime/worker.go | 13 +- 4 files changed, 224 insertions(+), 34 deletions(-) create mode 100644 internal/manager/biz/aiops/chatruntime/tool_lifecycle.go create mode 100644 internal/manager/biz/aiops/chatruntime/tool_lifecycle_test.go diff --git a/internal/manager/biz/aiops/chatruntime/runtime.go b/internal/manager/biz/aiops/chatruntime/runtime.go index 535d73c15..4f4518aa1 100644 --- a/internal/manager/biz/aiops/chatruntime/runtime.go +++ b/internal/manager/biz/aiops/chatruntime/runtime.go @@ -268,6 +268,7 @@ type ToolBagProvider interface { // keeps ToolSearch aligned with runtime-discovered MCP tools. type mutableToolBagProvider interface { ToolBagProvider + UpsertTools([]basetool.BaseTool) ReplaceToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) } @@ -302,7 +303,9 @@ type Runtime struct { // when the caller wires the runtime without calling SetToolBag — // not all code paths need it (legacy callers only feed // cfg.ToolBag through the graph). - bag ToolBagProvider + bag ToolBagProvider + toolVersions map[string]uint64 + nextToolVersion uint64 } // NewRuntime builds a Runtime. Returns an error when required deps @@ -321,7 +324,11 @@ func NewRuntime(cfg Config) (*Runtime, error) { if log == nil { log = slog.Default() } - return &Runtime{cfg: cfg, log: log}, nil + initial := cfg.ToolBag + cfg.ToolBag = nil + rt := &Runtime{cfg: cfg, log: log, toolVersions: map[string]uint64{}} + rt.AppendToolBag(initial) + return rt, nil } // SetMentionResolver wires the @-mention hydrator post-construction. @@ -373,23 +380,20 @@ func (rt *Runtime) SetToolBag(bag ToolBagProvider) { rt.bag = bag } -// AppendToolBag appends extra BaseTools to the coordinator tool bag. -// — used by main.go to bolt the coordinator-only AgentTool -// / SendMessageTool / TaskStopTool onto the Runtime AFTER construction -// (the chicken-and-egg: those tools need the Runtime as their spawner, -// the Runtime needs the tool bag at NewRuntime time). Workers do NOT -// observe these added tools — filterToolsForAgent strips them -// unconditionally via coordinatorOnlyTools (see worker.go). -// -// This is intentionally distinct from a generic "AddTool" method: it -// signals "these are coordinator-only" so a future reader of main.go -// understands why three more tools appear after NewRuntime returned. +// AppendToolBag 按名称注册或替换后初始化工具，并同步搜索目录。 +// Coordinator 专属工具仍由 worker 的角色过滤器排除。 func (rt *Runtime) AppendToolBag(tools []basetool.BaseTool) { if rt == nil || len(tools) == 0 { return } rt.toolMu.Lock() defer rt.toolMu.Unlock() + if rt.toolVersions == nil { + rt.toolVersions = map[string]uint64{} + } + if bag, ok := rt.bag.(mutableToolBagProvider); ok { + bag.UpsertTools(tools) + } index := map[string]int{} for i, t := range rt.cfg.ToolBag { if t == nil { @@ -405,6 +409,7 @@ func (rt *Runtime) AppendToolBag(tools []basetool.BaseTool) { } info, err := t.Info(context.Background()) if err == nil && info != nil { + t = rt.versionedToolLocked(info.Name, t) if i, ok := index[info.Name]; ok { rt.cfg.ToolBag[i] = t continue @@ -418,32 +423,45 @@ func (rt *Runtime) AppendToolBag(tools []basetool.BaseTool) { // ReplaceToolsByNamePrefix replaces tools in the graph-facing bag whose wire // names share prefix. MCP uses one prefix per server, preventing a refresh of // one registration from removing tools offered by other servers. +// ReplaceToolsByNamePrefix 是同 schema 工具的便捷更新入口。 func (rt *Runtime) ReplaceToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) { + rt.ReplaceToolSource(prefix, replacements, replacements) +} + +// ReplaceToolSource 在同一更新中替换执行视图与完整 schema 目录。 +func (rt *Runtime) ReplaceToolSource(prefix string, raw, decorated []basetool.BaseTool) { if rt == nil || strings.TrimSpace(prefix) == "" { return } rt.toolMu.Lock() defer rt.toolMu.Unlock() - next := make([]basetool.BaseTool, 0, len(rt.cfg.ToolBag)+len(replacements)) - for _, tool := range rt.cfg.ToolBag { - if toolNameHasPrefix(tool, prefix) { + if rt.toolVersions == nil { + rt.toolVersions = map[string]uint64{} + } + next := make([]basetool.BaseTool, 0, len(rt.cfg.ToolBag)+len(decorated)) + for _, t := range rt.cfg.ToolBag { + if toolNameHasPrefix(t, prefix) { + if info, err := t.Info(context.Background()); err == nil && info != nil { + delete(rt.toolVersions, info.Name) + } continue } - next = append(next, tool) + next = append(next, t) } - rt.cfg.ToolBag = append(next, replacements...) -} - -// ReplaceCatalogToolsByNamePrefix updates the unredacted catalog used by -// ToolSearch. It is separate from ReplaceToolsByNamePrefix because the graph -// sees decorated tools while ToolSearch must retain raw schemas. -func (rt *Runtime) ReplaceCatalogToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) { - if rt == nil || strings.TrimSpace(prefix) == "" { - return + for _, t := range decorated { + if t == nil { + continue + } + info, err := t.Info(context.Background()) + if err != nil || info == nil || !strings.HasPrefix(info.Name, prefix) { + continue + } + next = append(next, rt.versionedToolLocked(info.Name, t)) } if bag, ok := rt.bag.(mutableToolBagProvider); ok { - bag.ReplaceToolsByNamePrefix(prefix, replacements) + bag.ReplaceToolsByNamePrefix(prefix, raw) } + rt.cfg.ToolBag = next } func (rt *Runtime) toolBagSnapshot() []basetool.BaseTool { @@ -534,6 +552,7 @@ func (rt *Runtime) Handle(ctx context.Context, req *Request) (*Reply, error) { // (deep inside the graph) can fire a task_notification SSE frame // when a background worker terminates. / ctx = withEmit(ctx, req.Emit) + ctx = basetool.WithWebSearchAllowed(ctx, req.WebSearchEnabled) // 1. Ownership check. sess, err := rt.cfg.Sessions.GetSession(ctx, req.SessionID) @@ -624,7 +643,7 @@ func (rt *Runtime) Handle(ctx context.Context, req *Request) (*Reply, error) { // the coordinator path now honors it too. Stale agent ids fall back // to the global default with a log line — never crash a session. basePrompt := rt.cfg.BasePrompt - runtimeToolBag := rt.toolBagSnapshot() + runtimeToolBag := basetool.FilterWebSearch(ctx, rt.toolBagSnapshot()) sessionToolBag := runtimeToolBag agentReminderForPersona := "" // Coordinator iff session is pinned to "default" or has no diff --git a/internal/manager/biz/aiops/chatruntime/tool_lifecycle.go b/internal/manager/biz/aiops/chatruntime/tool_lifecycle.go new file mode 100644 index 000000000..839baec3a --- /dev/null +++ b/internal/manager/biz/aiops/chatruntime/tool_lifecycle.go @@ -0,0 +1,36 @@ +package chatruntime + +import ( + "context" + "fmt" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// runtimeTool 保留本轮 schema；撤销或替换后，旧引用不得开始新的调用。 +// 已经进入执行的调用继续沿用原超时/取消机制，不在持锁期间执行外部 IO。 +type runtimeTool struct { + inner basetool.BaseTool + runtime *Runtime + name string + version uint64 +} + +func (rt *Runtime) versionedToolLocked(name string, inner basetool.BaseTool) basetool.BaseTool { + rt.nextToolVersion++ + rt.toolVersions[name] = rt.nextToolVersion + return &runtimeTool{inner: inner, runtime: rt, name: name, version: rt.nextToolVersion} +} +func (t *runtimeTool) Info(ctx context.Context) (*basetool.ToolInfo, error) { return t.inner.Info(ctx) } +func (t *runtimeTool) InvokableRun(ctx context.Context, args string, opts ...basetool.InvokeOption) (string, error) { + t.runtime.toolMu.RLock() + current := t.runtime.toolVersions[t.name] + t.runtime.toolMu.RUnlock() + if current != t.version { + return "", fmt.Errorf("tool %q was removed or replaced; start a new turn", t.name) + } + if t.name == basetool.ToolWebSearch && !basetool.WebSearchAllowed(ctx) { + return "", fmt.Errorf("web_search disabled for this request") + } + return t.inner.InvokableRun(ctx, args, opts...) +} diff --git a/internal/manager/biz/aiops/chatruntime/tool_lifecycle_test.go b/internal/manager/biz/aiops/chatruntime/tool_lifecycle_test.go new file mode 100644 index 000000000..37935684b --- /dev/null +++ b/internal/manager/biz/aiops/chatruntime/tool_lifecycle_test.go @@ -0,0 +1,134 @@ +package chatruntime + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cloudwego/eino/schema" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" +) + +func TestRuntimeRemovedOrReplacedToolRejectsStaleReference(t *testing.T) { + rt, err := NewRuntime(Config{Sessions: newMemSessions(nil), ChatModel: newScriptedChatModel(), ToolBag: []basetool.BaseTool{&fakeTool{name: "mcp__test__search", schema: `{"type":"object"}`}}}) + if err != nil { + t.Fatal(err) + } + old := rt.toolBagSnapshot()[0] + rt.ReplaceToolSource("mcp__test__", nil, nil) + if _, err := old.InvokableRun(context.Background(), `{}`); err == nil { + t.Fatal("removed tool executed") + } + fresh := &fakeTool{name: "mcp__test__search", schema: `{"type":"object"}`} + rt.ReplaceToolSource("mcp__test__", []basetool.BaseTool{fresh}, []basetool.BaseTool{fresh}) + if _, err := old.InvokableRun(context.Background(), `{}`); err == nil { + t.Fatal("old generation became executable again") + } + if _, err := rt.toolBagSnapshot()[0].InvokableRun(context.Background(), `{}`); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeConcurrentToolRefreshAndSnapshot(t *testing.T) { + rt, err := NewRuntime(Config{Sessions: newMemSessions(nil), ChatModel: newScriptedChatModel()}) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + tool := &fakeTool{name: "mcp__test__search", schema: `{"type":"object"}`} + rt.ReplaceToolSource("mcp__test__", []basetool.BaseTool{tool}, []basetool.BaseTool{tool}) + for _, entry := range rt.toolBagSnapshot() { + // 并发替换允许明确拒绝旧引用，不能数据竞争或执行已撤销版本。 + _, err := entry.InvokableRun(context.Background(), `{}`) + if err != nil { + continue + } + } + } + }() + } + wg.Wait() +} + +type countingSearchTool struct{ calls atomic.Int32 } + +func (t *countingSearchTool) Info(context.Context) (*basetool.ToolInfo, error) { + return &basetool.ToolInfo{Name: basetool.ToolWebSearch, Description: "search", Class: "read", Parameters: []byte(`{"type":"object","properties":{"query":{"type":"string"}}}`)}, nil +} +func (t *countingSearchTool) InvokableRun(context.Context, string, ...basetool.InvokeOption) (string, error) { + t.calls.Add(1) + return `{"result":"found"}`, nil +} + +func TestRuntimeWebSearchToggleControlsActualDispatch(t *testing.T) { + for _, enabled := range []bool{false, true} { + t.Run(map[bool]string{false: "disabled", true: "enabled"}[enabled], func(t *testing.T) { + tool := &countingSearchTool{} + modelStub := newScriptedChatModel(&schema.Message{Role: schema.Assistant, ToolCalls: []schema.ToolCall{{ID: "tc-search", Type: "function", Function: schema.FunctionCall{Name: "web_search", Arguments: `{"query":"test"}`}}}}, &schema.Message{Role: schema.Assistant, Content: "done"}) + rt, err := NewRuntime(Config{Sessions: newMemSessions(&model.Session{ID: "s1", UserID: 7}), ChatModel: modelStub, ToolBag: []basetool.BaseTool{tool}}) + if err != nil { + t.Fatal(err) + } + if _, err := rt.Handle(context.Background(), &Request{SessionID: "s1", UserID: 7, Role: "admin", UserText: "search", WebSearchEnabled: enabled}); err != nil { + t.Fatal(err) + } + want := int32(0) + if enabled { + want = 1 + } + if tool.calls.Load() != want { + t.Fatalf("calls=%d want=%d", tool.calls.Load(), want) + } + // 即使持有旧 schema 或直接调用工具引用，也不能越过执行边界。 + if _, err := rt.toolBagSnapshot()[0].InvokableRun(context.Background(), `{}`); err == nil { + t.Fatal("missing execution gate") + } + }) + } +} + +func TestWorkerInheritsWebSearchGate(t *testing.T) { + for _, background := range []bool{false, true} { + for _, enabled := range []bool{false, true} { + t.Run(fmt.Sprintf("background=%t/enabled=%t", background, enabled), func(t *testing.T) { + tool := &countingSearchTool{} + scripted := newScriptedChatModel(&schema.Message{Role: schema.Assistant, ToolCalls: []schema.ToolCall{{ID: "worker-search", Type: "function", Function: schema.FunctionCall{Name: "web_search", Arguments: `{"query":"test"}`}}}}, &schema.Message{Role: schema.Assistant, Content: "done"}) + store, parent := newSeededRuntimeStore(t, 7) + rt := newRuntimeWithStore(t, "network", scripted, []basetool.BaseTool{tool}, store) + ctx := basetool.WithWebSearchAllowed(context.Background(), enabled) + done := make(chan struct{}, 1) + _, err := rt.SpawnWorker(ctx, SpawnRequest{AgentName: "network", ParentSession: parent, Prompt: "search", Background: background, ParentEmit: func(ev Event) { + if ev.Type == EventTaskNotification { + done <- struct{}{} + } + }}) + if err != nil { + t.Fatal(err) + } + if background { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("worker did not finish") + } + } + want := int32(0) + if enabled { + want = 1 + } + if tool.calls.Load() != want { + t.Fatalf("calls=%d want=%d", tool.calls.Load(), want) + } + }) + } + } +} diff --git a/internal/manager/biz/aiops/chatruntime/worker.go b/internal/manager/biz/aiops/chatruntime/worker.go index cfcb248f3..be2949c01 100644 --- a/internal/manager/biz/aiops/chatruntime/worker.go +++ b/internal/manager/biz/aiops/chatruntime/worker.go @@ -344,7 +344,7 @@ func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, // Sync spawn inherits the caller's ctx so a cancel propagates. parent := ctx if req.Background { - parent = context.Background() + parent = context.WithoutCancel(ctx) } workerCtx, cancel := context.WithCancel(parent) @@ -587,7 +587,7 @@ func (rt *Runtime) runWorker(ctx context.Context, agentDef *Agent, sessID, userT // Workers are not an authorization bypass. The coordinator stamps the // resolved gate on context; absent wiring fails closed so a direct spawn // cannot expose mutating tools unexpectedly. - workerTools := filterToolsForAgentRole(rt.toolBagSnapshot(), agentDef, false, !basetool.AgentWriteAllowedFromContext(ctx)) + workerTools := filterToolsForAgentRole(basetool.FilterWebSearch(ctx, rt.toolBagSnapshot()), agentDef, false, !basetool.AgentWriteAllowedFromContext(ctx)) // Thread the persona-filtered tool view onto ctx so ToolSearch // (which runs inside the worker graph) only returns tools the @@ -688,10 +688,11 @@ func (rt *Runtime) runWorker(ctx context.Context, agentDef *Agent, sessID, userT callbacks.FinalizeBatches(flushCtx, handlers) }() out, err := g.Invoke(workerCtx, &graph.Input{ - SystemPrompt: systemPrompt, - History: nil, - UserText: userText, - Locale: locale, + SystemPrompt: systemPrompt, + WebSearchEnabled: basetool.WebSearchAllowed(ctx), + History: nil, + UserText: userText, + Locale: locale, }, invokeOpts...) if err != nil { return "", err From 2cd3bdaa52a0025cbbe56d94921a898b37df035b Mon Sep 17 00:00:00 2001 From: singchia Date: Wed, 14 Oct 2026 19:20:51 +0800 Subject: [PATCH 04/24] refactor(aiops): retire legacy loop and unify runtime wiring (#499) --- Makefile | 4 + cmd/ongrid/main.go | 211 ++--- cmd/ongrid/main_kernel_test.go | 56 -- cmd/ongrid/tool_catalog_test.go | 38 + deploy/docker-compose.yml | 8 +- deploy/install/README.md | 6 + deploy/install/docker-compose.yml | 8 +- docs/rfc/RFC-005-unified-agent-runtime.md | 10 + internal/manager/biz/aiops/agent/agent.go | 878 ------------------ .../manager/biz/aiops/agent/agent_test.go | 502 ---------- .../biz/aiops/agent/build_messages_test.go | 149 --- .../biz/aiops/agent/legacy_k8s_action_test.go | 49 - .../manager/biz/aiops/chatprotocol/types.go | 8 +- internal/manager/biz/imbridge/sender.go | 2 +- internal/manager/service/aiops/service.go | 103 +- .../service/aiops/service_kernel_test.go | 77 +- .../aiops/service_mutating_proposal_test.go | 4 +- 17 files changed, 171 insertions(+), 1942 deletions(-) delete mode 100644 cmd/ongrid/main_kernel_test.go create mode 100644 cmd/ongrid/tool_catalog_test.go delete mode 100644 internal/manager/biz/aiops/agent/agent.go delete mode 100644 internal/manager/biz/aiops/agent/agent_test.go delete mode 100644 internal/manager/biz/aiops/agent/build_messages_test.go delete mode 100644 internal/manager/biz/aiops/agent/legacy_k8s_action_test.go diff --git a/Makefile b/Makefile index 394c401ff..aff43034c 100644 --- a/Makefile +++ b/Makefile @@ -912,3 +912,7 @@ test-web-ci-platform: ## Run the serial platform frontend CI tests (requires npm src/pages/settings/LLM.test.tsx src/pages/settings/CommunicationProviders.test.tsx \ src/components/icons/Provider.test.tsx src/pages/kubernetes/edgeAttachments.test.ts \ src/pages/kubernetes/registryCommands.test.ts + +.PHONY: test-agent-runtime +test-agent-runtime: ## 统一 Agent runtime 回归（tools 依赖 Linux cmdpolicy） + go test -race ./internal/manager/biz/aiops/... ./internal/manager/service/aiops ./internal/manager/server/aiops ./internal/manager/biz/imbridge ./internal/skill/... ./cmd/ongrid diff --git a/cmd/ongrid/main.go b/cmd/ongrid/main.go index da192da3e..19d887c54 100644 --- a/cmd/ongrid/main.go +++ b/cmd/ongrid/main.go @@ -100,7 +100,6 @@ import ( managermodeledge "github.com/ongridio/ongrid/internal/manager/model/edge" managerbizaiops "github.com/ongridio/ongrid/internal/manager/biz/aiops" - aiopsagent "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" aiopsgraph "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph" aiopsgraphcb "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph/callbacks" @@ -1470,33 +1469,12 @@ func main() { // find_topology_node show up in the BaseTool roster. nil-safe — the // two BaseTools are gated on this exact field. toolsReg.SetTopologyGraph(topologyUC) - aiopsAgent := aiopsagent.New( - llmClient, - toolsReg, - aiopsRepo, - aiopsagent.Config{Model: cfg.OpenAI.Model, Temperature: 0.1, MaxIterations: 30}, - log, - ) - aiopsAgent.SetAgentWriteEnabledProvider(func(ctx context.Context) bool { - return settingSvc.AgentWriteEnabled(ctx) - }) aiopsUsage := managerbizaiops.NewUsageUsecase(aiopsRepo, log) - // The graph-based AgentLoop is the default execution path. Operators can - // set ONGRID_AGENT_KERNEL=legacy for an explicit rollback while migration - // issues are investigated. - // When the env is set we build: - // - RoutingChatModel (PR-1) wrapping the existing llmRouter, one - // per provider id. - // - Decorated BaseTool slice via Registry.BuildBaseTools + - // AppendHostFilesTools, then Wrap'd with the standard chain. - // - SkillRegistry / AgentRegistry from ./skills + ./agents - // (silent on missing dirs — fresh installs boot fine). - // - chatruntime.Runtime, the cutover entry the service routes to. - // Mismatch / build errors fall back to legacy with a logged warning. - kernel := managersvcaiops.ParseKernel(os.Getenv("ONGRID_AGENT_KERNEL")) - log.Info("aiops agent kernel selected", slog.String("kernel", string(kernel))) - + // Graph 是唯一聊天运行时，旧配置保留一个版本的明确迁移提示。 + if old := strings.TrimSpace(os.Getenv("ONGRID_AGENT_KERNEL")); old != "" && old != "graph" { + log.Warn("ONGRID_AGENT_KERNEL is obsolete; using graph runtime", slog.String("configured", old)) + } // Knowledge base + git-repo integration (RAG Phase-1). Wire BEFORE // buildAIOpsRuntime so the BaseTool bag picks up query_knowledge — // SetKnowledgeSearcher only affects subsequent BuildBaseTools calls. @@ -1610,6 +1588,45 @@ func main() { } } + // L2 skill framework: builtin Executors registered via init() in + // internal/skill/builtin (imported above). Service dispatches via + // frontierbound.Client; audit goes to MySQL skill_executions. + skillSvc := managerbizskill.New( + fbClient, + managerbizskill.NewGormAuditSink(db), + log.With(slog.String("comp", "skill")), + ) + // Wire the multi-provider config resolver into the manager-scoped + // web_search built-in. Default provider is SearXNG (zero-config, + // docker-internal). The skill returns a skipped_reason envelope + // when the chosen provider is missing a key / unreachable, so this + // is safe to call even before any operator configures the integration. + skillbuiltin.SetWebSearchConfigResolver(managerbizsetting.NewWebSearchResolver(settingSvc)) + + // Subprocess skill loader: walks each allowlist root and registers + // SubprocessSkills for every skill.json found. Empty dir list = + // nothing loaded; missing dirs are logged and skipped so a fresh + // install with no /etc/ongrid/skills boots cleanly. + if loaded, err := skillcore.LoadDirs(skillcore.LoaderConfig{ + Dirs: cfg.Skills.ExternalDirs, + Logger: func(format string, args ...any) { + log.Info(fmt.Sprintf(format, args...), slog.String("comp", "skill-loader")) + }, + }); err != nil { + log.Warn("skill loader returned error", + slog.Int("loaded", loaded), + slog.Any("err", err), + ) + } else if loaded > 0 { + log.Info("subprocess skills loaded", slog.Int("count", loaded)) + } + + // Auto-register safe skills as LLM function-calling tools so the AI + // agent can invoke them through the same audit + permission path + // the HTTP layer uses. Mutating / dangerous classes are gated behind + // PR-G4 SOP signing and never auto-registered for the LLM. + toolsReg.SetSkillRunner(skillSvc) + // AgentRegistry + SkillRegistry are loaded UNCONDITIONALLY from // ./agents + ./skills + the marketplace root. They're metadata — a // persona description is just YAML+markdown — so the /v1/agents @@ -1632,11 +1649,10 @@ func main() { chatRT *aiopschatruntime.Runtime ) humanApprovalBroker := newHumanApprovalBroker() - if kernel == managersvcaiops.KernelGraph { + { rt, rterr := buildAIOpsRuntime(rootCtx, cfg, llmClient, llmRouter, toolsReg, aiopsRepo, mutatingProposalRepo, fbClient, edgeUC, deviceUC, reg, log, bootstrapSkillReg, bootstrapAgentReg, llmSettingsResolver, humanApprovalBroker) if rterr != nil { - log.Warn("aiops runtime build failed — falling back to legacy kernel", slog.Any("err", rterr)) - kernel = managersvcaiops.KernelLegacy + log.Error("chat runtime unavailable; fix configuration and restart Manager", slog.Any("err", rterr)) } else { aiopsRuntime = rt chatRT = rt @@ -1690,7 +1706,7 @@ func main() { } } - aiopsSvc := managersvcaiops.NewWithKernel(aiopsAgent, aiopsRuntime, kernel, aiopsRepo, aiopsUsage, log) + aiopsSvc := managersvcaiops.New(aiopsRuntime, aiopsRepo, aiopsUsage, log) aiopsSvc.SetMutatingProposalRepo(mutatingProposalRepo) aiopsSvc.WithAPMSourceResolver(apmService) aiopsSvc.SetAttachmentRepo(aiopsRepo) @@ -1819,13 +1835,8 @@ func main() { aiopsHandler.SetUserAgentManager(userAgentSvc) } // Hand the agent the mention resolver so @-mentions get inlined - // into the user message prelude on each Run. The agent uses its - // own Mention type to keep the agent package dep-light; an adapter - // shuttles between the two shapes here at the wiring site. - aiopsAgent.SetMentionResolver(mentionResolverAdapter{inner: mentionSearcher}) - // Mirror the same wiring on the new graph kernel runtime when it - // was successfully constructed. Same searcher, two adapter shapes - // — see the type definitions at the bottom of this file. + // into the user message prelude. The adapter converts the shared + // mention search types into the runtime request shape. if rt, ok := aiopsRuntime.(*aiopschatruntime.Runtime); ok && rt != nil { rt.SetMentionResolver(chatruntimeMentionAdapter{inner: mentionSearcher}) } @@ -1851,7 +1862,7 @@ func main() { if hasConfiguredLLMProvider(llmRouter) { legacy := aiopsinvestigator.New( llmClient, - toolsReg, + backgroundToolInvoker{reg: toolsReg}, alertRepo, aiopsinvestigator.Config{}, log, @@ -2081,14 +2092,6 @@ func main() { knowledgeHandler.SetAuthz(authzMW) } - // L2 skill framework: builtin Executors registered via init() in - // internal/skill/builtin (imported above). Service dispatches via - // frontierbound.Client; audit goes to MySQL skill_executions. - skillSvc := managerbizskill.New( - fbClient, - managerbizskill.NewGormAuditSink(db), - log.With(slog.String("comp", "skill")), - ) // HLD-017: surface chatruntime SKILL.md skills (built-in + marketplace- // installed) in the /v1/skills catalog. They live in a separate registry // from skillcore, so without this an installed pack (e.g. terrashark) is @@ -2461,8 +2464,7 @@ func main() { graphTools = append(graphTools, aiopstoolsdec.Wrap(raw, deps)) } } - chatRT.ReplaceCatalogToolsByNamePrefix(prefix, rawTools) - chatRT.ReplaceToolsByNamePrefix(prefix, graphTools) + chatRT.ReplaceToolSource(prefix, rawTools, graphTools) log.Info("mcp tools refreshed in chat runtime", slog.String("server", serverName), slog.Int("tools", len(graphTools)), @@ -2507,44 +2509,13 @@ func main() { slog.Bool("pinned_pubkey", mpPinnedKey != ""), ) - // Wire the multi-provider config resolver into the manager-scoped - // web_search built-in. Default provider is SearXNG (zero-config, - // docker-internal). The skill returns a skipped_reason envelope - // when the chosen provider is missing a key / unreachable, so this - // is safe to call even before any operator configures the integration. - skillbuiltin.SetWebSearchConfigResolver(managerbizsetting.NewWebSearchResolver(settingSvc)) - - // Subprocess skill loader: walks each allowlist root and registers - // SubprocessSkills for every skill.json found. Empty dir list = - // nothing loaded; missing dirs are logged and skipped so a fresh - // install with no /etc/ongrid/skills boots cleanly. - if loaded, err := skillcore.LoadDirs(skillcore.LoaderConfig{ - Dirs: cfg.Skills.ExternalDirs, - Logger: func(format string, args ...any) { - log.Info(fmt.Sprintf(format, args...), slog.String("comp", "skill-loader")) - }, - }); err != nil { - log.Warn("skill loader returned error", - slog.Int("loaded", loaded), - slog.Any("err", err), - ) - } else if loaded > 0 { - log.Info("subprocess skills loaded", slog.Int("count", loaded)) - } - - // Auto-register safe skills as LLM function-calling tools so the AI - // agent can invoke them through the same audit + permission path - // the HTTP layer uses. Mutating / dangerous classes are gated behind - // PR-G4 SOP signing and never auto-registered for the LLM. - toolsReg.RegisterSafeSkills(skillSvc) // Inventory bridge: register every BaseTool in the bag as a skill so // /skills surfaces the complete cloud-side capability inventory. - // Runs regardless of agent kernel (legacy / graph) so the page is + // Runs independently of runtime initialization so the page is // populated either way. Skipped tools (already exist as skills via // skill_bridge) are silently bypassed. Idempotent. { invBag := toolsReg.BuildBaseTools() - invBag = aiopstools.AppendHostFilesTools(invBag, fbClient, edgeUC, deviceUC, log) toolsReg.RegisterBaseToolsAsSkills(invBag, log.With(slog.String("comp", "inventory-bridge"))) // Re-merge so flow `tool` nodes can run tools registered after the // invoker was first built — cloud_bash (its proposer is wired above) @@ -3497,34 +3468,7 @@ func (c investigatorChain) InvestigateAsync(in *managermodelalert.Incident) { } } -// mentionResolverAdapter shuttles between agent.Mention (the type -// agent.go uses internally to keep its dependency surface narrow) and -// mentions.Mention (the biz layer's type). One copy on each Run is -// negligible; the alternative — leaking biz/aiops/mentions into the -// agent package — would invert the dep direction. -type mentionResolverAdapter struct { - inner *managerbizaiopsmentions.Searcher -} - -func (a mentionResolverAdapter) Resolve(ctx context.Context, in []aiopsagent.Mention) []string { - if a.inner == nil || len(in) == 0 { - return nil - } - out := make([]managerbizaiopsmentions.Mention, 0, len(in)) - for _, m := range in { - out = append(out, managerbizaiopsmentions.Mention{ - Type: managerbizaiopsmentions.Type(m.Type), - ID: m.ID, - Label: m.Label, - }) - } - return a.inner.Resolve(ctx, out) -} - -// chatruntimeMentionAdapter is the same translation as -// mentionResolverAdapter but for the chatruntime.Mention shape (the -// new graph kernel's local type — kept separate from agent.Mention so -// chatruntime doesn't import agent). +// chatruntimeMentionAdapter 将共享 mention 查询结果传给聊天运行时。 type chatruntimeMentionAdapter struct { inner *managerbizaiopsmentions.Searcher } @@ -3968,9 +3912,8 @@ func loadBootstrapRegistries(log *slog.Logger) (*aiopschatruntime.SkillRegistry, return skillReg, agentReg } -// buildAIOpsRuntime builds the chatruntime.Runtime when -// ONGRID_AGENT_KERNEL=graph. Returns (nil, err) on failure so the -// caller can fall back to the legacy kernel without a panic. +// buildAIOpsRuntime 构建唯一聊天运行时；失败时返回错误， +// 由调用方将聊天标记为不可用，不中断 Manager 的其他服务。 // // Keep the default persona aligned with graph.DefaultConfig's hard ceiling. // A persona override must never silently reopen a loop budget that the graph @@ -4123,7 +4066,6 @@ func buildAIOpsRuntime( // threshold the specialty tier auto-redacts and the LLM // fetches schemas via the always-loaded ToolSearch tool. bag := toolsReg.BuildBaseTools() - bag = aiopstools.AppendHostFilesTools(bag, fbClient, edgeUC, deviceUC, log) baseTools := bag.SchemasForLLM() reviewSpawner := &chatruntimeReviewSpawner{} reviewSink := newMutatingProposalSink(mutatingProposals) @@ -4864,17 +4806,6 @@ func (s k8sActionProposerShim) ProposeAndAwait(ctx context.Context, args aiopsto Command: command, }, }) - } else if emit := aiopsagent.EmitFromContext(ctx); emit != nil { - emit(aiopsagent.Event{ - Type: aiopsagent.EventApprovalPending, - Approval: &aiopsagent.ApprovalPendingEvent{ - ApprovalID: a.ID, - ToolCallID: toolCallID, - Kind: aiopstools.ToolNameExecuteK8sAction, - ToolName: aiopstools.ToolNameExecuteK8sAction, - Command: command, - }, - }) } return cloudBashProposerShim{uc: s.uc}.awaitDecision(ctx, a.ID) } @@ -5025,14 +4956,6 @@ func (s mcpProposerShim) ProposeMCPCallAndAwait(ctx context.Context, server, too ToolName: aiopstools.MCPToolName(server, tool), Command: cmd, }, }) - } else if emit := aiopsagent.EmitFromContext(ctx); emit != nil { - emit(aiopsagent.Event{ - Type: aiopsagent.EventApprovalPending, - Approval: &aiopsagent.ApprovalPendingEvent{ - ApprovalID: a.ID, ToolCallID: toolCallID, Kind: "mcp_call", - ToolName: aiopstools.MCPToolName(server, tool), Command: cmd, - }, - }) } return cloudBashProposerShim{uc: s.uc}.awaitDecision(ctx, a.ID) } @@ -5092,6 +5015,8 @@ func (s *flowToolInvoker) InvokeTool(ctx context.Context, name string, args json // Tag any artifact a tool node produces (serve_page) as workflow-sourced so // the operations UI's 生成来源 column distinguishes it from chat-generated pages. ctx = aiopstoolsbase.WithArtifactSource(ctx, aiopstoolsbase.ArtifactSourceWorkflow) + // 发布工作流中的显式工具节点即授权该步骤联网；聊天不使用此入口。 + ctx = aiopstoolsbase.WithWebSearchAllowed(ctx, true) // MCP tool nodes dispatch LIVE through the mcp source (resolve the current // server/tool, run directly without an approval card — placing the node in // a published flow IS the human authorization; a per-run inbox approval is @@ -6088,3 +6013,27 @@ func seedInfrastructureMenuDefaults(ctx context.Context, db *gorm.DB, settings * log.Warn("seed infrastructure menu defaults", slog.Any("err", err)) } } + +// backgroundToolInvoker 让确定性的告警初查复用统一工具目录，不依赖旧循环注册表。 +type backgroundToolInvoker struct{ reg *aiopstools.Registry } + +func (i backgroundToolInvoker) Invoke(ctx context.Context, name string, args json.RawMessage) (aiopstools.ExecuteResult, error) { + for _, t := range i.reg.BuildBaseTools().AllTools() { + info, err := t.Info(ctx) + if err != nil { + return aiopstools.ExecuteResult{}, err + } + if info == nil || info.Name != name { + continue + } + if info.Class != "read" { + return aiopstools.ExecuteResult{}, fmt.Errorf("background tool %q is not read-only", name) + } + result, err := t.InvokableRun(ctx, string(args)) + if err != nil { + return aiopstools.ExecuteResult{}, err + } + return aiopstools.ExecuteResult{ResultJSON: json.RawMessage(result)}, nil + } + return aiopstools.ExecuteResult{}, fmt.Errorf("background tool %q unavailable", name) +} diff --git a/cmd/ongrid/main_kernel_test.go b/cmd/ongrid/main_kernel_test.go deleted file mode 100644 index 591c7e9a7..000000000 --- a/cmd/ongrid/main_kernel_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package main - -import ( - "os" - "testing" - - managersvcaiops "github.com/ongridio/ongrid/internal/manager/service/aiops" -) - -// TestKernelEnvParsing covers the cmd-level boot-time decision: -// -// unset / empty → KernelLegacy (default = zero behavior change) -// "graph" → KernelGraph -// "garbage" → KernelLegacy (with warn — see service.NewWithKernel) -// -// The actual env wiring lives in main(); this test exercises the -// parser the env value flows through, plus emulates the env-set -// path via os.Setenv. / the default MUST be -// legacy so the kernel switch is opt-in. -func TestKernelEnvParsing(t *testing.T) { - cases := []struct { - name string - envVal string - setEnv bool - want managersvcaiops.Kernel - }{ - {"unset", "", false, managersvcaiops.KernelGraph}, - {"empty_string", "", true, managersvcaiops.KernelGraph}, - {"graph_lower", "graph", true, managersvcaiops.KernelGraph}, - {"graph_upper", "GRAPH", true, managersvcaiops.KernelGraph}, - {"graph_padded", " graph ", true, managersvcaiops.KernelGraph}, - {"legacy_explicit", "legacy", true, managersvcaiops.KernelLegacy}, - {"garbage", "this-is-not-a-kernel", true, managersvcaiops.KernelGraph}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - prev, hadPrev := os.LookupEnv("ONGRID_AGENT_KERNEL") - t.Cleanup(func() { - if hadPrev { - _ = os.Setenv("ONGRID_AGENT_KERNEL", prev) - } else { - _ = os.Unsetenv("ONGRID_AGENT_KERNEL") - } - }) - if c.setEnv { - _ = os.Setenv("ONGRID_AGENT_KERNEL", c.envVal) - } else { - _ = os.Unsetenv("ONGRID_AGENT_KERNEL") - } - got := managersvcaiops.ParseKernel(os.Getenv("ONGRID_AGENT_KERNEL")) - if got != c.want { - t.Errorf("ParseKernel(env=%q,set=%v) = %q, want %q", c.envVal, c.setEnv, got, c.want) - } - }) - } -} diff --git a/cmd/ongrid/tool_catalog_test.go b/cmd/ongrid/tool_catalog_test.go new file mode 100644 index 000000000..aef2def60 --- /dev/null +++ b/cmd/ongrid/tool_catalog_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "log/slog" + "testing" + + tools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" + devicebiz "github.com/ongridio/ongrid/internal/manager/biz/device" + edgebiz "github.com/ongridio/ongrid/internal/manager/biz/edge" +) + +type catalogCaller struct{} + +func (catalogCaller) Call(context.Context, uint64, string, []byte) ([]byte, error) { + return []byte(`{}`), nil +} + +func TestWorkflowPaletteAndInvokerIncludeHostFiles(t *testing.T) { + reg := tools.NewRegistry(catalogCaller{}, edgebiz.NewUsecase(nil, nil, nil, slog.Default()), &devicebiz.Usecase{}, nil, nil, nil, nil, slog.Default()) + catalog := flowToolCatalog{reg: reg} + palette := map[string]bool{} + for _, entry := range catalog.ListTools() { + palette[entry.Name] = true + } + invoker := newFlowToolInvoker(reg, nil) + for _, name := range []string{"host_find_large_files", "host_du_summary", "host_stat_file"} { + if !palette[name] { + t.Errorf("missing palette tool %s", name) + } + if invoker.tools[name] == nil { + t.Errorf("missing executable %s", name) + } + } + if palette[tools.ToolNameExecuteK8sAction] { + t.Fatal("workflow exposes unsupported approval action") + } +} diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index f6b015d97..393284b03 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -73,13 +73,7 @@ services: ONGRID_OPENAI_API_KEY: ${ONGRID_OPENAI_API_KEY:-} ONGRID_OPENAI_MODEL: ${ONGRID_OPENAI_MODEL:-gpt-4o} ONGRID_OPENAI_BASE_URL: ${ONGRID_OPENAI_BASE_URL:-} - # AIOps agent kernel. "graph" enables the eino graph kernel + - # SkillRegistry.Resolve activation-keyword filter (skills with - # activation:keyword stay out of the prompt unless the user query - # mentions a keyword) + ToolBag deferral pipeline. "legacy" = - # historical for-loop runner with all tools always full-schema. - # Default flipped to graph 2026-05-08; flip to legacy to bisect. - ONGRID_AGENT_KERNEL: ${ONGRID_AGENT_KERNEL:-graph} + # Graph/Eino is the only Agent runtime (RFC-005). # ToolBag deferral threshold. When tool count > this, specialty-tier # tools get redacted schemas (LLM must call ToolSearch to expand). # Default 30 is fine until marketplace grows the bag past it. diff --git a/deploy/install/README.md b/deploy/install/README.md index c290783bd..e1e1b94a7 100644 --- a/deploy/install/README.md +++ b/deploy/install/README.md @@ -401,3 +401,9 @@ sudo docker exec ongrid-prometheus wget -qO- \ `/opt/ongrid/prometheus.yml` 是 bind-mounted scrape config，改完跑 `docker compose restart prometheus` 或 `curl -XPOST http://prometheus:9090/-/reload`（`--web.enable-lifecycle` 已开）。 **备份**已迁移到宿主机目录（v0.7.45+），见上文「数据存储 → 数据备份」小节，不再需要 `docker run --rm -v prometheus_data:/data`。 + +### Agent runtime 迁移（统一 Graph） + +统一运行时版本仅使用 Graph/Eino。删除旧 `.env` 中的 `ONGRID_AGENT_KERNEL`；不再支持切换 legacy。手动设置旧变量的裸二进制部署会记录迁移告警并使用 Graph，Compose 不再向容器传递该变量。 + +升级前验证联网搜索、诊断工具、专家任务、IM、审批和历史会话续聊。Graph 初始化失败时聊天返回明确的不可用错误，其他 Manager 功能继续运行；根据启动日志修复配置并重启，不再隐式回退旧循环。首次发布先金丝雀验证，不变更会话表结构。回滚恢复上一版 Manager 镜像和对应 Compose 配置。 diff --git a/deploy/install/docker-compose.yml b/deploy/install/docker-compose.yml index f847c7405..d5b6f87c9 100644 --- a/deploy/install/docker-compose.yml +++ b/deploy/install/docker-compose.yml @@ -115,13 +115,7 @@ services: # default chat provider's model — no separate model config needed. ONGRID_INVESTIGATOR_ENABLED: ${ONGRID_INVESTIGATOR_ENABLED:-true} ONGRID_INVESTIGATOR_MIN_SEVERITY: ${ONGRID_INVESTIGATOR_MIN_SEVERITY:-warning} - # AIOps agent kernel. "graph" enables the eino graph kernel + - # SkillRegistry.Resolve activation-keyword filter (skills with - # activation:keyword stay out of the prompt unless the user query - # mentions a keyword) + ToolBag deferral pipeline. "legacy" = - # historical for-loop runner with all tools always full-schema. - # Default flipped to graph 2026-05-08; flip to legacy to bisect. - ONGRID_AGENT_KERNEL: ${ONGRID_AGENT_KERNEL:-graph} + # Graph/Eino is the only Agent runtime (RFC-005). # ToolBag deferral threshold. When tool count > this, specialty-tier # tools get redacted schemas (LLM must call ToolSearch to expand). # Default 30 is fine until marketplace grows the bag past it. diff --git a/docs/rfc/RFC-005-unified-agent-runtime.md b/docs/rfc/RFC-005-unified-agent-runtime.md index cc946825c..d31354823 100644 --- a/docs/rfc/RFC-005-unified-agent-runtime.md +++ b/docs/rfc/RFC-005-unified-agent-runtime.md @@ -51,3 +51,13 @@ Graph 默认运行路径与旧循环维护独立工具列表，安全 Skill 注 主要风险为漏迁工具、重复桥接、动态目录陈旧、权限弱化、共享类型误删。通过工具执行测试和调用边界硬校验控制。 各阶段保持可编译可测试，使用独立提交审查；部署先金丝雀验证聊天与历史会话。无需数据库迁移，回滚恢复上一版 Manager 镜像及对应 Compose 配置，不使用永久双循环。线上盘点、真实环境冒烟未经执行不得宣称完成。 + +## 实施验证（2026-10-14） + +- 已完成共享协议抽取、安全 Skill 与 host-files 目录接入、联网权限继承、动态工具旧引用撤销，以及旧循环和内核切换分支移除。 +- `make test-agent-runtime`：在本地已有 Linux/amd64 构建镜像中通过，Go 1.25.12，启用 `-race`；覆盖 AIOps 全部子包、Service、HTTP、IM、Skill 和 Manager 装配。 +- 新增确定性用例：真实 Graph 调用的联网开关、同步/后台 worker 权限继承、动态工具更新并发与旧引用拒绝、真实外部 subprocess Skill 执行、工作流文件工具目录与执行器一致性。 +- `make build-ongrid`、`git diff --check` 通过。 +- `make arch-lint` 未完成：本机缺少 `go-arch-lint`；目标虽打印 skipping，后续 recipe 仍调用缺失命令。未改动该既有行为，合并前需在已安装工具的环境补验。 +- 尚未执行生产 legacy 配置盘点、真实模型/MCP 集成冒烟、金丝雀升级与镜像回滚演练；这些是发布验收项，不能用单元测试替代。 +- 旧 Registry 中供审批兼容及既有测试使用的闭包适配器暂留；生产聊天只调用 Graph，告警初查也已改用 BaseTool。退役旧循环不等于删除历史审批协议。 diff --git a/internal/manager/biz/aiops/agent/agent.go b/internal/manager/biz/aiops/agent/agent.go deleted file mode 100644 index 253c9eef9..000000000 --- a/internal/manager/biz/aiops/agent/agent.go +++ /dev/null @@ -1,878 +0,0 @@ -// Package agent is the OpenAI tool-calling loop. -// -// The Run method: -// 1. loads the session's recent history -// 2. appends + persists the new user message -// 3. loops at most cfg.MaxIterations times: -// - calls llm.Chat with history + tool schemas -// - persists the assistant message (+ any tool_calls issued) -// - if no tool calls, returns the Reply -// - otherwise executes each tool with a per-call timeout, persists each -// tool-result message, and continues the loop -// 4. returns ErrMaxIterationsReached if the loop falls through -// -// MaxIterations defaults to 30. Per-tool timeout defaults to 15s. -package agent - -import ( - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" - "log/slog" - "strings" - "time" - - "go.opentelemetry.io/otel" - - biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/toolreplay" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" - model "github.com/ongridio/ongrid/internal/manager/model/aiops" - "github.com/ongridio/ongrid/internal/pkg/errs" - "github.com/ongridio/ongrid/internal/pkg/llm" - "github.com/ongridio/ongrid/internal/pkg/tenantctx" - "github.com/ongridio/ongrid/internal/pkg/tracing" -) - -// ErrMaxIterationsReached is returned from Run when cfg.MaxIterations elapse -// without the assistant producing a final (no-tool-calls) reply. -var ErrMaxIterationsReached = errors.New("agent: max iterations reached") - -var agentTracer = otel.Tracer("github.com/ongridio/ongrid/internal/manager/biz/aiops/agent") - -// Config tunes the agent loop. -type Config struct { - // Model is the OpenAI model slug; default "gpt-4o". - Model string - // Temperature is the sampling temperature; default 0.1. - Temperature float32 - // MaxIterations caps the outer tool-calling loop; default 30. - MaxIterations int - // SystemPrompt is optionally prepended to every run. - SystemPrompt string - // ToolTimeout is the per-tool deadline; default 15s. - ToolTimeout time.Duration - // HistoryLimit bounds how many past messages we replay; default 50. - HistoryLimit int -} - -type emitCtxKeyT struct{} - -var emitCtxKey = emitCtxKeyT{} - -func withEmit(ctx context.Context, emit Emit) context.Context { - if emit == nil { - return ctx - } - return context.WithValue(ctx, emitCtxKey, emit) -} - -// EmitFromContext returns the active legacy-kernel streaming callback. The -// cmd-layer approval adapter uses it to surface the same inline confirmation -// card that the graph kernel emits through chatruntime.EmitFromContext. -func EmitFromContext(ctx context.Context) Emit { - v, _ := ctx.Value(emitCtxKey).(Emit) - return v -} - -// MentionResolver hydrates a list of @-mention references into one -// string per mention, formatted as a markdown bullet. The agent inlines -// these into the user message prelude so the LLM has structured context -// without spending a tool round-trip. Optional — nil resolver = no -// inlining (single-tenant fallback that pre-mentions deployments use). -type MentionResolver interface { - Resolve(ctx context.Context, mentions []Mention) []string -} - -func attachmentIDs(items []model.Attachment) []string { - ids := make([]string, 0, len(items)) - for _, item := range items { - ids = append(ids, item.ID) - } - return ids -} - -// ToolWebSearch is the registered name of the manager-scoped Tavily / -// SearXNG-backed web_search skill. Centralised here so both the agent -// (filter) and the registry / skill code refer to the same string. -const ToolWebSearch = "web_search" - -// legacyKernelMutatingTools is the deny-list of mutating tool names -// that the legacy closure-based agent kernel refuses to execute. The -// new graph kernel (chatruntime.Runtime) wraps every BaseTool with the -// ReviewGate decorator (see decorators/review_gate.go); the legacy -// kernel does NOT have that decorator chain, so a mutating tool_call -// would skip the SOP review entirely. -// -// Rather than ship that loophole, the legacy kernel rejects mutating -// tool_calls by name. The set is small and exhaustive — every PR-N -// addition of a write/destructive tool MUST add its wire name here. -// -// Production deployments running ONGRID_AGENT_KERNEL=graph never hit -// this gate; it's strictly a safety net for the legacy default and -// for tests that exercise the legacy path. -var legacyKernelMutatingTools = map[string]struct{}{ - "host_restart_service": {}, - "capture_pcap": {}, -} - -const ( - legacyApprovalToolTimeout = 31 * time.Minute - legacyK8sDryRunTimeout = 45 * time.Second - legacyK8sDefaultDrainTimeout = 120 * time.Second - legacyK8sDryRunDrainTimeoutPadding = 30 * time.Second -) - -// Agent wires the LLM client, tool registry, and session repo. -type Agent struct { - llm llm.Client - tools *tools.Registry - sessions biz.SessionRepo - cfg Config - log *slog.Logger - resolver MentionResolver - writeEnabled func(ctx context.Context) bool -} - -// SetMentionResolver wires the @-mention hydrator. Optional — when -// unset the agent runs with no mention inlining (back-compat). -func (a *Agent) SetMentionResolver(r MentionResolver) { a.resolver = r } - -// SetAgentWriteEnabledProvider wires the live admin setting used by the -// default legacy kernel. A nil provider fails closed for write-tool exposure. -func (a *Agent) SetAgentWriteEnabledProvider(fn func(ctx context.Context) bool) { - a.writeEnabled = fn -} - -// New builds an Agent. Defaults are filled in here so callers can pass a -// zero-value Config and get sane behaviour. -func New(llmClient llm.Client, toolsReg *tools.Registry, sessions biz.SessionRepo, cfg Config, log *slog.Logger) *Agent { - if cfg.Model == "" { - cfg.Model = "gpt-5.4" - } - if cfg.Temperature == 0 { - cfg.Temperature = 0.1 - } - if cfg.MaxIterations <= 0 { - cfg.MaxIterations = 30 - } - if cfg.ToolTimeout <= 0 { - cfg.ToolTimeout = 15 * time.Second - } - if cfg.HistoryLimit <= 0 { - cfg.HistoryLimit = 50 - } - return &Agent{ - llm: llmClient, - tools: toolsReg, - sessions: sessions, - cfg: cfg, - log: log, - } -} - -// Run processes a new user message in an existing session and returns the -// terminal Reply once the agent loop completes. Ownership is checked -// against userID; admin bypass is delegated to the service layer which -// calls Run with the target session's owning userID. -// -// Run is the blocking convenience entrypoint; it is implemented as a -// no-emit invocation of RunStream, so the two share exactly one code path. -func (a *Agent) Run(ctx context.Context, sessionID string, userID uint64, userContent string) (*Reply, error) { - return a.RunStream(ctx, sessionID, userID, userContent, nil) -} - -// RunStreamWithOpts is the override-aware sibling of RunStream. The -// per-call provider/model + mention list let the SPA's model selector + -// @-mention popover round-trip cleanly without hijacking the broader -// Agent config. Empty fields fall back to a.cfg defaults. -func (a *Agent) RunStreamWithOpts(ctx context.Context, sessionID string, userID uint64, userContent string, emit Emit, opts RunOptions) (*Reply, error) { - return a.runInternal(ctx, sessionID, userID, userContent, emit, opts) -} - -// RunStream is the streaming variant of Run. emit (if non-nil) receives -// one Event per agent phase: assistant turn persisted, tool call started, -// tool call finished, done. emit must not block — callers should buffer -// or drop on a slow consumer. The function still returns the final Reply -// (or error) so non-streaming callers can use the same primitive. -func (a *Agent) RunStream(ctx context.Context, sessionID string, userID uint64, userContent string, emit Emit) (*Reply, error) { - return a.runInternal(ctx, sessionID, userID, userContent, emit, RunOptions{}) -} - -func (a *Agent) runInternal(ctx context.Context, sessionID string, userID uint64, userContent string, emit Emit, opts RunOptions) (reply *Reply, retErr error) { - ctx, span := agentTracer.Start(ctx, "aiops.Agent.Run") - defer func() { tracing.EndSpan(span, retErr) }() - if a.sessions == nil || a.llm == nil || a.tools == nil { - return nil, errs.ErrNotWiredYet - } - if emit == nil { - emit = func(Event) {} - } - ctx = withEmit(ctx, emit) - - sess, err := a.sessions.GetSession(ctx, sessionID) - if err != nil { - return nil, err - } - if sess.UserID != userID { - // Caller is not the owner. The service layer is responsible for - // admin-bypass bookkeeping; at this layer ownership is absolute. - // Returning ErrNotFound avoids leaking existence info (the HTTP - // layer wants 404 not 403 per spec). - return nil, errs.ErrNotFound - } - ctx = basetool.WithSessionID(ctx, sess.ID) - ctx = basetool.WithAPMSource(ctx, sess.APMSource) - writeEnabled := false - if a.writeEnabled != nil { - writeEnabled = a.writeEnabled(ctx) - } - ctx = basetool.WithAgentWriteAllowed(ctx, writeEnabled) - caller, hasCaller := tenantctx.From(ctx) - canExecuteK8sAction := writeEnabled && hasCaller && (caller.Role == "admin" || caller.IsSuperuser) - - // Persist the new user turn first so it survives a downstream crash. - // When the SPA chat input attached @-mentions, hydrate them into - // markdown bullets (read-only context expansion — no tool round- - // trip needed) and prepend the result to the user content. The - // persisted message stays the augmented form so on history replay - // the LLM still sees the same context block. (Edits are rare and - // the bullets are short; we accept the small storage cost for the - // reproducibility win.) - augmented := userContent - if len(opts.Mentions) > 0 && a.resolver != nil { - bullets := a.resolver.Resolve(ctx, opts.Mentions) - if len(bullets) > 0 { - augmented = "用户在消息中引用了以下平台对象，请在回答时考虑：\n" + - strings.Join(bullets, "\n") + "\n\n" + userContent - } - } - userMsgContent := augmented - userMsg := &model.Message{ - SessionID: sess.ID, - Role: model.RoleUser, - Content: &userMsgContent, - Attachments: opts.Attachments, - AttachmentIDs: attachmentIDs(opts.Attachments), - CreatedAt: time.Now().UTC(), - } - if err := a.sessions.AppendMessage(ctx, userMsg); err != nil { - return nil, fmt.Errorf("agent: persist user msg: %w", err) - } - - // Load history (includes the just-appended user msg). - history, err := a.sessions.ListMessages(ctx, sess.ID, a.cfg.HistoryLimit) - if err != nil { - return nil, fmt.Errorf("agent: load history: %w", err) - } - - // Build the llm message array. - llmMsgs := a.buildMessages(history) - - // Resolve effective provider + model. Per-call opts override config. - effectiveProvider := opts.Provider - effectiveModel := opts.Model - if effectiveModel == "" { - effectiveModel = a.cfg.Model - } - - var totalUsage llm.Usage - var toolCallRows []*model.ToolCall - - // Per-turn schema list: the manager-scoped web_search skill is only - // exposed when the SPA's globe toggle says so. By default the agent - // stays on internal data — the model otherwise tends to call - // web_search on every "what is X" question, blowing through the - // Tavily quota and slowing every chat by ~1s. - exposedSchemas := a.tools.Schemas() - if !opts.WebSearchEnabled { - exposedSchemas = filterToolSchemas(exposedSchemas, ToolWebSearch) - } - if !canExecuteK8sAction { - exposedSchemas = filterToolSchemas(exposedSchemas, tools.ToolNameExecuteK8sAction) - } - - for i := 0; i < a.cfg.MaxIterations; i++ { - resp, err := a.llm.Chat(ctx, llm.ChatReq{ - Model: effectiveModel, - Provider: effectiveProvider, - Messages: llmMsgs, - Tools: exposedSchemas, - Temperature: a.cfg.Temperature, - UserID: userID, - }) - if err != nil { - return nil, fmt.Errorf("agent: llm.Chat: %w", err) - } - totalUsage.PromptTokens += resp.Usage.PromptTokens - totalUsage.CompletionTokens += resp.Usage.CompletionTokens - totalUsage.TotalTokens += resp.Usage.TotalTokens - - // Persist the assistant message. If Content is empty and there are - // tool calls, we still write the row (Content is a *string so nil - // means "no textual reply"). - asstContent := resp.Assistant.Content - var asstContentPtr *string - if asstContent != "" { - asstContentPtr = &asstContent - } - pt := resp.Usage.PromptTokens - ct := resp.Usage.CompletionTokens - modelTag := effectiveModel - asstRow := &model.Message{ - SessionID: sess.ID, - Role: model.RoleAssistant, - Content: asstContentPtr, - ReasoningContent: stringPtrIfSet(resp.Assistant.ReasoningContent), - Model: stringPtrIfSet(modelTag), - PromptTokens: &pt, - CompletionTokens: &ct, - CreatedAt: time.Now().UTC(), - } - if err := a.sessions.AppendMessage(ctx, asstRow); err != nil { - return nil, fmt.Errorf("agent: persist assistant msg: %w", err) - } - - emit(Event{Type: EventAssistant, Assistant: &AssistantEvent{ - Iteration: i + 1, - MessageID: asstRow.ID, - Content: asstContent, - CreatedAt: asstRow.CreatedAt, - PendingToolCalls: len(resp.Assistant.ToolCalls), - }}) - - // Append the assistant message (with its tool_calls) to the llm - // working history so the next round sees it. - llmMsgs = append(llmMsgs, resp.Assistant) - - if len(resp.Assistant.ToolCalls) == 0 { - // Terminal: no tools requested, conversation is done. - if a.log != nil { - a.log.Info("agent run completed", - slog.String("session_id", sess.ID), - slog.Uint64("user_id", userID), - slog.Int("iterations", i+1), - slog.Int("prompt_tokens", totalUsage.PromptTokens), - slog.Int("completion_tokens", totalUsage.CompletionTokens), - ) - } - reply := &Reply{ - Message: asstRow, - Usage: totalUsage, - Iterations: i + 1, - ToolCalls: toolCallRows, - } - emit(Event{Type: EventDone, Done: reply}) - return reply, nil - } - - // Execute each requested tool sequentially. We persist a pending - // chat_tool_calls row, execute, then update the row with the - // result or error, and feed a role=tool message back to the LLM. - for _, tc := range resp.Assistant.ToolCalls { - startedAt := time.Now().UTC() - llmCallID := tc.ID - tcRow := &model.ToolCall{ - MessageID: asstRow.ID, - ToolName: tc.Name, - ArgumentsJSON: string(tc.Args), - LLMCallID: &llmCallID, - Status: model.StatusPending, - StartedAt: startedAt, - CreatedAt: startedAt, - } - if err := a.sessions.CreateToolCall(ctx, tcRow); err != nil { - return nil, fmt.Errorf("agent: persist tool_call row: %w", err) - } - - emit(Event{Type: EventToolStart, Tool: &ToolEvent{ - ToolCallID: tcRow.ID, - Name: tc.Name, - Status: model.StatusPending, - StartedAt: startedAt, - ArgsJSON: string(tc.Args), - }}) - - // Belt-and-braces: even though we filtered web_search out of - // the schema list above when WebSearchEnabled=false, a model - // occasionally tool_calls something it saw earlier in the - // transcript. Reject locally so a hijacked turn can't spend - // the operator's Tavily quota. - if tc.Name == ToolWebSearch && !opts.WebSearchEnabled { - execErr := errors.New("agent: web_search disabled — toggle the globe icon in the chat input to enable") - toolPayload := toolResultPayload(tools.ExecuteResult{}, execErr) - endedAt := time.Now().UTC() - status, errMsgPtr, resultPtr := classifyToolOutcome(ctx, tools.ExecuteResult{}, execErr, toolPayload) - if err := a.sessions.UpdateToolCallResult(ctx, tcRow.ID, status, resultPtr, errMsgPtr, endedAt); err != nil { - return nil, fmt.Errorf("agent: update tool_call row: %w", err) - } - tcRow.Status = status - tcRow.ResultJSON = resultPtr - tcRow.Error = errMsgPtr - tcRow.EndedAt = &endedAt - toolCallRows = append(toolCallRows, tcRow) - toolEvt := ToolEvent{ - ToolCallID: tcRow.ID, - Name: tc.Name, - Status: status, - StartedAt: startedAt, - EndedAt: &endedAt, - DurationMs: endedAt.Sub(startedAt).Milliseconds(), - ArgsJSON: string(tc.Args), - ResultJSON: truncateJSON(toolPayload, 8*1024), - } - if errMsgPtr != nil { - toolEvt.Error = *errMsgPtr - } - emit(Event{Type: EventToolEnd, Tool: &toolEvt}) - toolName := tc.Name - toolCallID := tc.ID - payloadStr := string(toolPayload) - toolRow := &model.Message{ - SessionID: sess.ID, - Role: model.RoleTool, - Content: &payloadStr, - ToolCallID: &toolCallID, - ToolName: &toolName, - CreatedAt: endedAt, - } - if err := a.sessions.AppendMessage(ctx, toolRow); err != nil { - return nil, fmt.Errorf("agent: persist tool msg: %w", err) - } - llmMsgs = append(llmMsgs, llm.Message{ - Role: model.RoleTool, - Content: string(toolPayload), - ToolCallID: tc.ID, - ToolName: tc.Name, - }) - continue - } - - // Legacy-kernel SOP gate. The graph kernel wraps every - // BaseTool with the ReviewGate decorator (SOP - // double-sign); the legacy closure kernel does not. To - // avoid silently letting mutating tool_calls slip through - // without review, we deny the call by name. Operators who - // want SOP gating must run with ONGRID_AGENT_KERNEL=graph - // (which lands in cmd/main.go's runtime wiring). - if _, mutating := legacyKernelMutatingTools[tc.Name]; mutating { - execErr := fmt.Errorf("agent: tool %q is mutating — not supported in legacy kernel; "+ - "set ONGRID_AGENT_KERNEL=graph (SOP gating lives in the new graph runtime)", tc.Name) - toolPayload := toolResultPayload(tools.ExecuteResult{}, execErr) - endedAt := time.Now().UTC() - status, errMsgPtr, resultPtr := classifyToolOutcome(ctx, tools.ExecuteResult{}, execErr, toolPayload) - if err := a.sessions.UpdateToolCallResult(ctx, tcRow.ID, status, resultPtr, errMsgPtr, endedAt); err != nil { - return nil, fmt.Errorf("agent: update tool_call row: %w", err) - } - tcRow.Status = status - tcRow.ResultJSON = resultPtr - tcRow.Error = errMsgPtr - tcRow.EndedAt = &endedAt - toolCallRows = append(toolCallRows, tcRow) - toolEvt := ToolEvent{ - ToolCallID: tcRow.ID, - Name: tc.Name, - Status: status, - StartedAt: startedAt, - EndedAt: &endedAt, - DurationMs: endedAt.Sub(startedAt).Milliseconds(), - ArgsJSON: string(tc.Args), - ResultJSON: truncateJSON(toolPayload, 8*1024), - } - if errMsgPtr != nil { - toolEvt.Error = *errMsgPtr - } - emit(Event{Type: EventToolEnd, Tool: &toolEvt}) - toolName := tc.Name - toolCallID := tc.ID - payloadStr := string(toolPayload) - toolRow := &model.Message{ - SessionID: sess.ID, - Role: model.RoleTool, - Content: &payloadStr, - ToolCallID: &toolCallID, - ToolName: &toolName, - CreatedAt: endedAt, - } - if err := a.sessions.AppendMessage(ctx, toolRow); err != nil { - return nil, fmt.Errorf("agent: persist tool msg: %w", err) - } - llmMsgs = append(llmMsgs, llm.Message{ - Role: model.RoleTool, - Content: string(toolPayload), - ToolCallID: tc.ID, - ToolName: tc.Name, - }) - continue - } - - toolCtx := basetool.WithToolCallID(ctx, tcRow.ID) - toolCtx, cancel := context.WithTimeout(toolCtx, legacyToolTimeout(a.cfg.ToolTimeout, tc.Name, tc.Args)) - execResult, execErr := a.tools.Invoke(toolCtx, tc.Name, tc.Args) - cancel() - - toolPayload := toolResultPayload(execResult, execErr) - endedAt := time.Now().UTC() - - status, errMsgPtr, resultPtr := classifyToolOutcome(toolCtx, execResult, execErr, toolPayload) - if execResult.DeviceID != nil { - tcRow.DeviceID = execResult.DeviceID - } - if err := a.sessions.UpdateToolCallResult(ctx, tcRow.ID, status, resultPtr, errMsgPtr, endedAt); err != nil { - return nil, fmt.Errorf("agent: update tool_call row: %w", err) - } - tcRow.Status = status - tcRow.ResultJSON = resultPtr - tcRow.Error = errMsgPtr - tcRow.EndedAt = &endedAt - toolCallRows = append(toolCallRows, tcRow) - - toolEvt := ToolEvent{ - ToolCallID: tcRow.ID, - Name: tc.Name, - DeviceID: tcRow.DeviceID, - Status: status, - StartedAt: startedAt, - EndedAt: &endedAt, - DurationMs: endedAt.Sub(startedAt).Milliseconds(), - ArgsJSON: string(tc.Args), - ResultJSON: truncateJSON(toolPayload, 8*1024), - } - if errMsgPtr != nil { - toolEvt.Error = *errMsgPtr - } - emit(Event{Type: EventToolEnd, Tool: &toolEvt}) - - // Persist a role=tool message with the payload we will also - // feed to the llm on the next iteration. - toolName := tc.Name - toolCallID := tc.ID - payloadStr := string(toolPayload) - toolRow := &model.Message{ - SessionID: sess.ID, - Role: model.RoleTool, - Content: &payloadStr, - ToolCallID: &toolCallID, - ToolName: &toolName, - CreatedAt: endedAt, - } - if err := a.sessions.AppendMessage(ctx, toolRow); err != nil { - return nil, fmt.Errorf("agent: persist tool msg: %w", err) - } - - llmMsgs = append(llmMsgs, llm.Message{ - Role: model.RoleTool, - Content: string(toolPayload), - ToolCallID: tc.ID, - ToolName: tc.Name, - }) - } - } - - // Loop ran the full budget without the model returning a final - // (no-tool-call) reply. Rather than surfacing a bare error in the - // SPA bubble, persist a friendly assistant message that summarises - // what was attempted; the user sees something actionable instead of - // "抱歉，处理消息时出错了". The chat thread keeps the tool cards we - // already wrote so the user can inspect what went wrong. - apology := fmt.Sprintf("我尝试了 %d 轮工具调用但还没拿到结论 — 看起来模型在重复同一个失败的调用。常见原因：\n• 设备 / 告警 ID 错误（用 @ 搜索更稳）\n• 工具不可用（检查 frontier / Prom / Loki 集成）\n• 问题超出当前工具集的覆盖\n\n请把请求换个方式再问一次，或直接看上面的工具调用结果定位原因。", a.cfg.MaxIterations) - finalMsg := &model.Message{ - SessionID: sess.ID, - Role: model.RoleAssistant, - Content: &apology, - Model: stringPtrIfSet(effectiveModel), - CreatedAt: time.Now().UTC(), - } - if err := a.sessions.AppendMessage(ctx, finalMsg); err == nil { - emit(Event{Type: EventAssistant, Assistant: &AssistantEvent{ - MessageID: finalMsg.ID, - Content: apology, - CreatedAt: finalMsg.CreatedAt, - }}) - return &Reply{ - Message: finalMsg, - Usage: totalUsage, - Iterations: a.cfg.MaxIterations, - ToolCalls: toolCallRows, - }, nil - } - return nil, ErrMaxIterationsReached -} - -// buildMessages translates persisted history rows + optional system prompt -// into the llm.Message shape. -// -// Tool-call replay: chat_messages stores assistant turns that emit -// tool_calls as Content=NULL rows (because the LLM gave us no natural- -// language text on that turn). The actual tool_calls metadata lives in -// chat_tool_calls and is hydrated onto Message.ToolCalls by the repo -// (see SessionRepo.hydrateToolCalls). We rebuild the assistant message -// as {role:assistant, content:"", tool_calls:[…]} so strict providers -// (DeepSeek v4+) don't reject the request with "tool must follow -// tool_calls" once the following role=tool messages appear. -// -// LLM call id resolution: NEW rows store the id the LLM assigned in -// chat_tool_calls.llm_call_id. OLD rows have NULL there; the -// pairing-by-order fallback walks forward through history matching -// the next N role=tool rows to the assistant's N tool_calls (in their -// stored order) and uses each tool row's tool_call_id. When pairing -// fails entirely (assistant has tool_calls but no usable id and no -// matching tool row), we drop both the assistant row and dependent -// tool rows — the same shape the pre-fix MVP emitted — so the request -// at worst regresses to the old behavior, never makes it worse. -func (a *Agent) buildMessages(history []*model.Message) []llm.Message { - callIDs := toolreplay.Resolve(history) - // See chatruntime.buildEinoHistory for rationale: index tool rows by - // tool_call_id so we can hoist responses next to their parent - // assistant regardless of created_at order. Same pattern, kept in - // lock-step with the graph kernel. - toolIdx := toolreplay.IndexToolMessagesByCallID(history) - skipTool := make(map[int]bool) - - out := make([]llm.Message, 0, len(history)+1) - if a.cfg.SystemPrompt != "" { - out = append(out, llm.Message{Role: "system", Content: a.cfg.SystemPrompt}) - } - - emitToolByCallID := func(callID string) { - j, ok := toolIdx[callID] - if !ok { - return - } - tm := history[j] - content := "" - if tm.Content != nil { - content = *tm.Content - } - var tcID, tname string - if tm.ToolCallID != nil { - tcID = *tm.ToolCallID - } - if tm.ToolName != nil { - tname = *tm.ToolName - } - out = append(out, llm.Message{ - Role: model.RoleTool, - Content: content, - ToolCallID: tcID, - ToolName: tname, - }) - skipTool[j] = true - } - for idx, m := range history { - switch m.Role { - case model.RoleUser, model.RoleSystem: - if m.Content == nil { - continue - } - msg := llm.Message{Role: m.Role, Content: *m.Content} - for _, image := range m.Attachments { - msg.Images = append(msg.Images, llm.ImageInput{Name: image.Name, MIMEType: image.MIMEType, Data: base64.StdEncoding.EncodeToString(image.Data)}) - } - out = append(out, msg) - case model.RoleAssistant: - calls, ok := callIDs[m.ID] - if len(m.ToolCalls) > 0 && !ok { - toolreplay.MarkDependentToolsForSkip(history, idx, len(m.ToolCalls), skipTool) - continue - } - // Precheck: every tool_call we'd emit must have a hoistable - // response. If any is missing (parallel ToolsNode dropped an - // OnEnd → chat_tool_calls row written but no role=tool - // chat_messages row), drop the whole assistant turn + its - // dependent tools rather than send an envelope that strict - // providers (DeepSeek v4+) reject with HTTP 400 - // "insufficient tool messages following tool_calls". - if ok && len(calls) > 0 { - complete := true - for _, tc := range calls { - if _, found := toolIdx[tc.ID]; !found { - complete = false - break - } - } - if !complete { - toolreplay.MarkDependentToolsForSkip(history, idx, len(calls), skipTool) - continue - } - } - content := "" - if m.Content != nil { - content = *m.Content - } - msg := llm.Message{Role: m.Role, Content: content} - if m.ReasoningContent != nil { - msg.ReasoningContent = *m.ReasoningContent - } - if ok { - msg.ToolCalls = make([]llm.ToolCall, 0, len(calls)) - for _, tc := range calls { - msg.ToolCalls = append(msg.ToolCalls, llm.ToolCall{ - ID: tc.ID, - Name: tc.Name, - Args: json.RawMessage(tc.ArgsJSON), - }) - } - } - if content == "" && len(msg.ToolCalls) == 0 { - // Polluted-data case: assistant turn with no replayable - // signal AND no hydrated tool_calls. Drop the assistant - // AND the dependent tool rows so they don't become - // orphans in the replay. - toolreplay.MarkAllFollowingToolsForSkip(history, idx, skipTool) - continue - } - out = append(out, msg) - // HOIST: pull each tool_call's response in by id so it - // follows the assistant immediately, regardless of where - // the natural created_at order put it. - for _, tc := range msg.ToolCalls { - emitToolByCallID(tc.ID) - } - case model.RoleTool: - if skipTool[idx] { - continue - } - var content string - if m.Content != nil { - content = *m.Content - } - var toolCallID, toolName string - if m.ToolCallID != nil { - toolCallID = *m.ToolCallID - } - if m.ToolName != nil { - toolName = *m.ToolName - } - out = append(out, llm.Message{ - Role: m.Role, - Content: content, - ToolCallID: toolCallID, - ToolName: toolName, - }) - } - } - return out -} - -// truncateJSON caps a payload bytes-string at max chars, appending an -// indicator suffix so the UI can label it as cut. We don't try to -// preserve JSON validity on truncation — the value is for human display -// only, never re-fed to the LLM. -func truncateJSON(b []byte, max int) string { - if max <= 0 || len(b) <= max { - return string(b) - } - return string(b[:max]) + "...(truncated)" -} - -// toolResultPayload renders the JSON bytes sent back to the LLM on a tool -// result message. On error we emit `{"error": "..."}` so the -// model can decide whether to retry, switch tools, or give up. A nil / -// zero-length ResultJSON is rendered as `{}` to avoid sending invalid JSON. -func toolResultPayload(r tools.ExecuteResult, err error) []byte { - if err != nil { - b, _ := json.Marshal(map[string]string{"error": err.Error()}) - return b - } - if len(r.ResultJSON) == 0 { - return []byte(`{}`) - } - return r.ResultJSON -} - -// filterToolSchemas returns a copy of schemas with any entries whose -// Name matches one of the excluded tools removed. Used to gate the -// web_search skill behind the SPA's globe toggle without rebuilding -// the whole tool registry per request. -func filterToolSchemas(schemas []llm.ToolSchema, excludeNames ...string) []llm.ToolSchema { - if len(excludeNames) == 0 { - return schemas - } - exclude := make(map[string]struct{}, len(excludeNames)) - for _, n := range excludeNames { - exclude[n] = struct{}{} - } - out := make([]llm.ToolSchema, 0, len(schemas)) - for _, s := range schemas { - if _, drop := exclude[s.Name]; drop { - continue - } - out = append(out, s) - } - return out -} - -func legacyToolTimeout(defaultTimeout time.Duration, name string, args json.RawMessage) time.Duration { - if name != tools.ToolNameExecuteK8sAction { - return defaultTimeout - } - var in struct { - Action string `json:"action"` - DryRun bool `json:"dry_run"` - DrainTimeoutSeconds int `json:"drain_timeout_seconds"` - } - if err := json.Unmarshal(args, &in); err != nil { - return defaultTimeout - } - if in.DryRun { - if strings.EqualFold(strings.TrimSpace(in.Action), "drain") { - drainTimeout := time.Duration(in.DrainTimeoutSeconds) * time.Second - if drainTimeout <= 0 { - drainTimeout = legacyK8sDefaultDrainTimeout - } - return drainTimeout + legacyK8sDryRunDrainTimeoutPadding - } - return legacyK8sDryRunTimeout - } - return legacyApprovalToolTimeout -} - -// stringPtrIfSet returns &s when s is non-empty, otherwise nil. Used to -// avoid writing zero-length values into nullable *string columns. -func stringPtrIfSet(s string) *string { - if s == "" { - return nil - } - return &s -} - -// classifyToolOutcome decides the status + error + resultJSON triple to -// persist in chat_tool_calls. toolCtx carries the per-call timeout; a -// cancelled/exceeded deadline maps to StatusTimeout. -func classifyToolOutcome(toolCtx context.Context, res tools.ExecuteResult, execErr error, payload []byte) (status string, errStr, resultJSON *string) { - if execErr != nil { - msg := execErr.Error() - if errors.Is(toolCtx.Err(), context.DeadlineExceeded) { - return model.StatusTimeout, &msg, nil - } - return model.StatusError, &msg, nil - } - // success - s := string(payload) - return model.StatusSuccess, nil, &s -} - -type Reply = chatprotocol.Reply -type EventType = chatprotocol.EventType -type Event = chatprotocol.Event -type ApprovalPendingEvent = chatprotocol.ApprovalPendingEvent -type TaskNotificationEvent = chatprotocol.TaskNotificationEvent -type AssistantEvent = chatprotocol.AssistantEvent -type ToolEvent = chatprotocol.ToolEvent -type Emit = chatprotocol.Emit -type Mention = chatprotocol.Mention -type RunOptions = chatprotocol.RunOptions - -const ( - EventAssistant = chatprotocol.EventAssistant - EventToolStart = chatprotocol.EventToolStart - EventToolEnd = chatprotocol.EventToolEnd - EventDone = chatprotocol.EventDone - EventTaskNotification = chatprotocol.EventTaskNotification - EventApprovalPending = chatprotocol.EventApprovalPending -) diff --git a/internal/manager/biz/aiops/agent/agent_test.go b/internal/manager/biz/aiops/agent/agent_test.go deleted file mode 100644 index dd37aea8d..000000000 --- a/internal/manager/biz/aiops/agent/agent_test.go +++ /dev/null @@ -1,502 +0,0 @@ -package agent - -import ( - "context" - "encoding/json" - "errors" - "log/slog" - "strconv" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" - edgebiz "github.com/ongridio/ongrid/internal/manager/biz/edge" - model "github.com/ongridio/ongrid/internal/manager/model/aiops" - edgemodel "github.com/ongridio/ongrid/internal/manager/model/edge" - "github.com/ongridio/ongrid/internal/pkg/errs" - "github.com/ongridio/ongrid/internal/pkg/llm" - "github.com/ongridio/ongrid/internal/pkg/tunnel" -) - -// ----- fakes ----- - -type fakeRepo struct { - mu sync.Mutex - sessions map[string]*model.Session - messages map[string][]*model.Message // keyed by session id - toolCalls map[string]*model.ToolCall - nextSess int - nextMsg int - nextTC int -} - -func newFakeRepo() *fakeRepo { - return &fakeRepo{ - sessions: map[string]*model.Session{}, - messages: map[string][]*model.Message{}, - toolCalls: map[string]*model.ToolCall{}, - } -} - -// fakeID returns a stable, predictable id of the form "fake--" -// so test assertions can use literal strings instead of UUIDs. Real -// production code uses uuid.NewString in the GORM BeforeCreate hook. -func fakeID(prefix string, n int) string { return prefix + "-" + strconv.Itoa(n) } - -func (r *fakeRepo) CreateSession(_ context.Context, s *model.Session) error { - r.mu.Lock() - defer r.mu.Unlock() - r.nextSess++ - if s.ID == "" { - s.ID = fakeID("sess", r.nextSess) - } - cp := *s - r.sessions[s.ID] = &cp - return nil -} -func (r *fakeRepo) GetSession(_ context.Context, id string) (*model.Session, error) { - r.mu.Lock() - defer r.mu.Unlock() - s, ok := r.sessions[id] - if !ok { - return nil, errs.ErrNotFound - } - cp := *s - return &cp, nil -} -func (r *fakeRepo) ListSessions(_ context.Context, userID uint64, _, _ int, _ *uint64) ([]*model.Session, error) { - r.mu.Lock() - defer r.mu.Unlock() - var out []*model.Session - for _, s := range r.sessions { - if s.UserID == userID { - cp := *s - out = append(out, &cp) - } - } - return out, nil -} -func (r *fakeRepo) ListByParent(_ context.Context, parentID string) ([]*model.Session, error) { - r.mu.Lock() - defer r.mu.Unlock() - out := make([]*model.Session, 0) - for _, s := range r.sessions { - if s.ParentSessionID != nil && *s.ParentSessionID == parentID { - cp := *s - out = append(out, &cp) - } - } - return out, nil -} -func (r *fakeRepo) RenameSession(_ context.Context, _, _ string) error { return nil } -func (r *fakeRepo) UpdateSessionModel(_ context.Context, _, _, _ string) error { return nil } -func (r *fakeRepo) CloseSession(_ context.Context, id string) error { - r.mu.Lock() - defer r.mu.Unlock() - s, ok := r.sessions[id] - if !ok { - return errs.ErrNotFound - } - now := time.Now().UTC() - s.ClosedAt = &now - return nil -} -func (r *fakeRepo) DeleteSession(_ context.Context, id string) error { - r.mu.Lock() - defer r.mu.Unlock() - if _, ok := r.sessions[id]; !ok { - return errs.ErrNotFound - } - delete(r.sessions, id) - return nil -} -func (r *fakeRepo) AppendMessage(_ context.Context, m *model.Message) error { - r.mu.Lock() - defer r.mu.Unlock() - r.nextMsg++ - if m.ID == "" { - m.ID = fakeID("msg", r.nextMsg) - } - cp := *m - r.messages[m.SessionID] = append(r.messages[m.SessionID], &cp) - return nil -} -func (r *fakeRepo) ListMessages(_ context.Context, sessionID string, limit int) ([]*model.Message, error) { - r.mu.Lock() - defer r.mu.Unlock() - msgs := r.messages[sessionID] - out := make([]*model.Message, len(msgs)) - for i, m := range msgs { - cp := *m - out[i] = &cp - } - if limit > 0 && len(out) > limit { - out = out[len(out)-limit:] - } - return out, nil -} -func (r *fakeRepo) CreateToolCall(_ context.Context, tc *model.ToolCall) error { - r.mu.Lock() - defer r.mu.Unlock() - r.nextTC++ - if tc.ID == "" { - tc.ID = fakeID("tc", r.nextTC) - } - cp := *tc - r.toolCalls[tc.ID] = &cp - return nil -} -func (r *fakeRepo) UpdateToolCallResult(_ context.Context, id string, status string, resultJSON, errStr *string, endedAt time.Time) error { - r.mu.Lock() - defer r.mu.Unlock() - tc, ok := r.toolCalls[id] - if !ok { - return errs.ErrNotFound - } - tc.Status = status - tc.ResultJSON = resultJSON - tc.Error = errStr - tc.EndedAt = &endedAt - return nil -} - -func (r *fakeRepo) SumTokensSince(_ context.Context, _ time.Time) (biz.TokenSums, error) { - return biz.TokenSums{}, nil -} - -var _ biz.SessionRepo = (*fakeRepo)(nil) - -type chatScript struct { - resps []llm.ChatResp - errs []error - idx int32 - calls atomic.Int32 -} - -type fakeLLM struct { - script *chatScript -} - -func (f *fakeLLM) Chat(_ context.Context, _ llm.ChatReq) (*llm.ChatResp, error) { - i := int(f.script.idx) - f.script.idx++ - f.script.calls.Add(1) - if i >= len(f.script.resps) { - return nil, errors.New("fakeLLM: out of script") - } - if i < len(f.script.errs) && f.script.errs[i] != nil { - return nil, f.script.errs[i] - } - r := f.script.resps[i] - return &r, nil -} - -// fakeCaller mimics the frontierbound.Client.Call surface. Tests preload -// a typed response value (marshaled on demand) or an error. -type fakeCaller struct { - resp any - err error -} - -func (f *fakeCaller) Call(_ context.Context, _ uint64, _ string, _ []byte) ([]byte, error) { - if f.err != nil { - return nil, f.err - } - b, err := json.Marshal(f.resp) - if err != nil { - return nil, err - } - return b, nil -} - -// buildRegistry wires a real tools.Registry around a fakeCaller that -// resolves every reverse Call with the JSON-encoded resp. -func buildRegistry(_ *testing.T, resp any) *tools.Registry { - edges := newFakeEdgeRepoAgent(&edgemodel.Edge{ID: 1, Name: "node-a"}) - uc := edgebiz.NewUsecase(edges, nil, nil, slog.Default()) - return tools.NewRegistry(&fakeCaller{resp: resp}, uc, nil, nil, nil, nil, nil, slog.Default()) -} - -// buildRegistryWithErr is like buildRegistry but every Call returns the -// supplied error. -func buildRegistryWithErr(_ *testing.T, errStr string) *tools.Registry { - edges := newFakeEdgeRepoAgent(&edgemodel.Edge{ID: 1, Name: "node-a"}) - uc := edgebiz.NewUsecase(edges, nil, nil, slog.Default()) - return tools.NewRegistry(&fakeCaller{err: errors.New(errStr)}, uc, nil, nil, nil, nil, nil, slog.Default()) -} - -// keep tunnel import alive (used by other tests in this file via Response types). -var _ = tunnel.MethodGetHostLoad - -// ---- edge repo fake (same shape as in tools/registry_test but local). ---- - -type fakeEdgeRepoAgent struct { - byName map[string]*edgemodel.Edge - byID map[uint64]*edgemodel.Edge -} - -func newFakeEdgeRepoAgent(es ...*edgemodel.Edge) *fakeEdgeRepoAgent { - r := &fakeEdgeRepoAgent{ - byName: map[string]*edgemodel.Edge{}, - byID: map[uint64]*edgemodel.Edge{}, - } - for _, e := range es { - r.byName[e.Name] = e - r.byID[e.ID] = e - } - return r -} - -func (r *fakeEdgeRepoAgent) Create(_ context.Context, _ *edgemodel.Edge) error { return nil } -func (r *fakeEdgeRepoAgent) GetByID(_ context.Context, id uint64) (*edgemodel.Edge, error) { - if e, ok := r.byID[id]; ok { - return e, nil - } - return nil, errs.ErrNotFound -} -func (r *fakeEdgeRepoAgent) GetByAccessKey(_ context.Context, _ string) (*edgemodel.Edge, error) { - return nil, errs.ErrNotFound -} -func (r *fakeEdgeRepoAgent) GetByName(_ context.Context, name string) (*edgemodel.Edge, error) { - if e, ok := r.byName[name]; ok { - return e, nil - } - return nil, errs.ErrNotFound -} -func (r *fakeEdgeRepoAgent) List(_ context.Context, _ edgebiz.ListFilter) ([]*edgemodel.Edge, error) { - out := make([]*edgemodel.Edge, 0, len(r.byID)) - for _, e := range r.byID { - out = append(out, e) - } - return out, nil -} -func (r *fakeEdgeRepoAgent) UpdateSecretHash(_ context.Context, _ uint64, _ string) error { - return nil -} -func (r *fakeEdgeRepoAgent) UpdateStatus(_ context.Context, _ uint64, _ string, _ time.Time) error { - return nil -} -func (r *fakeEdgeRepoAgent) MarkStaleOffline(_ context.Context, _ time.Time) (int64, error) { - return 0, nil -} -func (r *fakeEdgeRepoAgent) MarkRegistered(_ context.Context, _ uint64, _ time.Time) error { - return nil -} -func (r *fakeEdgeRepoAgent) UpdateRoles(_ context.Context, _ uint64, _ uint8) error { - return nil -} -func (r *fakeEdgeRepoAgent) UpdateName(_ context.Context, _ uint64, _ string) error { return nil } -func (r *fakeEdgeRepoAgent) SetDeviceID(_ context.Context, _ uint64, _ uint64) error { return nil } -func (r *fakeEdgeRepoAgent) ClearDeviceID(_ context.Context, _ uint64) error { return nil } -func (r *fakeEdgeRepoAgent) SetAgentVersion(_ context.Context, _ uint64, _ string) error { return nil } -func (r *fakeEdgeRepoAgent) Delete(_ context.Context, _ uint64) error { return nil } -func (r *fakeEdgeRepoAgent) Count(_ context.Context) (int64, error) { return 1, nil } - -// ----- tests ----- - -func createSession(t *testing.T, repo *fakeRepo, userID uint64) *model.Session { - t.Helper() - s := &model.Session{UserID: userID, Title: "t", CreatedAt: time.Now().UTC(), UpdatedAt: time.Now().UTC()} - if err := repo.CreateSession(context.Background(), s); err != nil { - t.Fatalf("CreateSession: %v", err) - } - return s -} - -func TestRun_SingleShotReplyNoTools(t *testing.T) { - repo := newFakeRepo() - sess := createSession(t, repo, 42) - reg := buildRegistry(t, tunnel.GetHostLoadResponse{CPUPct: 10}) - - script := &chatScript{ - resps: []llm.ChatResp{ - { - Assistant: llm.Message{Role: "assistant", Content: "node-a looks fine."}, - Usage: llm.Usage{PromptTokens: 5, CompletionTokens: 7, TotalTokens: 12}, - }, - }, - } - a := New(&fakeLLM{script: script}, reg, repo, Config{Model: "m", MaxIterations: 4}, slog.Default()) - - reply, err := a.Run(context.Background(), sess.ID, 42, "hi") - if err != nil { - t.Fatalf("Run: %v", err) - } - if reply.Iterations != 1 { - t.Errorf("Iterations = %d, want 1", reply.Iterations) - } - if reply.Message == nil || reply.Message.Content == nil || *reply.Message.Content != "node-a looks fine." { - t.Errorf("reply message = %+v", reply.Message) - } - if reply.Usage.TotalTokens != 12 { - t.Errorf("usage.total = %d, want 12", reply.Usage.TotalTokens) - } - if len(reply.ToolCalls) != 0 { - t.Errorf("tool calls = %d, want 0", len(reply.ToolCalls)) - } - if got := script.calls.Load(); got != 1 { - t.Errorf("llm called %d times, want 1", got) - } - // messages: user + assistant - msgs, _ := repo.ListMessages(context.Background(), sess.ID, 0) - if len(msgs) != 2 { - t.Errorf("persisted msgs = %d, want 2", len(msgs)) - } -} - -func TestRun_OneToolRound(t *testing.T) { - repo := newFakeRepo() - sess := createSession(t, repo, 42) - reg := buildRegistry(t, tunnel.GetHostLoadResponse{CPUPct: 77, MemPct: 44}) - - script := &chatScript{ - resps: []llm.ChatResp{ - { - Assistant: llm.Message{ - Role: "assistant", - ToolCalls: []llm.ToolCall{{ - ID: "call_1", Name: "get_host_load", - Args: json.RawMessage(`{"edge_name":"node-a"}`), - }}, - }, - Usage: llm.Usage{PromptTokens: 10, CompletionTokens: 2, TotalTokens: 12}, - }, - { - Assistant: llm.Message{Role: "assistant", Content: "cpu is 77%."}, - Usage: llm.Usage{PromptTokens: 15, CompletionTokens: 4, TotalTokens: 19}, - }, - }, - } - a := New(&fakeLLM{script: script}, reg, repo, Config{Model: "m", MaxIterations: 4}, slog.Default()) - - reply, err := a.Run(context.Background(), sess.ID, 42, "how is node-a?") - if err != nil { - t.Fatalf("Run: %v", err) - } - if reply.Iterations != 2 { - t.Errorf("Iterations = %d, want 2", reply.Iterations) - } - if len(reply.ToolCalls) != 1 { - t.Fatalf("tool calls = %d, want 1", len(reply.ToolCalls)) - } - if reply.ToolCalls[0].Status != model.StatusSuccess { - t.Errorf("tool call status = %q, want success", reply.ToolCalls[0].Status) - } - if reply.ToolCalls[0].DeviceID == nil || *reply.ToolCalls[0].DeviceID != 1 { - t.Errorf("tool call edge id = %v, want *1", reply.ToolCalls[0].DeviceID) - } - // Total usage sums both chat calls. - if reply.Usage.TotalTokens != 31 { - t.Errorf("total tokens = %d, want 31", reply.Usage.TotalTokens) - } - - // Persisted messages: user + assistant(tool_call) + tool(result) + assistant(final) = 4 - msgs, _ := repo.ListMessages(context.Background(), sess.ID, 0) - if len(msgs) != 4 { - t.Errorf("persisted msgs = %d, want 4", len(msgs)) - } -} - -func TestRun_MaxIterations(t *testing.T) { - repo := newFakeRepo() - sess := createSession(t, repo, 1) - reg := buildRegistry(t, tunnel.GetHostLoadResponse{}) - - // Script that always emits a tool call, never a final reply. - resp := llm.ChatResp{ - Assistant: llm.Message{Role: "assistant", ToolCalls: []llm.ToolCall{{ - ID: "call_x", Name: "get_host_load", - Args: json.RawMessage(`{"edge_name":"node-a"}`), - }}}, - } - script := &chatScript{resps: []llm.ChatResp{resp, resp, resp, resp, resp}} - a := New(&fakeLLM{script: script}, reg, repo, Config{Model: "m", MaxIterations: 3}, slog.Default()) - - reply, err := a.Run(context.Background(), sess.ID, 1, "go") - if err != nil { - t.Fatalf("Run returned err: %v", err) - } - if reply == nil || reply.Message == nil { - t.Fatalf("expected reply with apology message") - } - if reply.Iterations != 3 { - t.Errorf("iterations = %d, want 3", reply.Iterations) - } - if !strings.Contains(*reply.Message.Content, "尝试了") { - t.Errorf("apology missing zh hint, got %q", *reply.Message.Content) - } - if got := script.calls.Load(); got != 3 { - t.Errorf("llm calls = %d, want 3", got) - } -} - -func TestRun_ToolErrorFeedsIntoModel(t *testing.T) { - repo := newFakeRepo() - sess := createSession(t, repo, 1) - - // Stub frontier resolves every reverse call with errStr; the registry - // will wrap it and the agent should persist status=error and hand back - // a `{"error":...}` tool msg. - reg := buildRegistryWithErr(t, "tunnel blew up") - - script := &chatScript{ - resps: []llm.ChatResp{ - { - Assistant: llm.Message{Role: "assistant", ToolCalls: []llm.ToolCall{{ - ID: "call_e", Name: "get_host_load", - Args: json.RawMessage(`{"edge_name":"node-a"}`), - }}}, - }, - {Assistant: llm.Message{Role: "assistant", Content: "sorry, tool blew up"}}, - }, - } - a := New(&fakeLLM{script: script}, reg, repo, Config{Model: "m", MaxIterations: 4}, slog.Default()) - - reply, err := a.Run(context.Background(), sess.ID, 1, "go") - if err != nil { - t.Fatalf("Run: %v", err) - } - if len(reply.ToolCalls) != 1 || reply.ToolCalls[0].Status != model.StatusError { - t.Errorf("tool call = %+v, want status=error", reply.ToolCalls[0]) - } - msgs, _ := repo.ListMessages(context.Background(), sess.ID, 0) - // user + asst(tool) + tool(error) + asst(final) = 4 - if len(msgs) != 4 { - t.Errorf("msgs = %d, want 4", len(msgs)) - } - // Tool message content should contain error key. - var toolMsg *model.Message - for _, m := range msgs { - if m.Role == model.RoleTool { - toolMsg = m - break - } - } - if toolMsg == nil || toolMsg.Content == nil { - t.Fatal("tool msg not persisted") - } - var parsed map[string]string - if err := json.Unmarshal([]byte(*toolMsg.Content), &parsed); err != nil { - t.Fatalf("tool msg not JSON: %v", err) - } - if _, ok := parsed["error"]; !ok { - t.Errorf("tool msg lacks error key: %q", *toolMsg.Content) - } -} - -func TestRun_SessionOwnershipEnforced(t *testing.T) { - repo := newFakeRepo() - sess := createSession(t, repo, 42) - reg := buildRegistry(t, tunnel.GetHostLoadResponse{}) - a := New(&fakeLLM{script: &chatScript{}}, reg, repo, Config{Model: "m", MaxIterations: 2}, slog.Default()) - - // Caller user id != session owner -> ErrNotFound (no info leak). - _, err := a.Run(context.Background(), sess.ID, 43, "hi") - if !errors.Is(err, errs.ErrNotFound) { - t.Errorf("want ErrNotFound for non-owner, got %v", err) - } -} diff --git a/internal/manager/biz/aiops/agent/build_messages_test.go b/internal/manager/biz/aiops/agent/build_messages_test.go deleted file mode 100644 index 921ddad9f..000000000 --- a/internal/manager/biz/aiops/agent/build_messages_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package agent - -import ( - "encoding/json" - "testing" - - "github.com/ongridio/ongrid/internal/manager/model/aiops" -) - -func sp(s string) *string { return &s } - -// TestBuildMessages_ToolCallReplay covers the v0.7.170 fix: -// assistant turns with content=NULL but populated ToolCalls must -// replay as {role:assistant, content:"", tool_calls:[...]} so the -// following role=tool messages remain paired. DeepSeek v4+ rejected -// orphan tool messages with HTTP 400; OpenAI silently tolerated them -// pre-fix. -func TestBuildMessages_ToolCallReplay(t *testing.T) { - a := &Agent{cfg: Config{SystemPrompt: "S"}} - history := []*aiops.Message{ - {ID: "u1", Role: aiops.RoleUser, Content: sp("你好")}, - {ID: "a1", Role: aiops.RoleAssistant, Content: sp("你好,有什么可以帮你")}, - {ID: "u2", Role: aiops.RoleUser, Content: sp("查一下机器")}, - { - ID: "a2", Role: aiops.RoleAssistant, Content: nil, - ToolCalls: []aiops.ToolCall{ - {ToolName: "query_devices", ArgumentsJSON: `{}`, LLMCallID: sp("call_abc")}, - }, - }, - {ID: "t1", Role: aiops.RoleTool, Content: sp(`{"devices":[]}`), ToolCallID: sp("call_abc"), ToolName: sp("query_devices")}, - {ID: "a3", Role: aiops.RoleAssistant, Content: sp("你这边没有注册的设备")}, - {ID: "u3", Role: aiops.RoleUser, Content: sp("继续")}, - } - out := a.buildMessages(history) - // system + all 7 history rows (a2 now emits with tool_calls) - if len(out) != 8 { - t.Fatalf("len(out) = %d, want 8", len(out)) - } - if out[0].Role != "system" { - t.Errorf("out[0].Role = %q, want system", out[0].Role) - } - asst := out[4] - if asst.Role != aiops.RoleAssistant { - t.Fatalf("out[4].Role = %q, want assistant", asst.Role) - } - if asst.Content != "" { - t.Errorf("assistant tool-call turn content = %q, want empty", asst.Content) - } - if len(asst.ToolCalls) != 1 || asst.ToolCalls[0].ID != "call_abc" { - t.Errorf("assistant.ToolCalls = %+v, want one with id=call_abc", asst.ToolCalls) - } - if string(asst.ToolCalls[0].Args) != `{}` { - t.Errorf("assistant.ToolCalls[0].Args = %q, want {}", string(asst.ToolCalls[0].Args)) - } - tool := out[5] - if tool.Role != aiops.RoleTool || tool.ToolCallID != "call_abc" { - t.Errorf("tool message position/id wrong: %+v", tool) - } -} - -// TestBuildMessages_ToolCallReplay_BackcompatPairByOrder covers -// rows written before chat_tool_calls.llm_call_id existed: the call -// id is recoverable from the following role=tool message's -// tool_call_id column. -func TestBuildMessages_ToolCallReplay_BackcompatPairByOrder(t *testing.T) { - a := &Agent{cfg: Config{}} - history := []*aiops.Message{ - {ID: "u1", Role: aiops.RoleUser, Content: sp("查一下")}, - { - ID: "a1", Role: aiops.RoleAssistant, Content: nil, - ToolCalls: []aiops.ToolCall{ - {ToolName: "tool_a", ArgumentsJSON: `{"x":1}`}, // LLMCallID nil — legacy - {ToolName: "tool_b", ArgumentsJSON: `{"y":2}`}, - }, - }, - {ID: "t1", Role: aiops.RoleTool, Content: sp(`r1`), ToolCallID: sp("legacy_call_1"), ToolName: sp("tool_a")}, - {ID: "t2", Role: aiops.RoleTool, Content: sp(`r2`), ToolCallID: sp("legacy_call_2"), ToolName: sp("tool_b")}, - {ID: "a2", Role: aiops.RoleAssistant, Content: sp("好的")}, - } - out := a.buildMessages(history) - if len(out) != 5 { - t.Fatalf("len(out) = %d, want 5", len(out)) - } - asst := out[1] - if len(asst.ToolCalls) != 2 { - t.Fatalf("assistant.ToolCalls = %d entries, want 2", len(asst.ToolCalls)) - } - if asst.ToolCalls[0].ID != "legacy_call_1" || asst.ToolCalls[1].ID != "legacy_call_2" { - t.Errorf("pairing failed: got ids %q,%q", asst.ToolCalls[0].ID, asst.ToolCalls[1].ID) - } - if asst.ToolCalls[0].Name != "tool_a" || asst.ToolCalls[1].Name != "tool_b" { - t.Errorf("names = %q,%q", asst.ToolCalls[0].Name, asst.ToolCalls[1].Name) - } - // tool messages also kept - if out[2].ToolCallID != "legacy_call_1" || out[3].ToolCallID != "legacy_call_2" { - t.Errorf("tool order/id wrong") - } -} - -// TestBuildMessages_ToolCallReplay_UnresolvableDropsAssistantAndTools -// — if neither LLMCallID nor pairing can recover ids, drop the -// assistant + dependent tool rows so the LLM never sees an orphan -// tool. This is the pre-fix MVP behavior, kept as a safety net. -func TestBuildMessages_ToolCallReplay_UnresolvableDropsAssistantAndTools(t *testing.T) { - a := &Agent{cfg: Config{}} - history := []*aiops.Message{ - {ID: "u1", Role: aiops.RoleUser, Content: sp("hi")}, - { - ID: "a1", Role: aiops.RoleAssistant, Content: nil, - ToolCalls: []aiops.ToolCall{{ToolName: "x", ArgumentsJSON: `{}`}}, - }, - // no following role=tool — corrupt session - {ID: "u2", Role: aiops.RoleUser, Content: sp("are you there?")}, - } - out := a.buildMessages(history) - if len(out) != 2 { - t.Fatalf("len(out) = %d, want 2 (u1 + u2)", len(out)) - } - for _, m := range out { - if m.Role != aiops.RoleUser { - t.Errorf("unexpected %s in output: %+v", m.Role, m) - } - } -} - -// Smoke that ToolCalls Args JSON round-trips through json.RawMessage -// (a regression would be invalid JSON in the outgoing request body). -func TestBuildMessages_ToolCallArgsRoundtrip(t *testing.T) { - a := &Agent{cfg: Config{}} - args := `{"x":1,"nested":{"k":"v"}}` - history := []*aiops.Message{ - {ID: "u1", Role: aiops.RoleUser, Content: sp("go")}, - { - ID: "a1", Role: aiops.RoleAssistant, Content: nil, - ToolCalls: []aiops.ToolCall{ - {ToolName: "n", ArgumentsJSON: args, LLMCallID: sp("c1")}, - }, - }, - {ID: "t1", Role: aiops.RoleTool, Content: sp(`{}`), ToolCallID: sp("c1"), ToolName: sp("n")}, - } - out := a.buildMessages(history) - if len(out[1].ToolCalls) != 1 { - t.Fatalf("missing tool_call") - } - var probe any - if err := json.Unmarshal(out[1].ToolCalls[0].Args, &probe); err != nil { - t.Fatalf("args not valid JSON: %v (raw=%q)", err, string(out[1].ToolCalls[0].Args)) - } -} diff --git a/internal/manager/biz/aiops/agent/legacy_k8s_action_test.go b/internal/manager/biz/aiops/agent/legacy_k8s_action_test.go deleted file mode 100644 index a9faf2fda..000000000 --- a/internal/manager/biz/aiops/agent/legacy_k8s_action_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package agent - -import ( - "context" - "encoding/json" - "testing" - "time" - - "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" -) - -func TestLegacyToolTimeoutAllowsHumanApprovalWait(t *testing.T) { - defaultTimeout := 15 * time.Second - tests := []struct { - name string - tool string - args json.RawMessage - want time.Duration - }{ - {name: "ordinary tool", tool: "query_devices", args: json.RawMessage(`{}`), want: defaultTimeout}, - {name: "k8s dry-run", tool: tools.ToolNameExecuteK8sAction, args: json.RawMessage(`{"dry_run":true}`), want: legacyK8sDryRunTimeout}, - {name: "k8s drain dry-run default timeout", tool: tools.ToolNameExecuteK8sAction, args: json.RawMessage(`{"action":"drain","dry_run":true}`), want: legacyK8sDefaultDrainTimeout + legacyK8sDryRunDrainTimeoutPadding}, - {name: "k8s drain dry-run custom timeout", tool: tools.ToolNameExecuteK8sAction, args: json.RawMessage(`{"action":"drain","dry_run":true,"drain_timeout_seconds":300}`), want: 330 * time.Second}, - {name: "k8s approval", tool: tools.ToolNameExecuteK8sAction, args: json.RawMessage(`{"dry_run":false}`), want: legacyApprovalToolTimeout}, - {name: "k8s implicit write", tool: tools.ToolNameExecuteK8sAction, args: json.RawMessage(`{}`), want: legacyApprovalToolTimeout}, - {name: "malformed args", tool: tools.ToolNameExecuteK8sAction, args: json.RawMessage(`{`), want: defaultTimeout}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := legacyToolTimeout(defaultTimeout, tt.tool, tt.args); got != tt.want { - t.Fatalf("legacyToolTimeout()=%s want %s", got, tt.want) - } - }) - } -} - -func TestLegacyEmitIsAvailableToApprovalAdapter(t *testing.T) { - want := Event{Type: EventApprovalPending, Approval: &ApprovalPendingEvent{ApprovalID: "approval-1"}} - var got Event - ctx := withEmit(context.Background(), func(event Event) { got = event }) - emit := EmitFromContext(ctx) - if emit == nil { - t.Fatal("EmitFromContext returned nil") - } - emit(want) - if got.Type != EventApprovalPending || got.Approval == nil || got.Approval.ApprovalID != "approval-1" { - t.Fatalf("unexpected event: %+v", got) - } -} diff --git a/internal/manager/biz/aiops/chatprotocol/types.go b/internal/manager/biz/aiops/chatprotocol/types.go index deb02f700..cc289df5b 100644 --- a/internal/manager/biz/aiops/chatprotocol/types.go +++ b/internal/manager/biz/aiops/chatprotocol/types.go @@ -32,16 +32,12 @@ const ( EventToolEnd EventType = "tool_end" // EventDone fires once at terminal success with the final Reply. EventDone EventType = "done" - // EventTaskNotification fires when a background sub-agent worker - // reaches a terminal state. The legacy agent itself - // never spawns workers, so this constant exists purely so the SSE - // layer can carry the chatruntime kernel's task_notification frame - // through the same eventName / eventPayload translation as the rest. + // EventTaskNotification carries completed background-worker events. EventTaskNotification EventType = "task_notification" // EventApprovalPending fires when a synchronous-blocking tool // (HLD-021, e.g. execute_k8s_action) queues a human-approval proposal and is // about to block on the decision. Carries the live approve/reject card - // payload. Both kernels translate this shape through the same SSE frame. + // payload. The runtime adapter preserves the existing SSE frame. EventApprovalPending EventType = "approval_pending" ) diff --git a/internal/manager/biz/imbridge/sender.go b/internal/manager/biz/imbridge/sender.go index 300c103f0..0c2283e6e 100644 --- a/internal/manager/biz/imbridge/sender.go +++ b/internal/manager/biz/imbridge/sender.go @@ -164,7 +164,7 @@ func (e *streamEditor) flush() error { // can just take it. func assistantText(ev chatprotocol.Event) string { // chatprotocol.Event.Assistant is set on assistant events — see - // internal/manager/biz/aiops/agent/chatprotocol.go. We avoid a direct + // internal/manager/biz/aiops/chatprotocol/types.go. We avoid a direct // type assertion on the field shape to stay forward-compatible: // if the runtime adds richer payloads (citations, attachments) // they live on the same Event struct. diff --git a/internal/manager/service/aiops/service.go b/internal/manager/service/aiops/service.go index bb8f2734d..63fb6fd96 100644 --- a/internal/manager/service/aiops/service.go +++ b/internal/manager/service/aiops/service.go @@ -7,23 +7,12 @@ // existence. Admins bypass the ownership check — the handler passes the // caller's role through, and the service re-reads the session to pick the // owning user_id for the agent call. -// -// PR-9 of introduces the kernel switch. The service holds: -// -// - legacyAgent: the pre-PR-9 agent.Agent for-loop kernel. -// - runtime: the new chatruntime.Runtime graph kernel. -// - kernel: "legacy" | "graph" — picks which kernel runs. -// -// Default = "legacy" so the cutover is opt-in via ONGRID_AGENT_KERNEL. -// The HTTP handler is unchanged: the SSE frame names emitted by both -// kernels are byte-equal so the SPA round-trips without changes. package aiops import ( "context" "encoding/json" "fmt" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" "io" "log/slog" "reflect" @@ -48,31 +37,6 @@ const ( RoleViewer = "viewer" ) -// Kernel enumerates the two agent kernels the service can dispatch -// to. The graph runtime is the default; legacy remains an explicit rollback. -type Kernel string - -const ( - // KernelLegacy is the pre-PR-9 agent.Agent for-loop (chatprotocol.go). - KernelLegacy Kernel = "legacy" - // KernelGraph is the new eino + chatruntime + graph kernel - // (PR-1..PR-7 of). - KernelGraph Kernel = "graph" -) - -// ParseKernel normalises a string env value into a Kernel. Empty or -// unrecognised values default to KernelGraph. Used by cmd/ongrid/main.go. -func ParseKernel(s string) Kernel { - switch strings.ToLower(strings.TrimSpace(s)) { - case "legacy": - return KernelLegacy - case "graph": - return KernelGraph - default: - return KernelGraph - } -} - // Service bundles the agent + session repo. Handlers call into it with the // caller's user-id + role; ownership enforcement lives here. type APMSourceResolver interface { @@ -81,9 +45,7 @@ type APMSourceResolver interface { type Service struct { apmSource APMSourceResolver - legacyAgent *agent.Agent runtime RuntimeHandler - kernel Kernel sessions biz.SessionRepo attachments biz.AttachmentRepo modelCatalog modelCapabilityCatalog @@ -93,7 +55,7 @@ type Service struct { // cancels maps an in-flight turn's session id to its cancel func, so an // explicit user "stop" (Esc) can interrupt the turn. This is needed - // BECAUSE runWithKernel detaches the turn from the HTTP request ctx + // BECAUSE runChat detaches the turn from the HTTP request ctx // (WithoutCancel, so a refresh doesn't kill it, HLD-021) — passive // disconnect no longer cancels, so stopping requires an explicit signal. cancelMu sync.Mutex @@ -108,37 +70,13 @@ type RuntimeHandler interface { Handle(ctx context.Context, req *chatruntime.Request) (*chatruntime.Reply, error) } -// New builds the Service. runtime + kernel may be zero-valued; in -// that case the legacy kernel is the only path. NewWithKernel is the -// kernel-aware constructor introduced by PR-9 of -func New(a *agent.Agent, sessions biz.SessionRepo, usage *biz.UsageUsecase, log *slog.Logger) *Service { - return NewWithKernel(a, nil, KernelLegacy, sessions, usage, log) -} - -// NewWithKernel is the kernel-aware constructor. When kernel == graph -// AND runtime != nil, every chat-send path runs through -// chatruntime.Runtime; otherwise the legacy agent.Agent for-loop is -// used. Mismatched configurations (kernel=graph but runtime=nil) fall -// back to legacy with a logger warning at first PostMessage call. -func NewWithKernel(a *agent.Agent, runtime RuntimeHandler, kernel Kernel, sessions biz.SessionRepo, usage *biz.UsageUsecase, log *slog.Logger) *Service { - return &Service{ - legacyAgent: a, - runtime: runtime, - kernel: kernel, - sessions: sessions, - usage: usage, - log: log, - cancels: map[string]context.CancelFunc{}, - } +// New 构造唯一 Graph 聊天入口；runtime 不可用时调用明确失败，不回退旧循环。 +func New(runtime RuntimeHandler, sessions biz.SessionRepo, usage *biz.UsageUsecase, log *slog.Logger) *Service { + return &Service{runtime: runtime, sessions: sessions, usage: usage, log: log, cancels: map[string]context.CancelFunc{}} } -// Kernel returns the active kernel. Exposed so cmd/ongrid/main.go can -// log the resolved value at boot. -func (s *Service) Kernel() Kernel { return s.kernel } - // SetMutatingProposalRepo wires the ReviewGate proposal audit reader after -// the repo has been constructed in cmd. It is optional so legacy tests and -// deployments without graph ReviewGate keep starting; the HTTP endpoint fails +// the repo has been constructed in cmd. Without the dependency the HTTP endpoint fails // closed with ErrNotWiredYet until wired. func (s *Service) SetMutatingProposalRepo(repo biz.MutatingProposalRepo) { s.proposals = repo @@ -396,7 +334,7 @@ func (s *Service) PostMessage(ctx context.Context, caller Caller, sessionID stri // PostMessageWithOpts is the override-aware sibling of PostMessage. Per- // call provider/model + mentions flow into the agent unchanged. func (s *Service) PostMessageWithOpts(ctx context.Context, caller Caller, sessionID string, content string, opts chatprotocol.RunOptions) (*chatprotocol.Reply, error) { - return s.runWithKernel(ctx, caller, sessionID, content, nil, opts) + return s.runChat(ctx, caller, sessionID, content, nil, opts) } // PostMessageStream is the SSE variant of PostMessage. emit fires once per @@ -411,14 +349,11 @@ func (s *Service) PostMessageStream(ctx context.Context, caller Caller, sessionI // provider/model + mention overrides. Empty opts behaves identically to // PostMessageStream. func (s *Service) PostMessageStreamWithOpts(ctx context.Context, caller Caller, sessionID string, content string, emit chatprotocol.Emit, opts chatprotocol.RunOptions) (*chatprotocol.Reply, error) { - return s.runWithKernel(ctx, caller, sessionID, content, emit, opts) + return s.runChat(ctx, caller, sessionID, content, emit, opts) } -// runWithKernel is the single chokepoint that decides which kernel -// runs for this request. Both legacy and graph paths return the same -// chatprotocol.Reply DTO so the HTTP layer doesn't care which path served -// the response. -func (s *Service) runWithKernel(ctx context.Context, caller Caller, sessionID string, content string, emit chatprotocol.Emit, opts chatprotocol.RunOptions) (*chatprotocol.Reply, error) { +// runChat 是聊天请求的统一入口，完成鉴权、取消管理后调用 Graph runtime。 +func (s *Service) runChat(ctx context.Context, caller Caller, sessionID string, content string, emit chatprotocol.Emit, opts chatprotocol.RunOptions) (*chatprotocol.Reply, error) { content = strings.TrimSpace(content) if content == "" && len(opts.Attachments) == 0 { return nil, fmt.Errorf("%w: content required", errs.ErrInvalid) @@ -474,20 +409,10 @@ func (s *Service) runWithKernel(ctx context.Context, caller Caller, sessionID st s.registerCancel(sess.ID, cancel) defer s.unregisterCancel(sess.ID, cancel) - // Graph kernel — only when explicitly enabled AND wired. - if s.kernel == KernelGraph && s.runtime != nil { - return s.runGraph(ctx, sess, content, emit, opts) - } - // Legacy fallback. Logs once if kernel=graph but runtime is nil - // — ops misconfig that we want to be visible. - if s.kernel == KernelGraph && s.runtime == nil && s.log != nil { - s.log.Warn("aiops kernel=graph but runtime is nil — falling back to legacy agent", - slog.String("session_id", sess.ID)) - } - if s.legacyAgent == nil { - return nil, errs.ErrNotWiredYet + if s.runtime == nil { + return nil, fmt.Errorf("chat runtime unavailable: %w", errs.ErrNotWiredYet) } - return s.legacyAgent.RunStreamWithOpts(ctx, sessionID, sess.UserID, content, emit, opts) + return s.runGraph(ctx, sess, content, emit, opts) } func (s *Service) validateImageContext(ctx context.Context, sessionID string, opts *chatprotocol.RunOptions) error { @@ -674,7 +599,7 @@ func translateRuntimeEvent(ev chatruntime.Event) chatprotocol.Event { return out } -// translateMentionsToRuntime copies the legacy chatprotocol.Mention shape +// translateMentionsToRuntime copies the shared chatprotocol.Mention shape // into the chatruntime.Mention shape. One alloc per turn — fine. func translateMentionsToRuntime(in []chatprotocol.Mention) []chatruntime.Mention { if len(in) == 0 { @@ -688,7 +613,7 @@ func translateMentionsToRuntime(in []chatprotocol.Mention) []chatruntime.Mention } // runtimeReplyToAgentReply translates the graph kernel's Reply back -// into the legacy chatprotocol.Reply shape so the HTTP handler's DTO +// into the shared chatprotocol.Reply shape so the HTTP handler's DTO // transformer (toPostMessageResp) doesn't care which kernel produced // the answer. func runtimeReplyToAgentReply(r *chatruntime.Reply) *chatprotocol.Reply { diff --git a/internal/manager/service/aiops/service_kernel_test.go b/internal/manager/service/aiops/service_kernel_test.go index 00b987d20..ee6c90aed 100644 --- a/internal/manager/service/aiops/service_kernel_test.go +++ b/internal/manager/service/aiops/service_kernel_test.go @@ -119,57 +119,7 @@ func newSeededSessions(uid uint64) (*memSessions, string) { return store, id } -// TestParseKernel_Defaults exercises the env→Kernel parsing. -// -// unset / empty / garbage → legacy -// "graph" → graph -func TestParseKernel_Defaults(t *testing.T) { - cases := []struct { - in string - want Kernel - }{ - {"", KernelGraph}, - {"legacy", KernelLegacy}, - {"GRAPH", KernelGraph}, - {" graph ", KernelGraph}, - {"garbage", KernelGraph}, - } - for _, c := range cases { - if got := ParseKernel(c.in); got != c.want { - t.Errorf("ParseKernel(%q) = %q, want %q", c.in, got, c.want) - } - } -} - -// TestService_KernelLegacy_DispatchesToAgent verifies the default -// kernel routes through the legacy agent.Agent path. We can't easily -// stand up a real agent here (it needs an LLM client + tools.Registry) -// — instead, the test confirms the runtime is NOT consulted and the -// legacy path runs (errors with ErrNotWiredYet because we pass a nil -// agent on purpose). -func TestService_KernelLegacy_DispatchesToAgent(t *testing.T) { - store, sid := newSeededSessions(7) - rt := &fakeRuntime{} - svc := NewWithKernel(nil, rt, KernelLegacy, store, nil, nil) - - caller := Caller{UserID: 7} - _, err := svc.PostMessage(context.Background(), caller, sid, "hello") - // Legacy path is not wired (nil agent) — we expect a panic-free - // failure that is NOT runtime.Handle (rt.calls must stay 0). - if rt.calls.Load() != 0 { - t.Errorf("legacy kernel must NOT consult runtime, got %d calls", rt.calls.Load()) - } - if err == nil { - t.Errorf("expected error from nil legacy agent path") - } -} - -// TestService_KernelGraph_DispatchesToRuntime asserts that when -// kernel=graph and runtime != nil, every chat-send path goes through -// runtime.Handle. The fakeRuntime emits one of each event; we -// translate them to legacy frame names through the service's SSE -// adapter and assert the names + sequence. -func TestService_KernelGraph_DispatchesToRuntime(t *testing.T) { +func TestService_Graph_DispatchesToRuntime(t *testing.T) { store, sid := newSeededSessions(7) body := "all done" rt := &fakeRuntime{ @@ -184,7 +134,7 @@ func TestService_KernelGraph_DispatchesToRuntime(t *testing.T) { Iterations: 2, }, } - svc := NewWithKernel(nil, rt, KernelGraph, store, nil, nil) + svc := New(rt, store, nil, nil) var ( evtMu sync.Mutex @@ -238,30 +188,27 @@ func TestService_KernelGraph_DispatchesToRuntime(t *testing.T) { } } -// TestService_KernelGraph_NilRuntime_FallsBackToLegacy guards +// TestService_MissingRuntimeFailsExplicitly guards // against the misconfig where ONGRID_AGENT_KERNEL=graph but the // runtime fails to build. We must fall back to legacy; the test // confirms runtime is never called. -func TestService_KernelGraph_NilRuntime_FallsBackToLegacy(t *testing.T) { +// 运行时不可用时应返回稳定错误，不能隐式执行另一套循环。 +func TestService_MissingRuntimeFailsExplicitly(t *testing.T) { store, sid := newSeededSessions(7) - svc := NewWithKernel(nil, nil, KernelGraph, store, nil, nil) - caller := Caller{UserID: 7} - _, err := svc.PostMessage(context.Background(), caller, sid, "hello") - if err == nil { - t.Errorf("expected error from nil legacy agent path with graph kernel + nil runtime") + svc := New(nil, store, nil, nil) + _, err := svc.PostMessage(context.Background(), Caller{UserID: 7}, sid, "hello") + if !errors.Is(err, errs.ErrNotWiredYet) { + t.Fatalf("error=%v, want runtime unavailable", err) } - // Defensive: a fakeRuntime that's nil on the service AND a - // legacy agent that's nil yields a panic-free error path. - _ = errors.Is(err, errs.ErrNotWiredYet) // tolerate either err shape } -// TestService_KernelGraph_Ownership respects the owner check in +// TestService_Graph_Ownership respects the owner check in // the GetSession indirection — non-owner gets ErrNotFound BEFORE the // runtime is consulted. -func TestService_KernelGraph_Ownership(t *testing.T) { +func TestService_Graph_Ownership(t *testing.T) { store, sid := newSeededSessions(7) rt := &fakeRuntime{} - svc := NewWithKernel(nil, rt, KernelGraph, store, nil, nil) + svc := New(rt, store, nil, nil) caller := Caller{UserID: 99} _, err := svc.PostMessage(context.Background(), caller, sid, "hello") diff --git a/internal/manager/service/aiops/service_mutating_proposal_test.go b/internal/manager/service/aiops/service_mutating_proposal_test.go index 74f245ba2..b0f65943c 100644 --- a/internal/manager/service/aiops/service_mutating_proposal_test.go +++ b/internal/manager/service/aiops/service_mutating_proposal_test.go @@ -30,7 +30,7 @@ func (f *fakeMutatingProposalRepo) CountMutatingProposals(_ context.Context, fil } func TestServiceListMutatingProposalsRequiresAdmin(t *testing.T) { - s := NewWithKernel(nil, nil, KernelLegacy, nil, nil, nil) + s := New(nil, nil, nil, nil) s.SetMutatingProposalRepo(&fakeMutatingProposalRepo{}) _, _, err := s.ListMutatingProposals(context.Background(), Caller{UserID: 2, Role: "user"}, biz.MutatingProposalFilter{}) @@ -44,7 +44,7 @@ func TestServiceListMutatingProposalsNormalizesFilter(t *testing.T) { items: []*model.MutatingProposal{{ID: "proposal-1", ToolName: "execute_k8s_action"}}, total: 1, } - s := NewWithKernel(nil, nil, KernelLegacy, nil, nil, nil) + s := New(nil, nil, nil, nil) s.SetMutatingProposalRepo(repo) items, total, err := s.ListMutatingProposals(context.Background(), Caller{UserID: 1, Role: "admin"}, biz.MutatingProposalFilter{ From ad63664ff30ab395bb53d74bb959532e88949790 Mon Sep 17 00:00:00 2001 From: singchia Date: Wed, 14 Oct 2026 20:54:09 +0800 Subject: [PATCH 05/24] fix(aiops): preserve skill requirements and migrate incident prechecks --- .../biz/aiops/investigator/investigator.go | 25 ++++- .../aiops/investigator/investigator_test.go | 93 ++++++++++++++++--- .../biz/aiops/tools/skill_basetool_test.go | 36 +++++++ .../manager/biz/aiops/tools/skill_bridge.go | 14 +-- 4 files changed, 145 insertions(+), 23 deletions(-) diff --git a/internal/manager/biz/aiops/investigator/investigator.go b/internal/manager/biz/aiops/investigator/investigator.go index 60fbda118..cff9ea5d3 100644 --- a/internal/manager/biz/aiops/investigator/investigator.go +++ b/internal/manager/biz/aiops/investigator/investigator.go @@ -296,8 +296,8 @@ func (i *Investigator) runOne(incident *model.Incident) { // going through the chat session abstraction) and returns its raw JSON // result. Failures are surfaced so runOne can log + abort. func (i *Investigator) gatherBundle(ctx context.Context, incidentID uint64) ([]byte, error) { - args, err := json.Marshal(map[string]any{ - "incident_id": incidentID, + args, err := json.Marshal(aiopstools.CorrelateIncidentBatchArgs{ + IncidentIDs: []uint64{incidentID}, }) if err != nil { return nil, fmt.Errorf("marshal correlate args: %w", err) @@ -306,10 +306,25 @@ func (i *Investigator) gatherBundle(ctx context.Context, incidentID uint64) ([]b if err != nil { return nil, fmt.Errorf("invoke %s: %w", aiopstools.ToolNameCorrelateIncident, err) } - if len(res.ResultJSON) == 0 { - return nil, errors.New("correlate_incident returned empty result") + var batch aiopstools.CorrelateIncidentBatchResponse + if err := json.Unmarshal(res.ResultJSON, &batch); err != nil { + return nil, fmt.Errorf("decode correlate_incident batch: %w", err) } - return []byte(res.ResultJSON), nil + if len(batch.Results) != 1 || batch.Results[0].IncidentID != incidentID { + return nil, errors.New("correlate_incident returned unexpected incident results") + } + entry := batch.Results[0] + if entry.Error != "" { + return nil, fmt.Errorf("correlate_incident incident %d: %s", incidentID, entry.Error) + } + if entry.Bundle == nil { + return nil, errors.New("correlate_incident returned empty bundle") + } + bundle, err := json.Marshal(entry.Bundle) + if err != nil { + return nil, fmt.Errorf("marshal incident bundle: %w", err) + } + return bundle, nil } // capUserMessage trims the bundle JSON if it exceeds the configured diff --git a/internal/manager/biz/aiops/investigator/investigator_test.go b/internal/manager/biz/aiops/investigator/investigator_test.go index 774318d7c..9b06fa245 100644 --- a/internal/manager/biz/aiops/investigator/investigator_test.go +++ b/internal/manager/biz/aiops/investigator/investigator_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "log/slog" "strings" @@ -13,6 +14,8 @@ import ( "time" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + alertbiz "github.com/ongridio/ongrid/internal/manager/biz/alert" model "github.com/ongridio/ongrid/internal/manager/model/alert" "github.com/ongridio/ongrid/internal/pkg/llm" ) @@ -52,12 +55,13 @@ func (f *fakeLLM) Chat(ctx context.Context, req llm.ChatReq) (*llm.ChatResp, err // fakeTools stubs ToolInvoker; returns a canned bundle. type fakeTools struct { - mu sync.Mutex - bundle json.RawMessage - err error - gotName string - gotArgs json.RawMessage - callCount int + mu sync.Mutex + bundle json.RawMessage + rawResponse json.RawMessage + err error + gotName string + gotArgs json.RawMessage + callCount int } func (f *fakeTools) Invoke(_ context.Context, name string, args json.RawMessage) (aiopstools.ExecuteResult, error) { @@ -69,7 +73,18 @@ func (f *fakeTools) Invoke(_ context.Context, name string, args json.RawMessage) if f.err != nil { return aiopstools.ExecuteResult{}, f.err } - return aiopstools.ExecuteResult{ResultJSON: f.bundle}, nil + if f.rawResponse != nil { + return aiopstools.ExecuteResult{ResultJSON: f.rawResponse}, nil + } + var input aiopstools.CorrelateIncidentBatchArgs + if err := json.Unmarshal(args, &input); err != nil { + return aiopstools.ExecuteResult{}, err + } + if len(input.IncidentIDs) != 1 { + return aiopstools.ExecuteResult{}, errors.New("expected one incident_ids entry") + } + response := fmt.Sprintf(`{"success_count":1,"results":[{"incident_id":%d,"bundle":%s}]}`, input.IncidentIDs[0], f.bundle) + return aiopstools.ExecuteResult{ResultJSON: json.RawMessage(response)}, nil } // fakeWriter captures CreateEvent calls. @@ -153,10 +168,12 @@ func TestInvestigateAsyncWritesEvent(t *testing.T) { if toolsFake.gotName != aiopstools.ToolNameCorrelateIncident { t.Errorf("tool name=%q, want %q", toolsFake.gotName, aiopstools.ToolNameCorrelateIncident) } - var argMap map[string]any - _ = json.Unmarshal(toolsFake.gotArgs, &argMap) - if argMap["incident_id"] != float64(7) { - t.Errorf("expected incident_id=7 in args, got %v", argMap["incident_id"]) + var input aiopstools.CorrelateIncidentBatchArgs + if err := json.Unmarshal(toolsFake.gotArgs, &input); err != nil { + t.Fatal(err) + } + if len(input.IncidentIDs) != 1 || input.IncidentIDs[0] != 7 { + t.Errorf("expected incident_ids=[7], got %v", input.IncidentIDs) } // LLM round-trip sanity: system prompt + user bundle. @@ -324,3 +341,57 @@ func TestConcurrencyMultipleWorkers(t *testing.T) { t.Errorf("expected %d events, got %d", enqueued.Load(), got) } } + +func TestInvestigatorRejectsFailedOrMismatchedBatch(t *testing.T) { + for _, response := range []string{ + `{"error_count":1,"results":[{"incident_id":7,"error":"source unavailable"}]}`, + `{"results":[{"incident_id":8,"bundle":{}}]}`, + `{"results":[{"incident_id":7}]}`, + `{}`, `not json`, + } { + t.Run(response, func(t *testing.T) { + llmStub := &fakeLLM{content: "must not diagnose a missing bundle"} + writer := &fakeWriter{} + inv := New(llmStub, &fakeTools{rawResponse: json.RawMessage(response)}, writer, Config{Workers: 1}, newQuietLogger()) + inv.InvestigateAsync(&model.Incident{ID: 7}) + inv.Close() + if llmStub.calls != 0 || len(writer.snapshot()) != 0 { + t.Fatal("invalid bundle produced a diagnosis") + } + }) + } +} + +// 真实 BaseTool 契约回归：防止 fake 接受旧参数而掩盖运行时迁移错误。 +type realCorrelationInvoker struct{ tool basetool.BaseTool } + +func (i realCorrelationInvoker) Invoke(ctx context.Context, name string, args json.RawMessage) (aiopstools.ExecuteResult, error) { + if name != aiopstools.ToolNameCorrelateIncident { + return aiopstools.ExecuteResult{}, fmt.Errorf("unexpected tool %s", name) + } + result, err := i.tool.InvokableRun(ctx, string(args)) + return aiopstools.ExecuteResult{ResultJSON: json.RawMessage(result)}, err +} + +type correlationAlerts struct{ aiopstools.AlertUsecase } + +func (correlationAlerts) GetIncident(context.Context, uint64) (*model.Incident, error) { + return &model.Incident{ID: 7, Title: "batch contract incident", Rule: "cpu_high", FirstFiredAt: time.Now(), LabelsJSON: "{}", AnnotationsJSON: "{}"}, nil +} +func (correlationAlerts) ListIncidents(context.Context, alertbiz.IncidentFilter) ([]*model.Incident, error) { + return nil, nil +} +func TestInvestigatorUsesRealCorrelationToolContract(t *testing.T) { + llmStub := &fakeLLM{content: "diagnosis from real bundle"} + writer := &fakeWriter{} + tool := aiopstools.NewCorrelateIncidentTool(correlationAlerts{}, nil, nil, nil, nil, nil, newQuietLogger()) + inv := New(llmStub, realCorrelationInvoker{tool: tool}, writer, Config{Workers: 1}, newQuietLogger()) + inv.InvestigateAsync(&model.Incident{ID: 7}) + inv.Close() + if len(writer.snapshot()) != 1 || len(llmStub.gotMsgs) != 2 { + t.Fatal("real correlation did not produce a diagnosis") + } + if !strings.Contains(llmStub.gotMsgs[1].Content, "batch contract incident") { + t.Fatal("incident bundle missing from prompt") + } +} diff --git a/internal/manager/biz/aiops/tools/skill_basetool_test.go b/internal/manager/biz/aiops/tools/skill_basetool_test.go index 2bf2f82c0..c4dc77753 100644 --- a/internal/manager/biz/aiops/tools/skill_basetool_test.go +++ b/internal/manager/biz/aiops/tools/skill_basetool_test.go @@ -124,3 +124,39 @@ func writeCatalogFixture(t *testing.T, dir string) { t.Fatal(err) } } + +func TestHostSkillSchemaPreservesRequiredParameters(t *testing.T) { + for _, executor := range skillcore.AllByClass(skillcore.ClassSafe) { + if executor.Metadata().EffectiveScope() != skillcore.ScopeHost { + continue + } + t.Run(executor.Metadata().Key, func(t *testing.T) { + original, err := skillcore.BuildSchema(executor) + if err != nil { + t.Fatal(err) + } + adapted, err := buildSkillToolSchema(executor) + if err != nil { + t.Fatal(err) + } + var before, after struct { + Required []string `json:"required"` + } + if err := json.Unmarshal(original, &before); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(adapted, &after); err != nil { + t.Fatal(err) + } + counts := map[string]int{} + for _, field := range after.Required { + counts[field]++ + } + for _, field := range append(before.Required, "device_id") { + if counts[field] != 1 { + t.Errorf("required %s count=%d, schema=%s", field, counts[field], adapted) + } + } + }) + } +} diff --git a/internal/manager/biz/aiops/tools/skill_bridge.go b/internal/manager/biz/aiops/tools/skill_bridge.go index be7afac01..c3d44641c 100644 --- a/internal/manager/biz/aiops/tools/skill_bridge.go +++ b/internal/manager/biz/aiops/tools/skill_bridge.go @@ -5,13 +5,13 @@ // - Tool name = skill metadata Key (lower_snake, already unique) // - Tool description = skill Description // - Tool schema = skill.ParamSchema.ToJSONSchema() with one extra -// property `edge_id` (uint64) prepended — every -// skill execution targets a specific edge, so the -// LLM must always supply it. +// property `edge_id` (uint64) prepended — every +// skill execution targets a specific edge, so the +// LLM must always supply it. // - Tool execute = unmarshal args -> {edge_id, ...skillParams}, -// call manager/biz/skill.Service.Execute, return -// the raw result JSON (or an error envelope) so -// the LLM sees structured output. +// call manager/biz/skill.Service.Execute, return +// the raw result JSON (or an error envelope) so +// the LLM sees structured output. // // Only ClassSafe skills are registered; mutating / dangerous require a // human-in-the-loop workflow (PR-G4) before the agent can invoke them. @@ -128,7 +128,7 @@ func buildSkillToolSchema(e skillcore.Executor) (json.RawMessage, error) { "type": "integer", "description": "Device id to run this skill on (required). Same id as the @-mention chip and the Prom device_id label.", } - required, _ := base["required"].([]string) + required, _ := base["required"].([]any) hasDeviceID := false for _, name := range required { if name == "device_id" { From 322b35ab9b4deebf3fe14574fac00f783a514988 Mon Sep 17 00:00:00 2001 From: singchia Date: Wed, 14 Oct 2026 20:54:09 +0800 Subject: [PATCH 06/24] fix(aiops): validate tool updates and preserve turn schema snapshots --- cmd/ongrid/main.go | 15 +- docs/rfc/RFC-005-unified-agent-runtime.md | 10 ++ .../manager/biz/aiops/chatruntime/runtime.go | 148 +++--------------- .../biz/aiops/chatruntime/runtime_test.go | 6 +- .../biz/aiops/chatruntime/tool_lifecycle.go | 148 +++++++++++++++++- .../aiops/chatruntime/tool_lifecycle_test.go | 101 +++++++++++- .../aiops/tools/basetool/filtered_tools.go | 6 +- .../biz/aiops/tools/tool_search_tool.go | 16 ++ 8 files changed, 304 insertions(+), 146 deletions(-) diff --git a/cmd/ongrid/main.go b/cmd/ongrid/main.go index 19d887c54..fa3e92d9a 100644 --- a/cmd/ongrid/main.go +++ b/cmd/ongrid/main.go @@ -1698,7 +1698,9 @@ func main() { aiopstoolsdec.Wrap(aiopstools.NewSendMessageTool(chatruntimeSpawnerShim{rt: rt}, log), coordDepsFast), aiopstoolsdec.Wrap(aiopstools.NewTaskStopTool(chatruntimeSpawnerShim{rt: rt}, log), coordDepsFast), } - rt.AppendToolBag(wrappedCoord) + if err := rt.AppendToolBag(wrappedCoord); err != nil { + log.Error("register coordinator tools", slog.Any("err", err)) + } log.Info("aiops runtime wired", slog.Int("tool_count", rt.ToolCount()), slog.Any("tool_names", rt.ToolNames(rootCtx)), @@ -2411,14 +2413,16 @@ func main() { Registerer: reg, HumanApproval: humanApprovalBroker, } - chatRT.AppendToolBag([]aiopstoolsbase.BaseTool{ + if err := chatRT.AppendToolBag([]aiopstoolsbase.BaseTool{ aiopstoolsdec.Wrap(aiopstools.NewBashToolWithProposer(fbClient, edgeUC, deviceUC, hostBashProposerShim{uc: approvalUC}, log), cbDeps), aiopstoolsdec.Wrap(aiopstools.NewCloudBashTool(cloudBashProposerShim{uc: approvalUC}, log), cbDeps), aiopstoolsdec.Wrap(aiopstools.NewInstallSkillTool(installSkillProposerShim{uc: approvalUC}, log), cbDeps), aiopstoolsdec.Wrap(aiopstools.NewServePageTool(pageStore, log), quickDeps), aiopstoolsdec.Wrap(aiopstools.NewSendNotificationTool(notificationSenderShim{channels: alertRepo, router: notifyRouter}, log), quickDeps), aiopstoolsdec.Wrap(aiopstools.NewSendIMMessageTool(imMessageSenderShim{apps: imbridgeRepo}, log), quickDeps), - }) + }); err != nil { + log.Error("register late tools", slog.Any("err", err)) + } log.Info("host_bash + cloud_bash + install_skill + serve_page + messaging tools bolted onto chat runtime bag", slog.Int("tool_count", chatRT.ToolCount())) // HLD-017: wire the active-skill → bound-credentials resolver so // cloud_bash auto-injects the credentials an active skill was bound @@ -2464,7 +2468,10 @@ func main() { graphTools = append(graphTools, aiopstoolsdec.Wrap(raw, deps)) } } - chatRT.ReplaceToolSource(prefix, rawTools, graphTools) + if err := chatRT.ReplaceToolSource(prefix, rawTools, graphTools); err != nil { + log.Error("refresh MCP tool source", slog.String("server", serverName), slog.Any("err", err)) + return + } log.Info("mcp tools refreshed in chat runtime", slog.String("server", serverName), slog.Int("tools", len(graphTools)), diff --git a/docs/rfc/RFC-005-unified-agent-runtime.md b/docs/rfc/RFC-005-unified-agent-runtime.md index d31354823..b5d674af5 100644 --- a/docs/rfc/RFC-005-unified-agent-runtime.md +++ b/docs/rfc/RFC-005-unified-agent-runtime.md @@ -61,3 +61,13 @@ Graph 默认运行路径与旧循环维护独立工具列表，安全 Skill 注 - `make arch-lint` 未完成：本机缺少 `go-arch-lint`；目标虽打印 skipping，后续 recipe 仍调用缺失命令。未改动该既有行为，合并前需在已安装工具的环境补验。 - 尚未执行生产 legacy 配置盘点、真实模型/MCP 集成冒烟、金丝雀升级与镜像回滚演练；这些是发布验收项，不能用单元测试替代。 - 旧 Registry 中供审批兼容及既有测试使用的闭包适配器暂留；生产聊天只调用 Graph，告警初查也已改用 BaseTool。退役旧循环不等于删除历史审批协议。 + +### 集中复核补充 + +- 修复告警初查的批量参数迁移：使用 `incident_ids`，展开对应结果，拒绝失败/错配/空 bundle；用真实 `CorrelateIncidentTool` 验证诊断链路。 +- 修复安全 Skill schema 合并丢失原有 `required` 字段；覆盖全部内置 host-scope safe skills。 +- 工具注册整批校验名称唯一性、来源前缀以及目录/执行列表一致性；失败保留原有工具与目录。ToolSearch 读取本回合对应执行版本的完整 schema，不混用动态刷新的新参数。 +- 显式空工具视图清除继承的父回合权限列表。 +- 上述新增用例及 `make test-agent-runtime` 的 Linux race 回归通过。首版提交的 GitHub CI 已通过全仓 build/vet/race、部署清单及前端检查；补充提交需重新检查 CI。 +- 本地全仓测试尝试未通过：只读模块缓存缺少 APM 示例依赖，且 Docker amd64 仿真环境的 autoapm 进程发现测试失败。不能把本地全仓测试记为通过；原生 Ubuntu CI 的同项已通过。 +- 已在临时目录安装 `go-arch-lint v1.19.0` 并执行 `make arch-lint`：检查被原有配置错误阻断（不存在的 `internal/edgeagent/data/**`，以及 `shared_pkg`/`api` 的空依赖声明）。`.go-arch-lint.yml` 与基线相同；这替代此前“未安装工具”的原因，仍不能声称架构 lint 通过。 diff --git a/internal/manager/biz/aiops/chatruntime/runtime.go b/internal/manager/biz/aiops/chatruntime/runtime.go index 4f4518aa1..329bf9cc4 100644 --- a/internal/manager/biz/aiops/chatruntime/runtime.go +++ b/internal/manager/biz/aiops/chatruntime/runtime.go @@ -1,38 +1,7 @@ -// runtime.go is the in-process orchestration entry the new graph-based -// agent path goes through. PR-9 of — the cutover layer that -// pre-PR-9 callers (legacy agent.go) bypass entirely. After this PR -// the manager's HTTP service can pick which path runs based on the -// ONGRID_AGENT_KERNEL feature flag (default = legacy). -// -// Responsibilities: -// -// 1. Ownership check (caller user_id == session.user_id; admin bypass -// happens upstream — by the time Handle is called the service has -// already resolved the owning user_id). -// 2. Skill resolution against the user query (SkillRegistry.Resolve) -// — the active skills shape the system prompt. -// 3. System prompt assembly via ComposeSystemPrompt — base prompt + -// active skill prompts + (optional) agent persona. -// 4. @-mention inlining — same markdown-bullet preamble the legacy -// agent.go produces, so chat history replay is byte-identical -// across kernels (legacy mention text persists into the new path -// unchanged). -// 5. User-message persistence (chat_messages role=user). Mirrors the -// legacy "persist before LLM call" invariant from agent.go so a -// downstream crash leaves the user turn on disk. -// 6. Graph invoke with the default callback chain -// (callbacks.NewDefaultHandlers) wired via compose.WithCallbacks. -// The callback chain handles persistence (assistant + tool rows), -// SSE streaming, audit, metrics, and budget gating. -// 7. Reply translation — a chatruntime.Reply that the service layer -// translates back to agent.Reply for HTTP response shape parity. -// -// Behaviour parity with legacy agent.go is the explicit goal — the -// SPA, the persistence schema, and downstream reports must not see -// the kernel switch. Where exact parity isn't reachable (e.g. eino's -// graph runs the loop internally so we don't see per-iteration -// AssistantEvent counts), the value is best-effort and documented -// inline. +// runtime.go 是唯一聊天运行时入口，负责会话归属校验、技能与 persona +// 解析、提示词与 mention 装配、用户消息持久化和 Graph 调用。 +// callbacks 处理 assistant/tool 持久化、SSE、审计、指标与预算；Service +// 将内部事件映射为 chatprotocol，保持既有 HTTP/SSE/IM 协议兼容。 package chatruntime import ( @@ -61,17 +30,14 @@ import ( "github.com/ongridio/ongrid/internal/pkg/llm" ) -// Mention mirrors the legacy agent.Mention shape so chatruntime can -// be called without dragging the manager/biz/aiops/agent package as -// a dependency. The service-layer adapter does the type translation. +// Mention 是运行时的 mention 输入；Service 从 chatprotocol 转换。 type Mention struct { Type string ID string Label string } -// MentionResolver hydrates a list of @-mentions into one bullet line -// each. Mirrors agent.MentionResolver. nil = no inlining. +// MentionResolver hydrates each @-mention into one bullet line. nil disables inlining. type MentionResolver interface { Resolve(ctx context.Context, mentions []Mention) []string } @@ -327,7 +293,9 @@ func NewRuntime(cfg Config) (*Runtime, error) { initial := cfg.ToolBag cfg.ToolBag = nil rt := &Runtime{cfg: cfg, log: log, toolVersions: map[string]uint64{}} - rt.AppendToolBag(initial) + if err := rt.AppendToolBag(initial); err != nil { + return nil, fmt.Errorf("initialize tool catalog: %w", err) + } return rt, nil } @@ -377,91 +345,27 @@ func (rt *Runtime) SetToolBag(bag ToolBagProvider) { if rt == nil { return } - rt.bag = bag -} - -// AppendToolBag 按名称注册或替换后初始化工具，并同步搜索目录。 -// Coordinator 专属工具仍由 worker 的角色过滤器排除。 -func (rt *Runtime) AppendToolBag(tools []basetool.BaseTool) { - if rt == nil || len(tools) == 0 { - return - } rt.toolMu.Lock() defer rt.toolMu.Unlock() - if rt.toolVersions == nil { - rt.toolVersions = map[string]uint64{} - } - if bag, ok := rt.bag.(mutableToolBagProvider); ok { - bag.UpsertTools(tools) + rt.bag = bag + if bag == nil { + return } - index := map[string]int{} - for i, t := range rt.cfg.ToolBag { - if t == nil { - continue - } + full := map[string]*basetool.ToolInfo{} + for _, t := range bag.AllTools() { if info, err := t.Info(context.Background()); err == nil && info != nil { - index[info.Name] = i + full[info.Name] = info } } - for _, t := range tools { - if t == nil { - continue - } - info, err := t.Info(context.Background()) - if err == nil && info != nil { - t = rt.versionedToolLocked(info.Name, t) - if i, ok := index[info.Name]; ok { - rt.cfg.ToolBag[i] = t - continue - } - index[info.Name] = len(rt.cfg.ToolBag) - } - rt.cfg.ToolBag = append(rt.cfg.ToolBag, t) - } -} - -// ReplaceToolsByNamePrefix replaces tools in the graph-facing bag whose wire -// names share prefix. MCP uses one prefix per server, preventing a refresh of -// one registration from removing tools offered by other servers. -// ReplaceToolsByNamePrefix 是同 schema 工具的便捷更新入口。 -func (rt *Runtime) ReplaceToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) { - rt.ReplaceToolSource(prefix, replacements, replacements) -} - -// ReplaceToolSource 在同一更新中替换执行视图与完整 schema 目录。 -func (rt *Runtime) ReplaceToolSource(prefix string, raw, decorated []basetool.BaseTool) { - if rt == nil || strings.TrimSpace(prefix) == "" { - return - } - rt.toolMu.Lock() - defer rt.toolMu.Unlock() - if rt.toolVersions == nil { - rt.toolVersions = map[string]uint64{} - } - next := make([]basetool.BaseTool, 0, len(rt.cfg.ToolBag)+len(decorated)) - for _, t := range rt.cfg.ToolBag { - if toolNameHasPrefix(t, prefix) { - if info, err := t.Info(context.Background()); err == nil && info != nil { - delete(rt.toolVersions, info.Name) + for i, t := range rt.cfg.ToolBag { + if current, ok := t.(*runtimeTool); ok { + if info := full[current.name]; info != nil { + copy := *current + copy.fullInfo = cloneToolInfo(info) + rt.cfg.ToolBag[i] = © } - continue } - next = append(next, t) } - for _, t := range decorated { - if t == nil { - continue - } - info, err := t.Info(context.Background()) - if err != nil || info == nil || !strings.HasPrefix(info.Name, prefix) { - continue - } - next = append(next, rt.versionedToolLocked(info.Name, t)) - } - if bag, ok := rt.bag.(mutableToolBagProvider); ok { - bag.ReplaceToolsByNamePrefix(prefix, raw) - } - rt.cfg.ToolBag = next } func (rt *Runtime) toolBagSnapshot() []basetool.BaseTool { @@ -473,14 +377,6 @@ func (rt *Runtime) toolBagSnapshot() []basetool.BaseTool { return append([]basetool.BaseTool(nil), rt.cfg.ToolBag...) } -func toolNameHasPrefix(tool basetool.BaseTool, prefix string) bool { - if tool == nil { - return false - } - info, err := tool.Info(context.Background()) - return err == nil && info != nil && strings.HasPrefix(info.Name, prefix) -} - // AgentRegistry exposes the registry for narrow read-only callers // (cmd/main.go uses this when building AgentTool's subagent registry // shim). nil-safe. @@ -503,7 +399,7 @@ func (rt *Runtime) SkillRegistry() *SkillRegistry { // ToolCount returns how many tools are bound into the toolBag. Used // at boot time + tests for visibility (the spec asks us to log this -// when ONGRID_AGENT_KERNEL=graph). +// when the runtime is available). func (rt *Runtime) ToolCount() int { if rt == nil { return 0 diff --git a/internal/manager/biz/aiops/chatruntime/runtime_test.go b/internal/manager/biz/aiops/chatruntime/runtime_test.go index 04cf4a875..4bfedc12e 100644 --- a/internal/manager/biz/aiops/chatruntime/runtime_test.go +++ b/internal/manager/biz/aiops/chatruntime/runtime_test.go @@ -628,9 +628,11 @@ func TestRuntime_ReplaceToolsByNamePrefix(t *testing.T) { t.Fatalf("NewRuntime: %v", err) } - rt.ReplaceToolsByNamePrefix("mcp__github__", []basetool.BaseTool{ + if err := rt.ReplaceToolsByNamePrefix("mcp__github__", []basetool.BaseTool{ &fakeTool{name: "mcp__github__code_search", schema: `{"type":"object"}`}, - }) + }); err != nil { + t.Fatal(err) + } names := rt.ToolNames(context.Background()) got := map[string]bool{} for _, name := range names { diff --git a/internal/manager/biz/aiops/chatruntime/tool_lifecycle.go b/internal/manager/biz/aiops/chatruntime/tool_lifecycle.go index 839baec3a..6fbfb240a 100644 --- a/internal/manager/biz/aiops/chatruntime/tool_lifecycle.go +++ b/internal/manager/biz/aiops/chatruntime/tool_lifecycle.go @@ -3,6 +3,7 @@ package chatruntime import ( "context" "fmt" + "strings" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" ) @@ -10,18 +11,149 @@ import ( // runtimeTool 保留本轮 schema；撤销或替换后，旧引用不得开始新的调用。 // 已经进入执行的调用继续沿用原超时/取消机制，不在持锁期间执行外部 IO。 type runtimeTool struct { - inner basetool.BaseTool - runtime *Runtime - name string - version uint64 + inner basetool.BaseTool + runtime *Runtime + name string + version uint64 + info *basetool.ToolInfo + fullInfo *basetool.ToolInfo } -func (rt *Runtime) versionedToolLocked(name string, inner basetool.BaseTool) basetool.BaseTool { +func cloneToolInfo(info *basetool.ToolInfo) *basetool.ToolInfo { + copy := *info + copy.Parameters = append([]byte(nil), info.Parameters...) + return © +} + +func (rt *Runtime) versionedToolLocked(inner basetool.BaseTool, info, full *basetool.ToolInfo) basetool.BaseTool { rt.nextToolVersion++ - rt.toolVersions[name] = rt.nextToolVersion - return &runtimeTool{inner: inner, runtime: rt, name: name, version: rt.nextToolVersion} + rt.toolVersions[info.Name] = rt.nextToolVersion + return &runtimeTool{inner: inner, runtime: rt, name: info.Name, version: rt.nextToolVersion, info: cloneToolInfo(info), fullInfo: cloneToolInfo(full)} +} +func (t *runtimeTool) Info(context.Context) (*basetool.ToolInfo, error) { + return cloneToolInfo(t.info), nil +} + +// FullInfo 保留此执行版本的完整 schema，避免旧回合读到新版本参数。 +func (t *runtimeTool) FullInfo(context.Context) (*basetool.ToolInfo, error) { + return cloneToolInfo(t.fullInfo), nil +} + +// inspectToolBatch 在修改任何视图前校验整批元数据，错误不破坏现有工具。 +type toolBatch struct { + infos map[string]*basetool.ToolInfo + names []string +} + +func inspectToolBatch(tools []basetool.BaseTool, prefix string) (*toolBatch, error) { + infos := make(map[string]*basetool.ToolInfo, len(tools)) + names := make([]string, 0, len(tools)) + for _, t := range tools { + if t == nil { + return nil, fmt.Errorf("nil tool in registration") + } + info, err := t.Info(context.Background()) + if err != nil { + return nil, fmt.Errorf("read tool metadata: %w", err) + } + if info == nil || strings.TrimSpace(info.Name) == "" { + return nil, fmt.Errorf("tool name required") + } + if !strings.HasPrefix(info.Name, prefix) { + return nil, fmt.Errorf("tool %q is outside source %q", info.Name, prefix) + } + if infos[info.Name] != nil { + return nil, fmt.Errorf("duplicate tool %q", info.Name) + } + infos[info.Name] = cloneToolInfo(info) + names = append(names, info.Name) + } + return &toolBatch{infos: infos, names: names}, nil +} + +// AppendToolBag 同步注册执行与搜索视图；同名工具替换，整批重复名称被拒绝。 +func (rt *Runtime) AppendToolBag(tools []basetool.BaseTool) error { + if rt == nil { + return fmt.Errorf("runtime unavailable") + } + infos, err := inspectToolBatch(tools, "") + if err != nil { + return err + } + rt.toolMu.Lock() + defer rt.toolMu.Unlock() + if rt.toolVersions == nil { + rt.toolVersions = map[string]uint64{} + } + index := map[string]int{} + for i, t := range rt.cfg.ToolBag { + index[t.(*runtimeTool).name] = i + } + for j, t := range tools { + info := infos.infos[infos.names[j]] + wrapped := rt.versionedToolLocked(t, info, info) + if i, exists := index[info.Name]; exists { + rt.cfg.ToolBag[i] = wrapped + } else { + rt.cfg.ToolBag = append(rt.cfg.ToolBag, wrapped) + } + } + if bag, ok := rt.bag.(mutableToolBagProvider); ok { + bag.UpsertTools(tools) + } + return nil +} + +func (rt *Runtime) ReplaceToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) error { + return rt.ReplaceToolSource(prefix, replacements, replacements) +} + +// ReplaceToolSource 校验来源边界与两份视图，成功后一起替换并撤销旧版本。 +func (rt *Runtime) ReplaceToolSource(prefix string, raw, decorated []basetool.BaseTool) error { + if rt == nil || strings.TrimSpace(prefix) == "" { + return fmt.Errorf("runtime and tool source required") + } + full, err := inspectToolBatch(raw, prefix) + if err != nil { + return err + } + infos, err := inspectToolBatch(decorated, prefix) + if err != nil { + return err + } + if len(full.infos) != len(infos.infos) { + return fmt.Errorf("tool source %q catalog/executor mismatch", prefix) + } + for name := range infos.infos { + if full.infos[name] == nil { + return fmt.Errorf("tool %q has no catalog entry", name) + } + } + rt.toolMu.Lock() + defer rt.toolMu.Unlock() + if rt.toolVersions == nil { + rt.toolVersions = map[string]uint64{} + } + next := make([]basetool.BaseTool, 0, len(rt.cfg.ToolBag)+len(decorated)) + for _, t := range rt.cfg.ToolBag { + current := t.(*runtimeTool) + if strings.HasPrefix(current.name, prefix) { + delete(rt.toolVersions, current.name) + } else { + next = append(next, t) + } + } + for j, t := range decorated { + name := infos.names[j] + next = append(next, rt.versionedToolLocked(t, infos.infos[name], full.infos[name])) + } + if bag, ok := rt.bag.(mutableToolBagProvider); ok { + bag.ReplaceToolsByNamePrefix(prefix, raw) + } + rt.cfg.ToolBag = next + return nil } -func (t *runtimeTool) Info(ctx context.Context) (*basetool.ToolInfo, error) { return t.inner.Info(ctx) } + func (t *runtimeTool) InvokableRun(ctx context.Context, args string, opts ...basetool.InvokeOption) (string, error) { t.runtime.toolMu.RLock() current := t.runtime.toolVersions[t.name] diff --git a/internal/manager/biz/aiops/chatruntime/tool_lifecycle_test.go b/internal/manager/biz/aiops/chatruntime/tool_lifecycle_test.go index 37935684b..538523356 100644 --- a/internal/manager/biz/aiops/chatruntime/tool_lifecycle_test.go +++ b/internal/manager/biz/aiops/chatruntime/tool_lifecycle_test.go @@ -2,7 +2,10 @@ package chatruntime import ( "context" + "encoding/json" "fmt" + aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" + "strings" "sync" "sync/atomic" "testing" @@ -19,12 +22,16 @@ func TestRuntimeRemovedOrReplacedToolRejectsStaleReference(t *testing.T) { t.Fatal(err) } old := rt.toolBagSnapshot()[0] - rt.ReplaceToolSource("mcp__test__", nil, nil) + if err := rt.ReplaceToolSource("mcp__test__", nil, nil); err != nil { + t.Fatal(err) + } if _, err := old.InvokableRun(context.Background(), `{}`); err == nil { t.Fatal("removed tool executed") } fresh := &fakeTool{name: "mcp__test__search", schema: `{"type":"object"}`} - rt.ReplaceToolSource("mcp__test__", []basetool.BaseTool{fresh}, []basetool.BaseTool{fresh}) + if err := rt.ReplaceToolSource("mcp__test__", []basetool.BaseTool{fresh}, []basetool.BaseTool{fresh}); err != nil { + t.Fatal(err) + } if _, err := old.InvokableRun(context.Background(), `{}`); err == nil { t.Fatal("old generation became executable again") } @@ -45,7 +52,10 @@ func TestRuntimeConcurrentToolRefreshAndSnapshot(t *testing.T) { defer wg.Done() for j := 0; j < 100; j++ { tool := &fakeTool{name: "mcp__test__search", schema: `{"type":"object"}`} - rt.ReplaceToolSource("mcp__test__", []basetool.BaseTool{tool}, []basetool.BaseTool{tool}) + if err := rt.ReplaceToolSource("mcp__test__", []basetool.BaseTool{tool}, []basetool.BaseTool{tool}); err != nil { + t.Error(err) + return + } for _, entry := range rt.toolBagSnapshot() { // 并发替换允许明确拒绝旧引用，不能数据竞争或执行已撤销版本。 _, err := entry.InvokableRun(context.Background(), `{}`) @@ -132,3 +142,88 @@ func TestWorkerInheritsWebSearchGate(t *testing.T) { } } } + +func TestToolSearchKeepsTurnSchemaAcrossRefresh(t *testing.T) { + old := &fakeTool{name: "mcp__test__search", schema: `{"type":"object","properties":{"old_param":{"type":"string"}}}`} + bag := aiopstools.NewToolBag([]basetool.BaseTool{old}, 0) + rt, err := NewRuntime(Config{Sessions: newMemSessions(nil), ChatModel: newScriptedChatModel(), ToolBag: bag.SchemasForLLM()}) + if err != nil { + t.Fatal(err) + } + rt.SetToolBag(bag) + before := rt.toolBagSnapshot() + fresh := &fakeTool{name: old.name, schema: `{"type":"object","properties":{"new_param":{"type":"string"}}}`} + if err := rt.ReplaceToolSource("mcp__test__", []basetool.BaseTool{fresh}, []basetool.BaseTool{fresh}); err != nil { + t.Fatal(err) + } + search := aiopstools.NewToolSearchTool(bag, nil) + for _, tc := range []struct { + view []basetool.BaseTool + want string + }{{before, "old_param"}, {rt.toolBagSnapshot(), "new_param"}} { + result, err := search.InvokableRun(basetool.WithFilteredTools(context.Background(), tc.view), `{"query":"select:mcp__test__search"}`) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(result, tc.want) { + t.Fatalf("want %s, got %s", tc.want, result) + } + } + if _, err := before[0].InvokableRun(context.Background(), `{}`); err == nil { + t.Fatal("old executable survived refresh") + } + // An empty child view must clear the inherited parent view. + ctx := basetool.WithFilteredTools(context.Background(), rt.toolBagSnapshot()) + result, err := search.InvokableRun(basetool.WithFilteredTools(ctx, nil), `{"query":"select:mcp__test__search"}`) + if err != nil { + t.Fatal(err) + } + var response struct { + Tools []json.RawMessage `json:"tools"` + } + if err := json.Unmarshal([]byte(result), &response); err != nil { + t.Fatal(err) + } + if len(response.Tools) != 0 { + t.Fatalf("empty view leaked parent schemas: %s", result) + } +} + +func TestInvalidToolSourceUpdatePreservesExistingViews(t *testing.T) { + old := &fakeTool{name: "mcp__test__old", schema: `{"type":"object"}`} + other := &fakeTool{name: "mcp__other__tool", schema: `{"type":"object"}`} + for _, tc := range []struct { + name string + raw, executed []basetool.BaseTool + }{ + {"duplicate", []basetool.BaseTool{old, old}, []basetool.BaseTool{old, old}}, + {"outside source", []basetool.BaseTool{other}, []basetool.BaseTool{other}}, + {"missing execution", []basetool.BaseTool{old}, nil}, + {"missing schema", nil, []basetool.BaseTool{old}}, + {"nil entry", []basetool.BaseTool{nil}, []basetool.BaseTool{nil}}, + } { + t.Run(tc.name, func(t *testing.T) { + bag := aiopstools.NewToolBag([]basetool.BaseTool{old}, 30) + rt, err := NewRuntime(Config{Sessions: newMemSessions(nil), ChatModel: newScriptedChatModel(), ToolBag: bag.AllTools()}) + if err != nil { + t.Fatal(err) + } + rt.SetToolBag(bag) + ref := rt.toolBagSnapshot()[0] + if err := rt.ReplaceToolSource("mcp__test__", tc.raw, tc.executed); err == nil { + t.Fatal("invalid update accepted") + } + if _, err := ref.InvokableRun(context.Background(), `{}`); err != nil { + t.Fatalf("existing executable revoked: %v", err) + } + all := bag.AllTools() + info, err := all[0].Info(context.Background()) + if err != nil || len(all) != 1 || info.Name != old.name { + t.Fatal("catalog changed after rejected update") + } + }) + } + if _, err := NewRuntime(Config{Sessions: newMemSessions(nil), ChatModel: newScriptedChatModel(), ToolBag: []basetool.BaseTool{old, old}}); err == nil { + t.Fatal("startup accepted duplicate tools") + } +} diff --git a/internal/manager/biz/aiops/tools/basetool/filtered_tools.go b/internal/manager/biz/aiops/tools/basetool/filtered_tools.go index 8aa28ac74..c4b362b96 100644 --- a/internal/manager/biz/aiops/tools/basetool/filtered_tools.go +++ b/internal/manager/biz/aiops/tools/basetool/filtered_tools.go @@ -16,10 +16,10 @@ var filteredToolsCtxKey = filteredToolsCtxKeyT{} // WithFilteredTools returns ctx augmented with the persona-filtered tool // slice. The runtime calls this before graph.Invoke so ToolSearch (which // runs inside the graph) can scope its search to only whitelist-surviving -// tools. nil/empty is a no-op. +// tools. An empty view explicitly denies all tools, including inherited ones. func WithFilteredTools(ctx context.Context, tools []BaseTool) context.Context { - if len(tools) == 0 { - return ctx + if tools == nil { + tools = []BaseTool{} } return context.WithValue(ctx, filteredToolsCtxKey, tools) } diff --git a/internal/manager/biz/aiops/tools/tool_search_tool.go b/internal/manager/biz/aiops/tools/tool_search_tool.go index 2679da5f9..7e3a8c58e 100644 --- a/internal/manager/biz/aiops/tools/tool_search_tool.go +++ b/internal/manager/biz/aiops/tools/tool_search_tool.go @@ -198,6 +198,14 @@ func (t *ToolSearchTool) fullToolsAllowedByContext(ctx context.Context) []baseto continue } seen[info.Name] = struct{}{} + if snapshot, ok := tool.(interface { + FullInfo(context.Context) (*basetool.ToolInfo, error) + }); ok { + if full, err := snapshot.FullInfo(ctx); err == nil && full != nil { + out = append(out, &searchSchemaTool{BaseTool: tool, info: full}) + } + continue + } if full, ok := byName[info.Name]; ok { out = append(out, full) continue @@ -308,3 +316,11 @@ func splitNonEmpty(s, sep string) []string { } return out } + +// searchSchemaTool 只供 ToolSearch 读取完整元数据，执行仍委托受控引用。 +type searchSchemaTool struct { + basetool.BaseTool + info *basetool.ToolInfo +} + +func (t *searchSchemaTool) Info(context.Context) (*basetool.ToolInfo, error) { return t.info, nil } From be50baba27d00ee10bf3231f91dc8d25925efedc Mon Sep 17 00:00:00 2001 From: singchia Date: Wed, 14 Oct 2026 20:57:10 +0800 Subject: [PATCH 07/24] fix(aiops): snapshot background tools before starting investigations --- cmd/ongrid/main.go | 23 ++++++++++++----------- docs/rfc/RFC-005-unified-agent-runtime.md | 1 + 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cmd/ongrid/main.go b/cmd/ongrid/main.go index fa3e92d9a..46596ed72 100644 --- a/cmd/ongrid/main.go +++ b/cmd/ongrid/main.go @@ -1845,7 +1845,7 @@ func main() { // Two-tier proactive investigation wiring: // - // 1. The legacy ai_initial_diagnosis emitter (~3 paragraphs on + // 1. The ai_initial_diagnosis emitter (~3 paragraphs on // the alert timeline, lightweight, single LLM call via // correlate_incident) — fast read for operators glancing at // the incident timeline. @@ -1860,21 +1860,21 @@ func main() { // Both share the alert.Investigator interface and chain via // investigatorChain so each new-fire fans out to both. Either side // can be nil — the chain skips nil. - var legacyInv managerbizalert.Investigator + var initialInv managerbizalert.Investigator if hasConfiguredLLMProvider(llmRouter) { - legacy := aiopsinvestigator.New( + initial := aiopsinvestigator.New( llmClient, - backgroundToolInvoker{reg: toolsReg}, + backgroundToolInvoker{tools: toolsReg.BuildBaseTools().AllTools()}, alertRepo, aiopsinvestigator.Config{}, log, ) - defer legacy.Close() - legacyInv = legacy - log.Info("alert: legacy AI initial-diagnosis investigator wired", + defer initial.Close() + initialInv = initial + log.Info("alert: AI initial-diagnosis investigator wired", slog.String("model_source", "llm_default")) } else { - log.Info("alert: legacy AI investigator disabled (no LLM provider)") + log.Info("alert: AI initial-diagnosis investigator disabled (no LLM provider)") } var ( @@ -1940,7 +1940,7 @@ func main() { } } - if chained := chainInvestigators(legacyInv, rcaInv); chained != nil { + if chained := chainInvestigators(initialInv, rcaInv); chained != nil { alertUC.SetInvestigator(chained) } @@ -6022,10 +6022,11 @@ func seedInfrastructureMenuDefaults(ctx context.Context, db *gorm.DB, settings * } // backgroundToolInvoker 让确定性的告警初查复用统一工具目录，不依赖旧循环注册表。 -type backgroundToolInvoker struct{ reg *aiopstools.Registry } +// tools 是启动时快照，后台任务不读取后续装配仍在修改的 Registry。 +type backgroundToolInvoker struct{ tools []aiopstoolsbase.BaseTool } func (i backgroundToolInvoker) Invoke(ctx context.Context, name string, args json.RawMessage) (aiopstools.ExecuteResult, error) { - for _, t := range i.reg.BuildBaseTools().AllTools() { + for _, t := range i.tools { info, err := t.Info(ctx) if err != nil { return aiopstools.ExecuteResult{}, err diff --git a/docs/rfc/RFC-005-unified-agent-runtime.md b/docs/rfc/RFC-005-unified-agent-runtime.md index b5d674af5..4e8513a1c 100644 --- a/docs/rfc/RFC-005-unified-agent-runtime.md +++ b/docs/rfc/RFC-005-unified-agent-runtime.md @@ -71,3 +71,4 @@ Graph 默认运行路径与旧循环维护独立工具列表，安全 Skill 注 - 上述新增用例及 `make test-agent-runtime` 的 Linux race 回归通过。首版提交的 GitHub CI 已通过全仓 build/vet/race、部署清单及前端检查；补充提交需重新检查 CI。 - 本地全仓测试尝试未通过：只读模块缓存缺少 APM 示例依赖，且 Docker amd64 仿真环境的 autoapm 进程发现测试失败。不能把本地全仓测试记为通过；原生 Ubuntu CI 的同项已通过。 - 已在临时目录安装 `go-arch-lint v1.19.0` 并执行 `make arch-lint`：检查被原有配置错误阻断（不存在的 `internal/edgeagent/data/**`，以及 `shared_pkg`/`api` 的空依赖声明）。`.go-arch-lint.yml` 与基线相同；这替代此前“未安装工具”的原因，仍不能声称架构 lint 通过。 +- 告警初查使用启动时的 BaseTool 快照，避免后台任务与后续 Registry 装配并发读写；Manager 装配包 `go test -race ./cmd/ongrid` 通过。 From 0ed33ea20f7f754781f7f863ad498790ed5b39d8 Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 19:07:26 +0800 Subject: [PATCH 08/24] refactor(architecture): enforce domain boundaries and isolate shared audit contracts --- .github/workflows/ci.yml | 6 + .go-arch-lint.yml | 113 ++++++++------- Makefile | 16 ++- cmd/ongrid/main.go | 3 +- internal/iam/biz/membership/usecase.go | 9 +- internal/iam/data/membership/store/repo.go | 10 +- internal/iam/model/membership_view.go | 12 ++ internal/iam/server/http.go | 54 ++++---- internal/iam/service/service.go | 3 +- internal/manager/biz/audit/usecase.go | 34 +---- internal/manager/data/audit/store/repo.go | 11 +- internal/manager/model/audit/filters.go | 14 ++ internal/manager/model/audit/log.go | 129 +++++++----------- internal/manager/server/imbridge/http.go | 3 +- internal/manager/server/middleware/audit.go | 53 ++----- .../manager/server/middleware/audit_test.go | 56 ++++++++ .../{biz => service}/imbridge/adapter.go | 8 +- .../{biz => service}/imbridge/adapter_test.go | 0 internal/pkg/auditctx/context.go | 40 ++++++ internal/pkg/auditctx/context_test.go | 46 +++++++ internal/pkg/auditctx/event.go | 114 ++++++++++++++++ scripts/test-arch-policy.py | 74 ++++++++++ 22 files changed, 540 insertions(+), 268 deletions(-) create mode 100644 internal/iam/model/membership_view.go create mode 100644 internal/manager/model/audit/filters.go create mode 100644 internal/manager/server/middleware/audit_test.go rename internal/manager/{biz => service}/imbridge/adapter.go (93%) rename internal/manager/{biz => service}/imbridge/adapter_test.go (100%) create mode 100644 internal/pkg/auditctx/context.go create mode 100644 internal/pkg/auditctx/context_test.go create mode 100644 internal/pkg/auditctx/event.go create mode 100644 scripts/test-arch-policy.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c60b0633..34b6d80d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,6 +80,12 @@ jobs: if: steps.changes.outputs.docs_only != 'true' run: make lint-proto + - name: Architecture boundaries + if: steps.changes.outputs.docs_only != 'true' + run: | + make tools-arch + make arch-lint test-arch-policy + - name: go build if: steps.changes.outputs.docs_only != 'true' run: go build ./... diff --git a/.go-arch-lint.yml b/.go-arch-lint.yml index f972093b7..7e37c7154 100644 --- a/.go-arch-lint.yml +++ b/.go-arch-lint.yml @@ -7,12 +7,17 @@ # - 同一 BC 内部 service 层禁止越过 biz 直接 import 同 BC 的 data 层 # （go-arch-lint v3 支持以独立组件 + deps 表达，见下方 iam_service / manager_service 等） # -# ADR-005 枢轴备注：iam 下只剩 user 子域（org/membership 已删）；data 目录 -# 由 mysql/ 重命名为 sqlite/。组件 glob 使用 `**`，无需逐级更新。 +# v1.19.0 / schema v3。检查生产代码的 import 边界；测试允许组装真实 fixture。 +# deepScan 的实现类型流分析会把构造函数注入判为反向依赖；项目允许 +# 消费方接口 + cmd 装配，因此不启用该模式，禁止 import 的规则仍完整执行。 version: 3 workdir: . +allow: + depOnAnyVendor: true + deepScan: false + components: iam_service: { in: internal/iam/service/** } iam_biz: { in: internal/iam/biz/** } @@ -28,81 +33,97 @@ components: edgeagent_service: { in: internal/edgeagent/service/** } edgeagent_biz: { in: internal/edgeagent/biz/** } - edgeagent_data: { in: internal/edgeagent/data/** } edgeagent_model: { in: internal/edgeagent/model/** } + examples: { in: examples/** } + test_harness: { in: tests/** } + shared_skill: { in: internal/skill/** } + edgeagent_runtime: + in: + - internal/edgeagent/bash/** + - internal/edgeagent/cmdpolicy/** + - internal/edgeagent/collector/** + - internal/edgeagent/config/** + - internal/edgeagent/dpapi/** + - internal/edgeagent/edgedirs/** + - internal/edgeagent/host_files/** + - internal/edgeagent/install/** + - internal/edgeagent/k8s/** + - internal/edgeagent/operator/** + - internal/edgeagent/packetcapture/** + - internal/edgeagent/plugins/** + - internal/edgeagent/restart_service/** + - internal/edgeagent/skill/** + - internal/edgeagent/streamrouter/** + - internal/edgeagent/supervisorhealth/** + - internal/edgeagent/upgradebundle/** + - internal/edgeagent/upgrademachine/** + - internal/edgeagent/webshell/** + shared_pkg: { in: internal/pkg/** } api: { in: api/** } - cmd: { in: cmd/** } + cmd_manager: { in: cmd/ongrid/** } + cmd_edge: { in: [cmd/ongrid-edge/**, cmd/ongrid-edge-supervisor/**] } # 允许依赖的正向清单（未列出 = 禁止） deps: + examples: + anyVendorDeps: true + test_harness: + anyProjectDeps: true # ---------------- iam BC ---------------- iam_server: - mayDependOn: [iam_service, iam_biz, iam_model, shared_pkg, api] + mayDependOn: [iam_server, iam_service, iam_biz, iam_model, shared_pkg, api] iam_service: # 关键：service 不得直接依赖 iam_data（必须走 biz） - mayDependOn: [iam_biz, iam_model, shared_pkg, api] + mayDependOn: [iam_service, iam_biz, iam_model, shared_pkg, api] iam_biz: - mayDependOn: [iam_model, shared_pkg] - iam_data: mayDependOn: [iam_biz, iam_model, shared_pkg] + iam_data: + mayDependOn: [iam_data, iam_biz, iam_model, shared_pkg] iam_model: - mayDependOn: [shared_pkg] + mayDependOn: [iam_model, shared_pkg] # ---------------- manager BC ---------------- manager_server: - mayDependOn: [manager_service, manager_biz, manager_model, shared_pkg, api] + mayDependOn: [manager_server, manager_service, manager_biz, manager_model, shared_pkg, api] manager_service: - mayDependOn: [manager_biz, manager_model, shared_pkg, api] + mayDependOn: [manager_service, manager_biz, manager_model, shared_pkg, api] manager_biz: - mayDependOn: [manager_model, shared_pkg] + mayDependOn: [manager_biz, manager_model, shared_pkg, shared_skill] manager_data: - mayDependOn: [manager_biz, manager_model, shared_pkg] + mayDependOn: [manager_data, manager_biz, manager_model, shared_pkg] manager_model: - mayDependOn: [shared_pkg] + mayDependOn: [manager_model, shared_pkg] # ---------------- edgeagent BC ---------------- edgeagent_service: - mayDependOn: [edgeagent_biz, edgeagent_model, shared_pkg, api] + mayDependOn: [edgeagent_service, edgeagent_biz, edgeagent_model, shared_pkg, api] edgeagent_biz: - mayDependOn: [edgeagent_model, shared_pkg] - edgeagent_data: - mayDependOn: [edgeagent_biz, edgeagent_model, shared_pkg] + mayDependOn: [edgeagent_biz, edgeagent_model, edgeagent_runtime, shared_skill, shared_pkg] edgeagent_model: - mayDependOn: [shared_pkg] + mayDependOn: [edgeagent_model, shared_pkg] + + edgeagent_runtime: + mayDependOn: [edgeagent_runtime, edgeagent_biz, edgeagent_model, shared_pkg, shared_skill, api] + shared_skill: + mayDependOn: [shared_skill, shared_pkg] # ---------------- 共享 ---------------- shared_pkg: # 不得依赖任何 BC（业务无关） - mayDependOn: [] + mayDependOn: [shared_pkg] + anyVendorDeps: true api: - mayDependOn: [] + anyVendorDeps: true - # ---------------- 装配层 ---------------- - # cmd/ongrid 装配 iam + manager，cmd/ongrid-edge 装配 edgeagent； - # go-arch-lint v3 无"按子目录分 cmd"的原生表达，这里合并授权， - # 由人工 review 与 CODEOWNERS 把住"cmd/ongrid-edge 不 import iam/manager"。 - cmd: - mayDependOn: - - iam_server - - iam_service - - iam_biz - - iam_data - - iam_model - - manager_server - - manager_service - - manager_biz - - manager_data - - manager_model - - edgeagent_service - - edgeagent_biz - - edgeagent_data - - edgeagent_model - - shared_pkg - - api + # Manager 与 Edge 分别装配，禁止两个入口互相 import 对方业务域。 + cmd_manager: + mayDependOn: [cmd_manager, iam_server, iam_service, iam_biz, iam_data, iam_model, manager_server, manager_service, manager_biz, manager_data, manager_model, shared_pkg, shared_skill, api] + cmd_edge: + mayDependOn: [cmd_edge, edgeagent_service, edgeagent_biz, edgeagent_model, edgeagent_runtime, shared_pkg, shared_skill, api] -# 注：go-arch-lint 的 schema 字段（version / components / deps）按 v3 -# 主流文档书写；若 CI 报 schema 错，按实际版本微调字段名即可。 commonComponents: [] -excludeFiles: [] +# 包内测试允许组装真实存储/跨域 fixture；生产文件仍严格执行分层。 +excludeFiles: + - "_test\\.go$" diff --git a/Makefile b/Makefile index aff43034c..ac56b015b 100644 --- a/Makefile +++ b/Makefile @@ -152,9 +152,17 @@ test-e2e-live: ## E2E live mode（用 tests/e2e/secrets.local.env 打通真实 lint: ## 运行 golangci-lint golangci-lint run -arch-lint: ## 运行 go-arch-lint（校验 BC 边界） - @command -v go-arch-lint >/dev/null 2>&1 || { echo "go-arch-lint not installed; skipping"; exit 0; } - go-arch-lint check +ARCH_LINT ?= go-arch-lint +.PHONY: tools-arch test-arch-policy +tools-arch: ## 安装固定版本的架构检查工具 + go install github.com/fe3dback/go-arch-lint@v1.19.0 + +arch-lint: ## 校验生产代码 BC 边界（缺少工具时明确失败） + @command -v $(ARCH_LINT) >/dev/null 2>&1 || { echo "go-arch-lint not installed; run make tools-arch" >&2; exit 1; } + $(ARCH_LINT) check + +test-arch-policy: ## 验证架构规则确实拒绝跨域和跨层 import + python3 scripts/test-arch-policy.py --binary $(ARCH_LINT) # ---------------------------------------------------------------------------- # proto @@ -915,4 +923,4 @@ test-web-ci-platform: ## Run the serial platform frontend CI tests (requires npm .PHONY: test-agent-runtime test-agent-runtime: ## 统一 Agent runtime 回归（tools 依赖 Linux cmdpolicy） - go test -race ./internal/manager/biz/aiops/... ./internal/manager/service/aiops ./internal/manager/server/aiops ./internal/manager/biz/imbridge ./internal/skill/... ./cmd/ongrid + go test -race ./internal/manager/biz/aiops/... ./internal/manager/service/aiops ./internal/manager/server/aiops ./internal/manager/biz/imbridge ./internal/manager/service/imbridge ./internal/skill/... ./cmd/ongrid diff --git a/cmd/ongrid/main.go b/cmd/ongrid/main.go index 46596ed72..401535b09 100644 --- a/cmd/ongrid/main.go +++ b/cmd/ongrid/main.go @@ -146,6 +146,7 @@ import ( managerserverk8s "github.com/ongridio/ongrid/internal/manager/server/k8s" managerserverknowledge "github.com/ongridio/ongrid/internal/manager/server/knowledge" managerwebshellserver "github.com/ongridio/ongrid/internal/manager/server/webshell" + managersvcimbridge "github.com/ongridio/ongrid/internal/manager/service/imbridge" mcpclient "github.com/ongridio/ongrid/internal/pkg/mcpclient" "github.com/ongridio/ongrid/internal/pkg/tunnel" @@ -1776,7 +1777,7 @@ func main() { // Service-account user_id: superuser admin (id=1 on every install // thanks to bootstrap). Future: take from cfg. const imBridgeServiceUserID uint64 = 1 - imbridgeAgentAdapter := managerbizimbridge.NewAiopsAdapter(aiopsSvc, imBridgeServiceUserID, llmSettingsResolver, log) + imbridgeAgentAdapter := managersvcimbridge.NewAiopsAdapter(aiopsSvc, imBridgeServiceUserID, llmSettingsResolver, log) imbridgeSvc := managerbizimbridge.NewBridge(imbridgeRepo, imbridgeAgentAdapter, imBridgeServiceUserID, log) imbridgeHandler := managerserverimbridge.NewHandler(imbridgeSvc, imbridgeRepo, imbridgeUC, log) diff --git a/internal/iam/biz/membership/usecase.go b/internal/iam/biz/membership/usecase.go index c99297013..1aa36f6cb 100644 --- a/internal/iam/biz/membership/usecase.go +++ b/internal/iam/biz/membership/usecase.go @@ -8,7 +8,6 @@ import ( "context" "fmt" - "github.com/ongridio/ongrid/internal/iam/data/membership/store" "github.com/ongridio/ongrid/internal/iam/model" "github.com/ongridio/ongrid/internal/pkg/errs" ) @@ -19,8 +18,8 @@ type Repo interface { Delete(ctx context.Context, userID, orgID uint64) error DeleteByOrg(ctx context.Context, orgID uint64) error DeleteByUser(ctx context.Context, userID uint64) error - ListByOrg(ctx context.Context, orgID uint64) ([]store.MembershipWithUser, error) - ListByUser(ctx context.Context, userID uint64) ([]store.MembershipWithOrg, error) + ListByOrg(ctx context.Context, orgID uint64) ([]model.MembershipWithUser, error) + ListByUser(ctx context.Context, userID uint64) ([]model.MembershipWithOrg, error) All(ctx context.Context) ([]model.OrgMembership, error) } @@ -73,12 +72,12 @@ func (s *Service) Remove(ctx context.Context, userID, orgID uint64) error { } // ListByOrg returns members of an org, joined with user rows. -func (s *Service) ListByOrg(ctx context.Context, orgID uint64) ([]store.MembershipWithUser, error) { +func (s *Service) ListByOrg(ctx context.Context, orgID uint64) ([]model.MembershipWithUser, error) { return s.repo.ListByOrg(ctx, orgID) } // ListByUser returns orgs the user belongs to, joined with org rows. -func (s *Service) ListByUser(ctx context.Context, userID uint64) ([]store.MembershipWithOrg, error) { +func (s *Service) ListByUser(ctx context.Context, userID uint64) ([]model.MembershipWithOrg, error) { return s.repo.ListByUser(ctx, userID) } diff --git a/internal/iam/data/membership/store/repo.go b/internal/iam/data/membership/store/repo.go index 7509da09c..461ce0f82 100644 --- a/internal/iam/data/membership/store/repo.go +++ b/internal/iam/data/membership/store/repo.go @@ -78,10 +78,7 @@ func (r *Repo) DeleteByUser(ctx context.Context, userID uint64) error { // ListByOrg returns every membership in the given org with the // embedded user pre-joined. -type MembershipWithUser struct { - model.OrgMembership - User model.User `gorm:"-"` -} +type MembershipWithUser = model.MembershipWithUser // ListByOrg lists memberships joined to user rows. func (r *Repo) ListByOrg(ctx context.Context, orgID uint64) ([]MembershipWithUser, error) { @@ -116,10 +113,7 @@ func (r *Repo) ListByOrg(ctx context.Context, orgID uint64) ([]MembershipWithUse // ListByUser returns every org the user is a member of, with the // embedded org row. -type MembershipWithOrg struct { - model.OrgMembership - Org model.Org `gorm:"-"` -} +type MembershipWithOrg = model.MembershipWithOrg // ListByUser lists memberships joined to org rows. func (r *Repo) ListByUser(ctx context.Context, userID uint64) ([]MembershipWithOrg, error) { diff --git a/internal/iam/model/membership_view.go b/internal/iam/model/membership_view.go new file mode 100644 index 000000000..ee9329584 --- /dev/null +++ b/internal/iam/model/membership_view.go @@ -0,0 +1,12 @@ +package model + +// MembershipWithUser / MembershipWithOrg 是成员查询的领域视图，不绑定存储实现。 +type MembershipWithUser struct { + OrgMembership + User User `gorm:"-"` +} + +type MembershipWithOrg struct { + OrgMembership + Org Org `gorm:"-"` +} diff --git a/internal/iam/server/http.go b/internal/iam/server/http.go index a3e6d5e59..17f8ffd57 100644 --- a/internal/iam/server/http.go +++ b/internal/iam/server/http.go @@ -16,9 +16,7 @@ import ( "github.com/ongridio/ongrid/internal/iam/model" "github.com/ongridio/ongrid/internal/iam/service" - bizaudit "github.com/ongridio/ongrid/internal/manager/biz/audit" - auditmodel "github.com/ongridio/ongrid/internal/manager/model/audit" - auditmw "github.com/ongridio/ongrid/internal/manager/server/middleware" + "github.com/ongridio/ongrid/internal/pkg/auditctx" "github.com/ongridio/ongrid/internal/pkg/errs" "github.com/ongridio/ongrid/internal/pkg/tenantctx" ) @@ -43,8 +41,8 @@ type throttleSlot struct { const ( // IP-level limit: looser, since one IP may legitimately host multiple // users (NAT, office gateway). - loginIPLimit = 20 - loginIPWindow = 5 * time.Minute + loginIPLimit = 20 + loginIPWindow = 5 * time.Minute // Email-level limit: tighter — anyone trying 6+ passwords against // admin@x in 15min is hostile. loginEmailLimit = 6 @@ -135,9 +133,9 @@ func clientIP(r *http.Request) string { // Handler bundles the Service with its logger and exposes chi routers. type Handler struct { - svc *service.Service - log *slog.Logger - throttle *loginThrottle + svc *service.Service + log *slog.Logger + throttle *loginThrottle } // NewHandler builds the iam HTTP handler bundle. Callers register routes @@ -227,12 +225,12 @@ func (h *Handler) login(w http.ResponseWriter, r *http.Request) { ip := clientIP(r) emailKey := strings.ToLower(strings.TrimSpace(in.Email)) if err := h.throttle.check(ip, emailKey); err != nil { - auditmw.SetAuditEvent(r, bizaudit.Event{ - Action: auditmodel.ActionAuthLoginFailed, - ResourceType: auditmodel.ResourceAuth, + auditctx.SetAuditEvent(r, auditctx.Event{ + Action: auditctx.ActionAuthLoginFailed, + ResourceType: auditctx.ResourceAuth, ResourceID: emailKey, UserEmail: emailKey, - Status: auditmodel.StatusFailure, + Status: auditctx.StatusFailure, ErrorMessage: "rate limited", }) writeErr(w, err) @@ -243,12 +241,12 @@ func (h *Handler) login(w http.ResponseWriter, r *http.Request) { h.throttle.recordFailure(ip, emailKey) // HLD-010: record failed login. UserID stays nil; email comes // from request body so we can spot password-spraying patterns. - auditmw.SetAuditEvent(r, bizaudit.Event{ - Action: auditmodel.ActionAuthLoginFailed, - ResourceType: auditmodel.ResourceAuth, + auditctx.SetAuditEvent(r, auditctx.Event{ + Action: auditctx.ActionAuthLoginFailed, + ResourceType: auditctx.ResourceAuth, ResourceID: emailKey, UserEmail: emailKey, - Status: auditmodel.StatusFailure, + Status: auditctx.StatusFailure, ErrorMessage: err.Error(), }) writeErr(w, err) @@ -301,12 +299,12 @@ func (h *Handler) register(w http.ResponseWriter, r *http.Request) { writeErr(w, err) return } - auditmw.SetAuditEvent(r, bizaudit.Event{ - Action: auditmodel.ActionUserCreate, - ResourceType: auditmodel.ResourceUser, + auditctx.SetAuditEvent(r, auditctx.Event{ + Action: auditctx.ActionUserCreate, + ResourceType: auditctx.ResourceUser, ResourceID: strconv.FormatUint(u.ID, 10), ResourceName: u.Email, - Status: auditmodel.StatusSuccess, + Status: auditctx.StatusSuccess, Payload: map[string]any{"role": u.Role}, }) writeJSON(w, http.StatusCreated, userDTO{ID: u.ID, Email: u.Email, Role: u.Role}) @@ -355,11 +353,11 @@ func (h *Handler) deleteUser(w http.ResponseWriter, r *http.Request) { writeErr(w, err) return } - auditmw.SetAuditEvent(r, bizaudit.Event{ - Action: auditmodel.ActionUserDelete, - ResourceType: auditmodel.ResourceUser, + auditctx.SetAuditEvent(r, auditctx.Event{ + Action: auditctx.ActionUserDelete, + ResourceType: auditctx.ResourceUser, ResourceID: strconv.FormatUint(id, 10), - Status: auditmodel.StatusSuccess, + Status: auditctx.StatusSuccess, }) w.WriteHeader(http.StatusNoContent) } @@ -382,11 +380,11 @@ func (h *Handler) setRole(w http.ResponseWriter, r *http.Request) { writeErr(w, err) return } - auditmw.SetAuditEvent(r, bizaudit.Event{ - Action: auditmodel.ActionUserUpdate, - ResourceType: auditmodel.ResourceUser, + auditctx.SetAuditEvent(r, auditctx.Event{ + Action: auditctx.ActionUserUpdate, + ResourceType: auditctx.ResourceUser, ResourceID: strconv.FormatUint(id, 10), - Status: auditmodel.StatusSuccess, + Status: auditctx.StatusSuccess, Payload: map[string]any{"field": "role", "new_role": in.Role}, }) w.WriteHeader(http.StatusNoContent) diff --git a/internal/iam/service/service.go b/internal/iam/service/service.go index 567cd463e..b68d89b89 100644 --- a/internal/iam/service/service.go +++ b/internal/iam/service/service.go @@ -11,7 +11,6 @@ import ( "github.com/ongridio/ongrid/internal/iam/biz/membership" "github.com/ongridio/ongrid/internal/iam/biz/org" biz "github.com/ongridio/ongrid/internal/iam/biz/user" - "github.com/ongridio/ongrid/internal/iam/data/membership/store" "github.com/ongridio/ongrid/internal/iam/model" ) @@ -93,7 +92,7 @@ func (s *Service) SetRole(ctx context.Context, id uint64, role string) error { // MembershipsByUser returns the orgs a user is in (or nil when memberships // service isn't wired). -func (s *Service) MembershipsByUser(ctx context.Context, userID uint64) ([]store.MembershipWithOrg, error) { +func (s *Service) MembershipsByUser(ctx context.Context, userID uint64) ([]model.MembershipWithOrg, error) { if s.memberships == nil { return nil, nil } diff --git a/internal/manager/biz/audit/usecase.go b/internal/manager/biz/audit/usecase.go index fdb0bd4b4..1cfbc4da6 100644 --- a/internal/manager/biz/audit/usecase.go +++ b/internal/manager/biz/audit/usecase.go @@ -10,8 +10,8 @@ import ( "log/slog" "time" - store "github.com/ongridio/ongrid/internal/manager/data/audit/store" model "github.com/ongridio/ongrid/internal/manager/model/audit" + "github.com/ongridio/ongrid/internal/pkg/auditctx" ) // Repo is the persistence seam the usecase consumes. Implemented by @@ -22,38 +22,12 @@ type Repo interface { DeleteOlderThan(ctx context.Context, cutoff time.Time) (int64, error) } -// ListFilters is an alias to the store-level filter struct so handlers -// can depend only on biz/audit without importing data/audit/store. -type ListFilters = store.ListFilters +// ListFilters shares the model contract without depending on persistence. +type ListFilters = model.ListFilters // Event is the input shape for Emit. Caller fills what it knows; the // usecase stamps OccurredAt and serialises Payload. -type Event struct { - // Actor — filled by the middleware from JWT claims, by handlers - // for failed-auth or anon paths. - UserID *uint64 - UserEmail string - Role string - IP string - UserAgent string - RequestID string - - // Action — must be one of the canonical model.Action* constants. - Action string - ResourceType string - ResourceID string - ResourceName string - - // Outcome. - Status string // success|failure|denied - ErrorCode string - ErrorMessage string - - // Free-form structured detail. Caller is responsible for redacting - // secrets BEFORE passing in (LLM keys, passwords, tokens). Pass a - // map or struct; the usecase JSON-encodes. - Payload any -} +type Event = auditctx.Event // Usecase is the BC façade. type Usecase struct { diff --git a/internal/manager/data/audit/store/repo.go b/internal/manager/data/audit/store/repo.go index d0c6d02c9..621a46b94 100644 --- a/internal/manager/data/audit/store/repo.go +++ b/internal/manager/data/audit/store/repo.go @@ -35,16 +35,7 @@ func (r *Repo) Insert(ctx context.Context, log *model.Log) error { // ListFilters captures all the GET /v1/admin/audit-logs querystring // knobs in one struct so the http layer doesn't have to wire 7 args. // Zero values mean "no filter on this column". -type ListFilters struct { - UserEmail string - Action string - ResourceType string - Status string - From time.Time - To time.Time - Limit int // capped at 500 by the repo to bound payload size - Offset int -} +type ListFilters = model.ListFilters // List returns matching rows newest-first plus the total count for the // same filters (excluding limit/offset) so the UI can render diff --git a/internal/manager/model/audit/filters.go b/internal/manager/model/audit/filters.go new file mode 100644 index 000000000..7b72e911d --- /dev/null +++ b/internal/manager/model/audit/filters.go @@ -0,0 +1,14 @@ +package audit + +import "time" + +type ListFilters struct { + UserEmail string + Action string + ResourceType string + Status string + From time.Time + To time.Time + Limit int // capped at 500 by the repo to bound payload size + Offset int +} diff --git a/internal/manager/model/audit/log.go b/internal/manager/model/audit/log.go index 013189b46..5669724d9 100644 --- a/internal/manager/model/audit/log.go +++ b/internal/manager/model/audit/log.go @@ -5,7 +5,10 @@ // delete rows except via the retention job. package audit -import "time" +import ( + "github.com/ongridio/ongrid/internal/pkg/auditctx" + "time" +) // Log is one audit observation. // @@ -50,87 +53,47 @@ type Log struct { // parallel empty table. func (Log) TableName() string { return "audit_logs" } -// Status enum — bounded set used by both the writer and the list filter. -const ( - StatusSuccess = "success" - StatusFailure = "failure" - StatusDenied = "denied" -) - -// Canonical actions. The naming convention is verb_resource in -// snake_case — kept deliberately small. Sub-flavours (enable vs disable, -// role-change vs password-reset, single vs bulk) live in the payload, -// not the action name, so the UI's action dropdown stays short and the -// audit row's payload tells the operator exactly what changed. -// -// Operator feedback 2026-05-20: the prior 38-action enum mixed -// CRUD verbs with state-transition flavours (rule_enable / rule_disable) -// and per-setting names (llm_key_set / grafana_config_set), which made -// the action filter sprawl. The current set collapses those to the -// underlying verb + a payload that carries the specifics. -const ( - // Note 2026-05-21: auth_login / auth_logout / audit_view dropped. - // Operator flagged read-only / session-bookkeeping rows as drowning - // out the mutation signal. We keep auth_login_failed because a - // brute-force pattern still wants to be visible. - ActionAuthLoginFailed = "auth_login_failed" - - // User CRUD. role / password / profile field changes all surface as - // user_update — the payload field carries which field flipped. - ActionUserCreate = "user_create" - ActionUserUpdate = "user_update" - ActionUserDelete = "user_delete" - ActionUserExport = "user_export" - - // Device CRUD. enable / disable / bulk-delete fold into update / - // delete + a payload (e.g. {"enabled": false, "count": 3}). - ActionDeviceUpdate = "device_update" - ActionDeviceDelete = "device_delete" - - // Alert rule CRUD. enable / disable fold into update with payload - // {"enabled": }. - ActionRuleCreate = "rule_create" - ActionRuleUpdate = "rule_update" - ActionRuleDelete = "rule_delete" - - ActionIncidentAck = "incident_ack" - ActionIncidentResolve = "incident_resolve" - ActionIncidentSilence = "incident_silence" - - // Settings umbrella. LLM key / Grafana config / SSH key writes all - // land here; payload carries {"key": "...", "category": "..."}. - // Sensitive values are redacted upstream. - ActionSettingUpdate = "setting_update" - ActionSettingDelete = "setting_delete" - - ActionChannelCreate = "channel_create" - ActionChannelUpdate = "channel_update" - ActionChannelDelete = "channel_delete" - - ActionRepoCreate = "repo_create" - ActionRepoUpdate = "repo_update" - ActionRepoDelete = "repo_delete" - ActionRepoSync = "repo_sync" - - ActionSkillInstall = "skill_install" - ActionSkillUninstall = "skill_uninstall" -) - -// ResourceType buckets used in the resource_type column. Same flat-list -// convention as Action — group in the UI, not the data. +// 审计枚举沿用共享契约；兼容现有 Manager 调用方。 const ( - ResourceUser = "user" - ResourceDevice = "device" - ResourceIncident = "incident" - ResourceSetting = "setting" - ResourceRule = "rule" - ResourceChannel = "channel" - ResourceRepo = "repo" - ResourceSkill = "skill" - ResourceLLM = "llm" - ResourceGitKey = "git_ssh_key" - ResourceGrafana = "grafana" - ResourceRAG = "rag" - ResourceAudit = "audit" - ResourceAuth = "auth" + StatusSuccess = auditctx.StatusSuccess + StatusFailure = auditctx.StatusFailure + StatusDenied = auditctx.StatusDenied + ActionAuthLoginFailed = auditctx.ActionAuthLoginFailed + ActionUserCreate = auditctx.ActionUserCreate + ActionUserUpdate = auditctx.ActionUserUpdate + ActionUserDelete = auditctx.ActionUserDelete + ActionUserExport = auditctx.ActionUserExport + ActionDeviceUpdate = auditctx.ActionDeviceUpdate + ActionDeviceDelete = auditctx.ActionDeviceDelete + ActionRuleCreate = auditctx.ActionRuleCreate + ActionRuleUpdate = auditctx.ActionRuleUpdate + ActionRuleDelete = auditctx.ActionRuleDelete + ActionIncidentAck = auditctx.ActionIncidentAck + ActionIncidentResolve = auditctx.ActionIncidentResolve + ActionIncidentSilence = auditctx.ActionIncidentSilence + ActionSettingUpdate = auditctx.ActionSettingUpdate + ActionSettingDelete = auditctx.ActionSettingDelete + ActionChannelCreate = auditctx.ActionChannelCreate + ActionChannelUpdate = auditctx.ActionChannelUpdate + ActionChannelDelete = auditctx.ActionChannelDelete + ActionRepoCreate = auditctx.ActionRepoCreate + ActionRepoUpdate = auditctx.ActionRepoUpdate + ActionRepoDelete = auditctx.ActionRepoDelete + ActionRepoSync = auditctx.ActionRepoSync + ActionSkillInstall = auditctx.ActionSkillInstall + ActionSkillUninstall = auditctx.ActionSkillUninstall + ResourceUser = auditctx.ResourceUser + ResourceDevice = auditctx.ResourceDevice + ResourceIncident = auditctx.ResourceIncident + ResourceSetting = auditctx.ResourceSetting + ResourceRule = auditctx.ResourceRule + ResourceChannel = auditctx.ResourceChannel + ResourceRepo = auditctx.ResourceRepo + ResourceSkill = auditctx.ResourceSkill + ResourceLLM = auditctx.ResourceLLM + ResourceGitKey = auditctx.ResourceGitKey + ResourceGrafana = auditctx.ResourceGrafana + ResourceRAG = auditctx.ResourceRAG + ResourceAudit = auditctx.ResourceAudit + ResourceAuth = auditctx.ResourceAuth ) diff --git a/internal/manager/server/imbridge/http.go b/internal/manager/server/imbridge/http.go index 77a495173..77b25f7d7 100644 --- a/internal/manager/server/imbridge/http.go +++ b/internal/manager/server/imbridge/http.go @@ -17,7 +17,6 @@ import ( "github.com/go-chi/chi/v5" - iammodel "github.com/ongridio/ongrid/internal/iam/model" bizbridge "github.com/ongridio/ongrid/internal/manager/biz/imbridge" "github.com/ongridio/ongrid/internal/manager/biz/imbridge/provider/dingtalk" "github.com/ongridio/ongrid/internal/manager/biz/imbridge/provider/feishu" @@ -37,7 +36,7 @@ func requireAdmin(w http.ResponseWriter, r *http.Request) bool { http.Error(w, errs.ErrUnauthorized.Error(), errs.HTTPStatus(errs.ErrUnauthorized)) return false } - if t.Role != iammodel.RoleAdmin { + if t.Role != "admin" { http.Error(w, errs.ErrForbidden.Error(), errs.HTTPStatus(errs.ErrForbidden)) return false } diff --git a/internal/manager/server/middleware/audit.go b/internal/manager/server/middleware/audit.go index 14f0c8b52..36a238329 100644 --- a/internal/manager/server/middleware/audit.go +++ b/internal/manager/server/middleware/audit.go @@ -10,48 +10,13 @@ import ( "github.com/ongridio/ongrid/internal/manager/biz/audit" auditmodel "github.com/ongridio/ongrid/internal/manager/model/audit" + "github.com/ongridio/ongrid/internal/pkg/auditctx" "github.com/ongridio/ongrid/internal/pkg/tenantctx" ) -// auditContextKey points to a mutable *auditSlot in the request -// context. The slot is **installed by AuditMiddleware before the -// inner middleware chain runs**, so handlers down the chain can write -// to it even after intermediate middlewares (auth, otel, ...) have -// re-wrapped the request via r.WithContext — the pointer survives -// wrapping because the value type is `*auditSlot`, not the Event -// itself. Earlier impl (set Event by mutating *r) broke whenever any -// middleware between Audit and handler called r.WithContext, which -// is most of them. -type auditContextKey struct{} - -type auditSlot struct { - ev audit.Event - set bool -} - -// SetAuditEvent records the explicit Event the handler wants audited. -// Safe to call even outside an AuditMiddleware chain — it just no-ops -// if the slot isn't installed. -func SetAuditEvent(r *http.Request, ev audit.Event) { - if r == nil { - return - } - slot, ok := r.Context().Value(auditContextKey{}).(*auditSlot) - if !ok || slot == nil { - return - } - slot.ev = ev - slot.set = true -} - -// GetAuditEvent returns the stashed Event, if any. -func GetAuditEvent(ctx context.Context) (audit.Event, bool) { - slot, ok := ctx.Value(auditContextKey{}).(*auditSlot) - if !ok || slot == nil || !slot.set { - return audit.Event{}, false - } - return slot.ev, true -} +// SetAuditEvent / GetAuditEvent 保持现有 Handler API，实际槽由共享契约管理。 +func SetAuditEvent(r *http.Request, ev audit.Event) { auditctx.SetAuditEvent(r, ev) } +func GetAuditEvent(ctx context.Context) (audit.Event, bool) { return auditctx.GetAuditEvent(ctx) } // AuditMiddleware records HLD-010 audit_logs rows for **explicitly- // annotated user actions only**. The middleware no longer derives a @@ -78,8 +43,7 @@ func AuditMiddleware(uc *audit.Usecase) func(http.Handler) http.Handler { // SetAuditEvent. We hand the inner chain a new request // pointer carrying the slot; the slot pointer itself // survives every subsequent r.WithContext call. - slot := &auditSlot{} - ctx := context.WithValue(r.Context(), auditContextKey{}, slot) + ctx := auditctx.WithSlot(r.Context()) // tenantctx slot lives at the OUTER ctx so auth.Middleware // (deeper in the chain) can mutate it and we see the value // here post-handler. Without this, enrichFromRequest below @@ -89,7 +53,8 @@ func AuditMiddleware(uc *audit.Usecase) func(http.Handler) http.Handler { ctx = tenantctx.WithSlot(ctx) next.ServeHTTP(ww, r.WithContext(ctx)) - if uc == nil || !slot.set { + event, recorded := auditctx.GetAuditEvent(ctx) + if uc == nil || !recorded { return } // IMPORTANT: pass the wrapped ctx (the one carrying the @@ -97,8 +62,8 @@ func AuditMiddleware(uc *audit.Usecase) func(http.Handler) http.Handler { // outer r.Context() — the outer ctx doesn't have the slot // key. The slot itself is a pointer so the mutation by // auth.Middleware deeper in the chain is visible here. - enrichFromRequest(&slot.ev, r, ctx, ww.Status()) - uc.Emit(ctx, slot.ev) + enrichFromRequest(&event, r, ctx, ww.Status()) + uc.Emit(ctx, event) }) } } diff --git a/internal/manager/server/middleware/audit_test.go b/internal/manager/server/middleware/audit_test.go new file mode 100644 index 000000000..d7b5fded7 --- /dev/null +++ b/internal/manager/server/middleware/audit_test.go @@ -0,0 +1,56 @@ +package middleware + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/ongridio/ongrid/internal/manager/biz/audit" + model "github.com/ongridio/ongrid/internal/manager/model/audit" + "github.com/ongridio/ongrid/internal/pkg/auditctx" + "github.com/ongridio/ongrid/internal/pkg/tenantctx" +) + +type auditRecorder struct{ rows []model.Log } + +func (r *auditRecorder) Insert(_ context.Context, row *model.Log) error { + r.rows = append(r.rows, *row) + return nil +} +func (*auditRecorder) List(context.Context, audit.ListFilters) ([]model.Log, int64, error) { + return nil, 0, nil +} +func (*auditRecorder) DeleteOlderThan(context.Context, time.Time) (int64, error) { return 0, nil } + +// 跨域 Handler 只引用共享事件契约；原中间件仍必须写入相同的审计行。 +func TestSharedAuditEventSurvivesWrappedRequest(t *testing.T) { + for _, explicit := range []bool{false, true} { + repo := &auditRecorder{} + handler := AuditMiddleware(audit.New(repo, nil))(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tenantctx.SetOnSlot(r.Context(), tenantctx.Tenant{UserID: 17, Email: "audit@example.test", Role: "admin"}) + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + if explicit { + auditctx.SetAuditEvent(r.WithContext(ctx), auditctx.Event{Action: auditctx.ActionUserUpdate, ResourceType: auditctx.ResourceUser, ResourceID: "42"}) + } + w.WriteHeader(http.StatusForbidden) + })) + req := httptest.NewRequest(http.MethodPost, "/users/42", nil) + handler.ServeHTTP(httptest.NewRecorder(), req) + if !explicit { + if len(repo.rows) != 0 { + t.Fatal("unannotated request audited") + } + continue + } + if len(repo.rows) != 1 { + t.Fatalf("rows=%d", len(repo.rows)) + } + got := repo.rows[0] + if got.Action != model.ActionUserUpdate || got.Status != model.StatusDenied || got.UserID == nil || *got.UserID != 17 || got.ResourceID != "42" { + t.Fatalf("audit changed: %+v", got) + } + } +} diff --git a/internal/manager/biz/imbridge/adapter.go b/internal/manager/service/imbridge/adapter.go similarity index 93% rename from internal/manager/biz/imbridge/adapter.go rename to internal/manager/service/imbridge/adapter.go index 8d2259ee0..2724ad0ab 100644 --- a/internal/manager/biz/imbridge/adapter.go +++ b/internal/manager/service/imbridge/adapter.go @@ -13,7 +13,7 @@ import ( // LLMDefaultProvider returns the cluster-wide default LLM provider id + // model that web/SPA chats would pick. The IM path uses this so it // stays in sync with whatever the operator selected in the UI picker; -// otherwise chatprotocol.New's internal "" → "gpt-5.4" fallback would route IM +// otherwise a stale hard-coded fallback would route IM // traffic to a model that's not in the catalog and the LLM router // rejects it ("this model has beta-limitations…"), which surfaces in // chat apps as "助手执行失败". @@ -38,8 +38,7 @@ type AiopsServiceAdapter struct { } // NewAiopsAdapter wires the adapter. defaults is optional — when nil -// the adapter falls back to chatprotocol.RunOptions{} (which puts chatprotocol.New's -// hard-coded model default in charge). Pass the LLMSettingsResolver +// the adapter uses empty RunOptions and follows the router default. Pass the LLMSettingsResolver // so the IM path picks up the same default the SPA picker writes. func NewAiopsAdapter(svc *svcaiops.Service, serviceUserID uint64, defaults LLMDefaultProvider, log *slog.Logger) *AiopsServiceAdapter { if log == nil { @@ -90,8 +89,7 @@ func (a *AiopsServiceAdapter) StreamMessage(ctx context.Context, sessionID strin // runOptions resolves the cluster default provider+model so IM traffic // uses the same LLM the SPA picker writes. On any resolver error or // missing catalog, returns zero RunOptions and lets the agent layer -// apply its own fallback (which currently lands on gpt-5.4 — see -// chatprotocol.New). +// resolve the configured default. func (a *AiopsServiceAdapter) runOptions(ctx context.Context) chatprotocol.RunOptions { if a.defaults == nil { return chatprotocol.RunOptions{} diff --git a/internal/manager/biz/imbridge/adapter_test.go b/internal/manager/service/imbridge/adapter_test.go similarity index 100% rename from internal/manager/biz/imbridge/adapter_test.go rename to internal/manager/service/imbridge/adapter_test.go diff --git a/internal/pkg/auditctx/context.go b/internal/pkg/auditctx/context.go new file mode 100644 index 000000000..81255dad0 --- /dev/null +++ b/internal/pkg/auditctx/context.go @@ -0,0 +1,40 @@ +package auditctx + +import ( + "context" + "net/http" + "sync" +) + +type contextKey struct{} +type slot struct { + mu sync.RWMutex + event Event + set bool +} + +// WithSlot 为一次请求建立审计槽，供各域 Handler 与外层审计中间件交换事件。 +func WithSlot(ctx context.Context) context.Context { + return context.WithValue(ctx, contextKey{}, &slot{}) +} +func SetAuditEvent(r *http.Request, event Event) { + if r == nil { + return + } + s, _ := r.Context().Value(contextKey{}).(*slot) + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.event, s.set = event, true +} +func GetAuditEvent(ctx context.Context) (Event, bool) { + s, _ := ctx.Value(contextKey{}).(*slot) + if s == nil { + return Event{}, false + } + s.mu.RLock() + defer s.mu.RUnlock() + return s.event, s.set +} diff --git a/internal/pkg/auditctx/context_test.go b/internal/pkg/auditctx/context_test.go new file mode 100644 index 000000000..e70b89a19 --- /dev/null +++ b/internal/pkg/auditctx/context_test.go @@ -0,0 +1,46 @@ +package auditctx + +import ( + "context" + "net/http/httptest" + "sync" + "testing" +) + +func TestAuditSlotsAreRequestScoped(t *testing.T) { + req := httptest.NewRequest("POST", "/test", nil) + SetAuditEvent(req, Event{Action: ActionUserUpdate}) + if _, ok := GetAuditEvent(req.Context()); ok { + t.Fatal("uninstalled audit slot recorded an event") + } + first := req.WithContext(WithSlot(context.Background())) + second := req.WithContext(WithSlot(context.Background())) + SetAuditEvent(first, Event{Action: ActionUserUpdate, ResourceID: "first"}) + if _, ok := GetAuditEvent(second.Context()); ok { + t.Fatal("event leaked to another request") + } + derived, cancel := context.WithCancel(first.Context()) + defer cancel() + if event, ok := GetAuditEvent(derived); !ok || event.ResourceID != "first" { + t.Fatal("wrapped request lost its audit event") + } +} + +func TestConcurrentAuditSlotAccess(t *testing.T) { + req := httptest.NewRequest("POST", "/test", nil).WithContext(WithSlot(context.Background())) + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + SetAuditEvent(req, Event{Action: ActionUserUpdate}) + GetAuditEvent(req.Context()) + } + }() + } + wg.Wait() + if event, ok := GetAuditEvent(req.Context()); !ok || event.Action != ActionUserUpdate { + t.Fatal("event lost") + } +} diff --git a/internal/pkg/auditctx/event.go b/internal/pkg/auditctx/event.go new file mode 100644 index 000000000..78ae4a3d9 --- /dev/null +++ b/internal/pkg/auditctx/event.go @@ -0,0 +1,114 @@ +// Package auditctx 提供跨域审计事件契约，不依赖任何业务或存储包。 +package auditctx + +type Event struct { + // Actor — filled by the middleware from JWT claims, by handlers + // for failed-auth or anon paths. + UserID *uint64 + UserEmail string + Role string + IP string + UserAgent string + RequestID string + + // Action — must be one of the canonical model.Action* constants. + Action string + ResourceType string + ResourceID string + ResourceName string + + // Outcome. + Status string // success|failure|denied + ErrorCode string + ErrorMessage string + + // Free-form structured detail. Caller is responsible for redacting + // secrets BEFORE passing in (LLM keys, passwords, tokens). Pass a + // map or struct; the usecase JSON-encodes. + Payload any +} + +// Status enum — bounded set used by both the writer and the list filter. +const ( + StatusSuccess = "success" + StatusFailure = "failure" + StatusDenied = "denied" +) + +// Canonical actions. The naming convention is verb_resource in +// snake_case — kept deliberately small. Sub-flavours (enable vs disable, +// role-change vs password-reset, single vs bulk) live in the payload, +// not the action name, so the UI's action dropdown stays short and the +// audit row's payload tells the operator exactly what changed. +// +// Operator feedback 2026-05-20: the prior 38-action enum mixed +// CRUD verbs with state-transition flavours (rule_enable / rule_disable) +// and per-setting names (llm_key_set / grafana_config_set), which made +// the action filter sprawl. The current set collapses those to the +// underlying verb + a payload that carries the specifics. +const ( + // Note 2026-05-21: auth_login / auth_logout / audit_view dropped. + // Operator flagged read-only / session-bookkeeping rows as drowning + // out the mutation signal. We keep auth_login_failed because a + // brute-force pattern still wants to be visible. + ActionAuthLoginFailed = "auth_login_failed" + + // User CRUD. role / password / profile field changes all surface as + // user_update — the payload field carries which field flipped. + ActionUserCreate = "user_create" + ActionUserUpdate = "user_update" + ActionUserDelete = "user_delete" + ActionUserExport = "user_export" + + // Device CRUD. enable / disable / bulk-delete fold into update / + // delete + a payload (e.g. {"enabled": false, "count": 3}). + ActionDeviceUpdate = "device_update" + ActionDeviceDelete = "device_delete" + + // Alert rule CRUD. enable / disable fold into update with payload + // {"enabled": }. + ActionRuleCreate = "rule_create" + ActionRuleUpdate = "rule_update" + ActionRuleDelete = "rule_delete" + + ActionIncidentAck = "incident_ack" + ActionIncidentResolve = "incident_resolve" + ActionIncidentSilence = "incident_silence" + + // Settings umbrella. LLM key / Grafana config / SSH key writes all + // land here; payload carries {"key": "...", "category": "..."}. + // Sensitive values are redacted upstream. + ActionSettingUpdate = "setting_update" + ActionSettingDelete = "setting_delete" + + ActionChannelCreate = "channel_create" + ActionChannelUpdate = "channel_update" + ActionChannelDelete = "channel_delete" + + ActionRepoCreate = "repo_create" + ActionRepoUpdate = "repo_update" + ActionRepoDelete = "repo_delete" + ActionRepoSync = "repo_sync" + + ActionSkillInstall = "skill_install" + ActionSkillUninstall = "skill_uninstall" +) + +// ResourceType buckets used in the resource_type column. Same flat-list +// convention as Action — group in the UI, not the data. +const ( + ResourceUser = "user" + ResourceDevice = "device" + ResourceIncident = "incident" + ResourceSetting = "setting" + ResourceRule = "rule" + ResourceChannel = "channel" + ResourceRepo = "repo" + ResourceSkill = "skill" + ResourceLLM = "llm" + ResourceGitKey = "git_ssh_key" + ResourceGrafana = "grafana" + ResourceRAG = "rag" + ResourceAudit = "audit" + ResourceAuth = "auth" +) diff --git a/scripts/test-arch-policy.py b/scripts/test-arch-policy.py new file mode 100644 index 000000000..d817bcc0e --- /dev/null +++ b/scripts/test-arch-policy.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""在隔离源码副本中验证架构检查器不会放过禁止的 import。""" +import argparse +import pathlib +import shutil +import subprocess +import tempfile + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binary', default='go-arch-lint') + args = parser.parse_args() + binary = shutil.which(args.binary) + if not binary: + parser.error('go-arch-lint not found; run make tools-arch') + source = pathlib.Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory(prefix='ongrid-arch-policy-') as directory: + root = pathlib.Path(directory) + for filename in ('go.mod', '.go-arch-lint.yml'): + shutil.copy2(source / filename, root / filename) + for area in ('internal', 'cmd', 'api', 'examples', 'tests'): + (root / area).mkdir(parents=True, exist_ok=True) + for directory in (source / area).rglob('*'): + if directory.is_dir(): + (root / directory.relative_to(source)).mkdir(parents=True, exist_ok=True) + for filename in (source / area).rglob('*.go'): + if filename.name.endswith('_test.go'): + continue + target = root / filename.relative_to(source) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(filename, target) + + def check(expected, dependency=''): + result = subprocess.run( + [binary, 'check', '--output-color=false'], cwd=root, + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=60, + ) + if (result.returncode == 0) != expected: + raise AssertionError(result.stdout) + if not expected and dependency not in result.stdout: + raise AssertionError('Failed for an unrelated reason:\n' + result.stdout) + + check(True) + cases = [ + ('internal/manager/biz', 'internal/iam/model', False), + ('internal/iam/biz', 'internal/manager/model/audit', False), + ('internal/edgeagent/biz', 'internal/manager/model/audit', False), + ('internal/pkg', 'internal/manager/model/audit', False), + ('internal/skill', 'internal/manager/model/audit', False), + ('internal/manager/service', 'internal/manager/data/audit/store', False), + ('internal/manager/biz', 'internal/manager/data/audit/store', False), + ('cmd/ongrid-edge', 'internal/iam/model', False), + ('cmd/ongrid', 'internal/edgeagent/edgedirs', False), + ('internal/manager/biz', 'internal/manager/model/audit', True), + ('internal/manager/biz', 'internal/skill', True), + ] + for area, dependency, permitted in cases: + fixture = root / area / 'arch_policy_fixture' / 'policy.go' + fixture.parent.mkdir(parents=True, exist_ok=True) + fixture.write_text( + 'package arch_policy_fixture\nimport _ "github.com/ongridio/ongrid/' + + dependency + '"\n', encoding='utf-8', + ) + try: + check(permitted, dependency) + finally: + shutil.rmtree(fixture.parent) + print(f'Architecture policy: baseline and {len(cases)} boundary cases passed') + + +if __name__ == '__main__': + main() From 70586b5083dc53a544e9963a6f59e862f57af92f Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 19:07:26 +0800 Subject: [PATCH 09/24] docs(aiops): track capability acceptance and remaining live checks --- docs/rfc/RFC-005-unified-agent-runtime.md | 42 +++++++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/docs/rfc/RFC-005-unified-agent-runtime.md b/docs/rfc/RFC-005-unified-agent-runtime.md index 4e8513a1c..b55aa53ec 100644 --- a/docs/rfc/RFC-005-unified-agent-runtime.md +++ b/docs/rfc/RFC-005-unified-agent-runtime.md @@ -1,6 +1,6 @@ # RFC-005：统一 Agent runtime 与可执行工具目录 -- 状态：实施中（用户已确认分阶段方案） +- 状态：代码与自动化验证持续收尾；草稿 PR，禁止自动合并/部署 - 日期：2026-10-14 - 关联：#499、#498、ADR-035 - 基线：v0.18.1 / a6bc1039 @@ -58,7 +58,7 @@ Graph 默认运行路径与旧循环维护独立工具列表，安全 Skill 注 - `make test-agent-runtime`：在本地已有 Linux/amd64 构建镜像中通过，Go 1.25.12，启用 `-race`；覆盖 AIOps 全部子包、Service、HTTP、IM、Skill 和 Manager 装配。 - 新增确定性用例：真实 Graph 调用的联网开关、同步/后台 worker 权限继承、动态工具更新并发与旧引用拒绝、真实外部 subprocess Skill 执行、工作流文件工具目录与执行器一致性。 - `make build-ongrid`、`git diff --check` 通过。 -- `make arch-lint` 未完成：本机缺少 `go-arch-lint`；目标虽打印 skipping，后续 recipe 仍调用缺失命令。未改动该既有行为，合并前需在已安装工具的环境补验。 +- 首轮架构检查因工具缺失未完成；后续补装后暴露配置与依赖问题，现已修复，见下方收尾记录。 - 尚未执行生产 legacy 配置盘点、真实模型/MCP 集成冒烟、金丝雀升级与镜像回滚演练；这些是发布验收项，不能用单元测试替代。 - 旧 Registry 中供审批兼容及既有测试使用的闭包适配器暂留；生产聊天只调用 Graph，告警初查也已改用 BaseTool。退役旧循环不等于删除历史审批协议。 @@ -68,7 +68,43 @@ Graph 默认运行路径与旧循环维护独立工具列表，安全 Skill 注 - 修复安全 Skill schema 合并丢失原有 `required` 字段；覆盖全部内置 host-scope safe skills。 - 工具注册整批校验名称唯一性、来源前缀以及目录/执行列表一致性；失败保留原有工具与目录。ToolSearch 读取本回合对应执行版本的完整 schema，不混用动态刷新的新参数。 - 显式空工具视图清除继承的父回合权限列表。 -- 上述新增用例及 `make test-agent-runtime` 的 Linux race 回归通过。首版提交的 GitHub CI 已通过全仓 build/vet/race、部署清单及前端检查；补充提交需重新检查 CI。 +- 上述新增用例及 `make test-agent-runtime` 的 Linux race 回归通过。`be50baba` 的 GitHub CI 已通过全仓 build/vet/race、部署清单及前端检查；后续架构修复提交仍须重新检查 CI。 - 本地全仓测试尝试未通过：只读模块缓存缺少 APM 示例依赖，且 Docker amd64 仿真环境的 autoapm 进程发现测试失败。不能把本地全仓测试记为通过；原生 Ubuntu CI 的同项已通过。 - 已在临时目录安装 `go-arch-lint v1.19.0` 并执行 `make arch-lint`：检查被原有配置错误阻断（不存在的 `internal/edgeagent/data/**`，以及 `shared_pkg`/`api` 的空依赖声明）。`.go-arch-lint.yml` 与基线相同；这替代此前“未安装工具”的原因，仍不能声称架构 lint 通过。 - 告警初查使用启动时的 BaseTool 快照，避免后台任务与后续 Registry 装配并发读写；Manager 装配包 `go test -race ./cmd/ongrid` 通过。 + + +## 架构与验收收尾（2026-10-16） + +- 修复 `.go-arch-lint.yml`：补齐共享 Skill、Edge 运行时、示例和测试装配分类；删除不存在的 Edge data 目录；显式允许同层内部引用。Manager / Edge 命令分别装配，不能互相 import 对方业务域。 +- 生产 import 边界保留：三个业务域相互隔离、共享包不依赖业务、Service/Biz 不依赖 Data。包内 `_test.go` 允许组装存储 fixture，集成测试归独立测试组件；第三方库版本/漏洞由依赖检查管理。 +- `deepScan` 的实现类型流分析会将合法的消费方接口注入判为反向依赖，因此关闭该分析，使用 import 检查；不是给生产跨域依赖加白名单。新增 11 个正反向用例验证规则能拒绝越界，包括命令入口、共享包、业务层和服务层。 +- 修复实际逆向依赖：成员查询 DTO 与审计过滤类型下移到模型层；跨域审计事件、枚举和请求槽移入 `internal/pkg/auditctx`；保留旧 Manager 类型别名/函数封装以兼容调用方。IM 的 AIOps Service 适配器移至 Service 层。HTTP、数据库表和审计事件值不变。 +- `make arch-lint test-arch-policy` 通过（固定 `go-arch-lint v1.19.0`）；缺少工具时目标明确失败，不再打印虚假的 skipping。CI 安装固定版本并执行同样入口。 +- IAM、审计、IM、Manager 装配及 `make test-agent-runtime` 的 Linux race 回归通过。新增跨域审计事件在包装后的请求中仍正确持久化的回归。 + +### 能力验收矩阵 + +| 能力/入口 | 自动化证据 | 真实环境剩余验收 | +| --- | --- | --- | +| 默认聊天、安全内置 Skill | `TestGraphCatalogIncludesAndExecutesSafeSkills`；Graph 调用路径 | 真实模型正确选择并调用工具 | +| 联网开关 | `TestRuntimeWebSearchToggleControlsActualDispatch`：关闭时零调用；直接旧引用也拒绝 | 配置搜索服务，开/关各一轮 | +| Host Skill 参数 | `TestHostSkillSchemaPreservesRequiredParameters`：保留原必填项并添加 device_id | 测试 Edge 执行 DNS/TCP/HTTP/ping、日志与文件只读能力 | +| 外部安全 Skill | `TestGraphCatalogAdaptsExternalSubprocessSkill`：真实 manifest + 子进程 | 安装一个测试包，确认审计结果；危险类不自动注册 | +| 工作流文件工具 | `TestWorkflowPaletteAndInvokerIncludeHostFiles` | 画布选取、发布、运行、结果核对；K8s 审批动作仍明确排除 | +| 专家/后台子任务 | `TestWorkerInheritsWebSearchGate`；`TestSpawnWorker_*` | 真实专家路由、前后台联网权限一致 | +| MCP 生命周期 | `TestRuntimeRemovedOrReplacedToolRejectsStaleReference`；`TestToolSearchKeepsTurnSchemaAcrossRefresh`；`TestInvalidToolSourceUpdatePreservesExistingViews` | 实际 MCP 增删/禁用/替换，验证当前与下轮行为 | +| 只读/写开关/审批 | `TestFilterToolsForAgentRole_ViewerDropsSensitiveReadTool`；HumanApprovalGate/ReviewGate 测试 | 测试账号拒绝写操作；沙箱动作批准/拒绝/超时 | +| 停止与历史消息 | `TestStopWorker_Idempotent`；`TestBuildEinoHistory_*` | 停止长调用、重启后继续旧会话；核对 SSE 终止语义 | +| HTTP/IM 协议 | HTTP happy-path / stable DTO 测试；IM adapter/sender 回归 | 测试 IM 消息流完整、只发送一次终态 | +| 告警初查 | `TestInvestigatorUsesRealCorrelationToolContract`；失败/错配 batch 测试 | 测试告警生成正确初查记录 | +| 初始化失败 | `TestService_MissingRuntimeFailsExplicitly` | 确认聊天明确不可用、不切旧循环、其他 API 健康 | +| 架构与审计 | `make arch-lint test-arch-policy`；`TestSharedAuditEventSurvivesWrappedRequest` | 确认登录失败与用户修改的审计内容连续 | + +### 发布前仍需现场证据 + +1. 在指定测试环境记录 Manager 版本、模型/MCP 服务和测试账号；密钥留在环境配置，不写入报告。 +2. 按矩阵执行真实模型、测试 Edge 和 IM 冒烟，记录会话/工具调用/审计 ID，清理测试数据。 +3. 盘点部署中的 `ONGRID_AGENT_KERNEL=legacy`；确认已迁移到 Graph 及新版配置。 +4. 在隔离部署演练旧镜像升级到候选镜像，再恢复旧镜像与其 Compose 配置，检查会话可读、审批可查询、健康检查恢复。无数据库迁移。 +5. 仅在上述证据齐全后提交最终验收结论；当前不合并、不部署，不将模拟模型测试称为真实模型验收。 From 449d07078fdbacf7c0a968e80a60a2e29f42601d Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 19:25:06 +0800 Subject: [PATCH 10/24] test(aiops): include workflow adapters in runtime regression target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d684a9502..be46d98eb 100644 --- a/Makefile +++ b/Makefile @@ -914,4 +914,4 @@ test-web-ci-platform: ## Run the serial platform frontend CI tests (requires npm .PHONY: test-agent-runtime test-agent-runtime: ## 统一 Agent runtime 回归（tools 依赖 Linux cmdpolicy） - go test -race ./internal/manager/biz/aiops/... ./internal/manager/service/aiops ./internal/manager/server/aiops ./internal/manager/biz/imbridge ./internal/manager/service/imbridge ./internal/skill/... ./cmd/ongrid + go test -race ./internal/manager/biz/aiops/... ./internal/manager/service/aiops ./internal/manager/server/aiops ./internal/manager/biz/imbridge ./internal/manager/service/imbridge ./internal/manager/service/flow ./internal/skill/... ./cmd/ongrid From afd8111e5552a7d8493d7928d79f2d07ac519c99 Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 20:16:44 +0800 Subject: [PATCH 11/24] fix(aiops): retire legacy approval execution and invalidate stale requests --- Makefile | 2 +- cmd/ongrid/bootstrap_extensions.go | 3 - .../biz/aiops/tools/execute_k8s_action.go | 60 +------ .../aiops/tools/execute_k8s_action_test.go | 104 +----------- internal/manager/biz/aiops/tools/registry.go | 44 ------ internal/manager/biz/approval/usecase.go | 57 +++++-- internal/manager/biz/approval/usecase_test.go | 149 ++++++++++++++++++ .../service/aiops/approval_adapters.go | 80 +--------- .../service/aiops/approval_executors.go | 13 -- .../manager/service/aiops/human_approval.go | 14 +- .../service/aiops/mcp_approval_test.go | 10 ++ 11 files changed, 233 insertions(+), 303 deletions(-) create mode 100644 internal/manager/biz/approval/usecase_test.go diff --git a/Makefile b/Makefile index be46d98eb..f20532d8a 100644 --- a/Makefile +++ b/Makefile @@ -914,4 +914,4 @@ test-web-ci-platform: ## Run the serial platform frontend CI tests (requires npm .PHONY: test-agent-runtime test-agent-runtime: ## 统一 Agent runtime 回归（tools 依赖 Linux cmdpolicy） - go test -race ./internal/manager/biz/aiops/... ./internal/manager/service/aiops ./internal/manager/server/aiops ./internal/manager/biz/imbridge ./internal/manager/service/imbridge ./internal/manager/service/flow ./internal/skill/... ./cmd/ongrid + go test -race ./internal/manager/biz/aiops/... ./internal/manager/biz/approval ./internal/manager/service/aiops ./internal/manager/server/aiops ./internal/manager/biz/imbridge ./internal/manager/service/imbridge ./internal/manager/service/flow ./internal/skill/... ./cmd/ongrid diff --git a/cmd/ongrid/bootstrap_extensions.go b/cmd/ongrid/bootstrap_extensions.go index 4d6385fe0..fc0b0da1a 100644 --- a/cmd/ongrid/bootstrap_extensions.go +++ b/cmd/ongrid/bootstrap_extensions.go @@ -161,15 +161,12 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. Frontier: tunnel.fbClient, Edges: devices.edgeUC, Devices: devices.deviceUC, - Settings: settings.settingSvc, - Tools: aiops.toolsReg, Marketplace: mpUC, MCP: mcpUC, Log: log, }).Register(approvalUC) aiops.toolsReg.SetCloudBashProposer(managersvcaiops.CloudBashProposer{Usecase: approvalUC}) aiops.toolsReg.SetHostBashProposer(managersvcaiops.HostBashProposer{Usecase: approvalUC}) - aiops.toolsReg.SetK8sActionProposer(managersvcaiops.K8sActionProposer{Usecase: approvalUC}) // The Actions page is a read-only projection over two immutable audit // sources: graph ReviewGate decisions and legacy human approvals. k8sHandler := managerserverk8s.NewHandler(devices.k8sSvc, managersvck8s.NewActionAuditReader(aiops.mutatingProposalRepo, approvalUC)) diff --git a/internal/manager/biz/aiops/tools/execute_k8s_action.go b/internal/manager/biz/aiops/tools/execute_k8s_action.go index 7e50b5de1..6334fa32a 100644 --- a/internal/manager/biz/aiops/tools/execute_k8s_action.go +++ b/internal/manager/biz/aiops/tools/execute_k8s_action.go @@ -12,7 +12,6 @@ import ( "sync" "time" - "github.com/cloudwego/eino/compose" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" "github.com/ongridio/ongrid/internal/pkg/errs" "github.com/ongridio/ongrid/internal/pkg/tenantctx" @@ -131,18 +130,10 @@ const ( maxK8sGracePeriodSeconds = 3600 ) -// K8sActionProposer queues a preflight-validated Kubernetes write for human -// approval and returns the post-approval execution result. The legacy kernel -// wires this seam because it does not have the graph kernel's ReviewGate. -type K8sActionProposer interface { - ProposeAndAwait(ctx context.Context, args ExecuteK8sActionArgs, sessionID, toolCallID string, userID uint64) (string, error) -} - type ExecuteK8sActionTool struct { - caller Caller - reader K8sSnapshotReader - proposer K8sActionProposer - log *slog.Logger + caller Caller + reader K8sSnapshotReader + log *slog.Logger preflightMu sync.Mutex preflights map[string]executeK8sActionPreflightGrant @@ -156,20 +147,12 @@ type executeK8sActionPreflightGrant struct { } func NewExecuteK8sActionTool(caller Caller, reader K8sSnapshotReader, log *slog.Logger) *ExecuteK8sActionTool { - return NewExecuteK8sActionToolWithProposer(caller, reader, nil, log) -} - -// NewExecuteK8sActionToolWithProposer builds the legacy-compatible variant. -// A nil proposer preserves the graph-kernel path: ReviewGate authorizes the -// call and InvokableRun dispatches directly after consuming its preflight. -func NewExecuteK8sActionToolWithProposer(caller Caller, reader K8sSnapshotReader, proposer K8sActionProposer, log *slog.Logger) *ExecuteK8sActionTool { if log == nil { log = slog.Default() } return &ExecuteK8sActionTool{ caller: caller, reader: reader, - proposer: proposer, log: log, preflights: make(map[string]executeK8sActionPreflightGrant), } @@ -236,47 +219,12 @@ func (t *ExecuteK8sActionTool) InvokableRun(ctx context.Context, argsJSON string return "", err } if !req.DryRun { - if t.proposer != nil && !basetool.AgentWriteAllowedFromContext(ctx) { - return `{"status":"blocked","message":"Agent write actions are disabled; the Kubernetes action was not proposed or run."}`, nil - } if err := t.consumePreflight(strings.TrimSpace(in.PreflightToken), req, caller.UserID, basetool.SessionIDFromContext(ctx)); err != nil { return "", err } - if t.proposer != nil { - toolCallID := compose.GetToolCallID(ctx) - if toolCallID == "" { - toolCallID = basetool.ToolCallIDFromContext(ctx) - } - result, err := t.proposer.ProposeAndAwait(ctx, in, basetool.SessionIDFromContext(ctx), toolCallID, caller.UserID) - if err != nil { - return "", fmt.Errorf("%s: propose: %w", ToolNameExecuteK8sAction, err) - } - return result, nil - } - } - return t.dispatch(ctx, req, caller.UserID, basetool.SessionIDFromContext(ctx)) -} -// RunApproved dispatches the exact action payload stored in an approved inbox -// row. InvokableRun already consumed and bound the one-time preflight token -// before creating that row, so this method deliberately does not consume it a -// second time. The expected_resource_version still gives the controller an -// optimistic-lock guard if cluster state changed while approval was pending. -func (t *ExecuteK8sActionTool) RunApproved(ctx context.Context, in ExecuteK8sActionArgs, userID uint64, sessionID string) (string, error) { - if userID == 0 || strings.TrimSpace(sessionID) == "" { - return "", fmt.Errorf("%s: approved action is missing proposer identity", ToolNameExecuteK8sAction) - } - if strings.TrimSpace(in.PreflightToken) == "" { - return "", fmt.Errorf("%s: approved action is missing preflight_token", ToolNameExecuteK8sAction) - } - req, err := normalizeExecuteK8sActionArgs(in) - if err != nil { - return "", err - } - if req.DryRun { - return "", fmt.Errorf("%s: approved action must not be a dry-run", ToolNameExecuteK8sAction) } - return t.dispatch(ctx, req, userID, sessionID) + return t.dispatch(ctx, req, caller.UserID, basetool.SessionIDFromContext(ctx)) } func (t *ExecuteK8sActionTool) dispatch(ctx context.Context, req tunnel.KubernetesActionRequest, userID uint64, sessionID string) (string, error) { diff --git a/internal/manager/biz/aiops/tools/execute_k8s_action_test.go b/internal/manager/biz/aiops/tools/execute_k8s_action_test.go index 479b618c6..bc04a1f7a 100644 --- a/internal/manager/biz/aiops/tools/execute_k8s_action_test.go +++ b/internal/manager/biz/aiops/tools/execute_k8s_action_test.go @@ -168,103 +168,6 @@ func TestExecuteK8sActionToolCallsControllerEdge(t *testing.T) { } } -type recordingK8sActionProposer struct { - calls int - args ExecuteK8sActionArgs - sessionID string - toolCallID string - userID uint64 - result string -} - -func (p *recordingK8sActionProposer) ProposeAndAwait(_ context.Context, args ExecuteK8sActionArgs, sessionID, toolCallID string, userID uint64) (string, error) { - p.calls++ - p.args = args - p.sessionID = sessionID - p.toolCallID = toolCallID - p.userID = userID - return p.result, nil -} - -func TestExecuteK8sActionToolLegacyRequiresDryRunThenHumanApproval(t *testing.T) { - fc := &fakeCaller{respBody: mustMarshal(tunnel.KubernetesActionResponse{ - ClusterID: 1, Action: "scale", Kind: "Deployment", APIVersion: "apps/v1", - Namespace: "default", Name: "api", DryRun: true, - Preflight: tunnel.KubernetesActionPreflight{ - Kind: "Deployment", APIVersion: "apps/v1", Namespace: "default", Name: "api", - UID: "deploy-uid", ResourceVersion: "10", Exists: true, - }, - })} - proposer := &recordingK8sActionProposer{result: `{"status":"executed","result":"approved"}`} - tool := NewExecuteK8sActionToolWithProposer(fc, newFakeK8sSnapshotReader(), proposer, slog.Default()) - ctx := tenantctx.With(context.Background(), tenantctx.Tenant{UserID: 7, Role: "admin"}) - ctx = basetool.WithSessionID(ctx, "session-1") - ctx = basetool.WithToolCallID(ctx, "tool-row-1") - - dryRunOut, err := tool.InvokableRun(ctx, `{"cluster_id":1,"action":"scale","kind":"Deployment","namespace":"default","name":"api","replicas":3,"dry_run":true,"reason":"capacity"}`) - if err != nil { - t.Fatalf("dry-run InvokableRun: %v", err) - } - var preflight executeK8sActionResponse - if err := json.Unmarshal([]byte(dryRunOut), &preflight); err != nil { - t.Fatalf("decode preflight: %v", err) - } - if preflight.PreflightToken == "" || preflight.ExpectedResourceVersion != "10" { - t.Fatalf("missing preflight grant: %+v", preflight) - } - - replicas := 3 - applyArgs := ExecuteK8sActionArgs{ - ClusterID: 1, Action: "scale", Kind: "Deployment", Namespace: "default", Name: "api", - Replicas: &replicas, ExpectedResourceVersion: preflight.ExpectedResourceVersion, - PreflightToken: preflight.PreflightToken, Reason: "capacity", - } - applyBody := mustMarshal(applyArgs) - - blocked, err := tool.InvokableRun(ctx, string(applyBody)) - if err != nil { - t.Fatalf("write-disabled invocation: %v", err) - } - if !strings.Contains(blocked, `"status":"blocked"`) || proposer.calls != 0 || fc.calls != 1 { - t.Fatalf("write-disabled result=%s proposer.calls=%d controller.calls=%d", blocked, proposer.calls, fc.calls) - } - - ctx = basetool.WithHostWriteAllowed(ctx, true) - result, err := tool.InvokableRun(ctx, string(applyBody)) - if err != nil { - t.Fatalf("approval proposal: %v", err) - } - if result != proposer.result || proposer.calls != 1 { - t.Fatalf("proposal result=%s calls=%d", result, proposer.calls) - } - if proposer.sessionID != "session-1" || proposer.toolCallID != "tool-row-1" || proposer.userID != 7 { - t.Fatalf("proposal metadata: session=%q tool=%q user=%d", proposer.sessionID, proposer.toolCallID, proposer.userID) - } - if fc.calls != 1 { - t.Fatalf("write dispatched before approval: controller.calls=%d", fc.calls) - } - if _, err := tool.InvokableRun(ctx, string(applyBody)); err == nil || !strings.Contains(err.Error(), "already used") { - t.Fatalf("reused preflight error=%v", err) - } - - fc.respBody = mustMarshal(tunnel.KubernetesActionResponse{ - ClusterID: 1, Action: "scale", Kind: "Deployment", APIVersion: "apps/v1", - Namespace: "default", Name: "api", Applied: true, - Preflight: tunnel.KubernetesActionPreflight{ - Kind: "Deployment", APIVersion: "apps/v1", Namespace: "default", Name: "api", - UID: "deploy-uid", ResourceVersion: "10", Exists: true, - }, - ResultResourceVersion: "11", - }) - approvedOut, err := tool.RunApproved(ctx, proposer.args, proposer.userID, proposer.sessionID) - if err != nil { - t.Fatalf("RunApproved: %v", err) - } - if !strings.Contains(approvedOut, `"applied":true`) || fc.calls != 2 { - t.Fatalf("approved output=%s controller.calls=%d", approvedOut, fc.calls) - } -} - func TestExecuteK8sActionToolRejectsWriteWithoutPreflight(t *testing.T) { fc := &fakeCaller{} tool := NewExecuteK8sActionTool(fc, newFakeK8sSnapshotReader(), slog.Default()) @@ -422,17 +325,14 @@ func TestNormalizeExecuteK8sActionRejectsDrainOptionBounds(t *testing.T) { } } -func TestExecuteK8sActionRegisteredForLegacyOnlyWithApprovalProposer(t *testing.T) { +func TestExecuteK8sActionOnlyRegisteredInGraphCatalog(t *testing.T) { reg := NewRegistry(&fakeCaller{}, nil, nil, nil, nil, nil, nil, slog.Default()) reg.SetK8sSnapshotReader(newFakeK8sSnapshotReader()) if containsName(schemaNames(reg.Schemas()), ToolNameExecuteK8sAction) { t.Fatalf("legacy closure registry exposed %q without approval proposer", ToolNameExecuteK8sAction) } - reg.SetK8sActionProposer(&recordingK8sActionProposer{}) - if !containsName(schemaNames(reg.Schemas()), ToolNameExecuteK8sAction) { - t.Fatalf("legacy closure registry missing approval-protected %q", ToolNameExecuteK8sAction) - } + names := toolInfoNames(t, reg.BuildBaseTools().AllTools()) if !containsName(names, ToolNameExecuteK8sAction) { t.Fatalf("base tool registry missing %q: %v", ToolNameExecuteK8sAction, names) diff --git a/internal/manager/biz/aiops/tools/registry.go b/internal/manager/biz/aiops/tools/registry.go index fe2109724..e1e76a6d6 100644 --- a/internal/manager/biz/aiops/tools/registry.go +++ b/internal/manager/biz/aiops/tools/registry.go @@ -139,8 +139,6 @@ type Registry struct { // k8sSnapshot feeds query_k8s_snapshot. It reads manager DB snapshots, // not the live Kubernetes API. k8sSnapshot K8sSnapshotReader - k8sActionProposer K8sActionProposer - legacyK8sActionTool *ExecuteK8sActionTool packetCapture PacketCaptureCreator packetOperationCreate PacketCaptureOperationCreator @@ -155,15 +153,6 @@ func (r *Registry) SetCloudBashProposer(p CloudBashProposer) { r.cloudBashPropos // SetHostBashProposer wires mutating host_bash commands to the approval inbox. func (r *Registry) SetHostBashProposer(p HostBashProposer) { r.hostBashProposer = p } -// SetK8sActionProposer wires the default legacy kernel to the human approval -// inbox. The tool is registered only when caller, snapshot reader, and -// proposer are all available, so a partial boot never exposes an unfulfillable -// or approval-bypassing Kubernetes write surface. -func (r *Registry) SetK8sActionProposer(p K8sActionProposer) { - r.k8sActionProposer = p - r.registerLegacyK8sAction() -} - // SetNotificationSender wires send_notification → channel store + notify router. func (r *Registry) SetNotificationSender(s NotificationSender) { r.notificationSender = s } @@ -185,7 +174,6 @@ func (r *Registry) SetPacketCaptureOperationCreator(create PacketCaptureOperatio // NewRegistry's already-heavy constructor. func (r *Registry) SetK8sSnapshotReader(k K8sSnapshotReader) { r.k8sSnapshot = k - r.registerLegacyK8sAction() if k != nil { r.Register(Tool{ Name: ToolNameQueryK8sSnapshot, @@ -210,38 +198,6 @@ func (r *Registry) SetK8sSnapshotReader(k K8sSnapshotReader) { } } -func (r *Registry) registerLegacyK8sAction() { - delete(r.tools, ToolNameExecuteK8sAction) - r.legacyK8sActionTool = nil - if r.caller == nil || r.k8sSnapshot == nil || r.k8sActionProposer == nil { - return - } - tool := NewExecuteK8sActionToolWithProposer(r.caller, r.k8sSnapshot, r.k8sActionProposer, r.log) - r.legacyK8sActionTool = tool - r.Register(Tool{ - Name: ToolNameExecuteK8sAction, - Description: ExecuteK8sActionDescription, - Schema: ExecuteK8sActionSchema, - Execute: func(ctx context.Context, args json.RawMessage) (ExecuteResult, error) { - out, err := tool.InvokableRun(ctx, string(args)) - if err != nil { - return ExecuteResult{}, err - } - return ExecuteResult{ResultJSON: json.RawMessage(out)}, nil - }, - }) -} - -// RunApprovedK8sAction executes an exact payload after the approval usecase -// has recorded an admin decision. It deliberately reuses the same tool object -// that issued and consumed the legacy preflight grant. -func (r *Registry) RunApprovedK8sAction(ctx context.Context, args ExecuteK8sActionArgs, userID uint64, sessionID string) (string, error) { - if r.legacyK8sActionTool == nil { - return "", fmt.Errorf("%s: legacy approval executor is not configured", ToolNameExecuteK8sAction) - } - return r.legacyK8sActionTool.RunApproved(ctx, args, userID, sessionID) -} - // SetAuditLister wires the audit query seam consumed by // query_change_events. Call after NewRegistry (cmd/main.go). func (r *Registry) SetAuditLister(a AuditLister) { r.auditLister = a } diff --git a/internal/manager/biz/approval/usecase.go b/internal/manager/biz/approval/usecase.go index e863c3abd..87a8da8e6 100644 --- a/internal/manager/biz/approval/usecase.go +++ b/internal/manager/biz/approval/usecase.go @@ -9,6 +9,7 @@ package approval import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "strings" @@ -31,8 +32,8 @@ type Repo interface { // Executor runs an approved action's payload and returns a result blob. // Registered per Kind by the producer (e.g. cloud-shell registers -// "shell_command"). Absent executor → Approve just marks the row approved -// (no execution), which is safe. +// "shell_command"). Missing executors fail the action explicitly; historical +// approval records never imply that an unavailable action was executed. type Executor func(ctx context.Context, payloadJSON string) (resultJSON string, err error) // Usecase is the inbox facade. @@ -119,21 +120,32 @@ func (u *Usecase) Approve(ctx context.Context, approverID uint64, id string) (*m if err != nil { return nil, err } - exec, ok := u.executors[a.Kind] - if !ok { - u.log.Warn("approved but no executor for kind", slog.String("id", id), slog.String("kind", a.Kind)) - return a, nil + exec := u.executors[a.Kind] + var res string + var runErr error + if exec == nil { + runErr = fmt.Errorf("approval executor for %q is unavailable; submit a new request", a.Kind) + } else { + res, runErr = exec(ctx, a.PayloadJSON) } - res, runErr := exec(ctx, a.PayloadJSON) status := model.StatusExecuted if runErr != nil { status = model.StatusFailed - res = fmt.Sprintf(`{"error":%q}`, runErr.Error()) + payload, err := json.Marshal(struct { + Error string `json:"error"` + }{Error: runErr.Error()}) + if err != nil { + return nil, fmt.Errorf("encode approval failure: %w", err) + } + res = string(payload) } if err := u.repo.SetResult(ctx, id, status, res, time.Now().UTC()); err != nil { - u.log.Warn("set approval result failed", slog.String("id", id), slog.Any("err", err)) + return nil, fmt.Errorf("persist approval execution result: %w", err) + } + a, err = u.repo.Get(ctx, id) + if err != nil { + return nil, fmt.Errorf("read completed approval: %w", err) } - a, _ = u.repo.Get(ctx, id) return a, nil } @@ -145,3 +157,28 @@ func (u *Usecase) Reject(ctx context.Context, approverID uint64, id, reason stri "reason": strings.TrimSpace(reason), "decided_at": now, }) } + +// InvalidatePending retires a proposal whose originating request can no longer +// execute. The conditional decision never overwrites approval or rejection. +func (u *Usecase) InvalidatePending(ctx context.Context, id, reason string) error { + result, err := json.Marshal(struct { + Error string `json:"error"` + }{Error: reason}) + if err != nil { + return fmt.Errorf("encode invalidated approval: %w", err) + } + err = u.repo.Decide(ctx, id, map[string]any{"status": model.StatusFailed, "reason": reason, "result_json": string(result), "decided_at": time.Now().UTC()}) + if errors.Is(err, errs.ErrNotFound) { + row, readErr := u.repo.Get(ctx, id) + if readErr != nil { + return fmt.Errorf("read invalidated approval: %w", readErr) + } + if row.Status != model.StatusPending { + return nil + } + } + if err != nil { + return fmt.Errorf("invalidate pending approval: %w", err) + } + return nil +} diff --git a/internal/manager/biz/approval/usecase_test.go b/internal/manager/biz/approval/usecase_test.go new file mode 100644 index 000000000..867bc7ec7 --- /dev/null +++ b/internal/manager/biz/approval/usecase_test.go @@ -0,0 +1,149 @@ +package approval_test + +import ( + "context" + "errors" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/glebarez/sqlite" + "github.com/ongridio/ongrid/internal/manager/biz/approval" + "github.com/ongridio/ongrid/internal/manager/data/approval/store" + model "github.com/ongridio/ongrid/internal/manager/model/approval" + "github.com/ongridio/ongrid/internal/pkg/errs" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func approvalRepo(t *testing.T) *store.Repo { + t.Helper() + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "approval.db")), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatal(err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + sqlDB.SetMaxOpenConns(1) + t.Cleanup(func() { + if err := sqlDB.Close(); err != nil { + t.Error(err) + } + }) + if err := store.Migrate(db); err != nil { + t.Fatal(err) + } + return store.NewRepo(db) +} +func propose(t *testing.T, uc *approval.Usecase) string { + t.Helper() + a, err := uc.Propose(context.Background(), approval.ProposeInput{Kind: "action", Title: "test", Payload: struct{}{}, ProposedBy: 7}) + if err != nil { + t.Fatal(err) + } + return a.ID +} +func TestUnavailableExecutorFailsHistoricalApproval(t *testing.T) { + for _, registerNil := range []bool{false, true} { + repo := approvalRepo(t) + uc := approval.NewUsecase(repo, nil) + id := propose(t, uc) + if registerNil { + uc.RegisterExecutor("action", nil) + } + a, err := uc.Approve(context.Background(), 9, id) + if err != nil { + t.Fatal(err) + } + if a.Status != model.StatusFailed || a.ResultJSON == nil || !strings.Contains(*a.ResultJSON, "unavailable") { + t.Fatalf("approval=%+v", a) + } + if _, err := uc.Approve(context.Background(), 9, id); err == nil { + t.Fatal("failed action approved twice") + } + stored, err := repo.Get(context.Background(), id) + if err != nil || stored.ApprovedBy == nil || *stored.ApprovedBy != 9 || stored.PayloadJSON != "{}" { + t.Fatalf("history=%+v err=%v", stored, err) + } + } +} +func TestConcurrentApprovalExecutesOnce(t *testing.T) { + uc := approval.NewUsecase(approvalRepo(t), nil) + var executed atomic.Int32 + uc.RegisterExecutor("action", func(context.Context, string) (string, error) { executed.Add(1); return `{}`, nil }) + id := propose(t, uc) + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, err := uc.Approve(context.Background(), 9, id) + if err != nil && !errors.Is(err, errs.ErrNotFound) { + t.Error(err) + } + }() + } + wg.Wait() + if executed.Load() != 1 { + t.Fatalf("executed=%d", executed.Load()) + } + a, err := uc.Get(context.Background(), id) + if err != nil || a.Status != model.StatusExecuted { + t.Fatalf("approval=%+v err=%v", a, err) + } +} + +type failingResultRepo struct { + approval.Repo + err error +} + +func (r failingResultRepo) SetResult(context.Context, string, string, string, time.Time) error { + return r.err +} +func TestApprovalResultPersistenceFailureIsReturned(t *testing.T) { + failure := errors.New("result storage unavailable") + uc := approval.NewUsecase(failingResultRepo{Repo: approvalRepo(t), err: failure}, nil) + uc.RegisterExecutor("action", func(context.Context, string) (string, error) { return `{}`, nil }) + if _, err := uc.Approve(context.Background(), 9, propose(t, uc)); !errors.Is(err, failure) { + t.Fatalf("err=%v", err) + } +} + +func TestInvalidationCannotOverwriteDecisionOrPermitLaterExecution(t *testing.T) { + for _, decision := range []string{"pending", "approved", "rejected"} { + t.Run(decision, func(t *testing.T) { + uc := approval.NewUsecase(approvalRepo(t), nil) + uc.RegisterExecutor("action", func(context.Context, string) (string, error) { return `{}`, nil }) + id := propose(t, uc) + want := model.StatusFailed + switch decision { + case "approved": + if _, err := uc.Approve(context.Background(), 9, id); err != nil { + t.Fatal(err) + } + want = model.StatusExecuted + case "rejected": + if err := uc.Reject(context.Background(), 9, id, "denied"); err != nil { + t.Fatal(err) + } + want = model.StatusRejected + } + if err := uc.InvalidatePending(context.Background(), id, "expired"); err != nil { + t.Fatal(err) + } + a, err := uc.Get(context.Background(), id) + if err != nil || a.Status != want { + t.Fatalf("approval=%+v err=%v", a, err) + } + if _, err := uc.Approve(context.Background(), 9, id); err == nil { + t.Fatal("terminal proposal executed again") + } + }) + } +} diff --git a/internal/manager/service/aiops/approval_adapters.go b/internal/manager/service/aiops/approval_adapters.go index 986cbd86d..1b12ae435 100644 --- a/internal/manager/service/aiops/approval_adapters.go +++ b/internal/manager/service/aiops/approval_adapters.go @@ -205,76 +205,6 @@ func (s HostBashProposer) ProposeAndAwait(ctx context.Context, deviceIDs []uint6 return CloudBashProposer{Usecase: s.Usecase}.awaitDecision(ctx, a.ID) } -// K8sActionProposer gives the default legacy kernel the same synchronous -// propose-confirm UX used by host_bash. The tool has already consumed a -// matching dry-run token before this method is called; no Kubernetes write is -// dispatched until the approval executor runs. -type K8sActionProposer struct{ Usecase *managerbizapproval.Usecase } - -func (s K8sActionProposer) ProposeAndAwait(ctx context.Context, args aiopstools.ExecuteK8sActionArgs, sessionID, toolCallID string, userID uint64) (string, error) { - command := formatK8sActionApproval(args) - title := aiopstools.ToolNameExecuteK8sAction + " " + command - if len(title) > 120 { - title = title[:120] + "…" - } - summary := command - if reason := strings.TrimSpace(args.Reason); reason != "" { - summary += " reason=" + reason - } - a, err := s.Usecase.Propose(ctx, managerbizapproval.ProposeInput{ - Kind: aiopstools.ToolNameExecuteK8sAction, - Title: title, - Summary: summary, - Payload: K8sActionPayload{Args: args, UserID: userID, SessionID: sessionID}, - Source: "agent", - SessionID: sessionID, - ProposedBy: userID, - }) - if err != nil { - return "", err - } - if emit := aiopschatruntime.EmitFromContext(ctx); emit != nil { - emit(aiopschatruntime.Event{ - Type: aiopschatruntime.EventApprovalPending, - Approval: &aiopschatruntime.ApprovalPending{ - ApprovalID: a.ID, - ToolCallID: toolCallID, - Kind: aiopstools.ToolNameExecuteK8sAction, - ToolName: aiopstools.ToolNameExecuteK8sAction, - Command: command, - }, - }) - } - return CloudBashProposer{Usecase: s.Usecase}.awaitDecision(ctx, a.ID) -} - -func formatK8sActionApproval(args aiopstools.ExecuteK8sActionArgs) string { - target := strings.TrimSpace(args.Name) - if namespace := strings.TrimSpace(args.Namespace); namespace != "" { - target = namespace + "/" + target - } - parts := []string{fmt.Sprintf("cluster_id=%d", args.ClusterID)} - for _, value := range []string{args.Action, args.Kind, target} { - if value = strings.TrimSpace(value); value != "" { - parts = append(parts, value) - } - } - if args.Replicas != nil { - parts = append(parts, fmt.Sprintf("replicas=%d", *args.Replicas)) - } - if rv := strings.TrimSpace(args.ExpectedResourceVersion); rv != "" { - parts = append(parts, "resource_version="+rv) - } - return strings.Join(parts, " ") -} - -// awaitDecision blocks until the approval row reaches a terminal state, then -// returns the tool result string the ReAct loop continues with. The approve -// REST handler runs the executor synchronously and records the result, so a -// poll only ever reads it back — there is no double execution. "approved" -// (executor mid-run, result not yet stored) is treated as non-terminal: -// cloud_bash always has an executor, so it transitions to executed/failed -// shortly; the timeout is the backstop. func (s CloudBashProposer) awaitDecision(ctx context.Context, id string) (string, error) { deadline := time.Now().Add(ApprovalWaitTimeout) ticker := time.NewTicker(approvalPollInterval) @@ -282,9 +212,10 @@ func (s CloudBashProposer) awaitDecision(ctx context.Context, id string) (string for { select { case <-ctx.Done(): - // Stream closed (user navigated away) — stop waiting. The - // approval remains pending in the inbox for later decision. - return `{"status":"cancelled","message":"The approval wait was interrupted; the command was not run."}`, nil + if err := invalidateApproval(ctx, s.Usecase, id, "The originating request was cancelled; submit a new request."); err != nil { + return "", err + } + return `{"status":"cancelled","message":"The approval wait was interrupted. Check the approval record for the final execution status."}`, nil case <-ticker.C: a, err := s.Usecase.Get(ctx, id) if err != nil { @@ -305,6 +236,9 @@ func (s CloudBashProposer) awaitDecision(ctx context.Context, id string) (string return `{"status":"rejected","message":"The user rejected this command; it was not run. Do not retry it without new instructions."}`, nil } if time.Now().After(deadline) { + if err := invalidateApproval(ctx, s.Usecase, id, "The approval wait expired; submit a new request."); err != nil { + return "", err + } return `{"status":"timeout","message":"No approval within 30 minutes; the command was not run."}`, nil } } diff --git a/internal/manager/service/aiops/approval_executors.go b/internal/manager/service/aiops/approval_executors.go index 50f6b55b1..8a409171b 100644 --- a/internal/manager/service/aiops/approval_executors.go +++ b/internal/manager/service/aiops/approval_executors.go @@ -15,7 +15,6 @@ import ( managerbizmarketplace "github.com/ongridio/ongrid/internal/manager/biz/marketplace" managerbizmcp "github.com/ongridio/ongrid/internal/manager/biz/mcp" managerbizsecret "github.com/ongridio/ongrid/internal/manager/biz/secret" - managerbizsetting "github.com/ongridio/ongrid/internal/manager/biz/setting" managersvcfb "github.com/ongridio/ongrid/internal/manager/service/frontierbound" "github.com/ongridio/ongrid/internal/pkg/runner" "github.com/ongridio/ongrid/internal/pkg/workspace" @@ -30,8 +29,6 @@ type ApprovalExecutors struct { Frontier *managersvcfb.Client Edges *managerbizedge.Usecase Devices *managerbizdevice.Usecase - Settings *managerbizsetting.Service - Tools *aiopstools.Registry Marketplace *managerbizmarketplace.Usecase MCP *managerbizmcp.Usecase Log *slog.Logger @@ -115,16 +112,6 @@ func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { tool := aiopstools.NewBashToolWithProposer(e.Frontier, e.Edges, e.Devices, nil, e.Log) return tool.RunApproved(ctx, p.DeviceIDs, p.Command, p.TimeoutSeconds) }) - approvalUC.RegisterExecutor(aiopstools.ToolNameExecuteK8sAction, func(ctx context.Context, payloadJSON string) (string, error) { - var p K8sActionPayload - if err := json.Unmarshal([]byte(payloadJSON), &p); err != nil { - return "", fmt.Errorf("decode %s approval payload: %w", aiopstools.ToolNameExecuteK8sAction, err) - } - if !e.Settings.AgentWriteEnabled(ctx) { - return "", fmt.Errorf("%s: Agent write actions are disabled", aiopstools.ToolNameExecuteK8sAction) - } - return e.Tools.RunApprovedK8sAction(ctx, p.Args, p.UserID, p.SessionID) - }) // HLD-018 P2: mcp_call executor — on approve, connect the server and run // the tool. Trusted servers skip this and run synchronously in the tool. approvalUC.RegisterExecutor("mcp_call", func(ctx context.Context, payloadJSON string) (string, error) { diff --git a/internal/manager/service/aiops/human_approval.go b/internal/manager/service/aiops/human_approval.go index 6a62d2dac..4a3a654ca 100644 --- a/internal/manager/service/aiops/human_approval.go +++ b/internal/manager/service/aiops/human_approval.go @@ -95,7 +95,10 @@ func (b *HumanApprovalBroker) ProposeAndAwait(ctx context.Context, req aiopstool select { case <-ctx.Done(): cleanup() - return `{"status":"cancelled","message":"The approval wait was interrupted; the action was not run."}`, nil + if err := invalidateApproval(ctx, uc, a.ID, "The originating request was cancelled; submit a new request."); err != nil { + return "", err + } + return `{"status":"cancelled","message":"The approval wait was interrupted. Check the approval record for the final execution status."}`, nil case <-ticker.C: row, getErr := uc.Get(ctx, a.ID) if getErr != nil { @@ -114,6 +117,9 @@ func (b *HumanApprovalBroker) ProposeAndAwait(ctx context.Context, req aiopstool } if time.Now().After(deadline) { cleanup() + if err := invalidateApproval(ctx, uc, a.ID, "The approval wait expired; submit a new request."); err != nil { + return "", err + } return `{"status":"timeout","message":"No approval within 30 minutes; the action was not run."}`, nil } } @@ -141,3 +147,9 @@ func truncateApprovalTitle(title string) string { } return title } + +func invalidateApproval(ctx context.Context, uc *managerbizapproval.Usecase, id, reason string) error { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + return uc.InvalidatePending(cleanupCtx, id, reason) +} diff --git a/internal/manager/service/aiops/mcp_approval_test.go b/internal/manager/service/aiops/mcp_approval_test.go index 70cd5600e..fb4ef46a2 100644 --- a/internal/manager/service/aiops/mcp_approval_test.go +++ b/internal/manager/service/aiops/mcp_approval_test.go @@ -119,6 +119,16 @@ func TestMCPApprovalLifecycle(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("approval did not complete") } + + if decision == "cancel" { + row, err := uc.Get(context.Background(), id) + if err != nil || row.Status != "failed" { + t.Fatalf("cancelled approval=%+v err=%v", row, err) + } + if _, err := uc.Approve(context.Background(), 7, id); err == nil { + t.Fatal("cancelled proposal accepted a late approval") + } + } wantCalls := int32(1) if decision == "reject" || decision == "cancel" { wantCalls = 0 From 781fc1f0a25d9f6a495140e8596d932e8443747e Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 20:16:44 +0800 Subject: [PATCH 12/24] docs(aiops): propose isolated v2 and v3 runtime selection --- docs/adr/ADR-035-unified-agent-runtime.md | 1 + .../PRD-009-agent-runtime-selection.md | 27 ++++++++ docs/rfc/RFC-008-unified-agent-runtime.md | 12 ++++ docs/rfc/RFC-009-agent-runtime-versions.md | 67 +++++++++++++++++++ 4 files changed, 107 insertions(+) create mode 100644 docs/requirements/PRD-009-agent-runtime-selection.md create mode 100644 docs/rfc/RFC-009-agent-runtime-versions.md diff --git a/docs/adr/ADR-035-unified-agent-runtime.md b/docs/adr/ADR-035-unified-agent-runtime.md index f78687046..c7dd30254 100644 --- a/docs/adr/ADR-035-unified-agent-runtime.md +++ b/docs/adr/ADR-035-unified-agent-runtime.md @@ -3,6 +3,7 @@ - 状态：已接受（用户确认实施方案） - 日期：2026-10-14 - 关联：RFC-008、#499、#498 +- 后续提议：RFC-009 / PRD-009 增加 Graph v2/v3 隔离与显式选择；尚未实现，原决策保留作历史。 ## 背景 diff --git a/docs/requirements/PRD-009-agent-runtime-selection.md b/docs/requirements/PRD-009-agent-runtime-selection.md new file mode 100644 index 000000000..ff049b7da --- /dev/null +++ b/docs/requirements/PRD-009-agent-runtime-selection.md @@ -0,0 +1,27 @@ +# PRD-009：Agent Runtime 版本选择 + +- 状态：提议，尚未实现 +- 日期：2026-10-16 +- 优先级：P1 +- 关联：RFC-009、PR #503 + +## 用户目标 + +用户试用新版 Agent 后，可显式选择已发布版本继续工作。管理员可控制新会话默认版本，避免新行为一次影响所有人。 + +## 建议交互与验收 + +- v2 对应正式发布 v0.18.2 的 Graph 行为，v3 对应当前改进；不是模型版本或 Edge 版本。 +- 新建会话可选择版本，当前会话显示实际版本。管理员默认值只影响之后新建的会话。 +- 切换时创建新会话；旧会话、运行中任务和审批继续归属原版本。不自动复制待执行动作。 +- 升级前的正式发布会话继续使用 v2；非法版本明确报错，不静默切换。 +- 两个版本遵守相同的权限和安全底线。未配置模型/工具时如实提示，回退不恢复已撤销权限。 +- 测试覆盖所有聊天入口、子任务和审批；版本路由正确率 100%，跨版本执行次数为零。 + +## 范围与依赖 + +API、会话持久化、默认设置、聊天选择 UI 与版本路由一起交付，技术设计见 RFC-009。依赖 v2 基线验证、v3 当前审批清理和双版本回归。暂不提供同一会话原地切换、执行恢复或对模型输出完全一致的承诺。 + +## 风险、计划与复盘 + +主要风险是共享工具/模型变化导致回退行为漂移、历史数据错误归类、审批跨版本执行。通过版本标记、冻结策略与双版本验收控制。任务拆分和预计里程碑见 RFC-009；发布日期以验收为准。发布后四周内比较各版本工具成功率、用户回退比例与错误率，决定是否调整默认值。 diff --git a/docs/rfc/RFC-008-unified-agent-runtime.md b/docs/rfc/RFC-008-unified-agent-runtime.md index 683b1eeee..84d08b895 100644 --- a/docs/rfc/RFC-008-unified-agent-runtime.md +++ b/docs/rfc/RFC-008-unified-agent-runtime.md @@ -118,3 +118,15 @@ Graph 默认运行路径与旧循环维护独立工具列表，安全 Skill 注 - 真实模型/MCP、线上配置盘点与升级回滚仍未执行，PR 保持草稿且禁止合并。 - 最终定向 E2E 通过（25.167s）：真实 Manager + 临时 MySQL，覆盖 `TestAssistantChat_*`、`TestMCP_CRUD`、`TestSkills_Registry`；模型及外部服务为测试替身，不能视为真实模型验收。 + +## 变更记录 + +- 2026-10-16：用户新增 v2/v3 显式切换与代码隔离需求，见 RFC-009 / PRD-009（提议）。原文“唯一实现、不保留双版本”的范围需随新方案调整；当前 PR 尚未交付版本选择。旧 legacy for-loop 仍退役。 + +### 审批退役修正（2026-10-16） + +- 当前 Graph 使用的 shell/MCP/安装 Skill 执行器保留；删除 legacy K8s proposer、旧 Registry 注册/执行入口及专用批准后执行方法，Graph 的 dry-run/preflight/ReviewGate 路径不变。 +- `K8sActionPayload` 仍供历史动作审计读取，已完成审批数据不删除。旧待审批动作在缺少执行器时明确落为 failed，要求重新提交。 +- 更正此前分析：旧 K8s token 在创建审批前消费，旧 RunApproved 并不再次校验内存 grant；不能将其描述为重启必然失效。删除该执行路径是明确的退役政策。 +- 取消/等待超时通过 pending 条件更新使审批失效，不覆盖已批准或已拒绝的结果；并发批准只执行一次。执行结果保存失败向上返回错误，不再静默成功。 +- 本地审批/适配器/历史审计 race、完整 Agent Linux race、架构基线和 11 个边界 fixture 均通过。版本选择实现另见 RFC-009 提议，尚未完成。 diff --git a/docs/rfc/RFC-009-agent-runtime-versions.md b/docs/rfc/RFC-009-agent-runtime-versions.md new file mode 100644 index 000000000..e1be8248d --- /dev/null +++ b/docs/rfc/RFC-009-agent-runtime-versions.md @@ -0,0 +1,67 @@ +# RFC-009：Agent Runtime v2/v3 隔离与显式切换 + +- 状态：提议；版本化方向来自用户需求，以下切换交互待确认，尚未实现 +- 日期：2026-10-16 +- 关联：PR #503、RFC-008、ADR-035；PRD-009 +- 基线：正式发布标签 v0.18.2（`3b41874cae792329fd61cfe0450e57335efa3f1d`）的 Graph；不把已退役的 legacy for-loop 当作 v2 + +## 背景与目标 + +用户需要在新版 Agent 效果不佳时切回熟悉的行为，同时隔离实现。Agent Runtime 版本独立于 Ongrid 产品版本和模型版本：v2 是已发布 Graph 的行为基线，v3 是当前统一目录、权限和工具生命周期改进。当前代码仍只有一个 Graph 实现，本文不是已交付功能声明。 + +## 版本与隔离边界 + +拟定结构： + +``` +internal/manager/biz/aiops/ + chatprotocol/ # 稳定边界，不引用版本实现 + runtime/contract/ # 请求/回复、worker/目录管理契约 + runtime/v2/ # 发布基线的循环、上下文策略、prompt、装配策略 + runtime/v3/ # 本次改进，后续行为变更在此进行 +internal/manager/service/aiops/ + runtime_router.go # 依据会话记录选版本，禁止自动降级 +``` + +v2/v3 不互相 import；入口只依赖契约。代码位置仍需结合实际依赖图细化，不能仅给两个名字包一层相同实现。目录装配、工具暴露、上下文压缩、worker 策略及内置 prompt 均是行为版本的一部分。业务工具的底层 IO、存储、鉴权、审计和审批领域服务复用；版本对应的工具集合/装饰策略分别装配。 + +共享工具或 Eino 依赖升级可能改变两个版本，必须同时跑契约和行为回归；必要时保留版本适配器，不能声称共享依赖下逐字节复现旧发布。模型与外部 Skill/MCP 可变，版本回退保证的是运行时策略；记录实际模型、prompt 版本、工具目录摘要，方便解释差异。安全修复同时覆盖 v2/v3，不提供回退绕过安全约束的能力。 + +## 路由与切换（建议默认） + +- 管理员设置新会话默认版本；首发保持 v2 默认、v3 可选，完成灰度后再调整默认。 +- 用户创建会话时选择 v2/v3；已有会话不可原地切换。切换操作创建新会话，默认不复制工具历史或待审批动作。 +- `chat_sessions.runtime_version` 持久化选择。明确来自既有发布、且无版本标记的会话归 v2；PR 试验环境中已生成的 v3 会话需显式标记，不能统一回填为 v2。 +- coordinator、同步/后台 worker、报告、告警和工作流 Agent 节点均绑定版本。子任务继承父版本；无父会话的任务在创建时解析一次默认版本。 +- IM 既有会话保持版本，新会话使用默认；将来如加用户偏好也只影响新会话。 +- v3 不可用时明确失败，不静默路由到 v2。停用版本仅阻止新建，已有任务/审批不跨版本迁移。 + +## 审批与共享状态 + +沿用 Graph 审批链和 `approvals`。新动作记录来源 runtime 版本，审批始终回到原版本执行器/内存执行能力；修改默认版本不得改变待审批行为。缺失版本能力、已取消、超时或内存能力丢失时明确失败，要求重新发起。 + +不引入审批自动恢复或后台重放。旧 legacy K8s 执行入口删除，历史 payload 的只读解析保留。两个 Runtime 各自管理 worker、工具快照与 MCP 更新；共享注册器避免重复指标注册，低基数 runtime_version 标签用于区分。不能让两个实例共享可变工具目录或通过全局变量切换当前版本。 + +## API、数据与发布 + +先更新 proto，再增加会话创建/查询 DTO 的可选 runtime_version 和支持版本列表；非法或不可用版本明确拒绝。现有客户端不传时由服务端选默认。前端复用当前聊天设置组件，显示会话实际版本；不将 Edge Agent 版本与 Runtime 版本混用。 + +按项目当前 schema 管理方式做加法字段与回填，滚动升级兼容；读取端先兼容空值再写入明确版本。回滚默认版本只影响新会话；回滚整个旧镜像无法保证读取 v3 语义，必须单独验收，不把版本切换当镜像回滚替代。 + +## 实施与验收 + +开始：2026-10-16；完成时间由验证结果决定，里程碑各按 1–3 天工作量拆分。 + +1. 固定 v2 发布提交、记录行为样例，完成契约与依赖清单；先完成当前审批退役修复。 +2. 抽出版本无关协议，隔离 v2/v3 的编排及 prompt；所有入口通过统一路由。 +3. 加会话版本字段、默认设置、API 与选择界面；同步 worker、IM、报告、工作流入口。 +4. 双版本回归及真实模型对照验收，v3 灰度；保持 PR 草稿，不合并/部署。 + +必须验证：v2 发布行为样例；v3 safe Skill/web toggle 修复；跨版本不共享工具状态；worker 继承；默认版本变更不改变旧会话；待审批版本不漂移；非法版本拒绝；两版本并存时无指标注册 panic/数据竞争；历史会话升级与旧镜像回滚边界。分别记录延迟、工具成功率和任务完成结果，不能只比较最终文本。 + +## 备选与风险 + +- 只做模型或 prompt 开关：不能隔离循环和工具行为，不采用。 +- 通过镜像回滚选择版本：适合整个部署回退，不能满足用户级试用。 +- 完整复制所有工具和业务服务：隔离强，但安全修复易分叉、维护成本高；优先隔离策略并共享业务执行能力。 +- 长期同时维护两个版本增加测试成本；v2 仅做必要兼容和安全修复，后续退役需明确公告与迁移策略。 From 039bac1d4e08430c5bb5ba175733f026e696bd42 Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 20:17:26 +0800 Subject: [PATCH 13/24] fix(aiops): clarify approval wait timeout outcomes --- internal/manager/service/aiops/approval_adapters.go | 2 +- internal/manager/service/aiops/human_approval.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/manager/service/aiops/approval_adapters.go b/internal/manager/service/aiops/approval_adapters.go index 1b12ae435..293d0a586 100644 --- a/internal/manager/service/aiops/approval_adapters.go +++ b/internal/manager/service/aiops/approval_adapters.go @@ -239,7 +239,7 @@ func (s CloudBashProposer) awaitDecision(ctx context.Context, id string) (string if err := invalidateApproval(ctx, s.Usecase, id, "The approval wait expired; submit a new request."); err != nil { return "", err } - return `{"status":"timeout","message":"No approval within 30 minutes; the command was not run."}`, nil + return `{"status":"timeout","message":"The approval wait expired. Check the approval record for the final execution status."}`, nil } } } diff --git a/internal/manager/service/aiops/human_approval.go b/internal/manager/service/aiops/human_approval.go index 4a3a654ca..a47b6578c 100644 --- a/internal/manager/service/aiops/human_approval.go +++ b/internal/manager/service/aiops/human_approval.go @@ -120,7 +120,7 @@ func (b *HumanApprovalBroker) ProposeAndAwait(ctx context.Context, req aiopstool if err := invalidateApproval(ctx, uc, a.ID, "The approval wait expired; submit a new request."); err != nil { return "", err } - return `{"status":"timeout","message":"No approval within 30 minutes; the action was not run."}`, nil + return `{"status":"timeout","message":"The approval wait expired. Check the approval record for the final execution status."}`, nil } } } From cc25f29fd95a48b4f0a5bbf064feeacb6e56c8fa Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 22:44:18 +0800 Subject: [PATCH 14/24] docs(aiops): trace v1 history and limit runtime selection to settings --- .../PRD-009-agent-runtime-selection.md | 8 +++---- docs/rfc/RFC-009-agent-runtime-versions.md | 23 +++++++++++++++---- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/requirements/PRD-009-agent-runtime-selection.md b/docs/requirements/PRD-009-agent-runtime-selection.md index ff049b7da..4b3c8246e 100644 --- a/docs/requirements/PRD-009-agent-runtime-selection.md +++ b/docs/requirements/PRD-009-agent-runtime-selection.md @@ -7,20 +7,20 @@ ## 用户目标 -用户试用新版 Agent 后，可显式选择已发布版本继续工作。管理员可控制新会话默认版本，避免新行为一次影响所有人。 +管理员可在设置 → AI / Agent 设置选择已发布 Runtime 版本。聊天页不提供选择器；release 默认使用该发布包含的最新稳定版本。 ## 建议交互与验收 - v2 对应正式发布 v0.18.2 的 Graph 行为，v3 对应当前改进；不是模型版本或 Edge 版本。 -- 新建会话可选择版本，当前会话显示实际版本。管理员默认值只影响之后新建的会话。 -- 切换时创建新会话；旧会话、运行中任务和审批继续归属原版本。不自动复制待执行动作。 +- 仅管理员设置提供版本切换。建议设置只影响之后新建的会话，并保留管理员显式配置，避免升级覆盖回退意图。 +- 建议旧会话、运行中任务和审批继续归属原版本，新建会话使用切换后的版本。不自动复制待执行动作。 - 升级前的正式发布会话继续使用 v2；非法版本明确报错，不静默切换。 - 两个版本遵守相同的权限和安全底线。未配置模型/工具时如实提示，回退不恢复已撤销权限。 - 测试覆盖所有聊天入口、子任务和审批；版本路由正确率 100%，跨版本执行次数为零。 ## 范围与依赖 -API、会话持久化、默认设置、聊天选择 UI 与版本路由一起交付，技术设计见 RFC-009。依赖 v2 基线验证、v3 当前审批清理和双版本回归。暂不提供同一会话原地切换、执行恢复或对模型输出完全一致的承诺。 +API、会话持久化、默认设置、管理员设置 UI 与版本路由一起交付，技术设计见 RFC-009。依赖 v2 基线验证、v3 当前审批清理和双版本回归。暂不提供同一会话原地切换、执行恢复或对模型输出完全一致的承诺。 ## 风险、计划与复盘 diff --git a/docs/rfc/RFC-009-agent-runtime-versions.md b/docs/rfc/RFC-009-agent-runtime-versions.md index e1be8248d..ed0bd3986 100644 --- a/docs/rfc/RFC-009-agent-runtime-versions.md +++ b/docs/rfc/RFC-009-agent-runtime-versions.md @@ -1,6 +1,6 @@ # RFC-009：Agent Runtime v2/v3 隔离与显式切换 -- 状态：提议；版本化方向来自用户需求，以下切换交互待确认，尚未实现 +- 状态：提议；用户已确认仅管理员设置入口、release 默认最新稳定 Runtime，尚未实现 - 日期：2026-10-16 - 关联：PR #503、RFC-008、ADR-035；PRD-009 - 基线：正式发布标签 v0.18.2（`3b41874cae792329fd61cfe0450e57335efa3f1d`）的 Graph；不把已退役的 legacy for-loop 当作 v2 @@ -29,8 +29,8 @@ v2/v3 不互相 import；入口只依赖契约。代码位置仍需结合实际 ## 路由与切换（建议默认） -- 管理员设置新会话默认版本；首发保持 v2 默认、v3 可选，完成灰度后再调整默认。 -- 用户创建会话时选择 v2/v3；已有会话不可原地切换。切换操作创建新会话，默认不复制工具历史或待审批动作。 +- 仅在设置 → AI / Agent 设置提供管理员版本切换；每次 release 的默认值为该发布包含的最新稳定 Runtime。建议保留管理员显式指定的版本，不在升级时覆盖回退选择。 +- 聊天页不提供版本选择器；新会话采用管理员设置或 release 默认值。建议已有会话保持绑定版本，默认不复制工具历史或待审批动作。 - `chat_sessions.runtime_version` 持久化选择。明确来自既有发布、且无版本标记的会话归 v2；PR 试验环境中已生成的 v3 会话需显式标记，不能统一回填为 v2。 - coordinator、同步/后台 worker、报告、告警和工作流 Agent 节点均绑定版本。子任务继承父版本；无父会话的任务在创建时解析一次默认版本。 - IM 既有会话保持版本，新会话使用默认；将来如加用户偏好也只影响新会话。 @@ -44,7 +44,7 @@ v2/v3 不互相 import；入口只依赖契约。代码位置仍需结合实际 ## API、数据与发布 -先更新 proto，再增加会话创建/查询 DTO 的可选 runtime_version 和支持版本列表；非法或不可用版本明确拒绝。现有客户端不传时由服务端选默认。前端复用当前聊天设置组件，显示会话实际版本；不将 Edge Agent 版本与 Runtime 版本混用。 +先更新 proto，再增加会话创建/查询 DTO 的可选 runtime_version 和支持版本列表；非法或不可用版本明确拒绝。现有客户端不传时由服务端选默认。前端复用系统设置组件，仅管理员可以切换；聊天页不新增版本选择器。不将 Edge Agent 版本与 Runtime 版本混用。 按项目当前 schema 管理方式做加法字段与回填，滚动升级兼容；读取端先兼容空值再写入明确版本。回滚默认版本只影响新会话；回滚整个旧镜像无法保证读取 v3 语义，必须单独验收，不把版本切换当镜像回滚替代。 @@ -65,3 +65,18 @@ v2/v3 不互相 import；入口只依赖契约。代码位置仍需结合实际 - 通过镜像回滚选择版本：适合整个部署回退，不能满足用户级试用。 - 完整复制所有工具和业务服务：隔离强，但安全修复易分叉、维护成本高；优先隔离策略并共享业务执行能力。 - 长期同时维护两个版本增加测试成本；v2 仅做必要兼容和安全修复，后续退役需明确公告与迁移策略。 + +## v1 历史核对（2026-10-16） + +v1 是对旧 `internal/manager/biz/aiops/agent` 手写循环的追溯命名，历史代码称为 legacy，并没有原生的 v1 版本字段。 + +| 参照 | Git 基线 | 说明 | +| --- | --- | --- | +| 最早公开实现 | `efbda840fa2f3e927a4681a29229b716796b6d32`，2026-05-24 | 首次公开提交已同时包含 legacy 和 Graph，不能据此推断 Graph 的真实初创日期 | +| 中期发布参照 | `v0.12.0 / ba6f4628` | Service 默认 legacy，但两个 Compose 配置均默认 graph，部署默认和代码默认不同 | +| 代码默认切换 | `b94c8ae1`，2026-08-14，首次进入 `v0.13.0` 标签 | ParseKernel 的缺省值改为 Graph，仍可显式 legacy | +| 后期 v1 参照 | `v0.18.2 / 3b41874c` 的 `biz/aiops/agent/` | 发布时仍包含 legacy 实现，累积了历史回放、附件和审批等兼容修改 | + +v1 的核心路径是：加载历史 → 调用 llm.Chat → 串行 Registry.Invoke → 持久化工具结果 → 下一轮；默认最多 30 轮、工具超时 15 秒（有工具专属覆盖）。循环直接承担模型调用、回放、SSE 事件与持久化，未采用 Graph/Eino 编排。 + +后期 v1 不等于最初 v1：评估简单循环架构看最早参照；评估兼容成本看 v0.18.2 中的后期参照。v1 依赖旧 Registry 和专用审批适配，恢复运行需要单独迁移验证，不能仅复制 agent.go。目前仅保留历史参照，不恢复生产入口，也未承诺提供 v1 切换。 From ee61997a441ee05aff5d7f261918ab22844e9bdc Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 22:56:34 +0800 Subject: [PATCH 15/24] refactor(aiops): isolate release runtimes as v1 v2 and v3 --- cmd/ongrid/bootstrap_automation.go | 2 +- cmd/ongrid/bootstrap_extensions.go | 2 +- cmd/ongrid/bootstrap_imbridge.go | 2 +- cmd/ongrid/bootstrap_runtime.go | 2 +- cmd/ongrid/bootstrap_tracing.go | 2 +- cmd/ongrid/runtime_builder.go | 6 +- cmd/ongrid/skill_bootstrap_test.go | 2 +- .../PRD-009-agent-runtime-selection.md | 1 + docs/rfc/RFC-009-agent-runtime-versions.md | 24 +- internal/manager/biz/agenttrace/adapters.go | 2 +- .../manager/biz/agenttrace/content_test.go | 2 +- internal/manager/biz/aiops/runtime/README.md | 13 + .../biz/aiops/runtime/isolation_test.go | 68 + .../manager/biz/aiops/runtime/v1/README.md | 9 + .../runtime/v1/chatruntime/agent_parser.go | 109 + .../v1/chatruntime/agent_parser_test.go | 100 + .../runtime/v1/chatruntime/agent_registry.go | 175 ++ .../v1/chatruntime/agent_registry_test.go | 48 + .../v1}/chatruntime/bare_skills_test.go | 0 .../v1}/chatruntime/chatmodelopts_test.go | 0 .../v1}/chatruntime/command_parser.go | 0 .../v1}/chatruntime/command_parser_test.go | 0 .../aiops/runtime/v1/chatruntime/load_all.go | 245 +++ .../v1}/chatruntime/load_all_test.go | 0 .../v1/chatruntime/plugin_container.go | 680 +++++++ .../v1}/chatruntime/plugin_container_test.go | 0 .../aiops/runtime/v1/chatruntime/runtime.go | 1159 +++++++++++ .../runtime/v1/chatruntime/runtime_test.go | 545 +++++ .../v1/chatruntime/runtime_worker_test.go | 446 ++++ .../v1}/chatruntime/skill_parser.go | 0 .../v1}/chatruntime/skill_parser_test.go | 0 .../runtime/v1/chatruntime/skill_registry.go | 239 +++ .../v1}/chatruntime/skill_registry_test.go | 0 .../runtime/v1/chatruntime/system_prompt.go | 127 ++ .../v1/chatruntime/system_prompt_test.go | 102 + .../minimal/incident-investigator.md | 15 + .../agent_parser/missing_when_to_use/foo.md | 0 .../with_critical_reminder/auditor.md | 0 .../with_disallowed_tools/reviewer.md | 0 .../testdata/agent_registry/multi/README.md | 0 .../multi/incident-investigator.md | 0 .../testdata/agent_registry/multi/reviewer.md | 0 .../claude_pack/.claude-plugin/plugin.json | 0 .../.claude-plugin/plugin.json | 0 .../claude_pack_with_extras/.mcp.json | 0 .../hooks/PRE_PROMPT.md | 0 .../openclaw_pack/openclaw.plugin.json | 0 .../path_traversal/.claude-plugin/plugin.json | 0 .../path_traversal/skills/pwned | 0 .../path_traversal_target/SKILL.md | 0 .../.claude-plugin/plugin.json | 0 .../recursive_claude/.mcp.json | 0 .../recursive_claude/agents/pr-summary.md | 0 .../recursive_claude/commands/commit.md | 0 .../recursive_claude/hooks/pre-prompt/HOOK.md | 0 .../skills/hello-world/SKILL.md | 0 .../recursive_openclaw/openclaw.plugin.json | 0 .../recursive_openclaw/skills/bar/SKILL.md | 0 .../recursive_openclaw/skills/foo/SKILL.md | 0 .../testdata/skill_parser/minimal/SKILL.md | 0 .../skill_parser/missing_description/SKILL.md | 0 .../skill_parser/name_with_dash/SKILL.md | 0 .../testdata/skill_parser/typical/SKILL.md | 0 .../skill_parser/with_ongrid_ext/SKILL.md | 0 .../skill_parser/with_unknown_fields/SKILL.md | 0 .../skill_parser/with_when_to_use/SKILL.md | 0 .../multi/always_skill/SKILL.md | 0 .../multi/keyword_skill/SKILL.md | 0 .../multi/nested/deeper/buried_skill/SKILL.md | 0 .../with_bad_skill/bad/SKILL.md | 0 .../with_bad_skill/good/SKILL.md | 0 .../biz/aiops/runtime/v1/chatruntime/types.go | 388 ++++ .../aiops/runtime/v1/chatruntime/worker.go | 858 ++++++++ .../{ => runtime/v1}/graph/callbacks/audit.go | 0 .../v1}/graph/callbacks/audit_test.go | 0 .../aiops/runtime/v1/graph/callbacks/chain.go | 134 ++ .../v1}/graph/callbacks/chain_test.go | 8 +- .../v1}/graph/callbacks/metrics.go | 2 +- .../v1}/graph/callbacks/metrics_test.go | 0 .../runtime/v1/graph/callbacks/persistence.go | 435 ++++ .../v1/graph/callbacks/persistence_test.go | 300 +++ .../{ => runtime/v1}/graph/callbacks/sse.go | 14 +- .../v1}/graph/callbacks/sse_test.go | 0 .../biz/aiops/runtime/v1/graph/react.go | 305 +++ .../biz/aiops/runtime/v1/graph/react_test.go | 398 ++++ .../aiops/runtime/v1/graph/tool_adapter.go | 299 +++ .../runtime/v1/graph/tool_adapter_test.go | 290 +++ .../biz/aiops/runtime/v1/graph/types.go | 172 ++ .../manager/biz/aiops/runtime/v1/source.json | 376 ++++ .../manager/biz/aiops/runtime/v2/README.md | 9 + .../v2}/chatruntime/agent_parser.go | 0 .../v2}/chatruntime/agent_parser_test.go | 0 .../v2}/chatruntime/agent_registry.go | 0 .../v2}/chatruntime/agent_registry_test.go | 0 .../v2/chatruntime/bare_skills_test.go | 138 ++ .../v2/chatruntime/chatmodelopts_test.go | 23 + .../runtime/v2/chatruntime/command_parser.go | 151 ++ .../v2/chatruntime/command_parser_test.go | 159 ++ .../v2}/chatruntime/config_confirm.go | 0 .../{ => runtime/v2}/chatruntime/decision.go | 0 .../v2}/chatruntime/decision_test.go | 0 .../v2}/chatruntime/k8s_action_e2e_test.go | 2 +- .../aiops/runtime/v2/chatruntime/load_all.go | 245 +++ .../runtime/v2/chatruntime/load_all_test.go | 146 ++ .../v2}/chatruntime/plugin_container.go | 0 .../v2/chatruntime/plugin_container_test.go | 302 +++ .../v2}/chatruntime/reasoning_test.go | 0 .../aiops/runtime/v2/chatruntime/runtime.go | 1801 +++++++++++++++++ .../runtime/v2/chatruntime/runtime_test.go | 1096 ++++++++++ .../v2}/chatruntime/runtime_worker_test.go | 2 +- .../runtime/v2/chatruntime/skill_parser.go | 248 +++ .../v2/chatruntime/skill_parser_test.go | 230 +++ .../runtime/v2/chatruntime/skill_registry.go | 239 +++ .../v2/chatruntime/skill_registry_test.go | 166 ++ .../v2}/chatruntime/system_prompt.go | 0 .../v2}/chatruntime/system_prompt_test.go | 0 .../minimal/incident-investigator.md | 0 .../agent_parser/missing_when_to_use/foo.md | 6 + .../with_critical_reminder/auditor.md | 23 + .../with_disallowed_tools/reviewer.md | 23 + .../testdata/agent_registry/multi/README.md | 4 + .../multi/incident-investigator.md | 7 + .../testdata/agent_registry/multi/reviewer.md | 9 + .../claude_pack/.claude-plugin/plugin.json | 13 + .../.claude-plugin/plugin.json | 7 + .../claude_pack_with_extras/.mcp.json | 3 + .../hooks/PRE_PROMPT.md | 3 + .../openclaw_pack/openclaw.plugin.json | 18 + .../path_traversal/.claude-plugin/plugin.json | 6 + .../path_traversal/skills/pwned | 1 + .../path_traversal_target/SKILL.md | 8 + .../.claude-plugin/plugin.json | 6 + .../recursive_claude/.mcp.json | 8 + .../recursive_claude/agents/pr-summary.md | 7 + .../recursive_claude/commands/commit.md | 7 + .../recursive_claude/hooks/pre-prompt/HOOK.md | 3 + .../skills/hello-world/SKILL.md | 8 + .../recursive_openclaw/openclaw.plugin.json | 7 + .../recursive_openclaw/skills/bar/SKILL.md | 8 + .../recursive_openclaw/skills/foo/SKILL.md | 8 + .../testdata/skill_parser/minimal/SKILL.md | 8 + .../skill_parser/missing_description/SKILL.md | 7 + .../skill_parser/name_with_dash/SKILL.md | 9 + .../testdata/skill_parser/typical/SKILL.md | 29 + .../skill_parser/with_ongrid_ext/SKILL.md | 27 + .../skill_parser/with_unknown_fields/SKILL.md | 16 + .../skill_parser/with_when_to_use/SKILL.md | 16 + .../multi/always_skill/SKILL.md | 15 + .../multi/keyword_skill/SKILL.md | 20 + .../multi/nested/deeper/buried_skill/SKILL.md | 8 + .../with_bad_skill/bad/SKILL.md | 7 + .../with_bad_skill/good/SKILL.md | 8 + .../aiops/runtime/v2/chatruntime/tracing.go | 31 + .../v2}/chatruntime/tracing_test.go | 4 +- .../{ => runtime/v2}/chatruntime/types.go | 0 .../aiops/runtime/v2/chatruntime/worker.go | 1080 ++++++++++ .../v2}/chatruntime/worker_filter_test.go | 0 .../v2}/graph/budget_stop_model.go | 0 .../v2}/graph/callbacks/alert_draft_guard.go | 0 .../graph/callbacks/alert_draft_guard_test.go | 0 .../aiops/runtime/v2/graph/callbacks/audit.go | 288 +++ .../runtime/v2/graph/callbacks/audit_test.go | 166 ++ .../{ => runtime/v2}/graph/callbacks/chain.go | 0 .../runtime/v2/graph/callbacks/chain_test.go | 83 + .../runtime/v2/graph/callbacks/metrics.go | 305 +++ .../v2/graph/callbacks/metrics_test.go | 120 ++ .../v2}/graph/callbacks/persistence.go | 0 .../v2}/graph/callbacks/persistence_test.go | 0 .../v2}/graph/callbacks/reasoning_test.go | 0 .../aiops/runtime/v2/graph/callbacks/sse.go | 455 +++++ .../runtime/v2/graph/callbacks/sse_test.go | 222 ++ .../v2}/graph/callbacks/tracing.go | 0 .../v2}/graph/callbacks/tracing_test.go | 0 .../biz/aiops/{ => runtime/v2}/graph/react.go | 0 .../{ => runtime/v2}/graph/react_test.go | 0 .../v2}/graph/token_budget_test.go | 0 .../aiops/runtime/v2/graph/tool_adapter.go | 522 +++++ .../v2}/graph/tool_adapter_test.go | 0 .../biz/aiops/{ => runtime/v2}/graph/types.go | 0 .../manager/biz/aiops/runtime/v2/source.json | 451 +++++ .../manager/biz/aiops/runtime/v3/README.md | 5 + .../runtime/v3/chatruntime/agent_parser.go | 111 + .../v3/chatruntime/agent_parser_test.go | 106 + .../runtime/v3/chatruntime/agent_registry.go | 201 ++ .../v3/chatruntime/agent_registry_test.go | 64 + .../v3/chatruntime/bare_skills_test.go | 138 ++ .../v3/chatruntime/chatmodelopts_test.go | 23 + .../runtime/v3/chatruntime/command_parser.go | 151 ++ .../v3/chatruntime/command_parser_test.go | 159 ++ .../runtime/v3/chatruntime/config_confirm.go | 403 ++++ .../aiops/runtime/v3/chatruntime/decision.go | 129 ++ .../runtime/v3/chatruntime/decision_test.go | 159 ++ .../v3/chatruntime/k8s_action_e2e_test.go | 324 +++ .../{ => runtime/v3}/chatruntime/load_all.go | 0 .../runtime/v3/chatruntime/load_all_test.go | 146 ++ .../v3/chatruntime/plugin_container.go | 686 +++++++ .../v3/chatruntime/plugin_container_test.go | 302 +++ .../runtime/v3/chatruntime/reasoning_test.go | 23 + .../{ => runtime/v3}/chatruntime/runtime.go | 4 +- .../v3}/chatruntime/runtime_test.go | 4 +- .../v3/chatruntime/runtime_worker_test.go | 633 ++++++ .../runtime/v3/chatruntime/skill_parser.go | 248 +++ .../v3/chatruntime/skill_parser_test.go | 230 +++ .../v3}/chatruntime/skill_registry.go | 0 .../v3/chatruntime/skill_registry_test.go | 166 ++ .../runtime/v3/chatruntime/system_prompt.go | 275 +++ .../v3/chatruntime/system_prompt_test.go | 151 ++ .../minimal/incident-investigator.md | 21 + .../agent_parser/missing_when_to_use/foo.md | 6 + .../with_critical_reminder/auditor.md | 23 + .../with_disallowed_tools/reviewer.md | 23 + .../testdata/agent_registry/multi/README.md | 4 + .../multi/incident-investigator.md | 7 + .../testdata/agent_registry/multi/reviewer.md | 9 + .../claude_pack/.claude-plugin/plugin.json | 13 + .../.claude-plugin/plugin.json | 7 + .../claude_pack_with_extras/.mcp.json | 3 + .../hooks/PRE_PROMPT.md | 3 + .../openclaw_pack/openclaw.plugin.json | 18 + .../path_traversal/.claude-plugin/plugin.json | 6 + .../path_traversal/skills/pwned | 1 + .../path_traversal_target/SKILL.md | 8 + .../.claude-plugin/plugin.json | 6 + .../recursive_claude/.mcp.json | 8 + .../recursive_claude/agents/pr-summary.md | 7 + .../recursive_claude/commands/commit.md | 7 + .../recursive_claude/hooks/pre-prompt/HOOK.md | 3 + .../skills/hello-world/SKILL.md | 8 + .../recursive_openclaw/openclaw.plugin.json | 7 + .../recursive_openclaw/skills/bar/SKILL.md | 8 + .../recursive_openclaw/skills/foo/SKILL.md | 8 + .../testdata/skill_parser/minimal/SKILL.md | 8 + .../skill_parser/missing_description/SKILL.md | 7 + .../skill_parser/name_with_dash/SKILL.md | 9 + .../testdata/skill_parser/typical/SKILL.md | 29 + .../skill_parser/with_ongrid_ext/SKILL.md | 27 + .../skill_parser/with_unknown_fields/SKILL.md | 16 + .../skill_parser/with_when_to_use/SKILL.md | 16 + .../multi/always_skill/SKILL.md | 15 + .../multi/keyword_skill/SKILL.md | 20 + .../multi/nested/deeper/buried_skill/SKILL.md | 8 + .../with_bad_skill/bad/SKILL.md | 7 + .../with_bad_skill/good/SKILL.md | 8 + .../v3}/chatruntime/tool_lifecycle.go | 0 .../v3}/chatruntime/tool_lifecycle_test.go | 0 .../{ => runtime/v3}/chatruntime/tracing.go | 0 .../runtime/v3/chatruntime/tracing_test.go | 336 +++ .../biz/aiops/runtime/v3/chatruntime/types.go | 458 +++++ .../{ => runtime/v3}/chatruntime/worker.go | 4 +- .../v3}/chatruntime/worker_context_test.go | 0 .../v3/chatruntime/worker_filter_test.go | 103 + .../runtime/v3/graph/budget_stop_model.go | 577 ++++++ .../v3/graph/callbacks/alert_draft_guard.go | 310 +++ .../graph/callbacks/alert_draft_guard_test.go | 193 ++ .../aiops/runtime/v3/graph/callbacks/audit.go | 288 +++ .../runtime/v3/graph/callbacks/audit_test.go | 166 ++ .../aiops/runtime/v3/graph/callbacks/chain.go | 178 ++ .../runtime/v3/graph/callbacks/chain_test.go | 83 + .../runtime/v3/graph/callbacks/metrics.go | 305 +++ .../v3/graph/callbacks/metrics_test.go | 120 ++ .../runtime/v3/graph/callbacks/persistence.go | 1037 ++++++++++ .../v3/graph/callbacks/persistence_test.go | 704 +++++++ .../v3/graph/callbacks/reasoning_test.go | 34 + .../aiops/runtime/v3/graph/callbacks/sse.go | 455 +++++ .../runtime/v3/graph/callbacks/sse_test.go | 222 ++ .../runtime/v3/graph/callbacks/tracing.go | 97 + .../v3/graph/callbacks/tracing_test.go | 113 ++ .../biz/aiops/runtime/v3/graph/react.go | 356 ++++ .../biz/aiops/runtime/v3/graph/react_test.go | 670 ++++++ .../runtime/v3/graph/token_budget_test.go | 86 + .../{ => runtime/v3}/graph/tool_adapter.go | 0 .../runtime/v3/graph/tool_adapter_test.go | 690 +++++++ .../biz/aiops/runtime/v3/graph/types.go | 180 ++ .../manager/biz/alert/investigator/usecase.go | 2 +- .../biz/alert/investigator/usecase_test.go | 2 +- .../capability_credentials_test.go | 2 +- .../biz/marketplace/integration_test.go | 2 +- internal/manager/biz/marketplace/usecase.go | 2 +- .../manager/biz/marketplace/usecase_test.go | 3 +- internal/manager/biz/report/generator.go | 2 +- internal/manager/biz/report/generator_test.go | 16 +- internal/manager/server/aiops/http.go | 2 +- .../service/aiops/approval_adapters.go | 2 +- .../manager/service/aiops/human_approval.go | 2 +- .../aiops/mcp_approval_runtime_test.go | 4 +- internal/manager/service/aiops/mcp_refresh.go | 2 +- .../manager/service/aiops/mcp_refresh_test.go | 2 +- .../manager/service/aiops/runtime_adapters.go | 2 +- internal/manager/service/aiops/service.go | 2 +- .../service/aiops/service_kernel_test.go | 2 +- internal/manager/service/aiops/user_agent.go | 2 +- .../manager/service/aiops/worker_metrics.go | 2 +- internal/manager/service/flow/runners.go | 2 +- .../manager/service/skill/runtime_catalog.go | 2 +- scripts/test-ci-docs-only.py | 2 +- 295 files changed, 30125 insertions(+), 79 deletions(-) create mode 100644 internal/manager/biz/aiops/runtime/README.md create mode 100644 internal/manager/biz/aiops/runtime/isolation_test.go create mode 100644 internal/manager/biz/aiops/runtime/v1/README.md create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/agent_parser.go create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/agent_parser_test.go create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/agent_registry.go create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/agent_registry_test.go rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/bare_skills_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/chatmodelopts_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/command_parser.go (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/command_parser_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/load_all.go rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/load_all_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/plugin_container.go rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/plugin_container_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/runtime.go create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/runtime_test.go create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/runtime_worker_test.go rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/skill_parser.go (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/skill_parser_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/skill_registry.go rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/skill_registry_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/system_prompt.go create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/system_prompt_test.go create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_parser/minimal/incident-investigator.md rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/agent_registry/multi/README.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/agent_registry/multi/incident-investigator.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/agent_registry/multi/reviewer.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/path_traversal/skills/pwned (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_parser/minimal/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_parser/missing_description/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_parser/typical/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md (100%) create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/types.go create mode 100644 internal/manager/biz/aiops/runtime/v1/chatruntime/worker.go rename internal/manager/biz/aiops/{ => runtime/v1}/graph/callbacks/audit.go (100%) rename internal/manager/biz/aiops/{ => runtime/v1}/graph/callbacks/audit_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v1/graph/callbacks/chain.go rename internal/manager/biz/aiops/{ => runtime/v1}/graph/callbacks/chain_test.go (90%) rename internal/manager/biz/aiops/{ => runtime/v1}/graph/callbacks/metrics.go (99%) rename internal/manager/biz/aiops/{ => runtime/v1}/graph/callbacks/metrics_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v1/graph/callbacks/persistence.go create mode 100644 internal/manager/biz/aiops/runtime/v1/graph/callbacks/persistence_test.go rename internal/manager/biz/aiops/{ => runtime/v1}/graph/callbacks/sse.go (96%) rename internal/manager/biz/aiops/{ => runtime/v1}/graph/callbacks/sse_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v1/graph/react.go create mode 100644 internal/manager/biz/aiops/runtime/v1/graph/react_test.go create mode 100644 internal/manager/biz/aiops/runtime/v1/graph/tool_adapter.go create mode 100644 internal/manager/biz/aiops/runtime/v1/graph/tool_adapter_test.go create mode 100644 internal/manager/biz/aiops/runtime/v1/graph/types.go create mode 100644 internal/manager/biz/aiops/runtime/v1/source.json create mode 100644 internal/manager/biz/aiops/runtime/v2/README.md rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/agent_parser.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/agent_parser_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/agent_registry.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/agent_registry_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/bare_skills_test.go create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/chatmodelopts_test.go create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/command_parser.go create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/command_parser_test.go rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/config_confirm.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/decision.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/decision_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/k8s_action_e2e_test.go (99%) create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/load_all.go create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/load_all_test.go rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/plugin_container.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/plugin_container_test.go rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/reasoning_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/runtime.go create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_test.go rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/runtime_worker_test.go (99%) create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/skill_parser.go create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/skill_parser_test.go create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/skill_registry.go create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/skill_registry_test.go rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/system_prompt.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/system_prompt_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/testdata/agent_parser/minimal/incident-investigator.md (100%) create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/README.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/incident-investigator.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/reviewer.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json create mode 120000 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal/skills/pwned create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/minimal/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/missing_description/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/typical/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/tracing.go rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/tracing_test.go (98%) rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/types.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/worker.go rename internal/manager/biz/aiops/{ => runtime/v2}/chatruntime/worker_filter_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/graph/budget_stop_model.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/graph/callbacks/alert_draft_guard.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/graph/callbacks/alert_draft_guard_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v2/graph/callbacks/audit.go create mode 100644 internal/manager/biz/aiops/runtime/v2/graph/callbacks/audit_test.go rename internal/manager/biz/aiops/{ => runtime/v2}/graph/callbacks/chain.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v2/graph/callbacks/chain_test.go create mode 100644 internal/manager/biz/aiops/runtime/v2/graph/callbacks/metrics.go create mode 100644 internal/manager/biz/aiops/runtime/v2/graph/callbacks/metrics_test.go rename internal/manager/biz/aiops/{ => runtime/v2}/graph/callbacks/persistence.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/graph/callbacks/persistence_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/graph/callbacks/reasoning_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v2/graph/callbacks/sse.go create mode 100644 internal/manager/biz/aiops/runtime/v2/graph/callbacks/sse_test.go rename internal/manager/biz/aiops/{ => runtime/v2}/graph/callbacks/tracing.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/graph/callbacks/tracing_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/graph/react.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/graph/react_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/graph/token_budget_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v2/graph/tool_adapter.go rename internal/manager/biz/aiops/{ => runtime/v2}/graph/tool_adapter_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v2}/graph/types.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v2/source.json create mode 100644 internal/manager/biz/aiops/runtime/v3/README.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/agent_parser.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/agent_parser_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/bare_skills_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/chatmodelopts_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/command_parser.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/command_parser_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/config_confirm.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/decision.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/decision_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/k8s_action_e2e_test.go rename internal/manager/biz/aiops/{ => runtime/v3}/chatruntime/load_all.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/load_all_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/plugin_container.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/plugin_container_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/reasoning_test.go rename internal/manager/biz/aiops/{ => runtime/v3}/chatruntime/runtime.go (99%) rename internal/manager/biz/aiops/{ => runtime/v3}/chatruntime/runtime_test.go (99%) create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_worker_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/skill_parser.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/skill_parser_test.go rename internal/manager/biz/aiops/{ => runtime/v3}/chatruntime/skill_registry.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/skill_registry_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/system_prompt.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/system_prompt_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/minimal/incident-investigator.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/README.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/incident-investigator.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/reviewer.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json create mode 120000 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal/skills/pwned create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/minimal/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/missing_description/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/typical/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md rename internal/manager/biz/aiops/{ => runtime/v3}/chatruntime/tool_lifecycle.go (100%) rename internal/manager/biz/aiops/{ => runtime/v3}/chatruntime/tool_lifecycle_test.go (100%) rename internal/manager/biz/aiops/{ => runtime/v3}/chatruntime/tracing.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/tracing_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/types.go rename internal/manager/biz/aiops/{ => runtime/v3}/chatruntime/worker.go (99%) rename internal/manager/biz/aiops/{ => runtime/v3}/chatruntime/worker_context_test.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/worker_filter_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/budget_stop_model.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/alert_draft_guard.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/alert_draft_guard_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/audit.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/audit_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/chain.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/chain_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/metrics.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/metrics_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/persistence.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/persistence_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/reasoning_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/sse.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/sse_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/tracing.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/callbacks/tracing_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/react.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/react_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/token_budget_test.go rename internal/manager/biz/aiops/{ => runtime/v3}/graph/tool_adapter.go (100%) create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/tool_adapter_test.go create mode 100644 internal/manager/biz/aiops/runtime/v3/graph/types.go diff --git a/cmd/ongrid/bootstrap_automation.go b/cmd/ongrid/bootstrap_automation.go index 2c5fdd4cb..46478b532 100644 --- a/cmd/ongrid/bootstrap_automation.go +++ b/cmd/ongrid/bootstrap_automation.go @@ -9,8 +9,8 @@ import ( "time" "github.com/google/uuid" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" aiopsinvestigator "github.com/ongridio/ongrid/internal/manager/biz/aiops/investigator" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" managerbizalert "github.com/ongridio/ongrid/internal/manager/biz/alert" "github.com/ongridio/ongrid/internal/manager/biz/alert/investigator" managerbizflow "github.com/ongridio/ongrid/internal/manager/biz/flow" diff --git a/cmd/ongrid/bootstrap_extensions.go b/cmd/ongrid/bootstrap_extensions.go index fc0b0da1a..20ad22521 100644 --- a/cmd/ongrid/bootstrap_extensions.go +++ b/cmd/ongrid/bootstrap_extensions.go @@ -8,7 +8,7 @@ import ( "strings" "time" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" diff --git a/cmd/ongrid/bootstrap_imbridge.go b/cmd/ongrid/bootstrap_imbridge.go index ad127ab9a..c6b7ddb1b 100644 --- a/cmd/ongrid/bootstrap_imbridge.go +++ b/cmd/ongrid/bootstrap_imbridge.go @@ -5,8 +5,8 @@ import ( managersvcimbridge "github.com/ongridio/ongrid/internal/manager/service/imbridge" "log/slog" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" managerbizaiopsmentions "github.com/ongridio/ongrid/internal/manager/biz/aiops/mentions" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" managerbizimbridge "github.com/ongridio/ongrid/internal/manager/biz/imbridge" managerbizimbridgedingtalk "github.com/ongridio/ongrid/internal/manager/biz/imbridge/provider/dingtalk" managerbizimbridgefeishu "github.com/ongridio/ongrid/internal/manager/biz/imbridge/provider/feishu" diff --git a/cmd/ongrid/bootstrap_runtime.go b/cmd/ongrid/bootstrap_runtime.go index 0a28e26a3..e6f73b308 100644 --- a/cmd/ongrid/bootstrap_runtime.go +++ b/cmd/ongrid/bootstrap_runtime.go @@ -5,7 +5,7 @@ import ( "log/slog" "time" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" diff --git a/cmd/ongrid/bootstrap_tracing.go b/cmd/ongrid/bootstrap_tracing.go index 193ef8590..8528c509d 100644 --- a/cmd/ongrid/bootstrap_tracing.go +++ b/cmd/ongrid/bootstrap_tracing.go @@ -9,7 +9,7 @@ import ( "strings" "time" - aiopsgraphcb "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph/callbacks" + aiopsgraphcb "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph/callbacks" "github.com/ongridio/ongrid/internal/pkg/tracing" ) diff --git a/cmd/ongrid/runtime_builder.go b/cmd/ongrid/runtime_builder.go index d4ebf0783..4f7c966c5 100644 --- a/cmd/ongrid/runtime_builder.go +++ b/cmd/ongrid/runtime_builder.go @@ -10,9 +10,9 @@ import ( einomodel "github.com/cloudwego/eino/components/model" managerbizaiops "github.com/ongridio/ongrid/internal/manager/biz/aiops" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" - aiopsgraph "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph" - aiopsgraphcb "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph/callbacks" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" + aiopsgraph "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph" + aiopsgraphcb "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph/callbacks" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" diff --git a/cmd/ongrid/skill_bootstrap_test.go b/cmd/ongrid/skill_bootstrap_test.go index f40b75e78..55a3b2e1a 100644 --- a/cmd/ongrid/skill_bootstrap_test.go +++ b/cmd/ongrid/skill_bootstrap_test.go @@ -9,7 +9,7 @@ import ( ) func TestLoadBootstrapRegistriesSkillRoots(t *testing.T) { - fixtures, err := filepath.Abs("../../internal/manager/biz/aiops/chatruntime/testdata") + fixtures, err := filepath.Abs("../../internal/manager/biz/aiops/runtime/v3/chatruntime/testdata") if err != nil { t.Fatal(err) } diff --git a/docs/requirements/PRD-009-agent-runtime-selection.md b/docs/requirements/PRD-009-agent-runtime-selection.md index 4b3c8246e..364caa22c 100644 --- a/docs/requirements/PRD-009-agent-runtime-selection.md +++ b/docs/requirements/PRD-009-agent-runtime-selection.md @@ -11,6 +11,7 @@ ## 建议交互与验收 +- v1 固定为首个正式 release v0.7.158 的 Eino Graph；源码已提取，启用前仍需安全、装配和路由验收。 - v2 对应正式发布 v0.18.2 的 Graph 行为，v3 对应当前改进；不是模型版本或 Edge 版本。 - 仅管理员设置提供版本切换。建议设置只影响之后新建的会话，并保留管理员显式配置，避免升级覆盖回退意图。 - 建议旧会话、运行中任务和审批继续归属原版本，新建会话使用切换后的版本。不自动复制待执行动作。 diff --git a/docs/rfc/RFC-009-agent-runtime-versions.md b/docs/rfc/RFC-009-agent-runtime-versions.md index ed0bd3986..866f82034 100644 --- a/docs/rfc/RFC-009-agent-runtime-versions.md +++ b/docs/rfc/RFC-009-agent-runtime-versions.md @@ -1,13 +1,13 @@ -# RFC-009：Agent Runtime v2/v3 隔离与显式切换 +# RFC-009：Agent Runtime v1/v2/v3 隔离与显式切换 -- 状态：提议；用户已确认仅管理员设置入口、release 默认最新稳定 Runtime，尚未实现 +- 状态：提议；用户已确认仅管理员设置入口、release 默认最新稳定 Runtime，源码隔离已实现，版本切换尚未实现 - 日期：2026-10-16 - 关联：PR #503、RFC-008、ADR-035；PRD-009 - 基线：正式发布标签 v0.18.2（`3b41874cae792329fd61cfe0450e57335efa3f1d`）的 Graph；不把已退役的 legacy for-loop 当作 v2 ## 背景与目标 -用户需要在新版 Agent 效果不佳时切回熟悉的行为，同时隔离实现。Agent Runtime 版本独立于 Ongrid 产品版本和模型版本：v2 是已发布 Graph 的行为基线，v3 是当前统一目录、权限和工具生命周期改进。当前代码仍只有一个 Graph 实现，本文不是已交付功能声明。 +用户需要在新版 Agent 效果不佳时切回熟悉的行为，同时隔离实现。Agent Runtime 版本独立于 Ongrid 产品版本和模型版本：v2 是已发布 Graph 的行为基线，v3 是当前统一目录、权限和工具生命周期改进。三个版本源码已隔离，当前业务入口仅装配 v3；版本切换尚未交付。 ## 版本与隔离边界 @@ -17,13 +17,14 @@ internal/manager/biz/aiops/ chatprotocol/ # 稳定边界，不引用版本实现 runtime/contract/ # 请求/回复、worker/目录管理契约 + runtime/v1/ # 首个正式 release v0.7.158 的 Eino Graph runtime/v2/ # 发布基线的循环、上下文策略、prompt、装配策略 runtime/v3/ # 本次改进，后续行为变更在此进行 internal/manager/service/aiops/ runtime_router.go # 依据会话记录选版本，禁止自动降级 ``` -v2/v3 不互相 import；入口只依赖契约。代码位置仍需结合实际依赖图细化，不能仅给两个名字包一层相同实现。目录装配、工具暴露、上下文压缩、worker 策略及内置 prompt 均是行为版本的一部分。业务工具的底层 IO、存储、鉴权、审计和审批领域服务复用；版本对应的工具集合/装饰策略分别装配。 +v1/v2/v3 不互相 import；入口只依赖契约。代码位置仍需结合实际依赖图细化，不能仅给两个名字包一层相同实现。目录装配、工具暴露、上下文压缩、worker 策略及内置 prompt 均是行为版本的一部分。业务工具的底层 IO、存储、鉴权、审计和审批领域服务复用；版本对应的工具集合/装饰策略分别装配。 共享工具或 Eino 依赖升级可能改变两个版本，必须同时跑契约和行为回归；必要时保留版本适配器，不能声称共享依赖下逐字节复现旧发布。模型与外部 Skill/MCP 可变，版本回退保证的是运行时策略；记录实际模型、prompt 版本、工具目录摘要，方便解释差异。安全修复同时覆盖 v2/v3，不提供回退绕过安全约束的能力。 @@ -66,17 +67,10 @@ v2/v3 不互相 import；入口只依赖契约。代码位置仍需结合实际 - 完整复制所有工具和业务服务：隔离强，但安全修复易分叉、维护成本高；优先隔离策略并共享业务执行能力。 - 长期同时维护两个版本增加测试成本；v2 仅做必要兼容和安全修复，后续退役需明确公告与迁移策略。 -## v1 历史核对（2026-10-16） +## v1 发布基线与源码提取（2026-10-16） -v1 是对旧 `internal/manager/biz/aiops/agent` 手写循环的追溯命名，历史代码称为 legacy，并没有原生的 v1 版本字段。 +v1 固定为 GitHub 最早正式 release `v0.7.158`（2026-05-27，`04b2796319785df840d7a472569c380e0871730b`）的 Eino Graph。此前把 legacy 称作 v1 的追溯命名已撤销；最早公开提交同时存在 legacy 与 Graph，legacy 是独立历史旁支。 -| 参照 | Git 基线 | 说明 | -| --- | --- | --- | -| 最早公开实现 | `efbda840fa2f3e927a4681a29229b716796b6d32`，2026-05-24 | 首次公开提交已同时包含 legacy 和 Graph，不能据此推断 Graph 的真实初创日期 | -| 中期发布参照 | `v0.12.0 / ba6f4628` | Service 默认 legacy，但两个 Compose 配置均默认 graph，部署默认和代码默认不同 | -| 代码默认切换 | `b94c8ae1`，2026-08-14，首次进入 `v0.13.0` 标签 | ParseKernel 的缺省值改为 Graph，仍可显式 legacy | -| 后期 v1 参照 | `v0.18.2 / 3b41874c` 的 `biz/aiops/agent/` | 发布时仍包含 legacy 实现，累积了历史回放、附件和审批等兼容修改 | +已完成：`runtime/v1/{chatruntime,graph}` 提取首个 release；`runtime/v2/{chatruntime,graph}` 提取 v0.18.2；当前实现及测试迁移至 `runtime/v3/{chatruntime,graph}`，现有调用方指向 v3。v1/v2 保留源提交、文件 blob 清单和兼容改动说明。三版本均参与标准测试，禁止直接跨版本 import。 -v1 的核心路径是：加载历史 → 调用 llm.Chat → 串行 Registry.Invoke → 持久化工具结果 → 下一轮；默认最多 30 轮、工具超时 15 秒（有工具专属覆盖）。循环直接承担模型调用、回放、SSE 事件与持久化，未采用 Graph/Eino 编排。 - -后期 v1 不等于最初 v1：评估简单循环架构看最早参照；评估兼容成本看 v0.18.2 中的后期参照。v1 依赖旧 Registry 和专用审批适配，恢复运行需要单独迁移验证，不能仅复制 agent.go。目前仅保留历史参照，不恢复生产入口，也未承诺提供 v1 切换。 +本次仅完成源码隔离。版本契约、生产路由、会话版本字段、版本化工具装配及管理员设置仍未实现。外部 persona/skills、业务工具和依赖仍共享，不能将源码提取理解为完整复现旧 release，v1 接入前必须补齐现行安全与审批验收。 diff --git a/internal/manager/biz/agenttrace/adapters.go b/internal/manager/biz/agenttrace/adapters.go index b6ff2ffa8..f0543c295 100644 --- a/internal/manager/biz/agenttrace/adapters.go +++ b/internal/manager/biz/agenttrace/adapters.go @@ -50,7 +50,7 @@ func adapt(s *Step) { s.Provider = stringAttr(a, "gen_ai.provider.name", "llm.provider", "gen_ai.system", "llm.system") // LangSmith trace.session_id identifies a project, not a conversation. s.SessionID = stringAttr(a, "gen_ai.conversation.id", "session.id", "langsmith.metadata.thread_id", "langsmith.metadata.session_id", "langsmith.metadata.conversation_id", "traceloop.association.properties.session_id", "traceloop.association.properties.thread_id") - if s.Scope.Name == "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" || s.Scope.Name == "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" { + if s.Scope.Name == "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" || s.Scope.Name == "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" { s.ChatSessionID = stringAttr(a, "ongrid.agent.session.id") } if s.Identity.Environment == "" { diff --git a/internal/manager/biz/agenttrace/content_test.go b/internal/manager/biz/agenttrace/content_test.go index 6fd72ac5f..2d82a2235 100644 --- a/internal/manager/biz/agenttrace/content_test.go +++ b/internal/manager/biz/agenttrace/content_test.go @@ -14,7 +14,7 @@ func TestCapturedContentIsDecodedOnlyFromTrace(t *testing.T) { raw, _ := json.Marshal(strings.Repeat("执行快照😀", 10000)) snapshot := tracing.MaskAgentContent(string(raw)) body := testBody(t, testSpan(1, 0, map[string]any{"gen_ai.operation.name": "invoke_agent", "input.value": `"本轮问题"`, "output.value": snapshot, "ongrid.agent.session.id": "deleted-chat", "ongrid.agent.content.capture": true})) - body = bytes.ReplaceAll(body, []byte(`"name":"fixture"`), []byte(`"name":"github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime"`)) + body = bytes.ReplaceAll(body, []byte(`"name":"fixture"`), []byte(`"name":"github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime"`)) trace, err := Normalize(fixtureTraceID, body) if err != nil { t.Fatal(err) diff --git a/internal/manager/biz/aiops/runtime/README.md b/internal/manager/biz/aiops/runtime/README.md new file mode 100644 index 000000000..7d478aea1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/README.md @@ -0,0 +1,13 @@ +# Agent Runtime 版本源码 + +| 目录 | 基线 | 当前接线 | +| --- | --- | --- | +| [v1](v1/README.md) | GitHub 最早正式 release `v0.7.158` 的 Eino Graph | 编译、测试；未启用 | +| [v2](v2/README.md) | `v0.18.2` 的 Eino Graph | 编译、测试；未启用 | +| [v3](v3/README.md) | PR #503 当前统一运行时 | 所有现有调用方使用 | + +编号是本次引入的运行时版本，不是 release 当年的命名。legacy 手写循环是并存的历史旁支，不属于这里的 v1。 + +每个版本独立持有 `chatruntime`（上下文、prompt、Worker、目录策略）和 `graph`（Eino 编排、工具适配、回调）。禁止版本间 import，`isolation_test.go` 在标准 Go 测试中检查该边界，并验证三版本指标注册共存。业务执行能力与基础设施继续共享；尚未声称三版本已可切换。 + +验证：Linux 环境执行 `make test-agent-runtime`。不新增 schema/API/设置，不改变当前运行时选择；需要撤销本次隔离时回退该源码迁移提交即可。版本路由及管理设置的后续方案见 [RFC-009](../../../../../docs/rfc/RFC-009-agent-runtime-versions.md)。 diff --git a/internal/manager/biz/aiops/runtime/isolation_test.go b/internal/manager/biz/aiops/runtime/isolation_test.go new file mode 100644 index 000000000..60b1a07a2 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/isolation_test.go @@ -0,0 +1,68 @@ +package runtime_test + +import ( + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "strconv" + "strings" + "testing" + + v1 "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v1/graph/callbacks" + v2 "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph/callbacks" + v3 "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph/callbacks" + "github.com/prometheus/client_golang/prometheus" +) + +// Versioned policies must never silently delegate to another generation. +func TestRuntimeVersionsDoNotImportEachOther(t *testing.T) { + const base = "github.com/ongridio/ongrid/internal/manager/biz/aiops/" + for _, version := range []string{"v1", "v2", "v3"} { + t.Run(version, func(t *testing.T) { + err := filepath.WalkDir(version, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || !strings.HasSuffix(path, ".go") { + return nil + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly) + if err != nil { + return err + } + for _, spec := range file.Imports { + imported, err := strconv.Unquote(spec.Path.Value) + if err != nil { + return err + } + if strings.HasPrefix(imported, base+"runtime/") && !strings.HasPrefix(imported, base+"runtime/"+version+"/") { + t.Errorf("%s imports another runtime: %s", path, imported) + } + if imported == base+"graph" || strings.HasPrefix(imported, base+"graph/") || imported == base+"chatruntime" { + t.Errorf("%s imports an unversioned runtime: %s", path, imported) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + }) + } +} + +// All versions may live in one process without duplicate collector panics. +func TestRuntimeVersionsShareMetricsRegistry(t *testing.T) { + reg := prometheus.NewRegistry() + for i := 0; i < 2; i++ { + if v1.NewMetricsHandler(v1.MetricsDeps{Registerer: reg}) == nil || + v2.NewMetricsHandler(v2.MetricsDeps{Registerer: reg}) == nil || + v3.NewMetricsHandler(v3.MetricsDeps{Registerer: reg}) == nil { + t.Fatal("missing metrics handler") + } + } + if _, err := reg.Gather(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/manager/biz/aiops/runtime/v1/README.md b/internal/manager/biz/aiops/runtime/v1/README.md new file mode 100644 index 000000000..6decec827 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/README.md @@ -0,0 +1,9 @@ +# Agent Runtime v1 + +来源：[Ongrid v0.7.158](https://github.com/ongridio/ongrid/releases/tag/v0.7.158)，提交 `04b2796319785df840d7a472569c380e0871730b`。 + +提取该 release 的 `biz/aiops/chatruntime/` 与 `biz/aiops/graph/`，包括测试、testdata、prompt、Worker 和回调；不提取 legacy `agent/`。原文件清单与 Git blob 记录在 [source.json](source.json)。 + +适配：两个包的内部 import 改为本版本路径。v1 的两个仓储测试替身补充当前接口要求的 `UpdateSessionModel`，返回未接线错误，生产逻辑未修改。 + +当前与主模块共享 Eino/LLM 依赖、领域模型、业务工具和存储接口，因此不是旧 release 的完整可执行镜像。此目录参与编译和 race 测试，但尚未接入生产路由或管理员切换。接入前必须验证工具装配、权限、审批及后台任务生命周期，不能用历史策略绕过当前安全约束。 diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_parser.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_parser.go new file mode 100644 index 000000000..059f2a40f --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_parser.go @@ -0,0 +1,109 @@ +package chatruntime + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" +) + +// ParseAgentMd reads an agent persona file (agents/.md) at path. +// Same on-disk shape as SKILL.md but the body becomes the agent's +// system prompt. — required fields are name, description, +// when_to_use; the latter is strict per the spec because a coordinator +// can't decide whether to spawn an agent without it. +// +// Snake_case field names (this is the explicit +// rename of the original camelCase shape). +// +// Forward-compat: unknown frontmatter keys are preserved into +// Agent.UnknownFields so claude-code adding new persona fields (effort, +// memory, isolation, ...) doesn't break loading. +func ParseAgentMd(path string) (*Agent, []LoadWarning, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: read %s: %w", path, err) + } + frontmatter, body, err := splitFrontmatter(raw) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: %s: %w", path, err) + } + + var warnings []LoadWarning + + var ag Agent + if len(frontmatter) > 0 { + if err := yaml.Unmarshal(frontmatter, &ag); err != nil { + return nil, nil, fmt.Errorf("chatruntime: parse %s frontmatter: %w", path, err) + } + } + + rawMap := map[string]any{} + if len(frontmatter) > 0 { + if err := yaml.Unmarshal(frontmatter, &rawMap); err != nil { + return nil, nil, fmt.Errorf("chatruntime: parse %s frontmatter (raw): %w", path, err) + } + } + ag.UnknownFields = retainUnknownAgentFields(rawMap) + + // Required fields — strict. Missing when_to_use is an + // error not a warning: a coordinator can't choose this agent without + // it. + if strings.TrimSpace(ag.Name) == "" { + return nil, warnings, fmt.Errorf("chatruntime: %s: frontmatter missing required field 'name'", path) + } + if strings.TrimSpace(ag.Description) == "" { + return nil, warnings, fmt.Errorf("chatruntime: %s: frontmatter missing required field 'description'", path) + } + if strings.TrimSpace(ag.WhenToUse) == "" { + return nil, warnings, fmt.Errorf("chatruntime: %s: frontmatter missing required field 'when_to_use'", path) + } + + // Snake-case enforcement on name. We don't auto-normalize agent names + // (they're spawn keys; rewriting silently would break references). + if !skillNameRe.MatchString(ag.Name) { + // Allow dash for agents — claude-code uses `incident-investigator` + // style. Be permissive: warn rather than rewrite. + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "name_non_snake", + Reason: fmt.Sprintf("agent name %q is not snake_case (recommends underscores)", ag.Name), + }) + } + + ag.SystemPrompt = strings.TrimRight(string(body), "\n") + return &ag, warnings, nil +} + +// retainUnknownAgentFields keeps frontmatter keys we don't model on +// Agent. Drives forward compatibility with claude-code agent schema +// additions (effort, isolation, mcp_servers, hooks, ...). +func retainUnknownAgentFields(raw map[string]any) map[string]any { + known := map[string]struct{}{ + "name": {}, + "description": {}, + "when_to_use": {}, + "tools": {}, + "disallowed_tools": {}, + "permission_mode": {}, + "max_turns": {}, + "model": {}, + "critical_reminder": {}, + "initial_prompt": {}, + "background": {}, + "omit_claude_md": {}, + "metadata": {}, + } + out := map[string]any{} + for k, v := range raw { + if _, ok := known[k]; ok { + continue + } + out[k] = v + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_parser_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_parser_test.go new file mode 100644 index 000000000..d683e5a7a --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_parser_test.go @@ -0,0 +1,100 @@ +package chatruntime + +import ( + "path/filepath" + "strings" + "testing" +) + +// fixtureAgent returns the absolute path to an agent .md fixture. +func fixtureAgent(scenario, file string) string { + return filepath.Join("testdata", "agent_parser", scenario, file) +} + +func TestParseAgentMd_Minimal(t *testing.T) { + ag, warns, err := ParseAgentMd(fixtureAgent("minimal", "incident-investigator.md")) + if err != nil { + t.Fatalf("ParseAgentMd: %v", err) + } + // "incident-investigator" uses dash; we warn but don't rewrite. + foundDashWarn := false + for _, w := range warns { + if w.Code == "name_non_snake" { + foundDashWarn = true + } + } + if !foundDashWarn { + t.Errorf("expected name_non_snake warning; got %+v", warns) + } + if ag.Name != "incident-investigator" { + t.Errorf("Name = %q (should not be rewritten)", ag.Name) + } + if ag.Description == "" { + t.Error("Description empty") + } + if ag.WhenToUse == "" { + t.Error("WhenToUse empty") + } + if !strings.Contains(ag.SystemPrompt, "ongrid alert") { + t.Errorf("SystemPrompt body lost: %q", ag.SystemPrompt) + } +} + +func TestParseAgentMd_WithDisallowedTools(t *testing.T) { + ag, _, err := ParseAgentMd(fixtureAgent("with_disallowed_tools", "reviewer.md")) + if err != nil { + t.Fatalf("ParseAgentMd: %v", err) + } + if ag.Name != "reviewer" { + t.Errorf("Name = %q", ag.Name) + } + wantDis := []string{"*_skill", "run_shell"} + if !strSliceEq(ag.DisallowedTools, wantDis) { + t.Errorf("DisallowedTools = %v, want %v", ag.DisallowedTools, wantDis) + } + if ag.PermissionMode != "read-only" { + t.Errorf("PermissionMode = %q", ag.PermissionMode) + } + if ag.MaxTurns != 5 { + t.Errorf("MaxTurns = %d, want 5", ag.MaxTurns) + } + if !strings.Contains(ag.Model, "opus") { + t.Errorf("Model = %q", ag.Model) + } + if len(ag.Tools) != 4 { + t.Errorf("Tools len = %d, want 4", len(ag.Tools)) + } +} + +func TestParseAgentMd_MissingWhenToUse(t *testing.T) { + _, _, err := ParseAgentMd(fixtureAgent("missing_when_to_use", "foo.md")) + if err == nil { + t.Fatal("expected error for missing when_to_use, got nil") + } + if !strings.Contains(err.Error(), "when_to_use") { + t.Errorf("error should mention when_to_use; got %v", err) + } +} + +func TestParseAgentMd_WithCriticalReminder(t *testing.T) { + ag, _, err := ParseAgentMd(fixtureAgent("with_critical_reminder", "auditor.md")) + if err != nil { + t.Fatalf("ParseAgentMd: %v", err) + } + if ag.CriticalReminder == "" { + t.Fatal("CriticalReminder empty") + } + if !strings.Contains(ag.CriticalReminder, "Do not approve") { + t.Errorf("CriticalReminder content unexpected: %q", ag.CriticalReminder) + } + if ag.MaxTurns != 8 { + t.Errorf("MaxTurns = %d", ag.MaxTurns) + } +} + +func TestParseAgentMd_FileMissing(t *testing.T) { + _, _, err := ParseAgentMd(fixtureAgent("does_not_exist", "x.md")) + if err == nil { + t.Fatal("expected error for missing file") + } +} diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_registry.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_registry.go new file mode 100644 index 000000000..fa1e4474d --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_registry.go @@ -0,0 +1,175 @@ +package chatruntime + +import ( + "fmt" + "sync" +) + +// AgentRegistry holds parsed agent personas. Loaded +// once at startup from the configured agents root; coordinator looks up +// by name at spawn time. +// +// Concurrency: a sync.RWMutex guards the slices. marketplace +// Install/Uninstall calls Reload while in-flight coordinator turns may +// be running; Reload swaps a freshly-built slice atomically and +// returning slices are value copies so callers already holding a result +// don't observe mutations. +type AgentRegistry struct { + mu sync.RWMutex + agents []*Agent + warnings []LoadWarning +} + +// NewAgentRegistry returns an empty registry. +func NewAgentRegistry() *AgentRegistry { return &AgentRegistry{} } + +// Load walks agentsRoot for *.md files and plugin containers, parsing +// each agent persona it finds. Subdirectories are walked recursively; +// an `agents/team/.md` layout is fine. +// +// As of PR-skill-load (simplified loader) Load delegates +// to LoadAll, which also recurses into `.claude-plugin/` and +// `openclaw.plugin.json` containers under agentsRoot and pulls their +// agents (skills from packs in the agents root are kept on the result, +// but ignored here — AgentRegistry only owns agents). +// +// Per-file parse errors land as warnings rather than aborting the walk +// — same policy as SkillRegistry.Load. Non-existent agentsRoot is not +// an error. +func (r *AgentRegistry) Load(agentsRoot string) error { + res, err := LoadAll(LoadAllConfig{AgentsRoot: agentsRoot}) + if err != nil { + return fmt.Errorf("chatruntime: agent registry load: %w", err) + } + r.mu.Lock() + r.agents = append([]*Agent(nil), res.Agents...) + r.warnings = append([]LoadWarning(nil), res.Warnings...) + r.mu.Unlock() + return nil +} + +// Reload replaces the registry contents with a fresh scan of agentsRoot +// (primary) plus optional extras. Used by marketplace Install/Uninstall +// to hot-reload without restart. Atomic: builds the new slice outside +// the lock, then swaps under r.mu.Lock() in O(1). +// +// Variadic extras are treated as additional agent roots — each walked +// with the loose-*.md → persona rule so persona drops outside the +// primary AgentsRoot still load (used to preserve image-baked built-in +// agents through a marketplace install reload). +func (r *AgentRegistry) Reload(agentsRoot string, extras ...string) error { + cfg := LoadAllConfig{ + AgentsRoot: agentsRoot, + ExtraAgentsRoots: extras, + } + res, err := LoadAll(cfg) + if err != nil { + return fmt.Errorf("chatruntime: agent registry reload: %w", err) + } + newAgents := append([]*Agent(nil), res.Agents...) + newWarnings := append([]LoadWarning(nil), res.Warnings...) + r.mu.Lock() + r.agents = newAgents + r.warnings = newWarnings + r.mu.Unlock() + return nil +} + +// All returns a copy of every loaded agent. +func (r *AgentRegistry) All() []*Agent { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]*Agent, len(r.agents)) + copy(out, r.agents) + return out +} + +// Warnings returns a copy of every warning recorded during Load. +func (r *AgentRegistry) Warnings() []LoadWarning { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]LoadWarning, len(r.warnings)) + copy(out, r.warnings) + return out +} + +// ByName returns the agent with the given name (frontmatter `name` +// field). Returns (nil, false) when not found. Name match is exact — +// callers (the coordinator AgentTool) should already know the spawn key. +func (r *AgentRegistry) ByName(name string) (*Agent, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + for _, ag := range r.agents { + if ag.Name == name { + return ag, true + } + } + return nil, false +} + +// Add inserts a programmatically-constructed agent. Used by built-in +// agents (general-purpose) that ship in the binary rather than on disk. +func (r *AgentRegistry) Add(ag *Agent) { + if ag == nil { + return + } + r.mu.Lock() + r.agents = append(r.agents, ag) + r.mu.Unlock() +} + +// AddAll inserts a batch of agents. Nil entries are skipped. Used by +// LoadAll to merge plugin-container output into the registry. +func (r *AgentRegistry) AddAll(agents []*Agent) { + for _, ag := range agents { + r.Add(ag) + } +} + +// Replace swaps the agent with the same Name. When no row matches it +// behaves like Add (so callers can use it as upsert). Used by the +// user-agent edit path so a save mutates the live registry without a +// restart. +func (r *AgentRegistry) Replace(ag *Agent) { + if ag == nil || ag.Name == "" { + return + } + r.mu.Lock() + defer r.mu.Unlock() + for i, existing := range r.agents { + if existing != nil && existing.Name == ag.Name { + r.agents[i] = ag + return + } + } + r.agents = append(r.agents, ag) +} + +// Remove drops the agent with this name. Returns true when a row was +// removed. Used by the user-agent delete path. +func (r *AgentRegistry) Remove(name string) bool { + if name == "" { + return false + } + r.mu.Lock() + defer r.mu.Unlock() + for i, existing := range r.agents { + if existing != nil && existing.Name == name { + r.agents = append(r.agents[:i], r.agents[i+1:]...) + return true + } + } + return false +} + +// AddWarnings appends loader warnings (e.g. from LoadAll) so +// AgentRegistry.Warnings() exposes them alongside per-agent parse +// warnings. +func (r *AgentRegistry) AddWarnings(ws []LoadWarning) { + if len(ws) == 0 { + return + } + r.mu.Lock() + r.warnings = append(r.warnings, ws...) + r.mu.Unlock() +} diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_registry_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_registry_test.go new file mode 100644 index 000000000..d6935c463 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/agent_registry_test.go @@ -0,0 +1,48 @@ +package chatruntime + +import ( + "path/filepath" + "testing" +) + +func fixtureAgentRoot(scenario string) string { + return filepath.Join("testdata", "agent_registry", scenario) +} + +func TestAgentRegistry_LoadMulti(t *testing.T) { + r := NewAgentRegistry() + if err := r.Load(fixtureAgentRoot("multi")); err != nil { + t.Fatalf("Load: %v", err) + } + all := r.All() + if len(all) != 2 { + t.Fatalf("All() len = %d, want 2 (README.md skipped)", len(all)) + } + if _, ok := r.ByName("incident_investigator"); !ok { + t.Errorf("ByName(incident_investigator) miss") + } + if _, ok := r.ByName("reviewer"); !ok { + t.Errorf("ByName(reviewer) miss") + } + if _, ok := r.ByName("does_not_exist"); ok { + t.Errorf("ByName should miss for unknown name") + } +} + +func TestAgentRegistry_LoadNonExistent(t *testing.T) { + r := NewAgentRegistry() + if err := r.Load(fixtureAgentRoot("does_not_exist")); err != nil { + t.Fatalf("Load: %v", err) + } + if len(r.All()) != 0 { + t.Errorf("expected zero agents") + } +} + +func TestAgentRegistry_AddAndByName(t *testing.T) { + r := NewAgentRegistry() + r.Add(&Agent{Name: "general_purpose", Description: "default", WhenToUse: "fallback"}) + if got, ok := r.ByName("general_purpose"); !ok || got.Name != "general_purpose" { + t.Errorf("ByName failed for added agent") + } +} diff --git a/internal/manager/biz/aiops/chatruntime/bare_skills_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/bare_skills_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/bare_skills_test.go rename to internal/manager/biz/aiops/runtime/v1/chatruntime/bare_skills_test.go diff --git a/internal/manager/biz/aiops/chatruntime/chatmodelopts_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/chatmodelopts_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/chatmodelopts_test.go rename to internal/manager/biz/aiops/runtime/v1/chatruntime/chatmodelopts_test.go diff --git a/internal/manager/biz/aiops/chatruntime/command_parser.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/command_parser.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/command_parser.go rename to internal/manager/biz/aiops/runtime/v1/chatruntime/command_parser.go diff --git a/internal/manager/biz/aiops/chatruntime/command_parser_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/command_parser_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/command_parser_test.go rename to internal/manager/biz/aiops/runtime/v1/chatruntime/command_parser_test.go diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/load_all.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/load_all.go new file mode 100644 index 000000000..304ef3933 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/load_all.go @@ -0,0 +1,245 @@ +package chatruntime + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// LoadAllConfig is the input to LoadAll — the unified entry point for +// populating SkillRegistry + AgentRegistry from disk. PR-skill-load +// (simplified loader) collapses what used to be two +// separate walks (SkillRegistry.Load + AgentRegistry.Load) into one. +// +// Both roots are optional. A non-existent root is not an error (a +// fresh install with no skills/agents boots fine). +type LoadAllConfig struct { + // SkillsRoot is the directory walked for SKILL.md files and plugin + // containers. Plugin containers found here contribute their skills, + // agents, and command-converted skills to the result. + SkillsRoot string + + // AgentsRoot is the directory walked for *.md agent personas. + // Plugin containers found here contribute their agents (and their + // skills + command-skills, since a pack drop in either root is + // equally valid). + AgentsRoot string + + // ExtraSkillsRoots is a list of additional directories to walk + // alongside SkillsRoot. Used to keep image-baked built-in skills + // in one root and marketplace-installed packs in another, so + // marketplace Reload() doesn't have to drop the built-ins to refresh + // the user-installed set. Empty / non-existent entries are skipped + // silently. + ExtraSkillsRoots []string + + // ExtraAgentsRoots mirrors ExtraSkillsRoots for the agents side — + // each entry is walked with the "loose *.md = agent persona" rule + // so additional persona drops pile on top of AgentsRoot without + // the loose .md files being silently filtered out as non-SKILL.md. + ExtraAgentsRoots []string +} + +// LoadAll walks SkillsRoot + AgentsRoot once, dispatching to the right +// parser per file / directory: +// +// - /.../SKILL.md → ParseSkillMd +// - /...//.claude-plugin/plugin.json +// - /...//openclaw.plugin.json → LoadPluginContainer (recursive) +// - /.../*.md (not under a pack) → ParseAgentMd +// +// When a plugin container is detected, the walk SkipDir's into it and +// hands off to LoadPluginContainer; this prevents double-loading the +// same SKILL.md once from the outer walk and again from the recursive +// container load. +// +// The LoadResult.Pack field is left nil — LoadAll aggregates packs +// across multiple roots and the field is reserved for the +// single-container LoadPluginContainer flow. +func LoadAll(cfg LoadAllConfig) (*LoadResult, error) { + res := &LoadResult{} + + if cfg.SkillsRoot != "" { + more, err := walkLoadRoot(cfg.SkillsRoot, true) + if err != nil { + return nil, fmt.Errorf("chatruntime: load skills root %q: %w", cfg.SkillsRoot, err) + } + res.Skills = append(res.Skills, more.Skills...) + res.Agents = append(res.Agents, more.Agents...) + res.Warnings = append(res.Warnings, more.Warnings...) + } + + for _, extra := range cfg.ExtraSkillsRoots { + if extra == "" { + continue + } + more, err := walkLoadRoot(extra, true) + if err != nil { + return nil, fmt.Errorf("chatruntime: load extra skills root %q: %w", extra, err) + } + res.Skills = append(res.Skills, more.Skills...) + res.Agents = append(res.Agents, more.Agents...) + res.Warnings = append(res.Warnings, more.Warnings...) + } + + for _, extra := range cfg.ExtraAgentsRoots { + if extra == "" { + continue + } + more, err := walkLoadRoot(extra, false) + if err != nil { + return nil, fmt.Errorf("chatruntime: load extra agents root %q: %w", extra, err) + } + res.Skills = append(res.Skills, more.Skills...) + res.Agents = append(res.Agents, more.Agents...) + res.Warnings = append(res.Warnings, more.Warnings...) + } + + if cfg.AgentsRoot != "" { + more, err := walkLoadRoot(cfg.AgentsRoot, false) + if err != nil { + return nil, fmt.Errorf("chatruntime: load agents root %q: %w", cfg.AgentsRoot, err) + } + res.Skills = append(res.Skills, more.Skills...) + res.Agents = append(res.Agents, more.Agents...) + res.Warnings = append(res.Warnings, more.Warnings...) + } + return res, nil +} + +// walkLoadRoot walks root once. wantSkills selects whether we treat +// loose SKILL.md (true) or loose *.md (false → agent personas) as the +// primary content type — i.e. the root's "purpose". Plugin containers +// found in either root contribute everything they ship. +func walkLoadRoot(root string, wantSkills bool) (*LoadResult, error) { + res := &LoadResult{} + + info, err := os.Stat(root) + if err != nil { + if os.IsNotExist(err) { + return res, nil + } + return nil, fmt.Errorf("stat root %q: %w", root, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("root %q is not a directory", root) + } + + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return nil, fmt.Errorf("eval symlinks %q: %w", root, err) + } + + walkErr := filepath.Walk(root, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + res.Warnings = append(res.Warnings, LoadWarning{ + Path: path, Code: "walk_error", Reason: walkErr.Error(), + }) + return nil + } + if info == nil { + return nil + } + + // A symlink at the top of a tree is a common drop pattern (e.g. + // /var/lib/ongrid/system/skills/ → real-bundle). filepath.Walk + // won't follow it on its own, so we Stat-resolve it: if it lands + // inside resolvedRoot AND points to a directory that hosts a + // plugin container, treat it as a container. + if info.Mode()&os.ModeSymlink != 0 { + if !pathSafeUnderRoot(path, resolvedRoot, &res.Warnings) { + return nil + } + target, err := os.Stat(path) + if err != nil || !target.IsDir() { + return nil + } + kind, _, _ := DetectContainer(path) + if kind != ContainerNone { + lr, err := LoadPluginContainer(path) + if err != nil { + res.Warnings = append(res.Warnings, LoadWarning{ + Path: path, Code: "container_load_failed", Reason: err.Error(), + }) + return nil + } + res.Skills = append(res.Skills, lr.Skills...) + res.Agents = append(res.Agents, lr.Agents...) + res.Warnings = append(res.Warnings, lr.Warnings...) + } + return nil + } + + if info.IsDir() { + // Plugin container marker: hand off to LoadPluginContainer + // and SkipDir so the outer walk doesn't double-process the + // same SKILL.md / agent .md. + kind, _, _ := DetectContainer(path) + if kind != ContainerNone { + lr, err := LoadPluginContainer(path) + if err != nil { + res.Warnings = append(res.Warnings, LoadWarning{ + Path: path, Code: "container_load_failed", Reason: err.Error(), + }) + return filepath.SkipDir + } + res.Skills = append(res.Skills, lr.Skills...) + res.Agents = append(res.Agents, lr.Agents...) + res.Warnings = append(res.Warnings, lr.Warnings...) + return filepath.SkipDir + } + return nil + } + + // File handlers — only the root's "primary" content type is + // loaded loose. Skills root walks SKILL.md; agents root walks + // loose *.md (one persona per file). + base := info.Name() + if wantSkills { + if base != "SKILL.md" { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &res.Warnings) { + return nil + } + sk, ws, err := ParseSkillMd(path) + res.Warnings = append(res.Warnings, ws...) + if err != nil { + res.Warnings = append(res.Warnings, LoadWarning{ + Path: path, Code: "parse_failed", Reason: err.Error(), + }) + return nil + } + sk.Dir = filepath.Dir(path) + res.Skills = append(res.Skills, sk) + return nil + } + + // Agent root — *.md files are personas; README.md skipped. + if !strings.EqualFold(filepath.Ext(base), ".md") { + return nil + } + if strings.EqualFold(base, "README.md") { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &res.Warnings) { + return nil + } + ag, ws, err := ParseAgentMd(path) + res.Warnings = append(res.Warnings, ws...) + if err != nil { + res.Warnings = append(res.Warnings, LoadWarning{ + Path: path, Code: "parse_failed", Reason: err.Error(), + }) + return nil + } + ag.Dir = filepath.Dir(path) + res.Agents = append(res.Agents, ag) + return nil + }) + if walkErr != nil { + return nil, walkErr + } + return res, nil +} diff --git a/internal/manager/biz/aiops/chatruntime/load_all_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/load_all_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/load_all_test.go rename to internal/manager/biz/aiops/runtime/v1/chatruntime/load_all_test.go diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/plugin_container.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/plugin_container.go new file mode 100644 index 000000000..55bd09118 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/plugin_container.go @@ -0,0 +1,680 @@ +package chatruntime + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// PluginManifest is the on-disk shape of `.claude-plugin/plugin.json` +// (claude-code) and `openclaw.plugin.json` (openclaw). +// ongrid recognizes both: the former is treated as the canonical, the +// latter is treated as a superset whose ongrid-irrelevant fields are +// preserved into UIMetadata["openclaw_legacy"]. +// +// The struct is deliberately permissive — we model only the fields +// ongrid currently consumes; any other field is captured into Extras +// and (for openclaw) routed into Pack.UIMetadata["openclaw_legacy"]. +type PluginManifest struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + ConfigSchema json.RawMessage `json:"configSchema"` + + // Extras captures every JSON field not modeled above. We keep it + // raw so we can stash openclaw-specific keys (providers / channels + // / cliBackends / providerAuthChoices, ...) into Pack.UIMetadata. + Extras map[string]json.RawMessage `json:"-"` +} + +// ContainerKind labels which marker file we found in a candidate dir. +type ContainerKind string + +const ( + // ContainerClaude is the `.claude-plugin/plugin.json` form. + ContainerClaude ContainerKind = "claude" + // ContainerOpenclaw is the `openclaw.plugin.json` form. + ContainerOpenclaw ContainerKind = "openclaw" + // ContainerBareSkills is the skills.sh / vercel-labs/skills form: + // a directory containing one or more `skills//SKILL.md` + // (or a root-level SKILL.md) and NO manifest file. The pack ID + // and display name are synthesized from the directory name. This + // is what `npx skills add owner/repo` produces. + ContainerBareSkills ContainerKind = "bare_skills" + // ContainerNone means no recognized layout was found. + ContainerNone ContainerKind = "" +) + +// DetectContainer probes a directory for plugin container markers. +// Returns the marker kind and absolute path to the manifest file +// (empty for ContainerBareSkills, which has no manifest). Resolution +// order: openclaw > claude-plugin > bare-skills > none. +// +// ongrid recognizes the openclaw and claude-plugin manifest forms +// plus the bare-skills (skills.sh / vercel-labs/skills) layout +// where there's no manifest and the pack is just `skills//SKILL.md` +// drops under a parent dir. +func DetectContainer(dir string) (ContainerKind, string, error) { + openclawPath := filepath.Join(dir, "openclaw.plugin.json") + if info, err := os.Stat(openclawPath); err == nil && !info.IsDir() { + return ContainerOpenclaw, openclawPath, nil + } + claudePath := filepath.Join(dir, ".claude-plugin", "plugin.json") + if info, err := os.Stat(claudePath); err == nil && !info.IsDir() { + return ContainerClaude, claudePath, nil + } + if hasBareSkills(dir) { + return ContainerBareSkills, "", nil + } + return ContainerNone, "", nil +} + +// hasBareSkills reports whether dir looks like a skills.sh-style pack — +// either a root-level SKILL.md or at least one `skills//SKILL.md`. +// Scans at most one level deep under `skills/` to keep the probe cheap. +func hasBareSkills(dir string) bool { + if info, err := os.Stat(filepath.Join(dir, "SKILL.md")); err == nil && !info.IsDir() { + return true + } + skillsDir := filepath.Join(dir, "skills") + entries, err := os.ReadDir(skillsDir) + if err != nil { + return false + } + for _, e := range entries { + if !e.IsDir() { + continue + } + if info, err := os.Stat(filepath.Join(skillsDir, e.Name(), "SKILL.md")); err == nil && !info.IsDir() { + return true + } + } + return false +} + +// LoadPluginContainer parses the marker manifest in dir, then recursively +// loads every SKILL.md / agent .md / command .md found under the pack +// root. / — claude-code-style bundle dropped into the +// skills root has all sub-content auto-registered. +// +// Sub-roots: +// - claude pack: /skills/, /agents/, /commands/ +// - openclaw pack: paths from manifest.skills[] (relative); also +// /skills/, /agents/, /commands/ when present +// +// Path safety: every sub-path is resolved with filepath.EvalSymlinks +// and verified to land back inside the pack root before parse — symlink +// hops out of the tree and `..` components are rejected with an +// "escapes_root" warning. The pack itself is loaded; only the offending +// file is dropped. +// +// Per-file parse failures are recorded as warnings (code "parse_failed") +// rather than aborting the load — a single bad SKILL.md should not take +// the whole pack offline. +// +// hooks/ subdirs are walked once for diagnostics (per-hook warning code +// "hooks_dropped"); .mcp.json triggers a single "mcp_unsupported" +// warning. Neither is acted upon +func LoadPluginContainer(dir string) (*LoadResult, error) { + kind, manifestPath, err := DetectContainer(dir) + if err != nil { + return nil, fmt.Errorf("chatruntime: detect container in %s: %w", dir, err) + } + if kind == ContainerNone { + return nil, fmt.Errorf("chatruntime: no recognized pack layout in %s (need .claude-plugin/plugin.json, openclaw.plugin.json, or at least one skills//SKILL.md)", dir) + } + + var ( + pack *Pack + warnings []LoadWarning + raw []byte + ) + if kind == ContainerBareSkills { + // skills.sh-style drop: no manifest, synthesize one from the + // directory name. SHA-256 over the empty manifest stays + // deterministic across re-installs of the same dir. + pack = synthesizeBareSkillsPack(dir) + } else { + raw, err = os.ReadFile(manifestPath) + if err != nil { + return nil, fmt.Errorf("chatruntime: read %s: %w", manifestPath, err) + } + pack, warnings, err = parsePluginManifest(raw, kind) + if err != nil { + return nil, fmt.Errorf("chatruntime: parse %s: %w", manifestPath, err) + } + } + pack.Dir = mustAbs(dir) + pack.ManifestSHA256 = sha256Hex(raw) + + // Resolve the pack root — used as the allowlist boundary for every + // sub-file we touch. EvalSymlinks lets the user place the pack + // itself behind a symlink (cp -r dance) without breaking the root, + // while still rejecting symlinks under it that escape outward. + resolvedRoot, rootErr := filepath.EvalSymlinks(dir) + if rootErr != nil { + // If we can't even resolve the root we can't safely walk; + // return what we have (metadata only) plus a warning. + warnings = append(warnings, LoadWarning{ + Path: dir, + Code: "root_resolve_failed", + Reason: fmt.Sprintf("eval symlinks on pack root: %v; sub-tree skipped", rootErr), + }) + return &LoadResult{Pack: pack, Warnings: warnings}, nil + } + + // Skills sub-tree. + skillRoots := resolveSkillSubRoots(dir, kind, raw, &warnings) + var skills []*Skill + for _, sr := range skillRoots { + more, w := walkSkillsRoot(sr, resolvedRoot) + skills = append(skills, more...) + warnings = append(warnings, w...) + } + + // Agents sub-tree (claude only — openclaw doesn't ship agents/). + agentsDir := filepath.Join(dir, "agents") + agents, agentWarns := walkAgentsRoot(agentsDir, resolvedRoot) + warnings = append(warnings, agentWarns...) + + // Commands sub-tree → keyword skills. + commandsDir := filepath.Join(dir, "commands") + commandSkills, commandWarns := walkCommandsRoot(commandsDir, resolvedRoot) + skills = append(skills, commandSkills...) + warnings = append(warnings, commandWarns...) + + // hooks/ + .mcp.json — diagnostics only. + warnings = append(warnings, scanHooksDir(filepath.Join(dir, "hooks"), resolvedRoot)...) + if info, err := os.Stat(filepath.Join(dir, ".mcp.json")); err == nil && !info.IsDir() { + warnings = append(warnings, LoadWarning{ + Path: filepath.Join(dir, ".mcp.json"), + Code: "mcp_unsupported", + Reason: ".mcp.json found; ongrid does not run MCP servers yet", + }) + } + + return &LoadResult{ + Pack: pack, + Skills: skills, + Agents: agents, + Warnings: warnings, + }, nil +} + +// resolveSkillSubRoots returns the directories we should walk for +// SKILL.md files. Claude packs always use /skills; openclaw packs +// MAY declare a manifest.skills[] array of relative paths — we follow +// those, plus the conventional /skills if it exists. +// +// Every returned path is verified to be a child of `dir` (string-level +// guard against `..` in manifest.skills[]); deeper symlink-vs-root +// validation happens during the walk itself. +func resolveSkillSubRoots(dir string, kind ContainerKind, manifestRaw []byte, warnings *[]LoadWarning) []string { + var roots []string + seen := map[string]bool{} + + add := func(p string) { + clean := filepath.Clean(p) + if seen[clean] { + return + } + seen[clean] = true + if info, err := os.Stat(clean); err == nil && info.IsDir() { + roots = append(roots, clean) + } + } + + if kind == ContainerOpenclaw { + // Pull the optional skills[] array. When present it's authoritative — + // we follow only those paths (treats manifest.skills[] + // and the conventional skills/ directory as equivalent forms; a pack + // that lists both would otherwise double-load). + var generic struct { + Skills []string `json:"skills"` + } + hasManifestSkills := false + if err := json.Unmarshal(manifestRaw, &generic); err == nil { + for _, raw := range generic.Skills { + if strings.TrimSpace(raw) == "" { + continue + } + hasManifestSkills = true + // Disallow absolute paths and `..` components in the + // manifest declaration itself — purely string check; + // EvalSymlinks during the walk catches symlink escape. + if filepath.IsAbs(raw) || strings.Contains(raw, "..") { + *warnings = append(*warnings, LoadWarning{ + Path: raw, + Code: "skills_path_rejected", + Reason: fmt.Sprintf("manifest skills[] entry %q is absolute or contains '..'; rejected", raw), + }) + continue + } + add(filepath.Join(dir, raw)) + } + } + // Fall through to the conventional dir only if the manifest + // didn't declare its own skills[] list. + if !hasManifestSkills { + add(filepath.Join(dir, "skills")) + } + } else if kind == ContainerBareSkills { + // skills.sh layout: either a multi-skill repo with + // skills//SKILL.md or a single-skill repo with a + // root-level SKILL.md. Pick the right root — adding BOTH + // causes the walker to double-load every skills/* file + // (once via skills/, once via the recursive dir walk). + if info, err := os.Stat(filepath.Join(dir, "skills")); err == nil && info.IsDir() { + add(filepath.Join(dir, "skills")) + } else { + add(dir) + } + } else { + // Claude pack: conventional /skills only. + add(filepath.Join(dir, "skills")) + } + return roots +} + +// walkSkillsRoot walks skillsDir for SKILL.md files, returning the +// parsed skills + non-fatal warnings. resolvedRoot is the eval-symlinked +// pack root used to detect escape via symlinks. +func walkSkillsRoot(skillsDir, resolvedRoot string) ([]*Skill, []LoadWarning) { + info, err := os.Stat(skillsDir) + if err != nil || !info.IsDir() { + return nil, nil + } + var skills []*Skill + var warnings []LoadWarning + + walkErr := filepath.Walk(skillsDir, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: path, Code: "walk_error", Reason: walkErr.Error()}) + return nil + } + if info == nil { + return nil + } + // Symlinks (Lstat: ModeSymlink) need explicit policy — filepath.Walk + // does NOT follow them. We must validate that the link target is + // inside resolvedRoot; if it escapes, emit an escapes_root warning + // so the test (and humans) see the rejection. If it lands inside, + // we still don't auto-recurse (avoid loops); operators can land + // SKILL.md files at real paths. + if info.Mode()&os.ModeSymlink != 0 { + if !pathSafeUnderRoot(path, resolvedRoot, &warnings) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + return nil + } + if info.IsDir() { + return nil + } + if filepath.Base(path) != "SKILL.md" { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &warnings) { + return nil + } + sk, ws, err := ParseSkillMd(path) + warnings = append(warnings, ws...) + if err != nil { + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "parse_failed", + Reason: err.Error(), + }) + return nil + } + sk.Dir = filepath.Dir(path) + // Default activation = always when neither + // top-level nor metadata.ongrid.activation set a mode. + if sk.Activation.Mode == "" { + sk.Activation.Mode = "always" + } + if sk.Metadata.Ongrid.Activation.Mode == "" { + sk.Metadata.Ongrid.Activation.Mode = sk.Activation.Mode + } + skills = append(skills, sk) + return nil + }) + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: skillsDir, Code: "walk_error", Reason: walkErr.Error()}) + } + return skills, warnings +} + +// walkAgentsRoot walks agentsDir for *.md agent personas. +// README.md is skipped (documentation, not a persona) — same policy as +// AgentRegistry.Load. +func walkAgentsRoot(agentsDir, resolvedRoot string) ([]*Agent, []LoadWarning) { + info, err := os.Stat(agentsDir) + if err != nil || !info.IsDir() { + return nil, nil + } + var agents []*Agent + var warnings []LoadWarning + + walkErr := filepath.Walk(agentsDir, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: path, Code: "walk_error", Reason: walkErr.Error()}) + return nil + } + if info == nil || info.IsDir() { + return nil + } + if !strings.EqualFold(filepath.Ext(info.Name()), ".md") { + return nil + } + if strings.EqualFold(info.Name(), "README.md") { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &warnings) { + return nil + } + ag, ws, err := ParseAgentMd(path) + warnings = append(warnings, ws...) + if err != nil { + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "parse_failed", + Reason: err.Error(), + }) + return nil + } + ag.Dir = filepath.Dir(path) + agents = append(agents, ag) + return nil + }) + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: agentsDir, Code: "walk_error", Reason: walkErr.Error()}) + } + return agents, warnings +} + +// walkCommandsRoot walks commandsDir for *.md command files and +// converts each into a keyword skill. Nested subdirs +// are flattened — `commands/git/commit.md` becomes `cmd_commit`. +// README.md is skipped. +func walkCommandsRoot(commandsDir, resolvedRoot string) ([]*Skill, []LoadWarning) { + info, err := os.Stat(commandsDir) + if err != nil || !info.IsDir() { + return nil, nil + } + var skills []*Skill + var warnings []LoadWarning + + walkErr := filepath.Walk(commandsDir, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: path, Code: "walk_error", Reason: walkErr.Error()}) + return nil + } + if info == nil || info.IsDir() { + return nil + } + if !strings.EqualFold(filepath.Ext(info.Name()), ".md") { + return nil + } + if strings.EqualFold(info.Name(), "README.md") { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &warnings) { + return nil + } + sk, ws, err := ConvertCommandFile(path) + warnings = append(warnings, ws...) + if err != nil { + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "command_parse_failed", + Reason: err.Error(), + }) + return nil + } + skills = append(skills, sk) + return nil + }) + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: commandsDir, Code: "walk_error", Reason: walkErr.Error()}) + } + return skills, warnings +} + +// scanHooksDir walks the hooks/ subdir (if present) and emits one +// warning per hook file plus one summary warning. ongrid never executes +// hooks. +func scanHooksDir(hooksDir, resolvedRoot string) []LoadWarning { + info, err := os.Stat(hooksDir) + if err != nil || !info.IsDir() { + return nil + } + var warnings []LoadWarning + // Top-level summary warning so callers grepping for "hooks_unsupported" + // keep working (back-compat with PR-2 test). + warnings = append(warnings, LoadWarning{ + Path: hooksDir, + Code: "hooks_unsupported", + Reason: "hooks/ subdir found; ongrid does not run plugin hooks", + }) + + var dropped []string + _ = filepath.Walk(hooksDir, func(path string, info os.FileInfo, _ error) error { + if info == nil || info.IsDir() { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &warnings) { + return nil + } + dropped = append(dropped, path) + return nil + }) + sort.Strings(dropped) + for _, p := range dropped { + warnings = append(warnings, LoadWarning{ + Path: p, + Code: "hooks_dropped", + Reason: "hook file ignored; ongrid never executes plugin hooks", + }) + } + return warnings +} + +// pathSafeUnderRoot validates that path, after resolving symlinks, sits +// inside resolvedRoot. Failures append an "escapes_root" or +// "symlink_error" warning to *warnings and return false so the caller +// can skip the offending file. +// +// Implementation note: we use filepath.EvalSymlinks rather than a pure +// string-prefix check. A string check would miss symlink hops (the +// classic /tmp/evil → /etc dodge), and absent EvalSymlinks any +// `/skills/` symlink that resolves outside the pack would be +// silently followed by ParseSkillMd. The trade-off is one extra +// stat-per-file plus a hard fail when the path itself can't be +// resolved (broken symlink, race-deleted file) — surfaced as +// symlink_error rather than crashing the load. +func pathSafeUnderRoot(path, resolvedRoot string, warnings *[]LoadWarning) bool { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + *warnings = append(*warnings, LoadWarning{ + Path: path, + Code: "symlink_error", + Reason: err.Error(), + }) + return false + } + if !pathHasPrefix(resolved, resolvedRoot) { + *warnings = append(*warnings, LoadWarning{ + Path: path, + Code: "escapes_root", + Reason: fmt.Sprintf("path traversal: resolved path %s escapes pack root %s", resolved, resolvedRoot), + }) + return false + } + return true +} + +// parsePluginManifest decodes the manifest JSON and produces a Pack. +// openclawLegacyKeys are the openclaw-only fields ongrid stuffs into +// UIMetadata["openclaw_legacy"] for SPA display. +func parsePluginManifest(raw []byte, kind ContainerKind) (*Pack, []LoadWarning, error) { + var manifest PluginManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, nil, fmt.Errorf("decode: %w", err) + } + + // Re-decode into a generic map to capture extras. + var generic map[string]json.RawMessage + if err := json.Unmarshal(raw, &generic); err != nil { + return nil, nil, fmt.Errorf("decode generic: %w", err) + } + + known := map[string]struct{}{ + "id": {}, + "name": {}, + "version": {}, + "description": {}, + "configSchema": {}, + } + extras := map[string]json.RawMessage{} + for k, v := range generic { + if _, ok := known[k]; ok { + continue + } + extras[k] = v + } + manifest.Extras = extras + + var warnings []LoadWarning + + pack := &Pack{ + ID: manifest.ID, + DisplayName: manifest.Name, + Version: manifest.Version, + Description: manifest.Description, + ConfigSchema: manifest.ConfigSchema, + SignatureState: "unsigned", + } + + // id/name/version/description go straight into Pack. + // Required: id is the only must-have. + if pack.ID == "" { + // Some claude-code plugin.json files use `name` as the id when + // id is absent — fall back to that for compat. + if manifest.Name != "" { + pack.ID = manifest.Name + warnings = append(warnings, LoadWarning{ + Code: "id_fallback_from_name", + Reason: "plugin manifest missing 'id'; using 'name' as id", + }) + } else { + return nil, warnings, fmt.Errorf("manifest missing required field 'id'") + } + } + + // openclaw-only fields are routed into ui_metadata.openclaw_legacy + // so the SPA can render them, but ongrid's runtime ignores them. + if kind == ContainerOpenclaw && len(extras) > 0 { + legacy := map[string]any{} + for k, v := range extras { + // Keep raw JSON; SPA may want the structure verbatim. + var decoded any + if err := json.Unmarshal(v, &decoded); err == nil { + legacy[k] = decoded + } else { + legacy[k] = string(v) + } + } + pack.UIMetadata = map[string]any{ + "openclaw_legacy": legacy, + } + warnings = append(warnings, LoadWarning{ + Code: "openclaw_legacy_preserved", + Reason: fmt.Sprintf("preserved %d openclaw-specific manifest fields under ui_metadata.openclaw_legacy", len(legacy)), + }) + } else if len(extras) > 0 { + // Claude-plugin manifest with extra unrecognized keys — keep + // them too, but under ui_metadata.extra. + extraMap := map[string]any{} + for k, v := range extras { + var decoded any + if err := json.Unmarshal(v, &decoded); err == nil { + extraMap[k] = decoded + } else { + extraMap[k] = string(v) + } + } + pack.UIMetadata = map[string]any{ + "extra": extraMap, + } + } + + return pack, warnings, nil +} + +// synthesizeBareSkillsPack builds a minimal Pack for a skills.sh-style +// drop where there's no plugin.json / openclaw.plugin.json manifest. +// ID = sanitized directory basename (lowercase, hyphens), DisplayName +// = same, Version = "0.0.0". The pack-level ConfigSchema / UIMetadata +// stay empty — the individual SKILL.md files are the canonical metadata +// source for this layout. +func synthesizeBareSkillsPack(dir string) *Pack { + base := strings.TrimSpace(filepath.Base(strings.TrimRight(dir, string(filepath.Separator)))) + id := bareSkillsPackID(base) + return &Pack{ + ID: id, + DisplayName: base, + Version: "0.0.0", + Description: "Bare skills.sh-style pack (no manifest).", + } +} + +// bareSkillsPackID normalises a directory basename into a pack id. +// Rules mirror the SKILL.md skillNameRe: lowercase, digits, hyphens; +// non-conformant chars collapse to a single hyphen; leading/trailing +// hyphens trimmed; empty fallback = "untitled-skill-pack". +func bareSkillsPackID(base string) string { + if base == "" { + return "untitled-skill-pack" + } + var b strings.Builder + prevDash := false + for _, r := range strings.ToLower(base) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + prevDash = false + default: + if !prevDash && b.Len() > 0 { + b.WriteByte('-') + prevDash = true + } + } + } + out := strings.Trim(b.String(), "-") + if out == "" { + return "untitled-skill-pack" + } + return out +} + +func sha256Hex(raw []byte) string { + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) +} + +func mustAbs(dir string) string { + abs, err := filepath.Abs(dir) + if err != nil { + return dir + } + return abs +} diff --git a/internal/manager/biz/aiops/chatruntime/plugin_container_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/plugin_container_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/plugin_container_test.go rename to internal/manager/biz/aiops/runtime/v1/chatruntime/plugin_container_test.go diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/runtime.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/runtime.go new file mode 100644 index 000000000..c2947c2a1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/runtime.go @@ -0,0 +1,1159 @@ +// runtime.go is the in-process orchestration entry the new graph-based +// agent path goes through. PR-9 of — the cutover layer that +// pre-PR-9 callers (legacy agent.go) bypass entirely. After this PR +// the manager's HTTP service can pick which path runs based on the +// ONGRID_AGENT_KERNEL feature flag (default = legacy). +// +// Responsibilities: +// +// 1. Ownership check (caller user_id == session.user_id; admin bypass +// happens upstream — by the time Handle is called the service has +// already resolved the owning user_id). +// 2. Skill resolution against the user query (SkillRegistry.Resolve) +// — the active skills shape the system prompt. +// 3. System prompt assembly via ComposeSystemPrompt — base prompt + +// active skill prompts + (optional) agent persona. +// 4. @-mention inlining — same markdown-bullet preamble the legacy +// agent.go produces, so chat history replay is byte-identical +// across kernels (legacy mention text persists into the new path +// unchanged). +// 5. User-message persistence (chat_messages role=user). Mirrors the +// legacy "persist before LLM call" invariant from agent.go so a +// downstream crash leaves the user turn on disk. +// 6. Graph invoke with the default callback chain +// (callbacks.NewDefaultHandlers) wired via compose.WithCallbacks. +// The callback chain handles persistence (assistant + tool rows), +// SSE streaming, audit, metrics, and budget gating. +// 7. Reply translation — a chatruntime.Reply that the service layer +// translates back to agent.Reply for HTTP response shape parity. +// +// Behaviour parity with legacy agent.go is the explicit goal — the +// SPA, the persistence schema, and downstream reports must not see +// the kernel switch. Where exact parity isn't reachable (e.g. eino's +// graph runs the loop internally so we don't see per-iteration +// AssistantEvent counts), the value is best-effort and documented +// inline. +package chatruntime + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/schema" + + biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v1/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v1/graph/callbacks" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/errs" + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// Mention mirrors the legacy agent.Mention shape so chatruntime can +// be called without dragging the manager/biz/aiops/agent package as +// a dependency. The service-layer adapter does the type translation. +type Mention struct { + Type string + ID string + Label string +} + +// MentionResolver hydrates a list of @-mentions into one bullet line +// each. Mirrors agent.MentionResolver. nil = no inlining. +type MentionResolver interface { + Resolve(ctx context.Context, mentions []Mention) []string +} + +// EventType enumerates streaming events emitted by the runtime. The +// values are byte-equal to the legacy agent.EventType strings so the +// SSE adapter at the service layer can route both kernels through +// the same wire format. New event names introduced in PR-6 +// (assistant_start / assistant_delta / assistant_end) are NOT +// surfaced through this enum yet — the cutover keeps the legacy +// frame names so the SPA round-trips without changes. +type EventType string + +const ( + // EventAssistant fires after every assistant turn is persisted. + EventAssistant EventType = "assistant" + // EventToolStart fires after a tool_call row is persisted as + // pending, before the tool actually runs. + EventToolStart EventType = "tool_start" + // EventToolEnd fires once the tool finishes (success/error/timeout). + EventToolEnd EventType = "tool_end" + // EventDone fires once at terminal success with the final Reply. + EventDone EventType = "done" + // EventError fires on terminal failure. + EventError EventType = "error" +) + +// AssistantEvent describes one assistant turn's persisted row. Same +// shape as legacy agent.AssistantEvent. +type AssistantEvent struct { + Iteration int + MessageID string + Content string + CreatedAt time.Time + PendingToolCalls int +} + +// ToolEvent describes one tool_call lifecycle event. Same shape as +// legacy agent.ToolEvent. +type ToolEvent struct { + ToolCallID string + Name string + DeviceID *uint64 + Status string + StartedAt time.Time + EndedAt *time.Time + DurationMs int64 + Error string + ArgsJSON string + ResultJSON string +} + +// Event is the streaming envelope. Mirrors legacy agent.Event so the +// service-layer adapter can copy fields one-to-one without a switch. +// +// Notification carries the task_notification payload — set when +// Type == EventTaskNotification and a background worker reaches a +// terminal state. The SPA / SSE adapter renders it as a separate frame +// type next to assistant / tool / done. +type Event struct { + Type EventType + Assistant *AssistantEvent + Tool *ToolEvent + Done *Reply + Error string + Notification *TaskNotification +} + +// Emit is the streaming callback. nil-safe. +type Emit func(Event) + +// Request bundles every per-call input the runtime needs. +type Request struct { + SessionID string + UserID uint64 + // Role is the caller's system role (admin | user | viewer). + // gates the toolbag against this — viewer drops Class!=read so the + // LLM cannot reach mutating tools no matter what the persona allows. + Role string + UserText string + Mentions []Mention + Provider string + Model string + WebSearchEnabled bool + // Locale is the UI language ("en-US"/"zh-CN") the reply should use; + // threaded into graph.Input so the assembler adds a language directive. + Locale string + // Emit is the streaming sink. nil = no streaming (blocking call). + Emit Emit +} + +// Reply is the terminal result returned by Handle. Mirrors the +// legacy agent.Reply shape so the service layer can fan it back out +// through the same DTO without translation losses. +type Reply struct { + Message *aiopsmodel.Message + Usage llm.Usage + Iterations int + ToolCalls []*aiopsmodel.ToolCall +} + +// Config tunes runtime behaviour. ToolBag is the wrapped (decorated) +// tool list; the runtime does NOT wrap tools itself — that's the +// caller's job at construction time so the decorator deps (audit +// sink, limiter, registerer) live exactly once per process. +type Config struct { + // SkillRegistry holds the parsed SKILL.md descriptors. May be + // empty (nil-safe ComposeSystemPrompt). + SkillRegistry *SkillRegistry + + // AgentRegistry holds the parsed agent personas. May be empty. + AgentRegistry *AgentRegistry + + // Sessions is the persistence binding (chat_sessions / + // chat_messages / chat_tool_calls). Required. + Sessions biz.SessionRepo + + // ChatModel is the eino model.ToolCallingChatModel. Required. + // Production wiring passes RoutingChatModel from PR-1; tests + // inject a scriptedChatModel. + ChatModel model.ToolCallingChatModel + + // ToolBag is the pre-decorated BaseTool list the graph exposes + // to the LLM. cmd/ongrid/main.go assembles this once via + // Registry.BuildBaseTools + AppendHostFilesTools + Wrap. + ToolBag []basetool.BaseTool + + // CoordinatorStubs are name-matching redirect stubs visible only + // to the coordinator agent. They occupy hallucination-prone tool + // names (host_bash, get_host_load, ...) and return a "use + // AgentTool to dispatch to specialist-X" message instead of + // executing the real query. Without them, the eino runtime + // aborts with "tool not found in toolsNode indexes" the moment + // the LLM picks a name not actually in the coordinator's filtered + // bag — see internal/manager/biz/aiops/tools/redirect_stub.go. + // + // Workers (specialists) never see this slice. They have the real + // tool under the same name in their own filtered bag. + CoordinatorStubs []basetool.BaseTool + + // MentionResolver hydrates @-mentions into markdown bullets. + // Optional. + MentionResolver MentionResolver + + // BasePrompt is the universal preamble prepended to every + // system prompt. Empty = none. + BasePrompt string + + // HistoryLimit caps how many past messages we replay. 0 → 50 + // (legacy default). + HistoryLimit int + + // GraphCfg tunes the graph engine (max iterations, temperature, + // per-tool timeout). + GraphCfg graph.Config + + // CallbackDeps are the cross-cutting deps wired into every graph + // run. Persistence.SessionID is filled per-request from + // Request.SessionID; the rest of Persistence + Audit + Metrics + // + Budget are filled at construction. + CallbackDeps callbacks.Deps + + // Logger may be nil. + Logger *slog.Logger +} + +// ToolBagProvider is the local mirror of tools.DeferredToolBagProvider — +// chatruntime cannot import the tools package (cyclic: tools imports +// nothing from chatruntime, but cmd-side wiring goes the other +// direction and would close the loop). The shape is intentionally +// identical so a *tools.ToolBag value satisfies this interface +// directly without an adapter. +// +// — the runtime needs a handle on the unredacted tool +// universe so future code paths (e.g. logging the active toolBag at +// boot, exposing a /metrics list of registered tools) can introspect +// what the LLM has access to even when deferral is on. Today the +// handle is read-only — set once via SetToolBag, never mutated. +type ToolBagProvider interface { + DeferredTools() []basetool.BaseTool + AllTools() []basetool.BaseTool +} + +// Runtime is the cutover entry. Construct once at boot via +// NewRuntime; call Handle per HTTP request. +// +// workers holds the in-process map of spawned sub-agents keyed by +// worker id. Lifetime: bounded by the Runtime's lifetime — there's no +// auto-eviction yet (a follow-up PR adds a TTL sweeper once we move +// chat_sessions to track parent_session_id; until then workers persist +// in memory until process restart, which matches a fresh sandbox per +// chat in dev/test). +type Runtime struct { + cfg Config + log *slog.Logger + + workersMu sync.Mutex + workers map[string]*Worker + + // bag is the unredacted ToolBag handle wired by SetToolBag. nil + // when the caller wires the runtime without calling SetToolBag — + // not all code paths need it (legacy callers only feed + // cfg.ToolBag through the graph). + bag ToolBagProvider +} + +// NewRuntime builds a Runtime. Returns an error when required deps +// are missing — Sessions and ChatModel. +func NewRuntime(cfg Config) (*Runtime, error) { + if cfg.Sessions == nil { + return nil, errors.New("chatruntime: Sessions is required") + } + if cfg.ChatModel == nil { + return nil, errors.New("chatruntime: ChatModel is required") + } + if cfg.HistoryLimit <= 0 { + cfg.HistoryLimit = 50 + } + log := cfg.Logger + if log == nil { + log = slog.Default() + } + return &Runtime{cfg: cfg, log: log}, nil +} + +// SetMentionResolver wires the @-mention hydrator post-construction. +// Optional — when unset the runtime runs with no mention inlining +// (matches the legacy agent's nil-resolver behaviour). +func (rt *Runtime) SetMentionResolver(r MentionResolver) { + if rt == nil { + return + } + rt.cfg.MentionResolver = r +} + +// SetToolBag wires the unredacted ToolBag handle onto the Runtime so +// downstream introspection (ToolSearch, future /metrics +// surfaces) can query the full tool universe even when the LLM-facing +// view is the deferred / redacted one. Idempotent; nil is a no-op so +// tests can opt out without a panic. +// +// Note: the bag is INFORMATIONAL only. The graph still feeds the +// LLM via cfg.ToolBag, which the caller assembles from +// bag.SchemasForLLM() before calling NewRuntime — see +// cmd/ongrid/main.go's buildAIOpsRuntime for the wiring. +func (rt *Runtime) SetToolBag(bag ToolBagProvider) { + if rt == nil { + return + } + rt.bag = bag +} + +// AppendToolBag appends extra BaseTools to the coordinator tool bag. +// — used by main.go to bolt the coordinator-only AgentTool +// / SendMessageTool / TaskStopTool onto the Runtime AFTER construction +// (the chicken-and-egg: those tools need the Runtime as their spawner, +// the Runtime needs the tool bag at NewRuntime time). Workers do NOT +// observe these added tools — filterToolsForAgent strips them +// unconditionally via coordinatorOnlyTools (see worker.go). +// +// This is intentionally distinct from a generic "AddTool" method: it +// signals "these are coordinator-only" so a future reader of main.go +// understands why three more tools appear after NewRuntime returned. +func (rt *Runtime) AppendToolBag(tools []basetool.BaseTool) { + if rt == nil || len(tools) == 0 { + return + } + rt.cfg.ToolBag = append(rt.cfg.ToolBag, tools...) +} + +// AgentRegistry exposes the registry for narrow read-only callers +// (cmd/main.go uses this when building AgentTool's subagent registry +// shim). nil-safe. +func (rt *Runtime) AgentRegistry() *AgentRegistry { + if rt == nil { + return nil + } + return rt.cfg.AgentRegistry +} + +// SkillRegistry exposes the skill registry for narrow read-only +// callers — today the marketplace usecase wires Reload calls +// against it on install/uninstall. nil-safe. +func (rt *Runtime) SkillRegistry() *SkillRegistry { + if rt == nil { + return nil + } + return rt.cfg.SkillRegistry +} + +// ToolCount returns how many tools are bound into the toolBag. Used +// at boot time + tests for visibility (the spec asks us to log this +// when ONGRID_AGENT_KERNEL=graph). +func (rt *Runtime) ToolCount() int { + if rt == nil { + return 0 + } + return len(rt.cfg.ToolBag) +} + +// ToolNames returns the resolved tool names (best-effort — Info() +// errors are skipped). Production wiring logs this at boot so the +// operator sees what the LLM is exposed to. +func (rt *Runtime) ToolNames(ctx context.Context) []string { + if rt == nil { + return nil + } + out := make([]string, 0, len(rt.cfg.ToolBag)) + for _, t := range rt.cfg.ToolBag { + if t == nil { + continue + } + info, err := t.Info(ctx) + if err != nil || info == nil { + continue + } + out = append(out, info.Name) + } + return out +} + +// Handle runs one user turn end-to-end. Mirrors legacy +// agent.runInternal: ownership check → mention inline → user message +// persistence → graph invoke with callback chain → reply. +func (rt *Runtime) Handle(ctx context.Context, req *Request) (*Reply, error) { + if rt == nil || rt.cfg.Sessions == nil || rt.cfg.ChatModel == nil { + return nil, errs.ErrNotWiredYet + } + if req == nil { + return nil, fmt.Errorf("%w: request required", errs.ErrInvalid) + } + + emit := req.Emit + if emit == nil { + emit = func(Event) {} + } + // Thread the active emitter into ctx so AgentTool's InvokableRun + // (deep inside the graph) can fire a task_notification SSE frame + // when a background worker terminates. / + ctx = withEmit(ctx, req.Emit) + + // 1. Ownership check. + sess, err := rt.cfg.Sessions.GetSession(ctx, req.SessionID) + if err != nil { + return nil, err + } + if sess.UserID != req.UserID { + // Match legacy behaviour: surface ErrNotFound (not + // ErrForbidden) so non-owners can't fingerprint sessions. + return nil, errs.ErrNotFound + } + + // 2. Render @-mention bullets. + mentionsRendered := "" + augmentedUserText := req.UserText + if len(req.Mentions) > 0 && rt.cfg.MentionResolver != nil { + bullets := rt.cfg.MentionResolver.Resolve(ctx, req.Mentions) + if len(bullets) > 0 { + // Same prefix the legacy agent.go uses — keeps history + // replay byte-identical across kernels. + mentionsRendered = "用户在消息中引用了以下平台对象，请在回答时考虑：\n" + + strings.Join(bullets, "\n") + augmentedUserText = mentionsRendered + "\n\n" + req.UserText + } + } + + // 3. Persist the user message first (legacy invariant — survives + // a graph crash). + userContent := augmentedUserText + userMsg := &aiopsmodel.Message{ + SessionID: sess.ID, + Role: aiopsmodel.RoleUser, + Content: &userContent, + CreatedAt: time.Now().UTC(), + } + if err := rt.cfg.Sessions.AppendMessage(ctx, userMsg); err != nil { + return nil, fmt.Errorf("chatruntime: persist user msg: %w", err) + } + + // 4. Load history (includes the just-appended user message). + history, err := rt.cfg.Sessions.ListMessages(ctx, sess.ID, rt.cfg.HistoryLimit) + if err != nil { + return nil, fmt.Errorf("chatruntime: load history: %w", err) + } + + // 5. Resolve active skills + compose the system prompt. + policy := Policy{AllowedClasses: []string{"*"}} + var activeSkills []*Skill + if rt.cfg.SkillRegistry != nil { + activeSkills = rt.cfg.SkillRegistry.Resolve(req.UserText, policy) + } + // 5a. Top-level persona (Phase 2 of the user-Agent initiative): when + // the session was created with a chosen agent (sess.AgentID set), use + // that persona's SystemPrompt as the base instead of the global + // BasePrompt and filter the ToolBag through its allowed/disallowed + // list. Worker spawn already does this for sub-agents (see worker.go); + // the coordinator path now honors it too. Stale agent ids fall back + // to the global default with a log line — never crash a session. + basePrompt := rt.cfg.BasePrompt + sessionToolBag := rt.cfg.ToolBag + agentReminderForPersona := "" + // Coordinator iff session is pinned to "default" or has no + // AgentID at all. Specialist personas (= anything else in the + // registry) are workers, not coordinators. + isCoordinator := sess.AgentID == nil || *sess.AgentID == "" || *sess.AgentID == "default" + // viewer downgrade — drop every Class!=read tool out of the + // toolbag before persona / control filtering. This applies even when + // no persona is loaded, so the LLM never sees host_bash / AgentTool / + // restart_service via the viewer's chat session. + viewerOnly := req.Role == "viewer" + if viewerOnly { + sessionToolBag = filterToolsForAgentRole(rt.cfg.ToolBag, nil, isCoordinator, true) + } + // Resolve the active persona. Sessions with no AgentID still + // route through the "default" persona — that's where the curated + // coordinator toolbag lives. Without this lookup, a CreateSession + // without an explicit agent_id would see the full toolbag and + // the rule-1 dispatch policy becomes advisory again. + personaName := "default" + if sess.AgentID != nil && *sess.AgentID != "" { + personaName = *sess.AgentID + } + if rt.cfg.AgentRegistry != nil { + if persona, ok := rt.cfg.AgentRegistry.ByName(personaName); ok { + if strings.TrimSpace(persona.SystemPrompt) != "" { + basePrompt = persona.SystemPrompt + } + // Apply the persona's Tools whitelist even for the + // "default" coordinator — so a curated coordinator + // toolbag (AgentTool + a handful of triage tools) can + // force dispatch instead of letting the LLM see and + // reach for every deep-dive tool itself. The + // coordinatorOnlyTools (AgentTool/SendMessage/TaskStop) + // survive the strip via isCoordinator=true. + sessionToolBag = filterToolsForAgentRole(rt.cfg.ToolBag, persona, isCoordinator, viewerOnly) + agentReminderForPersona = persona.CriticalReminder + } else if rt.log != nil && personaName != "default" { + rt.log.Info("chatruntime: session agent_id not in registry — using default", + slog.String("session_id", sess.ID), + slog.String("agent_id", personaName)) + } + } + // Coordinator gets the redirect-stub overlay: same-name shadows + // for hallucination-prone tool names that hand the LLM a + // "dispatch to specialist-X via AgentTool" message instead of + // crashing the graph with "tool not found in toolsNode". Stubs + // must come AFTER filterToolsForAgent so they survive any name + // collision with the (already-stripped) real tools. + if isCoordinator && len(rt.cfg.CoordinatorStubs) > 0 { + sessionToolBag = append(sessionToolBag, rt.cfg.CoordinatorStubs...) + } + // AgentID="default" is the virtual top-level persona — same wiring + // as the no-agent coordinator (BasePrompt + full toolBag + agent + // catalog), but the session keeps "default" so the SPA shows the + // persona badge and the user can see / pick it on /agents. + // 5b. Multi-agent catalog: when the session is the coordinator + // (no specific persona pinned), append a markdown list of available + // specialist personas so the LLM knows what subagent_type values + // AgentTool will accept. Workers don't get the catalog (they can't + // spawn nested workers — see coordinatorOnlyTools). + if isCoordinator && rt.cfg.AgentRegistry != nil { + if catalog := buildAgentCatalog(rt.cfg.AgentRegistry); catalog != "" { + basePrompt = basePrompt + "\n\n" + catalog + } + } + systemPrompt := ComposeSystemPrompt(basePrompt, activeSkills, nil) + + // 6. Build the eino history slice. Convert persisted rows into + // schema.Message so the graph assembler can replay them. + einoHistory := buildEinoHistory(history) + + // 7. Build the graph (per-request — eino graphs are cheap to + // compose and the inner ReAct agent is rebuilt on every call + // today; an in-memory cache keyed on (toolBag identity, cfg) + // is a future optimisation). + graphCfg := rt.cfg.GraphCfg + // Persona-level MaxTurns override — already honoured for workers + // via worker.go runWorker; mirror it here for the coordinator + // path so the default persona can have a tighter cap. Without + // this, the global 30-iteration cap lets a misbehaving + // coordinator chain 120+ tool calls (we observed exactly this + // pattern in the E2E eval before adding the override). + if rt.cfg.AgentRegistry != nil { + if persona, ok := rt.cfg.AgentRegistry.ByName(personaName); ok && persona.MaxTurns > 0 { + graphCfg.MaxIterations = persona.MaxTurns + } + } + g, err := graph.BuildReActGraph(rt.cfg.ChatModel, sessionToolBag, graphCfg) + if err != nil { + return nil, fmt.Errorf("chatruntime: build graph: %w", err) + } + + // 8. Wire the per-request callback chain. Persistence's + // SessionID is filled in here; everything else lives on + // rt.cfg.CallbackDeps. The SSE handler is appended via the + // cutover layer's emitter when req.Emit != nil. + deps := rt.cfg.CallbackDeps + deps.Persistence.SessionID = sess.ID + if deps.Persistence.Repo == nil { + deps.Persistence.Repo = rt.cfg.Sessions + } + if req.Emit != nil { + deps.SSE = rt.toCallbackEmitter(req.Emit, sess.ID) + } else { + deps.SSE = nil + } + handlers := callbacks.NewDefaultHandlers(deps) + + // 9. Compute per-turn dynamic hints from history + persona's + // critical_reminder (if a worker persona is active for this turn). + // — these get inlined into the per-turn + // block by graph.assembleMessages. + dynamicHints := rt.calcDynamicHints(history) + // AgentReminder is populated when the session is pinned to a + // persona (Phase 2). — anti-drift reminder gets + // inlined into the per-turn block. Worker spawns + // have always passed CriticalReminder; the coordinator path now + // does too when sess.AgentID is set. + agentReminder := agentReminderForPersona + + // 10. Invoke. The graph's outer Output carries the assistant + // message + usage; the persistence callback already wrote the + // assistant + tool rows by the time Invoke returns. + // + // Per-request model selection (SPA model picker): thread + // req.Provider/req.Model into the ChatModel node as eino model + // options. RoutingChatModel.pick consumes WithProvider; the inner + // clientChatModel honours WithModel. Without this the graph kernel + // silently ignored the picker and always used the routing default + // — so a user who chose e.g. claude-opus still got the default + // model. When both are empty no option is added (default path + // unchanged). + invokeOpts := []compose.Option{compose.WithCallbacks(handlers...)} + if mopts := chatModelOpts(req); len(mopts) > 0 { + invokeOpts = append(invokeOpts, compose.WithChatModelOption(mopts...)) + } + out, invokeErr := g.Invoke(ctx, &graph.Input{ + SystemPrompt: systemPrompt, + History: einoHistory, + UserText: req.UserText, + WebSearchEnabled: req.WebSearchEnabled, + MentionsRendered: mentionsRendered, + AgentReminder: agentReminder, + DynamicHints: dynamicHints, + Locale: req.Locale, + }, invokeOpts...) + if invokeErr != nil { + // Soft-fail graph-level errors: write an apology assistant + // message, emit it like a normal turn, and emit Done. Keeps + // SPA experience consistent with legacy agent.go's + // MaxIterations apology path. Without this, errors like + // eino's ErrExceededMaxSteps surface as "stream error" in + // the user's chat instead of an actionable message. + apology := buildGraphErrorApology(invokeErr) + fallbackMsg := &aiopsmodel.Message{ + SessionID: sess.ID, + Role: aiopsmodel.RoleAssistant, + Content: &apology, + CreatedAt: time.Now().UTC(), + } + // Best-effort persist; if it fails we still emit so the user + // sees something instead of a stream error. + if persistErr := rt.cfg.Sessions.AppendMessage(ctx, fallbackMsg); persistErr != nil && rt.log != nil { + rt.log.Warn("chatruntime: persist apology failed", + slog.String("err", persistErr.Error())) + } + emit(Event{Type: EventAssistant, Assistant: &AssistantEvent{ + MessageID: fallbackMsg.ID, + Content: apology, + CreatedAt: fallbackMsg.CreatedAt, + }}) + reply := &Reply{ + Message: fallbackMsg, + } + emit(Event{Type: EventDone, Done: reply}) + if rt.log != nil { + rt.log.Warn("chatruntime: graph invoke failed (apology emitted)", + slog.String("err", invokeErr.Error())) + } + return reply, nil + } + + // 10. Translate Output → Reply. The graph's Output owns the + // assistant message and best-effort iterations + usage; the + // persistence handler has already written the chat_messages + // row, so we re-fetch the recently-persisted assistant message + // ID for handoff to the SSE adapter when needed. + reply := &Reply{Usage: out.Usage, Iterations: out.Iterations} + if out.AssistantMessage != nil { + // Build a synthetic *aiopsmodel.Message so the upper layer + // can re-render the legacy postMessageResp shape. The + // canonical row written by PersistenceHandler lives in + // chat_messages; we mirror its fields here for the wire DTO. + content := out.AssistantMessage.Content + var contentPtr *string + if content != "" { + contentPtr = &content + } + reply.Message = &aiopsmodel.Message{ + SessionID: sess.ID, + Role: aiopsmodel.RoleAssistant, + Content: contentPtr, + CreatedAt: time.Now().UTC(), + } + } + + // EventDone — keep parity with legacy agent.go's terminal + // emission. PersistenceHandler may have already written the + // assistant row; the legacy contract is "Done frame fires + // exactly once at terminal success". + emit(Event{Type: EventDone, Done: reply}) + return reply, nil +} + +// toCallbackEmitter adapts a chatruntime.Emit into the +// callbacks.SSEEmitter the persistence/SSE chain consumes. The +// adapter normalises the new event names back to the legacy ones +// (SSEEventAssistantStart / SSEEventAssistantEnd → "assistant"; +// SSEEventAssistantDelta dropped — see PR-9 spec note about +// IncludeDelta=false until the SPA catches up). +func (rt *Runtime) toCallbackEmitter(emit Emit, sessionID string) callbacks.SSEEmitter { + return func(ev callbacks.SSEEvent) { + switch ev.Type { + case callbacks.SSEEventAssistantEnd: + if ev.Assistant == nil { + return + } + emit(Event{Type: EventAssistant, Assistant: &AssistantEvent{ + Iteration: ev.Assistant.Iteration, + MessageID: ev.Assistant.MessageID, + Content: ev.Assistant.Content, + CreatedAt: ev.Assistant.CreatedAt, + PendingToolCalls: ev.Assistant.PendingToolCalls, + }}) + case callbacks.SSEEventAssistantStart: + // Legacy frontend doesn't have an "assistant_start" + // frame; suppress it so the SPA doesn't see a phantom + // empty assistant bubble. The assistant_end frame + // (above) carries the full content + pending tool + // count which matches the legacy "assistant" frame. + return + case callbacks.SSEEventAssistantDelta: + // Token-level streaming is gated behind a feature flag + // pending SPA support. Drop for now — see PR-9 spec. + return + case callbacks.SSEEventToolStart: + if ev.Tool == nil { + return + } + emit(Event{Type: EventToolStart, Tool: &ToolEvent{ + ToolCallID: ev.Tool.ToolCallID, + Name: ev.Tool.Name, + ArgsJSON: ev.Tool.ArgsJSON, + Status: ev.Tool.Status, + StartedAt: ev.Tool.StartedAt, + }}) + case callbacks.SSEEventToolEnd: + if ev.Tool == nil { + return + } + emit(Event{Type: EventToolEnd, Tool: &ToolEvent{ + ToolCallID: ev.Tool.ToolCallID, + Name: ev.Tool.Name, + ArgsJSON: ev.Tool.ArgsJSON, + Status: ev.Tool.Status, + StartedAt: ev.Tool.StartedAt, + EndedAt: ev.Tool.EndedAt, + DurationMs: ev.Tool.DurationMs, + Error: ev.Tool.Error, + ResultJSON: ev.Tool.ResultJSON, + }}) + case callbacks.SSEEventDone: + // Done is emitted by Handle directly so the chatruntime + // caller sees a single, well-typed Done event with the + // full Reply. The graph's terminal "done" callback fires + // without a Reply payload, which is unhelpful for the + // adapter. Drop the empty version to avoid double-fire. + return + case callbacks.SSEEventError: + msg := "" + if ev.Error != nil { + msg = ev.Error.Message + } + emit(Event{Type: EventError, Error: msg}) + } + } +} + +// buildEinoHistory translates persisted aiopsmodel.Message rows into +// schema.Message. Tool-only assistant rows (Content == nil) are +// dropped — same rule legacy agent.go's buildMessages applies. Tool +// result messages keep their tool_call_id + name so the model can +// thread them back to the original call. +func buildEinoHistory(rows []*aiopsmodel.Message) []*schema.Message { + if len(rows) == 0 { + return nil + } + // The graph assembler will append the user turn separately (it + // receives Input.UserText); to avoid the LLM seeing the same + // turn twice we strip the trailing user message from the + // persisted history. + end := len(rows) + if end > 0 && rows[end-1].Role == aiopsmodel.RoleUser { + end-- + } + out := make([]*schema.Message, 0, end) + for i := 0; i < end; i++ { + m := rows[i] + switch m.Role { + case aiopsmodel.RoleUser, aiopsmodel.RoleSystem: + if m.Content == nil { + continue + } + out = append(out, &schema.Message{ + Role: schema.RoleType(m.Role), + Content: *m.Content, + }) + case aiopsmodel.RoleAssistant: + if m.Content == nil { + // Legacy parity — tool-only assistant rows are not + // replayable from our persistence schema and the + // following role=tool rows already capture the + // outcome. + continue + } + out = append(out, &schema.Message{ + Role: schema.RoleType(m.Role), + Content: *m.Content, + }) + case aiopsmodel.RoleTool: + content := "" + if m.Content != nil { + content = *m.Content + } + tcID := "" + if m.ToolCallID != nil { + tcID = *m.ToolCallID + } + tname := "" + if m.ToolName != nil { + tname = *m.ToolName + } + out = append(out, &schema.Message{ + Role: schema.RoleType(m.Role), + Content: content, + ToolCallID: tcID, + ToolName: tname, + }) + } + } + return out +} + +// chatModelOpts turns the per-request model selection (SPA picker) into eino +// model options for the graph's ChatModel node. Empty fields add no option, +// so the routing default applies (unchanged default path). Provider routes +// via RoutingChatModel.pick (WithProvider); model name is honoured by the +// inner clientChatModel (WithModel). +func chatModelOpts(req *Request) []model.Option { + if req == nil { + return nil + } + var opts []model.Option + if p := strings.TrimSpace(req.Provider); p != "" { + opts = append(opts, llm.WithProvider(p)) + } + if m := strings.TrimSpace(req.Model); m != "" { + opts = append(opts, model.WithModel(m)) + } + return opts +} + +// calcDynamicHints produces the per-turn hint bullets that get injected +// into the block. Pure: depends only +// on the persisted history rows; no LLM, no I/O. Returns nil when no +// hint applies for this turn. +// +// Current heuristics (kept intentionally narrow — false positives on a +// reminder we re-show every turn corrode the LLM's trust in the block): +// +// (a) consecutive same-tool failures ≥ 2 → "switch tools or ask user" +// (b) ≥ 8 assistant turns → "give a partial conclusion now" +// (c) ≥ 15 assistant turns → "you must summarize next turn" +// (d) ≥ 25 assistant turns → "stop tool calls, answer directly" +// (e) repeated (tool, args) call ≥ 3 times → "stop repeating the same call" +func (rt *Runtime) calcDynamicHints(history []*aiopsmodel.Message) []string { + if rt == nil { + return nil + } + var hints []string + if name, n := consecutiveFailedTool(history, 2); n >= 2 { + hints = append(hints, fmt.Sprintf("%s 已连续失败 %d 次：换工具，或问用户澄清", name, n)) + } + // Repeat-call detection — same tool with similar args ≥ 3 times in + // the trailing window. This was the dominant failure mode in the + // "self-loop diagnose" 30-iter loop: query_promql kept getting called + // with slightly different metric names but no narrative progress. + if name, args, n := repeatedToolCall(history, 3); n >= 3 { + hints = append(hints, fmt.Sprintf("%s 已重复调用 %d 次（args: %s）：从你已有的数据下结论，不要再调用同款工具", name, n, args)) + } + iter := countAssistantTurns(history) + switch { + case iter >= 25: + hints = append(hints, fmt.Sprintf("已经 %d 轮：停止调用工具，下一轮直接给最终回答", iter)) + case iter >= 15: + hints = append(hints, fmt.Sprintf("已经 %d 轮：下一轮必须给最终回答，不再调用工具", iter)) + case iter >= 8: + hints = append(hints, fmt.Sprintf(`已经 %d 轮：现在给一段阶段性结论（"目前看到 X / Y，初步判断是 Z"），再决定是否继续`, iter)) + } + // "Promise-without-execution" detection — see ongridBasePrompt() + // 中等档位的 LLM (e.g. glm-4-plus) 偶尔写 "让我..." / "我先..." 这种 + // 计划句但没真发出 tool_call，导致 graph 提前 END，用户体验是消息 + // 戛然而止。这条 hint 在下一轮把 LLM 重新拽回来 — 但只在用户给了 + // 续聊指令（如 "继续"）的时候触发，避免在用户主动终止对话时打扰。 + if excerpt, found := detectUnfollowedPromise(history); found { + hints = append(hints, fmt.Sprintf(`上一轮你说"%s"但没真发 tool_call。如要继续探索请直接发 tool_call，不要再写计划句；如已有结论请直接给用户答复`, excerpt)) + } + return hints +} + +// repeatedToolCall walks the trailing tool messages looking for the most +// frequent tool. Returns (name, argsExcerpt, n) where n is occurrence +// count. Args excerpt = first 80 bytes of the latest call's args. n < +// minN is reported as 0. +func repeatedToolCall(history []*aiopsmodel.Message, minN int) (string, string, int) { + if minN <= 0 { + minN = 3 + } + // Look at last 12 tool messages + const window = 12 + counts := map[string]int{} + args := map[string]string{} + tail := 0 + for i := len(history) - 1; i >= 0 && tail < window; i-- { + m := history[i] + if m == nil || m.Role != aiopsmodel.RoleTool { + continue + } + tail++ + name := "" + if m.ToolName != nil { + name = *m.ToolName + } + if name == "" { + continue + } + counts[name]++ + // snapshot args for highest-count latest occurrence + if _, ok := args[name]; !ok && m.Content != nil { + c := *m.Content + if len(c) > 80 { + c = c[:80] + } + args[name] = c + } + } + var topName string + var topN int + for name, n := range counts { + if n > topN { + topName = name + topN = n + } + } + if topN < minN { + return "", "", 0 + } + return topName, args[topName], topN +} + +// consecutiveFailedTool walks history backwards looking at the trailing +// run of tool messages. Returns (toolName, n) where n is how many of the +// most-recent tool turns were the same tool AND each one's persisted +// content carries a failure marker. n < minN is reported as 0. +// +// Failure marker (deliberately simple): the persisted Content JSON +// contains the substring `"error"` — every tool decorator (PR-3) writes +// `{"error":"..."}` on tool failure (timeout, panic, returned error), +// and successful payloads do not happen to ship an `"error"` field. We +// keep the parse loose so a future tool that reports partial failures +// inside a richer JSON still gets caught. +// +// Boundary cases this honours: +// - the trailing tool block must be unbroken — interleaved +// user/assistant turns reset the counter (n = 0) +// - if the trailing tool block has mixed tool names, only the +// most-recent tool's run is counted (we walk backwards; on a +// name-mismatch we stop) +// - a single failure (n == 1) returns ("", 0) when minN == 2 — we do +// NOT want to nag on every transient error +func consecutiveFailedTool(history []*aiopsmodel.Message, minN int) (string, int) { + if minN <= 0 { + minN = 2 + } + name := "" + n := 0 + for i := len(history) - 1; i >= 0; i-- { + m := history[i] + if m == nil { + continue + } + if m.Role != aiopsmodel.RoleTool { + break + } + // Tool message — extract name + check failure. + curName := "" + if m.ToolName != nil { + curName = *m.ToolName + } + if curName == "" { + break + } + if name == "" { + name = curName + } else if curName != name { + // different tool — stop counting; we only flag a single + // stuck-loop tool, not a pattern across multiple tools. + break + } + content := "" + if m.Content != nil { + content = *m.Content + } + if !looksLikeToolFailure(content) { + break + } + n++ + } + if n < minN { + return "", 0 + } + return name, n +} + +// looksLikeToolFailure is the heuristic used by consecutiveFailedTool. +// True when the persisted tool message Content carries an "error" JSON +// key — the decorator chain's standard failure shape. +func looksLikeToolFailure(content string) bool { + if content == "" { + return false + } + // Cheap substring scan — JSON validity is enforced by the + // decorator path, and we only need a yes/no signal here. + return strings.Contains(content, `"error"`) +} + +// promiseMarkers are the substrings that indicate the assistant wrote a +// plan句 ("let me...") but did not actually emit a tool_call. Detected +// in the most recent assistant message; if matched + no tool message +// follows before the next user, calcDynamicHints injects a nudge for +// the next turn (see ongridBasePrompt). +var promiseMarkers = []string{ + "让我", "我先", "我来", "我会", "我将", "接下来", "首先", "先让我", "让我先", +} + +// detectUnfollowedPromise walks history backwards looking for the last +// assistant message. Returns (excerpt, true) when: +// +// 1. that message contains any promiseMarker token, AND +// 2. no role=tool message follows it before the next role=user message +// (or before end of history). +// +// Returns ("", false) otherwise. The excerpt is a 30-rune prefix of the +// matched assistant content for inclusion in the hint. +func detectUnfollowedPromise(history []*aiopsmodel.Message) (string, bool) { + if len(history) == 0 { + return "", false + } + // Find last assistant message. + asstIdx := -1 + for i := len(history) - 1; i >= 0; i-- { + if history[i].Role == aiopsmodel.RoleAssistant { + asstIdx = i + break + } + } + if asstIdx < 0 || history[asstIdx].Content == nil { + return "", false + } + content := strings.TrimSpace(*history[asstIdx].Content) + if content == "" || !containsAnyPromiseMarker(content) { + return "", false + } + // Look forward from asstIdx+1: did a tool message follow before the + // next user message? + for j := asstIdx + 1; j < len(history); j++ { + switch history[j].Role { + case aiopsmodel.RoleTool: + return "", false // promise WAS executed + case aiopsmodel.RoleUser: + // User came in before any tool ran — promise unfollowed. + break + } + } + // Build a clean excerpt: strip leading bullet/whitespace, take 30 runes. + excerpt := content + if r := []rune(excerpt); len(r) > 30 { + excerpt = string(r[:30]) + "..." + } + return excerpt, true +} + +func containsAnyPromiseMarker(content string) bool { + for _, m := range promiseMarkers { + if strings.Contains(content, m) { + return true + } + } + return false +} + +// buildGraphErrorApology turns a graph-level error into a user-facing +// markdown apology, picking different wording per error class so the user +// (and downstream Ant audit) can tell what happened. Mirrors the apology +// style of legacy agent.go's MaxIterations fallback. +func buildGraphErrorApology(err error) string { + if err == nil { + return "抱歉，处理消息时遇到未知错误。请换个问法再问一次。" + } + msg := err.Error() + low := strings.ToLower(msg) + switch { + // LLM hallucinated a tool name not in its actual schema (e.g. + // model "remembers" `get_host_load` from training even though + // only `query_devices / query_knowledge / ...` were presented). + // Eino's tool router can't dispatch → whole graph aborts. Map + // to a friendly retry message that nudges toward the right + // flow (AgentTool dispatch). + case strings.Contains(low, "not found in toolsnode"), + strings.Contains(low, "tool") && strings.Contains(low, "not found"): + return "这个问题需要的深度查询能力不在我（协调员）的工具范围内 — 那些工具都在专家手里。请直接告诉我你的具体诉求（例如「看 CPU/内存」「检查磁盘容量」「网络连通性」），我会派对应的 specialist 处理。" + // Match both eino's wording shapes: "exceeds max steps" (current + // runtime) and the legacy "exceeded max iterations" / "maxsteps" + // variants. The bug we ate before: matcher said "exceeded" (past + // tense) but eino emits "exceeds" (present tense), so every + // max-steps abort dumped raw "[GraphRunError]…" to the user. + case strings.Contains(low, "exceeds max"), + strings.Contains(low, "exceeded max"), + strings.Contains(low, "max steps"), + strings.Contains(low, "maxstep"), + strings.Contains(low, "max iterations"): + return "我跑了很多轮工具调用还是没收敛 — 大概率是搜索路径走偏了。上面工具调用区已经有阶段性数据，可以直接看；也可以这样收敛：\n• 把问题缩小成「看 X 指标的 Y 现象」——给具体 metric / device 让我聚焦\n• 或者告诉我「已经看到 A / B / C，下一步从 D 这条线索查」\n• 也可以直接 @ 一个 specialist（如 @specialist-compute 看 CPU/内存）让专家接手" + case strings.Contains(low, "context canceled"), strings.Contains(low, "context deadline"): + return "本次请求超时或被取消。一般是上游 LLM / 设备响应慢。请稍后重试，或换简单的问法。" + case strings.Contains(low, "budget"): + return "今日 LLM 预算已用完。请联系 admin 调整配额或明天再试。" + // LLM provider quota / rate-limit / insufficient balance. Substring + // covers OpenAI ("rate limit reached"), Zhipu ("余额不足"), generic + // "429 Too Many Requests" wording. All of these are operator-side + // (refill / change provider), not user-side — make that clear. + case strings.Contains(low, "429"), + strings.Contains(low, "too many requests"), + strings.Contains(low, "余额不足"), + strings.Contains(low, "insufficient"), + strings.Contains(low, "rate limit"), + strings.Contains(low, "quota"): + return "LLM provider 当前不可用——可能是配额用尽 / 限流 / 余额不足。请到「设置 → 模型」检查 provider 的状态和余额；问题在配置层面，重试不会解决。" + // Other LLM provider errors (auth failure, model not found, etc.) + // land here so the user gets a hint to check provider config + // instead of a 200-char raw API error dump. + case strings.Contains(low, "llm: chat completion"), + strings.Contains(low, "openai api"), + strings.Contains(low, "api error"): + return "LLM provider 报错（可能是 API key / 模型名 / 网络）。请到「设置 → 模型」检查 provider 配置；详细 raw 错误请看 manager 日志。" + default: + // 非已知 class：给一段保守的原文摘要，让用户能截图反馈。 + short := msg + if len(short) > 200 { + short = short[:200] + "..." + } + return "抱歉，处理消息时遇到错误：\n```\n" + short + "\n```\n请换个问法再试，或截图反馈给我们。" + } +} + +// countAssistantTurns returns the number of assistant role rows in +// history. Tool-only assistant rows (Content == nil) still count — they +// represent a ChatModel turn the loop took. +func countAssistantTurns(history []*aiopsmodel.Message) int { + n := 0 + for _, m := range history { + if m == nil { + continue + } + if m.Role == aiopsmodel.RoleAssistant { + n++ + } + } + return n +} diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/runtime_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/runtime_test.go new file mode 100644 index 000000000..b51b4249a --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/runtime_test.go @@ -0,0 +1,545 @@ +package chatruntime + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + einomodel "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + + biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v1/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +// scriptedChatModel returns one *schema.Message per Generate call, +// tracking generateCalls so tests can assert how many turns ran. +// Mirrors the pattern in graph/react_test.go. +type scriptedChatModel struct { + mu sync.Mutex + replies []*schema.Message + idx int + calls atomic.Int32 +} + +func newScriptedChatModel(replies ...*schema.Message) *scriptedChatModel { + return &scriptedChatModel{replies: replies} +} + +func (s *scriptedChatModel) Generate(_ context.Context, _ []*schema.Message, _ ...einomodel.Option) (*schema.Message, error) { + s.calls.Add(1) + s.mu.Lock() + defer s.mu.Unlock() + if len(s.replies) == 0 { + return &schema.Message{Role: schema.Assistant, Content: "ok"}, nil + } + if s.idx < len(s.replies) { + out := s.replies[s.idx] + s.idx++ + return out, nil + } + return s.replies[len(s.replies)-1], nil +} + +func (s *scriptedChatModel) Stream(ctx context.Context, input []*schema.Message, opts ...einomodel.Option) (*schema.StreamReader[*schema.Message], error) { + msg, err := s.Generate(ctx, input, opts...) + if err != nil { + return nil, err + } + return schema.StreamReaderFromArray([]*schema.Message{msg}), nil +} + +func (s *scriptedChatModel) BindTools(_ []*schema.ToolInfo) error { return nil } + +func (s *scriptedChatModel) WithTools(_ []*schema.ToolInfo) (einomodel.ToolCallingChatModel, error) { + return s, nil +} + +// memSessions is an in-memory SessionRepo for runtime tests. Only the +// methods runtime.Handle exercises are implemented; the rest panic on +// purpose so a future refactor can't silently slip past coverage. +type memSessions struct { + mu sync.Mutex + sessions map[string]*model.Session + messages []*model.Message + toolCalls []*model.ToolCall +} + +func newMemSessions(seed *model.Session) *memSessions { + m := &memSessions{sessions: map[string]*model.Session{}} + if seed != nil { + m.sessions[seed.ID] = seed + } + return m +} + +func (m *memSessions) CreateSession(_ context.Context, s *model.Session) error { + m.mu.Lock() + defer m.mu.Unlock() + m.sessions[s.ID] = s + return nil +} +func (m *memSessions) GetSession(_ context.Context, id string) (*model.Session, error) { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.sessions[id] + if !ok { + return nil, errs.ErrNotFound + } + return s, nil +} +func (m *memSessions) ListSessions(_ context.Context, _ uint64, _, _ int, _ *uint64) ([]*model.Session, error) { + return nil, nil +} +func (m *memSessions) ListByParent(_ context.Context, parentID string) ([]*model.Session, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]*model.Session, 0) + for _, s := range m.sessions { + if s.ParentSessionID != nil && *s.ParentSessionID == parentID { + cp := *s + out = append(out, &cp) + } + } + return out, nil +} +func (m *memSessions) CloseSession(_ context.Context, id string) error { + m.mu.Lock() + defer m.mu.Unlock() + if s, ok := m.sessions[id]; ok { + now := time.Now().UTC() + s.ClosedAt = &now + } + return nil +} +func (m *memSessions) RenameSession(_ context.Context, _, _ string) error { return nil } +func (m *memSessions) DeleteSession(_ context.Context, _ string) error { return nil } +func (m *memSessions) AppendMessage(_ context.Context, msg *model.Message) error { + m.mu.Lock() + defer m.mu.Unlock() + if msg.ID == "" { + msg.ID = "m" + idgen() + } + m.messages = append(m.messages, msg) + return nil +} +func (m *memSessions) ListMessages(_ context.Context, sessionID string, _ int) ([]*model.Message, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]*model.Message, 0, len(m.messages)) + for _, mg := range m.messages { + if mg.SessionID == sessionID { + out = append(out, mg) + } + } + return out, nil +} +func (m *memSessions) CreateToolCall(_ context.Context, tc *model.ToolCall) error { + m.mu.Lock() + defer m.mu.Unlock() + if tc.ID == "" { + tc.ID = "tc" + idgen() + } + m.toolCalls = append(m.toolCalls, tc) + return nil +} +func (m *memSessions) UpdateToolCallResult(_ context.Context, _ string, _ string, _, _ *string, _ time.Time) error { + return nil +} +func (m *memSessions) SumTokensSince(_ context.Context, _ time.Time) (biz.TokenSums, error) { + return biz.TokenSums{}, nil +} + +var idCounter atomic.Int64 + +func idgen() string { + n := idCounter.Add(1) + return time.Now().UTC().Format("20060102") + "-" + atoi(int(n)) +} + +func atoi(n int) string { + if n == 0 { + return "0" + } + out := []byte{} + for n > 0 { + out = append([]byte{byte('0' + n%10)}, out...) + n /= 10 + } + return string(out) +} + +// TestRuntime_NewRuntime_RequiresDeps confirms NewRuntime fails fast +// when ChatModel or Sessions is missing — production wiring must not +// silently drop a key dep. +func TestRuntime_NewRuntime_RequiresDeps(t *testing.T) { + if _, err := NewRuntime(Config{}); err == nil { + t.Errorf("NewRuntime{} returned nil error — expected dep check") + } + if _, err := NewRuntime(Config{Sessions: newMemSessions(nil)}); err == nil { + t.Errorf("NewRuntime sans ChatModel should error") + } + if _, err := NewRuntime(Config{ChatModel: newScriptedChatModel()}); err == nil { + t.Errorf("NewRuntime sans Sessions should error") + } +} + +// TestRuntime_Handle_OwnershipCheck enforces the "non-owner gets +// ErrNotFound" invariant. Mirrors the legacy agent's behaviour +// . +func TestRuntime_Handle_OwnershipCheck(t *testing.T) { + sess := &model.Session{ID: "s1", UserID: 7} + rt, err := NewRuntime(Config{ + Sessions: newMemSessions(sess), + ChatModel: newScriptedChatModel(), + ToolBag: nil, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + _, err = rt.Handle(context.Background(), &Request{ + SessionID: "s1", + UserID: 99, // not the owner + UserText: "hi", + }) + if !errors.Is(err, errs.ErrNotFound) { + t.Errorf("non-owner err = %v, want ErrNotFound", err) + } +} + +// TestRuntime_Handle_HappyPath_FinalReply runs the graph once +// against a scriptedChatModel that returns a no-tools assistant +// message. Asserts: +// - user message persisted +// - Reply.Message non-nil with the model's content +// - terminal Done event fires once +func TestRuntime_Handle_HappyPath_FinalReply(t *testing.T) { + sess := &model.Session{ID: "s1", UserID: 7} + store := newMemSessions(sess) + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "all good", + }) + rt, err := NewRuntime(Config{ + Sessions: store, + ChatModel: scripted, + ToolBag: nil, + GraphCfg: graph.Config{MaxIterations: 5}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + var ( + mu sync.Mutex + events []Event + ) + emit := func(ev Event) { + mu.Lock() + defer mu.Unlock() + events = append(events, ev) + } + + reply, err := rt.Handle(context.Background(), &Request{ + SessionID: "s1", + UserID: 7, + UserText: "what's up", + Emit: emit, + }) + if err != nil { + t.Fatalf("Handle: %v", err) + } + if reply == nil || reply.Message == nil { + t.Fatalf("expected non-nil reply.Message") + } + if reply.Message.Content == nil || *reply.Message.Content != "all good" { + t.Errorf("reply content = %v, want \"all good\"", reply.Message.Content) + } + + // User message must be persisted. + store.mu.Lock() + defer store.mu.Unlock() + if len(store.messages) == 0 { + t.Errorf("expected at least one persisted message (user turn)") + } + if store.messages[0].Role != model.RoleUser { + t.Errorf("first persisted message role = %q, want user", store.messages[0].Role) + } + + // Done event fires once at terminal success. + mu.Lock() + defer mu.Unlock() + if len(events) == 0 { + t.Fatalf("expected at least one event") + } + if events[len(events)-1].Type != EventDone { + t.Errorf("last event type = %q, want done", events[len(events)-1].Type) + } +} + +// TestRuntime_ToolCount + ToolNames provides the per-spec visibility. +// (a) ONGRID_AGENT_KERNEL=graph startup logs how many BaseTools are +// bound — this is the seam main.go logs against. +func TestRuntime_ToolCountAndNames(t *testing.T) { + sess := &model.Session{ID: "s1", UserID: 7} + rt, err := NewRuntime(Config{ + Sessions: newMemSessions(sess), + ChatModel: newScriptedChatModel(), + ToolBag: []basetool.BaseTool{ + &fakeTool{name: "echo", schema: `{"type":"object","properties":{}}`}, + &fakeTool{name: "ping", schema: `{"type":"object","properties":{}}`}, + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + if rt.ToolCount() != 2 { + t.Errorf("ToolCount = %d, want 2", rt.ToolCount()) + } + names := rt.ToolNames(context.Background()) + t.Logf("toolBag names = %v (count=%d)", names, len(names)) + if len(names) != 2 { + t.Errorf("ToolNames len = %d, want 2", len(names)) + } +} + +// strPtr is a small helper for tool-message content/name pointer +// fields in the test fixtures below. Inline helpers everywhere bloats +// the test fixture noise and obscures the assertions. +func strPtr(s string) *string { return &s } + +// makeToolMsg builds a chat_messages row in the persisted shape the +// runtime sees from ListMessages. role=tool, ToolName + Content are +// non-nil; failure flag is "is the JSON content carrying an error +// field" (looksLikeToolFailure). +func makeToolMsg(toolName string, fail bool) *model.Message { + body := `{"ok":true}` + if fail { + body = `{"error":"timeout"}` + } + return &model.Message{ + Role: model.RoleTool, + ToolName: strPtr(toolName), + Content: strPtr(body), + } +} + +// TestConsecutiveFailedTool covers the boundary cases the spec calls +// out: below threshold / different tool / mixed success+fail. +func TestConsecutiveFailedTool(t *testing.T) { + t.Parallel() + + t.Run("two_in_a_row_same_tool", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + } + name, n := consecutiveFailedTool(hist, 2) + if name != "query_logql" || n != 2 { + t.Errorf("got (%q, %d), want (query_logql, 2)", name, n) + } + }) + + t.Run("three_in_a_row_same_tool", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + } + name, n := consecutiveFailedTool(hist, 2) + if name != "query_logql" || n != 3 { + t.Errorf("got (%q, %d), want (query_logql, 3)", name, n) + } + }) + + t.Run("below_minN", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + } + name, n := consecutiveFailedTool(hist, 2) + if name != "" || n != 0 { + t.Errorf("got (%q, %d), want zero result below minN", name, n) + } + }) + + t.Run("different_tool_resets", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + makeToolMsg("query_promql", true), + } + name, n := consecutiveFailedTool(hist, 2) + // trailing tool is query_promql with one fail — below minN. + if name != "" || n != 0 { + t.Errorf("got (%q, %d), want zero (different tool breaks the run)", name, n) + } + }) + + t.Run("success_in_middle_breaks_run", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", false), // success -> resets + makeToolMsg("query_logql", true), + } + name, n := consecutiveFailedTool(hist, 2) + // only the trailing single fail counts. + if name != "" || n != 0 { + t.Errorf("got (%q, %d), want zero (success in middle breaks)", name, n) + } + }) + + t.Run("non_tool_role_at_tail", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + {Role: model.RoleAssistant, Content: strPtr("ok then")}, + } + // The trailing message is assistant, so the tool block isn't + // the most-recent thing — we should NOT report a stuck loop. + name, n := consecutiveFailedTool(hist, 2) + if name != "" || n != 0 { + t.Errorf("got (%q, %d), want zero (non-tool role at tail)", name, n) + } + }) + + t.Run("empty_history", func(t *testing.T) { + name, n := consecutiveFailedTool(nil, 2) + if name != "" || n != 0 { + t.Errorf("got (%q, %d), want zero", name, n) + } + }) +} + +// TestCalcDynamicHints checks the high-level wiring of both heuristics +// — consecutive-failed-tool + iteration-cap. +func TestCalcDynamicHints(t *testing.T) { + t.Parallel() + rt, err := NewRuntime(Config{ + Sessions: newMemSessions(&model.Session{ID: "s", UserID: 1}), + ChatModel: newScriptedChatModel(), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + t.Run("trailing_3_failed_logql_emits_failure_hint", func(t *testing.T) { + hist := []*model.Message{ + {Role: model.RoleAssistant, Content: strPtr("trying logql")}, + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + } + hints := rt.calcDynamicHints(hist) + if len(hints) == 0 { + t.Fatalf("expected at least one hint, got none") + } + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if !contains(joined, "query_logql") || !contains(joined, "连续失败") { + t.Errorf("missing failure hint: %q", joined) + } + }) + + t.Run("over_20_assistant_turns_emits_iteration_hint", func(t *testing.T) { + hist := make([]*model.Message, 0, 25) + for i := 0; i < 25; i++ { + hist = append(hist, &model.Message{Role: model.RoleAssistant, Content: strPtr("step")}) + } + hints := rt.calcDynamicHints(hist) + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if !contains(joined, "已经 25 轮") { + t.Errorf("missing iteration hint: %q", joined) + } + }) + + t.Run("clean_history_no_hints", func(t *testing.T) { + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("hi")}, + {Role: model.RoleAssistant, Content: strPtr("hi back")}, + } + hints := rt.calcDynamicHints(hist) + if len(hints) != 0 { + t.Errorf("expected no hints, got %v", hints) + } + }) + + t.Run("unfollowed_promise_emits_nudge", func(t *testing.T) { + // Last assistant said "让我..." but no tool message follows. + // Reproduces the d9fa4f42 session 17:00:36 trail-off. + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("看一下磁盘")}, + {Role: model.RoleAssistant, Content: strPtr("让我先查看 /data 目录的磁盘使用情况")}, + } + hints := rt.calcDynamicHints(hist) + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if !contains(joined, "没真发 tool_call") { + t.Errorf("missing unfollowed-promise hint: %q", joined) + } + }) + + t.Run("promise_followed_by_tool_no_hint", func(t *testing.T) { + // Same promise but tool ran — no hint expected. + toolName := "host_du_summary" + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("看一下磁盘")}, + {Role: model.RoleAssistant, Content: strPtr("让我先查看 /data")}, + {Role: model.RoleTool, ToolName: &toolName, Content: strPtr(`{"subpaths":[]}`)}, + } + hints := rt.calcDynamicHints(hist) + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if contains(joined, "没真发 tool_call") { + t.Errorf("hint should not fire when tool followed promise: %q", joined) + } + }) +} + +// contains is a tiny test helper so we don't import strings in this +// file just for one substring check. +func contains(s, sub string) bool { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false +} + +// fakeTool is a minimal BaseTool used by TestRuntime_ToolCountAndNames. +type fakeTool struct { + name string + schema string +} + +func (f *fakeTool) Info(_ context.Context) (*basetool.ToolInfo, error) { + return &basetool.ToolInfo{ + Name: f.name, + Description: "fake", + Parameters: []byte(f.schema), + Class: "read", + }, nil +} + +func (f *fakeTool) InvokableRun(_ context.Context, _ string, _ ...basetool.InvokeOption) (string, error) { + return `{"ok":true}`, nil +} + +// Compatibility with the current shared repository contract; v1 never calls this method. +func (r *memSessions) UpdateSessionModel(context.Context, string, string, string) error { + return errs.ErrNotWiredYet +} diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/runtime_worker_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/runtime_worker_test.go new file mode 100644 index 000000000..cd006bb83 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/runtime_worker_test.go @@ -0,0 +1,446 @@ +package chatruntime + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v1/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" +) + +// TestSpawnWorker_Sync drives SpawnWorker with Background=false and +// verifies the returned worker is in WorkerStatusCompleted with the +// scripted assistant content as Result. Also asserts the worker's +// chat_sessions row was persisted with the audit columns +// (agent_id / parent_session_id / background). +func TestSpawnWorker_Sync(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "worker-result", + }) + store, parentID := newSeededRuntimeStore(t, 7) + rt := newRuntimeWithStore(t, "incident-investigator", scripted, nil, store) + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "incident-investigator", + Prompt: "diagnose host-01", + ParentSession: parentID, + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + if w == nil { + t.Fatalf("expected non-nil worker") + } + if w.Status != WorkerStatusCompleted { + t.Errorf("status = %q, want completed", w.Status) + } + if w.Result != "worker-result" { + t.Errorf("result = %q, want worker-result", w.Result) + } + if w.AgentName != "incident-investigator" { + t.Errorf("agent = %q", w.AgentName) + } + if !strings.HasPrefix(w.ID, "agent-") { + t.Errorf("id = %q, want agent-* prefix", w.ID) + } + if !strings.HasPrefix(w.SessionID, "worker-") { + t.Errorf("session_id = %q, want worker-* prefix", w.SessionID) + } + + // chat_sessions audit row — assert all three columns are set. + row, err := store.GetSession(context.Background(), w.SessionID) + if err != nil || row == nil { + t.Fatalf("worker chat_sessions row missing: %v", err) + } + if row.AgentID == nil || *row.AgentID != "incident-investigator" { + t.Errorf("row.AgentID = %v, want incident-investigator", row.AgentID) + } + if row.ParentSessionID == nil || *row.ParentSessionID != parentID { + t.Errorf("row.ParentSessionID = %v, want %q", row.ParentSessionID, parentID) + } + if row.Background { + t.Errorf("row.Background = true, want false (sync spawn)") + } + if row.UserID != 7 { + t.Errorf("row.UserID = %d, want 7 (inherited from parent)", row.UserID) + } + + // ListByParent surfaces the worker session under the parent. + kids, err := store.ListByParent(context.Background(), parentID) + if err != nil { + t.Fatalf("ListByParent: %v", err) + } + if len(kids) != 1 || kids[0].ID != w.SessionID { + t.Errorf("ListByParent = %+v, want one row %q", kids, w.SessionID) + } +} + +// TestSpawnWorker_Async drives SpawnWorker with Background=true and +// verifies (a) immediate return with status running OR completed (race) +// + (b) eventual TaskNotification fires through ParentEmit. +func TestSpawnWorker_Async(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "async-result", + }) + rt := newRuntimeWithAgent(t, "general-purpose", scripted, nil) + + var ( + mu sync.Mutex + events []Event + done = make(chan struct{}, 1) + ) + emit := func(ev Event) { + mu.Lock() + events = append(events, ev) + mu.Unlock() + if ev.Type == EventTaskNotification { + select { + case done <- struct{}{}: + default: + } + } + } + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "general-purpose", + Prompt: "go", + Background: true, + ParentEmit: emit, + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + if w == nil { + t.Fatalf("expected worker") + } + // On the goroutine race the snapshot can already be running OR + // completed (scripted model is fast). Both are valid post-spawn + // states. + if w.Status != WorkerStatusRunning && + w.Status != WorkerStatusPending && + w.Status != WorkerStatusCompleted { + t.Errorf("immediate status = %q", w.Status) + } + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for task_notification") + } + + mu.Lock() + defer mu.Unlock() + var notif *TaskNotification + for _, e := range events { + if e.Type == EventTaskNotification { + notif = e.Notification + break + } + } + if notif == nil { + t.Fatalf("no task_notification in events: %+v", events) + } + if notif.TaskID != w.ID { + t.Errorf("notif.TaskID = %q, worker.ID = %q", notif.TaskID, w.ID) + } + if notif.Status != WorkerStatusCompleted { + t.Errorf("notif.status = %q, want completed", notif.Status) + } + if notif.Result != "async-result" { + t.Errorf("notif.Result = %q", notif.Result) + } + if !strings.Contains(notif.Summary, "general-purpose") { + t.Errorf("notif.Summary = %q (expected agent name)", notif.Summary) + } +} + +// TestSendToWorker_Continuation continues a completed worker with a +// follow-up message and checks the Result is updated. +func TestSendToWorker_Continuation(t *testing.T) { + scripted := newScriptedChatModel( + &schema.Message{Role: schema.Assistant, Content: "first"}, + &schema.Message{Role: schema.Assistant, Content: "second"}, + ) + rt := newRuntimeWithAgent(t, "general-purpose", scripted, nil) + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "general-purpose", + Prompt: "step 1", + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + if w.Result != "first" { + t.Fatalf("first result = %q, want first", w.Result) + } + if err := rt.SendToWorker(context.Background(), w.ID, "step 2 please"); err != nil { + t.Fatalf("SendToWorker: %v", err) + } + got, ok := rt.GetWorker(w.ID) + if !ok || got == nil { + t.Fatalf("GetWorker after send: not found") + } + if got.Result != "second" { + t.Errorf("after send, result = %q, want second", got.Result) + } + if got.Status != WorkerStatusCompleted { + t.Errorf("status = %q", got.Status) + } +} + +// TestStopWorker_Idempotent confirms StopWorker is safe to call on a +// terminal worker. +func TestStopWorker_Idempotent(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "x", + }) + rt := newRuntimeWithAgent(t, "general-purpose", scripted, nil) + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "general-purpose", + Prompt: "go", + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + if err := rt.StopWorker(context.Background(), w.ID); err != nil { + t.Errorf("Stop on completed: %v", err) + } + // Stop a non-existent id returns an error per contract. + if err := rt.StopWorker(context.Background(), "agent-nonsuch"); err == nil { + t.Errorf("expected error for unknown worker id") + } +} + +// TestSpawnWorker_ClosesSession is the regression test for the orphan +// worker session bug that accumulated 161 closed_at=NULL rows on the +// test env before the defer-close was added in worker.go. Both the +// sync and async termination paths must call CloseSession; this drives +// both and asserts ClosedAt is non-nil after termination. +func TestSpawnWorker_ClosesSession(t *testing.T) { + t.Run("sync", func(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, Content: "done", + }) + store, parentID := newSeededRuntimeStore(t, 1) + rt := newRuntimeWithStore(t, "incident-investigator", scripted, nil, store) + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "incident-investigator", + Prompt: "go", + ParentSession: parentID, + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + row, err := store.GetSession(context.Background(), w.SessionID) + if err != nil || row == nil { + t.Fatalf("session row missing: %v", err) + } + if row.ClosedAt == nil { + t.Fatalf("closed_at = nil after sync spawn; want non-nil so the row stops looking active") + } + }) + + t.Run("async", func(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, Content: "done", + }) + store, parentID := newSeededRuntimeStore(t, 1) + rt := newRuntimeWithStore(t, "general-purpose", scripted, nil, store) + + done := make(chan struct{}, 1) + emit := func(ev Event) { + if ev.Type == EventTaskNotification { + select { + case done <- struct{}{}: + default: + } + } + } + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "general-purpose", + Prompt: "go", + Background: true, + ParentSession: parentID, + ParentEmit: emit, + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for task_notification") + } + // CloseSession is deferred after the notify fires, so give the + // goroutine a tick to return. + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + row, _ := store.GetSession(context.Background(), w.SessionID) + if row != nil && row.ClosedAt != nil { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("closed_at still nil after async spawn — defer didn't fire") + }) +} + +// TestSpawnWorker_UnknownAgent confirms SpawnWorker rejects a name not +// in the AgentRegistry. +func TestSpawnWorker_UnknownAgent(t *testing.T) { + scripted := newScriptedChatModel() + rt := newRuntimeWithAgent(t, "general-purpose", scripted, nil) + _, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "no-such-agent", + Prompt: "x", + }) + if err == nil { + t.Fatalf("expected error for unknown agent") + } +} + +// TestFilterToolsForAgent verifies whitelist + blacklist + the +// implicit coordinator-only strip. +func TestFilterToolsForAgent(t *testing.T) { + bag := []basetool.BaseTool{ + &fakeTool{name: "AgentTool", schema: `{"type":"object"}`}, // coordinator-only — always stripped + &fakeTool{name: "SendMessage", schema: `{"type":"object"}`}, // coordinator-only — always stripped + &fakeTool{name: "TaskStop", schema: `{"type":"object"}`}, // coordinator-only — always stripped + &fakeTool{name: "query_promql", schema: `{"type":"object"}`}, // whitelisted + &fakeTool{name: "query_logql", schema: `{"type":"object"}`}, // not whitelisted + &fakeTool{name: "host_restart_service", schema: `{"type":"object"}`}, // blacklisted + &fakeTool{name: "kill_process", schema: `{"type":"object"}`}, // blacklisted via *_process + } + ag := &Agent{ + Name: "test-worker", + Tools: []string{"query_promql"}, + DisallowedTools: []string{"host_restart_service", "*_process"}, + } + // Worker path: isCoordinator=false → AgentTool/SendMessage/TaskStop stripped. + out := filterToolsForAgent(bag, ag, false) + names := make([]string, 0, len(out)) + for _, t := range out { + info, _ := t.Info(context.Background()) + names = append(names, info.Name) + } + if len(names) != 1 || names[0] != "query_promql" { + t.Errorf("worker filtered names = %v, want [query_promql]", names) + } + + // Coordinator path: isCoordinator=true → coordinator-only tools + // survive even when the persona's Tools whitelist doesn't include + // them. (The whitelist still gates non-control-plane tools.) + out = filterToolsForAgent(bag, ag, true) + names = names[:0] + for _, t := range out { + info, _ := t.Info(context.Background()) + names = append(names, info.Name) + } + gotCoord := map[string]bool{} + for _, n := range names { + gotCoord[n] = true + } + if !gotCoord["AgentTool"] || !gotCoord["SendMessage"] || !gotCoord["TaskStop"] || !gotCoord["query_promql"] { + t.Errorf("coordinator filtered names = %v; want all of AgentTool / SendMessage / TaskStop / query_promql", names) + } + if gotCoord["query_logql"] || gotCoord["host_restart_service"] || gotCoord["kill_process"] { + t.Errorf("coordinator filtered names = %v; whitelist or blacklist leaked", names) + } +} + +// TestNotificationFor_FieldShape locks down the wire shape consumers +// (the SPA's task_notification renderer) depend on. +func TestNotificationFor_FieldShape(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "shape-test", + }) + rt := newRuntimeWithAgent(t, "general-purpose", scripted, nil) + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "general-purpose", + Prompt: "shape", + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + // Reach into the internal map to invoke notificationFor on the + // canonical worker ptr (snapshots have nil mu). + rt.workersMu.Lock() + internal := rt.workers[w.ID] + rt.workersMu.Unlock() + ev := rt.notificationFor(internal) + if ev.Type != EventTaskNotification { + t.Errorf("type = %q", ev.Type) + } + if ev.Notification == nil { + t.Fatalf("Notification nil") + } + if ev.Notification.TaskID != w.ID { + t.Errorf("TaskID = %q", ev.Notification.TaskID) + } + if ev.Notification.Status != WorkerStatusCompleted { + t.Errorf("Status = %q", ev.Notification.Status) + } + if ev.Notification.Result != "shape-test" { + t.Errorf("Result = %q", ev.Notification.Result) + } + if _, ok := ev.Notification.Usage["duration_ms"]; !ok { + t.Errorf("usage.duration_ms missing") + } +} + +// newRuntimeWithAgent builds a Runtime with one persona registered and +// no tool bag. cfg.AgentRegistry's Add is used to inject the persona +// inline so the test doesn't need a fixture file. Default +// graph.MaxIterations is bumped down to keep tests snappy. +func newRuntimeWithAgent(t *testing.T, name string, cm *scriptedChatModel, toolBag []basetool.BaseTool) *Runtime { + t.Helper() + sess := &model.Session{ID: "s1", UserID: 7} + return newRuntimeWithStore(t, name, cm, toolBag, newMemSessions(sess)) +} + +// newRuntimeWithStore is the shape used by tests that need to inspect +// the SessionRepo after the spawn (worker session row, ListByParent). +// Caller passes the pre-seeded *memSessions so the test can read it back. +func newRuntimeWithStore(t *testing.T, name string, cm *scriptedChatModel, toolBag []basetool.BaseTool, store *memSessions) *Runtime { + t.Helper() + reg := NewAgentRegistry() + reg.Add(&Agent{ + Name: name, + Description: "test agent", + WhenToUse: "test", + MaxTurns: 3, + Tools: []string{}, // empty whitelist = inherit all tools (minus coordinator-only) + }) + rt, err := NewRuntime(Config{ + Sessions: store, + ChatModel: cm, + ToolBag: toolBag, + AgentRegistry: reg, + GraphCfg: graph.Config{MaxIterations: 3}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + return rt +} + +// newSeededRuntimeStore returns a memSessions with one parent (coordinator) +// session pre-inserted, plus that parent's id. UserID is the owner. +func newSeededRuntimeStore(t *testing.T, userID uint64) (*memSessions, string) { + t.Helper() + parentID := "parent-sess-1" + store := newMemSessions(&model.Session{ID: parentID, UserID: userID, Title: "parent"}) + return store, parentID +} diff --git a/internal/manager/biz/aiops/chatruntime/skill_parser.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/skill_parser.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/skill_parser.go rename to internal/manager/biz/aiops/runtime/v1/chatruntime/skill_parser.go diff --git a/internal/manager/biz/aiops/chatruntime/skill_parser_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/skill_parser_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/skill_parser_test.go rename to internal/manager/biz/aiops/runtime/v1/chatruntime/skill_parser_test.go diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/skill_registry.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/skill_registry.go new file mode 100644 index 000000000..d7ffc79c9 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/skill_registry.go @@ -0,0 +1,239 @@ +package chatruntime + +import ( + "fmt" + "path/filepath" + "strings" + "sync" +) + +// SkillRegistry holds every Skill discovered under a skills root plus the +// non-fatal warnings accumulated during the walk. Mirrors the shape of +// sealsuite-agent's SkillRegistry but adapted to SKILL.md. +// +// PR-2 scope: load + Resolve. Tool-class filter logic happens here at +// the skill level (we drop tools whose Class isn't allowed by the +// policy); the actual factory binding will land in PR-3. +// +// Concurrency: a sync.RWMutex protects the slices. marketplace +// Install / Uninstall calls Reload while in-flight chats may be calling +// Resolve / All — Reload swaps a freshly-built slice atomically under +// the write lock and returning slices are *value copies* so callers +// already holding a result don't observe mutations. +type SkillRegistry struct { + mu sync.RWMutex + skills []*Skill + warnings []LoadWarning +} + +// NewSkillRegistry returns an empty registry. +func NewSkillRegistry() *SkillRegistry { return &SkillRegistry{} } + +// Load walks skillsRoot looking for SKILL.md files and plugin containers +// and parses each one. Walks recursively; any depth under skillsRoot is OK. +// A non-existent skillsRoot is not an error (a fresh install with no skills +// boots fine). +// +// As of PR-skill-load (simplified loader) Load delegates to +// LoadAll, which also recurses into `.claude-plugin/` and +// `openclaw.plugin.json` containers and pulls their skills + commands +// (agents from packs in the skills root are also kept on the result, but +// they are ignored by this method since SkillRegistry only owns skills — +// callers wanting agents from packs should use LoadAll directly). +// +// Path safety: symlinks are resolved and verified to land back inside +// skillsRoot before parse. Per-file parse errors land as warnings rather +// than aborting; one bad SKILL.md should not take down boot. +func (r *SkillRegistry) Load(skillsRoot string) error { + res, err := LoadAll(LoadAllConfig{SkillsRoot: skillsRoot}) + if err != nil { + return fmt.Errorf("chatruntime: skill registry load: %w", err) + } + r.mu.Lock() + r.skills = append([]*Skill(nil), res.Skills...) + r.warnings = append([]LoadWarning(nil), res.Warnings...) + r.mu.Unlock() + return nil +} + +// Reload replaces the registry contents with a fresh scan of skillsRoot +// (delegating through LoadAll so packs / plugin containers / warnings +// behave identically to the boot-time path). Used by marketplace +// Install/Uninstall to hot-reload without restart. +// +// Atomicity: builds the new slice outside the lock, then swaps under +// r.mu.Lock() in O(1). All previously-returned slices from All() / +// Warnings() / Resolve() are value copies and remain unchanged — an +// in-flight chat that already grabbed a Resolve() result keeps running +// against the snapshot it was given. +// +// Empty skillsRoot wipes the registry (i.e. behaves like Load("") which +// is a no-op, but explicitly: skills slice goes to nil so a subsequent +// All() observes "nothing"). A non-existent skillsRoot is not an error +// — same convention as Load. +// +// Variadic extras lets the marketplace pass (builtinRoot, userRoot) so +// the image-baked built-in skills survive a hot-reload triggered by a +// pack install — a single-root Reload would otherwise drop them. +func (r *SkillRegistry) Reload(skillsRoot string, extras ...string) error { + res, err := LoadAll(LoadAllConfig{SkillsRoot: skillsRoot, ExtraSkillsRoots: extras}) + if err != nil { + return fmt.Errorf("chatruntime: skill registry reload: %w", err) + } + newSkills := append([]*Skill(nil), res.Skills...) + newWarnings := append([]LoadWarning(nil), res.Warnings...) + r.mu.Lock() + r.skills = newSkills + r.warnings = newWarnings + r.mu.Unlock() + return nil +} + +// All returns a copy of every loaded skill (so callers can't mutate +// the registry's internal slice). +func (r *SkillRegistry) All() []*Skill { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]*Skill, len(r.skills)) + copy(out, r.skills) + return out +} + +// Warnings returns a copy of every warning recorded during Load. +func (r *SkillRegistry) Warnings() []LoadWarning { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]LoadWarning, len(r.warnings)) + copy(out, r.warnings) + return out +} + +// Add inserts a programmatically-constructed skill. Used by feature-flag +// gated skills that don't exist as SKILL.md on disk (e.g. a runtime +// proposal skill). +func (r *SkillRegistry) Add(sk *Skill) { + if sk == nil { + return + } + r.mu.Lock() + r.skills = append(r.skills, sk) + r.mu.Unlock() +} + +// AddAll inserts a batch of skills. Nil entries are skipped. Used by +// LoadAll to merge plugin-container output into the registry. +func (r *SkillRegistry) AddAll(skills []*Skill) { + for _, sk := range skills { + r.Add(sk) + } +} + +// AddWarnings appends loader warnings (e.g. from LoadAll) so +// SkillRegistry.Warnings() exposes them alongside per-skill parse +// warnings. +func (r *SkillRegistry) AddWarnings(ws []LoadWarning) { + if len(ws) == 0 { + return + } + r.mu.Lock() + r.warnings = append(r.warnings, ws...) + r.mu.Unlock() +} + +// Resolve filters loaded skills against (query, policy): +// +// 1. Activation filter: mode "always" → include; mode "keyword" → +// include only when query (lowercased) contains any keyword. +// 2. Policy filter: drop tools whose Class is not in policy.AllowedClasses. +// The skill survives as long as at least one tool survives. A skill +// declaring zero tools is always kept after the activation filter +// (it may be a pure-prompt skill). +// +// Returned skills are *copies* with their Tools slice already filtered. +// The original registry skills are not mutated. +func (r *SkillRegistry) Resolve(query string, policy Policy) []*Skill { + queryLower := strings.ToLower(strings.TrimSpace(query)) + r.mu.RLock() + skills := make([]*Skill, len(r.skills)) + copy(skills, r.skills) + r.mu.RUnlock() + var out []*Skill + for _, sk := range skills { + if !activationMatches(sk.Activation, queryLower) { + continue + } + filtered := filterToolsByPolicy(sk.Tools, policy) + if len(sk.Tools) > 0 && len(filtered) == 0 { + // Skill had tools but none survived policy — drop the skill. + continue + } + clone := *sk + clone.Tools = filtered + out = append(out, &clone) + } + return out +} + +// activationMatches reports whether a skill's Activation matches the +// (already lowercased) query. +func activationMatches(a Activation, queryLower string) bool { + mode := strings.ToLower(strings.TrimSpace(a.Mode)) + switch mode { + case "", "always": + return true + case "keyword": + for _, kw := range a.Keywords { + kw = strings.ToLower(strings.TrimSpace(kw)) + if kw == "" { + continue + } + if strings.Contains(queryLower, kw) { + return true + } + } + return false + default: + // Unknown mode — fail open; the parse-time warning is the place + // to surface this instead of silently disabling the skill. + return true + } +} + +// filterToolsByPolicy drops tools whose Class is not allowed. +func filterToolsByPolicy(tools []ToolDecl, policy Policy) []ToolDecl { + if len(tools) == 0 { + return nil + } + out := make([]ToolDecl, 0, len(tools)) + for _, t := range tools { + // Empty class → assume read-only (safe default). + class := t.Class + if class == "" { + class = ClassRead + } + if !policy.Allows(class) { + continue + } + out = append(out, t) + } + return out +} + +// pathHasPrefix returns true when child sits inside parent (or is the +// parent itself). Cleans both paths first so trailing slashes don't +// cause false negatives. Mirrors internal/skill/loader.go for consistency. +func pathHasPrefix(child, parent string) bool { + child = filepath.Clean(child) + parent = filepath.Clean(parent) + if child == parent { + return true + } + rel, err := filepath.Rel(parent, child) + if err != nil { + return false + } + if strings.HasPrefix(rel, "..") { + return false + } + return true +} diff --git a/internal/manager/biz/aiops/chatruntime/skill_registry_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/skill_registry_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/skill_registry_test.go rename to internal/manager/biz/aiops/runtime/v1/chatruntime/skill_registry_test.go diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/system_prompt.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/system_prompt.go new file mode 100644 index 000000000..421f9c2f4 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/system_prompt.go @@ -0,0 +1,127 @@ +package chatruntime + +import ( + "strings" +) + +// ComposeSystemPrompt assembles the SystemPrompt block ChatRuntime feeds +// to the LLM. Layered (the ComposeSystemPrompt arrow +// in the main reference figure): +// +// 1. basePrompt (the runtime's universal preamble; can be empty) +// 2. agentProfile.SystemPrompt + agentProfile.CriticalReminder +// (only when a worker is being constructed; nil for coordinator +// unless the coordinator itself is also persona-driven) +// 3. for each active skill, a `[能力: ]` header + skill PromptBody +// +// PR-2 scope: this is a pure string assembly — no per-turn system-reminder +// injection. That happens in the graph layer; the TODO +// inline below points to where it lands. +func ComposeSystemPrompt(basePrompt string, activeSkills []*Skill, agentProfile *Agent) string { + var parts []string + + if base := strings.TrimSpace(basePrompt); base != "" { + parts = append(parts, base) + } + + if agentProfile != nil { + if body := strings.TrimSpace(agentProfile.SystemPrompt); body != "" { + parts = append(parts, body) + } + if reminder := strings.TrimSpace(agentProfile.CriticalReminder); reminder != "" { + // — the per-turn injection wraps this in + // ...; here we just plant + // it once into the system prompt as the persona-level + // constant. The graph layer will additionally re-inject + // per-turn. + parts = append(parts, "\n"+reminder+"\n") + } + } + + for _, sk := range activeSkills { + if sk == nil { + continue + } + header := "[能力: " + sk.Name + "]" + body := strings.TrimSpace(sk.PromptBody) + // PromptBody is already H1-normalized at parse time ( + // If the body already starts with the canonical header + // we skip prepending again. + if strings.HasPrefix(body, header) { + parts = append(parts, body) + continue + } + if body == "" { + parts = append(parts, header) + continue + } + parts = append(parts, header+"\n\n"+body) + } + + // TODO(PR-graph): per-turn injection happens at + // the graph MessageAssembler stage, not here. + // ComposeSystemPrompt only owns the system-message body. + + return strings.Join(parts, "\n\n") +} + +// buildAgentCatalog renders a markdown list of the registered agent +// personas the coordinator can spawn via AgentTool. Injected into the +// coordinator's system prompt at runtime so the LLM knows the valid +// subagent_type values + when to pick which. +// +// Only "user" + "disk" sourced personas are listed. The coordinator +// never spawns reviewer (only review_gate decorator does); skipping it +// here keeps the LLM from accidentally trying to use it for ad-hoc +// reviews. +func buildAgentCatalog(reg *AgentRegistry) string { + if reg == nil { + return "" + } + all := reg.All() + if len(all) == 0 { + return "" + } + type row struct{ name, when, desc string } + rows := make([]row, 0, len(all)) + for _, ag := range all { + if ag == nil || ag.Name == "" { + continue + } + // reviewer is reserved for the SOP twin-sign decorator; don't + // expose it to the coordinator's free-form AgentTool routing. + // "default" is the virtual top-level persona — listing it as a + // spawnable sub-agent would let the coordinator recursively + // spawn itself. + if ag.Name == "reviewer" || ag.Name == "default" { + continue + } + rows = append(rows, row{name: ag.Name, when: strings.TrimSpace(ag.WhenToUse), desc: strings.TrimSpace(ag.Description)}) + } + if len(rows) == 0 { + return "" + } + var sb strings.Builder + sb.WriteString("## 可用的 specialist 助理（AgentTool 的 subagent_type）\n\n") + sb.WriteString("当任务专门属于以下领域时，**优先用 AgentTool 派给对应专家**而不是自己硬刚 — 专家的 toolBag 更聚焦、token 更省、推理更深：\n\n") + for _, r := range rows { + sb.WriteString("- `") + sb.WriteString(r.name) + sb.WriteString("` — ") + if r.desc != "" { + sb.WriteString(r.desc) + } + if r.when != "" { + // Take just the first non-empty line of when_to_use so + // the prompt stays compact. + firstLine := strings.SplitN(r.when, "\n", 2)[0] + if firstLine != "" && firstLine != r.desc { + sb.WriteString("。何时派：") + sb.WriteString(firstLine) + } + } + sb.WriteString("\n") + } + sb.WriteString("\n派活时：在 prompt 里写清 incident_id / device_id / 用户原话——worker 看不到你的上下文。") + return sb.String() +} diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/system_prompt_test.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/system_prompt_test.go new file mode 100644 index 000000000..7c0c606ae --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/system_prompt_test.go @@ -0,0 +1,102 @@ +package chatruntime + +import ( + "strings" + "testing" +) + +func TestComposeSystemPrompt_Empty(t *testing.T) { + got := ComposeSystemPrompt("", nil, nil) + if got != "" { + t.Errorf("expected empty string, got %q", got) + } +} + +func TestComposeSystemPrompt_BaseOnly(t *testing.T) { + got := ComposeSystemPrompt("you are ongrid.", nil, nil) + if got != "you are ongrid." { + t.Errorf("got %q", got) + } +} + +func TestComposeSystemPrompt_MultipleSkills(t *testing.T) { + skills := []*Skill{ + {Name: "alpha", PromptBody: "[能力: alpha]\n\nalpha guide"}, + {Name: "beta", PromptBody: "beta guide"}, + } + got := ComposeSystemPrompt("base prompt", skills, nil) + if !strings.Contains(got, "base prompt") { + t.Error("base missing") + } + if !strings.Contains(got, "[能力: alpha]") { + t.Error("alpha header missing") + } + // beta wasn't pre-tagged in PromptBody, ComposeSystemPrompt should + // prepend the canonical tag. + if !strings.Contains(got, "[能力: beta]") { + t.Error("beta header missing") + } + if !strings.Contains(got, "alpha guide") || !strings.Contains(got, "beta guide") { + t.Error("skill bodies missing") + } + // Order: base before alpha before beta. + if strings.Index(got, "base prompt") > strings.Index(got, "alpha guide") { + t.Error("base should come before alpha") + } + if strings.Index(got, "alpha guide") > strings.Index(got, "beta guide") { + t.Error("alpha should come before beta") + } +} + +func TestComposeSystemPrompt_AgentProfile(t *testing.T) { + ag := &Agent{ + Name: "reviewer", + SystemPrompt: "you review proposals.", + CriticalReminder: "do not approve mutations on your own.", + } + got := ComposeSystemPrompt("base", nil, ag) + if !strings.Contains(got, "base") { + t.Error("base missing") + } + if !strings.Contains(got, "you review proposals.") { + t.Error("system prompt body missing") + } + if !strings.Contains(got, "") { + t.Error("critical-reminder block missing") + } + if !strings.Contains(got, "do not approve mutations") { + t.Error("reminder content missing") + } +} + +func TestComposeSystemPrompt_AgentAndSkills(t *testing.T) { + ag := &Agent{Name: "worker", SystemPrompt: "worker prompt", CriticalReminder: ""} + skills := []*Skill{{Name: "tool_x", PromptBody: "x guide"}} + got := ComposeSystemPrompt("base", skills, ag) + if !strings.Contains(got, "base") || !strings.Contains(got, "worker prompt") || !strings.Contains(got, "[能力: tool_x]") { + t.Errorf("missing parts; got %q", got) + } + // Order: base, then agent, then skills. + bIdx := strings.Index(got, "base") + aIdx := strings.Index(got, "worker prompt") + sIdx := strings.Index(got, "[能力: tool_x]") + if !(bIdx < aIdx && aIdx < sIdx) { + t.Errorf("order should be base→agent→skills; bIdx=%d aIdx=%d sIdx=%d", bIdx, aIdx, sIdx) + } +} + +func TestComposeSystemPrompt_NilSkillEntrySkipped(t *testing.T) { + skills := []*Skill{nil, {Name: "z", PromptBody: "z body"}} + got := ComposeSystemPrompt("", skills, nil) + if !strings.Contains(got, "[能力: z]") { + t.Errorf("z skill should be present despite preceding nil; got %q", got) + } +} + +func TestComposeSystemPrompt_EmptyPromptBody(t *testing.T) { + skills := []*Skill{{Name: "empty"}} + got := ComposeSystemPrompt("", skills, nil) + if got != "[能力: empty]" { + t.Errorf("got %q, want '[能力: empty]'", got) + } +} diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_parser/minimal/incident-investigator.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_parser/minimal/incident-investigator.md new file mode 100644 index 000000000..ab3c0adf5 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_parser/minimal/incident-investigator.md @@ -0,0 +1,15 @@ +--- +name: incident-investigator +description: Alert root-cause investigator. +when_to_use: | + Use when the user asks "what is the root cause / how to investigate / blast + radius" of an alert. Provide incident_id or device_id; returns three + sections: phenomenon / signals / hypotheses. +--- + +You are the ongrid alert root-cause investigation agent. + +Workflow: +1. Fetch the incident detail. +2. Correlate metric / log / trace. +3. Output three sections: phenomenon, related signals, hypotheses. diff --git a/internal/manager/biz/aiops/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/agent_registry/multi/README.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_registry/multi/README.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/agent_registry/multi/README.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_registry/multi/README.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/agent_registry/multi/incident-investigator.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_registry/multi/incident-investigator.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/agent_registry/multi/incident-investigator.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_registry/multi/incident-investigator.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/agent_registry/multi/reviewer.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_registry/multi/reviewer.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/agent_registry/multi/reviewer.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/agent_registry/multi/reviewer.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/path_traversal/skills/pwned b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/path_traversal/skills/pwned similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/path_traversal/skills/pwned rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/path_traversal/skills/pwned diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_parser/minimal/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/minimal/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_parser/minimal/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/minimal/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_parser/missing_description/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/missing_description/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_parser/missing_description/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/missing_description/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_parser/typical/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/typical/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_parser/typical/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/typical/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md diff --git a/internal/manager/biz/aiops/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md b/internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md rename to internal/manager/biz/aiops/runtime/v1/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/types.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/types.go new file mode 100644 index 000000000..cb3b52762 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/types.go @@ -0,0 +1,388 @@ +// Package chatruntime is the in-process orchestration layer for the ongrid +// AIOps agent. It owns the data shape of a "skill bundle" and an +// "agent persona": both are SKILL.md / *.md files with YAML +// frontmatter + markdown body. This file defines just the in-memory structs; +// parsing lives in skill_parser.go / agent_parser.go. +// +// PR-2 of / / scaffolding only — no graph, no LLM +// wiring, no tool execution. Old internal/skill stays running for compat. +// +// Field naming convention: every YAML / JSON tag uses snake_case per +// — the spec explicitly aligns ongrid with claude-code agent +// frontmatter, openclaw SKILL.md, and the SKILL.md body we ship to users. +package chatruntime + +import ( + "encoding/json" + "time" +) + +// ToolClass categorizes a tool's blast radius. Policies filter tools by +// class so the LLM never sees tools that aren't allowed by the channel / +// agent profile. Mirrors sealsuite-agent/internal/chatruntime.ToolClass. +type ToolClass string + +const ( + // ClassRead is read-only — no side effects on the system or device. + ClassRead ToolClass = "read" + // ClassWrite mutates state (creates / updates resources) — requires a + // permissionMode >= mutating-with-confirm. + ClassWrite ToolClass = "write" + // ClassDestructive is irreversible (deletes, restarts, exec-arbitrary) + // — requires dual-sign-required (SOP) + ClassDestructive ToolClass = "destructive" +) + +// Policy is the per-request capability gate. The HTTP layer / agent +// persona constructs a Policy; ChatRuntime feeds it into ToolRegistry.Build +// and SkillRegistry.Resolve. PR-2 only defines the data shape — the +// actual filtering logic lives in skill_registry.go (and a future +// tool_registry.go in PR-3). +type Policy struct { + // AllowedClasses lists which ToolClass values are permitted. + // ["*"] means unrestricted. Empty defaults to read-only (matches + // sealsuite-agent semantics — see Allows()). + AllowedClasses []string + + // ConfirmationRequired lists classes that need a two-phase commit + // before execution. PR-2 defines the field but does not implement + // the gate (reviewer flow lands in a later PR). + ConfirmationRequired []string + + // MaxOutputChars caps the final reply length. 0 = no cap. + MaxOutputChars int + + // MaxIterations bounds the ReAct loop. 0 = use agent default. + MaxIterations int +} + +// Allows reports whether class is permitted under this policy. +// Empty AllowedClasses defaults to read-only (safe default). +func (p Policy) Allows(class ToolClass) bool { + if len(p.AllowedClasses) == 0 { + return class == ClassRead + } + for _, c := range p.AllowedClasses { + if c == "*" || c == string(class) { + return true + } + } + return false +} + +// RequiresConfirmation reports whether class must pass a confirmation +// gate (permissionMode = mutating-with-confirm). +func (p Policy) RequiresConfirmation(class ToolClass) bool { + for _, c := range p.ConfirmationRequired { + if c == "*" || c == string(class) { + return true + } + } + return false +} + +// Activation controls when a skill is mounted into the toolBag. See +// (metadata.ongrid.activation) and +type Activation struct { + // Mode = "always" (default) | "keyword". + // + sealsuite chatruntime semantics. + Mode string `yaml:"mode" json:"mode"` + + // Keywords trigger activation when Mode == "keyword". Case-insensitive + // substring match against the user query. + Keywords []string `yaml:"keywords" json:"keywords"` +} + +// Requires expresses host-level dependencies of a skill. Matches openclaw +// SKILL.md `metadata.requires` shape. ongrid reads this verbatim per +type Requires struct { + // Bins lists native binaries the skill expects on PATH (e.g. ["find", "du"]). + // They go into the sandbox process whitelist. + Bins []string `yaml:"bins" json:"bins"` + + // Config lists settings keys that must be filled by an admin before + // the skill can run (e.g. ["accountsPath"]). Lands in system_settings. + Config []string `yaml:"config" json:"config"` +} + +// OngridExt is the ongrid-private extension subtree under metadata.ongrid. +// — these fields are ignored by openclaw / claude-code, so +// the same SKILL.md file runs on multiple platforms. +type OngridExt struct { + // Scope = "manager" (default) | "edge". Edge-scope skills run on the + // edge agent via frontier tunnel (plugin runtime). + Scope string `yaml:"scope" json:"scope"` + + // EdgeRuntime is a free-form hint for the edge plugin runtime + // (e.g. "subprocess", "wasm"). Empty = subprocess default. + EdgeRuntime string `yaml:"edge_runtime" json:"edge_runtime"` + + // EdgeCapabilities is the edge-side sandbox declaration — a list of + // capability statements Kept as raw map slice so + // future capability shapes don't break the parser. + EdgeCapabilities []map[string]any `yaml:"edge_capabilities" json:"edge_capabilities"` + + // Activation overrides the skill-level Activation (one is canonical, + // the other is for ergonomics). Loader prefers OngridExt.Activation + // when both are set, with a warning. + Activation Activation `yaml:"activation" json:"activation"` + + // MinOngridVersion gates loading on the running ongrid build (e.g. + // ">=0.7.30"). Failed gate produces a warning, not an error, in PR-2. + MinOngridVersion string `yaml:"min_ongrid_version" json:"min_ongrid_version"` +} + +// SkillMetadata is the `metadata` frontmatter subtree common to both +// openclaw and claude-code SKILL.md files plus the ongrid extension. +type SkillMetadata struct { + // OS gates loading on GOOS (— openclaw field). Empty = any. + OS []string `yaml:"os" json:"os"` + + // Requires is the openclaw `metadata.requires` shape. + Requires Requires `yaml:"requires" json:"requires"` + + // Ongrid is the ongrid-private extension subtree. + Ongrid OngridExt `yaml:"ongrid" json:"ongrid"` +} + +// ToolDecl is a single tool declared by a skill. PR-2 stores the +// declaration only — actual factory binding lives in PR-3. +type ToolDecl struct { + // Name is the wire-level tool name the LLM sees. + Name string `yaml:"name" json:"name"` + + // Impl is the registered factory key (e.g. "builtin:filetools.Find"). + Impl string `yaml:"impl" json:"impl"` + + // Class is the blast-radius classification — see ToolClass constants. + Class ToolClass `yaml:"class" json:"class"` + + // Description is the LLM-facing summary. + Description string `yaml:"description" json:"description"` + + // WhenToUse is the LLM-facing decision hint — + // claude-code splits "what is it" (description) from "when to pick it" + // (when_to_use). ongrid follows the same split at tool granularity. + WhenToUse string `yaml:"when_to_use" json:"when_to_use"` + + // Confirmation = "required" | "" — flips the per-tool confirmation + // gate even if the policy class would otherwise allow it. + Confirmation string `yaml:"confirmation" json:"confirmation"` + + // TimeoutSeconds caps the per-call wall time. 0 = use runtime default. + TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"` +} + +// Provenance records where a Skill / Agent / Pack came from. Filled by +// the marketplace install path; empty for filesystem drops. +type Provenance struct { + // Source = "fs" | "tgz" | "git" | "marketplace" | "" (unknown). + Source string `yaml:"source" json:"source"` + + // SourceURL is the original location (git URL, tgz fetch URL, + // marketplace slug). Diagnostic only. + SourceURL string `yaml:"source_url" json:"source_url"` + + // InstalledAt is the wall time of install. Zero for built-ins. + InstalledAt time.Time `yaml:"installed_at" json:"installed_at"` + + // ManifestSHA256 is hex of the SKILL.md / agent .md / plugin.json + // content — used by signature verification. + ManifestSHA256 string `yaml:"manifest_sha256" json:"manifest_sha256"` + + // SignatureState = "unsigned" | "valid" | "invalid" | "unknown". + SignatureState string `yaml:"signature_state" json:"signature_state"` +} + +// Skill is a parsed SKILL.md descriptor in memory. The on-disk format +// is YAML frontmatter + markdown body. The body is captured +// verbatim into PromptBody and gets prepended with `[能力: ]` +// during system-prompt assembly. +type Skill struct { + // Name is the snake_case unique key. Required. + Name string `yaml:"name" json:"name"` + + // Version is optional; "" if author didn't declare one. + Version string `yaml:"version" json:"version"` + + // Description is the one-liner shown to the LLM at listing time. + // Required. + Description string `yaml:"description" json:"description"` + + // WhenToUse is the longer "when should the LLM pick this" text. + // Recommended. + WhenToUse string `yaml:"when_to_use" json:"when_to_use"` + + // PromptBody is the markdown body of SKILL.md verbatim (after H1 + // normalization ). Loaded outside frontmatter so + // the YAML tag is `-`. + PromptBody string `yaml:"-" json:"prompt_body"` + + // Activation is the top-level activation hint (matches sealsuite + // shape). also allows it under metadata.ongrid; the + // ongrid one wins when both are set. + Activation Activation `yaml:"activation" json:"activation"` + + // ConfigSection is the legacy openclaw-style settings binding key + // (e.g. for "config_section: feilian.openapi"). Kept for compat. + ConfigSection string `yaml:"config_section" json:"config_section"` + + // Tools is the per-skill tool declaration list. + Tools []ToolDecl `yaml:"tools" json:"tools"` + + // Metadata is the `metadata` subtree (os, requires, ongrid.*). + Metadata SkillMetadata `yaml:"metadata" json:"metadata"` + + // Provenance is filled by the install path; empty for built-ins + // and direct fs drops. + Provenance Provenance `yaml:"-" json:"provenance"` + + // Dir is the absolute directory the skill was loaded from. Used + // by factories that need to resolve relative paths (openapi.yaml, + // domains/*.md, etc.). + Dir string `yaml:"-" json:"dir"` + + // UnknownFields preserves any frontmatter keys we don't recognize. + // explicitly calls for this so that openclaw / claude-code + // adding new fields doesn't break ongrid loading. Map values are + // kept as decoded YAML scalars / sequences / mappings. + UnknownFields map[string]any `yaml:"-" json:"unknown_fields"` +} + +// Agent is a parsed agent persona (frontmatter markdown). +// — same on-disk shape as SKILL.md but the body is the agent's system +// prompt rather than skill prose. Snake_case YAML tags +type Agent struct { + // Name is the agent identifier used at spawn time. Required. + Name string `yaml:"name" json:"name"` + + // Description is the human listing string. Required. + Description string `yaml:"description" json:"description"` + + // WhenToUse is the coordinator's spawn-decision hint. Required — + // calls this the strict field a coordinator reads. + WhenToUse string `yaml:"when_to_use" json:"when_to_use"` + + // Tools is the explicit tool whitelist. Empty = inherit from policy. + Tools []string `yaml:"tools" json:"tools"` + + // DisallowedTools is the blacklist (applied after whitelist). Black + // wins over white + DisallowedTools []string `yaml:"disallowed_tools" json:"disallowed_tools"` + + // PermissionMode = "read-only" | "mutating-with-confirm" | "dual-sign-required". + PermissionMode string `yaml:"permission_mode" json:"permission_mode"` + + // MaxTurns caps the worker's internal ReAct loop. + MaxTurns int `yaml:"max_turns" json:"max_turns"` + + // Model is the LLM identifier (e.g. "anthropic/claude-sonnet-4-6"). + // Empty = inherit coordinator default. + Model string `yaml:"model" json:"model"` + + // CriticalReminder is the system-reminder block injected on every + // turn. — anti-drift mechanism. + CriticalReminder string `yaml:"critical_reminder" json:"critical_reminder"` + + // InitialPrompt is prepended to the first user message at spawn. + InitialPrompt string `yaml:"initial_prompt" json:"initial_prompt"` + + // Background = true forces async execution (long-running workers). + Background bool `yaml:"background" json:"background"` + + // OmitClaudeMd skips inheriting the global system context. Used + // for tightly-scoped reviewer agents. + OmitClaudeMd bool `yaml:"omit_claude_md" json:"omit_claude_md"` + + // SystemPrompt is the markdown body — the actual system prompt + // the worker LLM sees. Loaded outside frontmatter. + SystemPrompt string `yaml:"-" json:"system_prompt"` + + // Source tells the API/UI where this persona came from: + // "builtin" — shipped in the binary (programmatic Add) + // "disk" — loaded from agents/*.md + // "user" — created by the user via /v1/agents/custom (Phase 3) + // "user" agents are editable + deletable from the UI; the others + // stay read-only. + Source string `yaml:"-" json:"source,omitempty"` + + // Metadata mirrors the SKILL.md metadata subtree — agents may + // declare os / requires / ongrid extensions too. + Metadata SkillMetadata `yaml:"metadata" json:"metadata"` + + // Provenance fills on install. + Provenance Provenance `yaml:"-" json:"provenance"` + + // Dir is the directory of the .md file. + Dir string `yaml:"-" json:"dir"` + + // UnknownFields preserves unrecognized frontmatter keys. + UnknownFields map[string]any `yaml:"-" json:"unknown_fields"` +} + +// Pack is a parsed plugin container — a directory shipping zero or more +// skills + agents + commands together. ongrid recognizes both +// claude-code's `.claude-plugin/plugin.json` and openclaw's +// `openclaw.plugin.json`. PR-2 only parses the manifest +// metadata; recursive load of skills/ agents/ commands/ subdirs is a +// later PR (TODO in plugin_container.go). +type Pack struct { + // ID is the pack key (e.g. "acme-tools"). Required. + ID string `yaml:"id" json:"id"` + + // DisplayName is the human-friendly title. + DisplayName string `yaml:"name" json:"display_name"` + + // Version follows semver (e.g. "1.2.3"). + Version string `yaml:"version" json:"version"` + + // Description is a one-liner. + Description string `yaml:"description" json:"description"` + + // ConfigSchema is a JSON Schema describing tenant-level config that + // admins fill in via the SPA install dialog. Optional. + ConfigSchema json.RawMessage `yaml:"config_schema" json:"config_schema"` + + // UIMetadata holds platform-specific extras the SPA may show. We + // stuff openclaw legacy fields under UIMetadata["openclaw_legacy"] + // so future SPA can render them as-is. + UIMetadata map[string]any `yaml:"ui_metadata" json:"ui_metadata"` + + // Source is the install source ("fs" | "tgz" | "git" | "marketplace"). + Source string `yaml:"source" json:"source"` + + // ManifestSHA256 is hex of the manifest content. + ManifestSHA256 string `yaml:"manifest_sha256" json:"manifest_sha256"` + + // SignatureState = "unsigned" | "valid" | "invalid" | "unknown". + SignatureState string `yaml:"signature_state" json:"signature_state"` + + // Dir is the absolute pack root. + Dir string `yaml:"-" json:"dir"` +} + +// LoadResult is what plugin_container / registry loaders return after a +// directory walk: the discovered Pack (if any), the skills + agents +// found inside, and accumulated warnings (one per skipped / normalized +// entry). +type LoadResult struct { + Pack *Pack `json:"pack"` + Skills []*Skill `json:"skills"` + Agents []*Agent `json:"agents"` + Warnings []LoadWarning `json:"warnings"` +} + +// LoadWarning is a non-fatal load issue. Code is a stable identifier +// (e.g. "name_normalized", "missing_when_to_use") so SPA / tests can +// filter without scraping Reason text. +type LoadWarning struct { + // Path is the file the warning applies to (absolute or relative + // to the loader's root — loader convention). + Path string `json:"path"` + + // Reason is human-readable English text. + Reason string `json:"reason"` + + // Code is the stable identifier. + Code string `json:"code"` +} diff --git a/internal/manager/biz/aiops/runtime/v1/chatruntime/worker.go b/internal/manager/biz/aiops/runtime/v1/chatruntime/worker.go new file mode 100644 index 000000000..7466e2f7b --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/chatruntime/worker.go @@ -0,0 +1,858 @@ +// worker.go implements — the coordinator/worker multi-agent +// path. The Runtime owns a small in-memory map of spawned workers; each +// worker is an independent graph.Invoke against a tool-bag filtered by +// the worker agent's persona (whitelist + disallowed_tools — black wins). +// +// Non-goals (first version): +// - Worker nesting: a worker cannot spawn another worker. SpawnWorker +// is exposed only on the Runtime, and AgentTool's wiring keeps it on +// the coordinator. +// - Cross-session memory sharing: each worker has its own logical +// session id so its message history is opaque to its parent. The +// parent → worker relationship is now persisted on chat_sessions +// (agent_id / parent_session_id / background columns); the in-memory +// map is still authoritative for live status until a worker tile UI +// queries those columns directly. +// +// State machine (figure): +// +// pending — set transiently between SpawnRequest accept and goroutine +// start (effectively never observable for sync spawns) +// ↓ +// running — goroutine has begun graph.Invoke; status flips here +// BEFORE the child graph emits its first event +// ↓ +// completed — graph.Invoke returned a non-nil AssistantMessage with +// no error; Result holds the final assistant content +// failed — graph.Invoke returned err != nil OR the agent name was +// unknown / agent registry was nil +// killed — StopWorker called while status was running; cancel() +// fires and the goroutine observes ctx.Done() then sets +// EndedAt + status=killed (without overwriting err) +// +// Background semantics — when SpawnRequest.Background is true SpawnWorker +// returns immediately with status=running and the graph.Invoke runs in a +// detached goroutine; the goroutine emits a "task_notification" SSE +// envelope back through req.ParentEmit upon terminal status. Callers +// (AgentTool) get the worker_id from the synchronous return and can +// answer the LLM with `{task_id, status: "pending"}` while the SPA +// renders a live tile that gets updated when the notification fires. +package chatruntime + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + einocallbacks "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/compose" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v1/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v1/graph/callbacks" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" +) + +// WorkerStatus enumerates the worker state machine. See package header +// for the transition diagram. +type WorkerStatus string + +const ( + // WorkerStatusPending — accepted but not yet started. Observable + // only in tests that intercept between SpawnWorker accept and goroutine + // scheduling; in production this state is essentially transient. + WorkerStatusPending WorkerStatus = "pending" + // WorkerStatusRunning — graph.Invoke is in flight. + WorkerStatusRunning WorkerStatus = "running" + // WorkerStatusCompleted — graph.Invoke returned a final assistant + // message with no error. + WorkerStatusCompleted WorkerStatus = "completed" + // WorkerStatusFailed — graph.Invoke returned an error, or the agent + // name was unknown. + WorkerStatusFailed WorkerStatus = "failed" + // WorkerStatusKilled — explicitly stopped via StopWorker. + WorkerStatusKilled WorkerStatus = "killed" +) + +// Worker tracks a spawned sub-agent. Unique by ID within a Runtime +// instance. +type Worker struct { + // ID is "agent-<8 hex>" — unique per Runtime, stable for the + // worker's lifetime. + ID string + // AgentName is the persona name (frontmatter `name` field). + AgentName string + // SessionID is the worker's own session id (distinct from the + // coordinator's session). For PR-1 of this is a synthetic + // in-memory id; the chat_sessions row + parent_session_id column + // migration is follow-up. + SessionID string + // ParentSessionID is the coordinator session that requested the + // spawn. Tracked here so a future audit query can rebuild the parent + // → worker tree without a schema change. + ParentSessionID string + // Prompt is the initial user message handed to the worker. + Prompt string + // Status is the lifecycle state. Read-only outside the runtime; + // callers should always pull through GetWorker for a snapshot. + Status WorkerStatus + // Background indicates whether this is a fire-and-forget spawn. + Background bool + // StartedAt is the wall time the worker entered "running". + StartedAt time.Time + // EndedAt is the wall time of any terminal transition. nil while + // running. + EndedAt *time.Time + // Result is the worker's final assistant content (when terminal = + // completed). Empty for failed / killed. + Result string + // Err is the failure reason (when terminal = failed / killed). + Err string + + // Internal fields — not surfaced to callers via copies. + cancel context.CancelFunc + mu sync.Mutex +} + +// SpawnRequest describes a worker to spawn. See AgentTool.InvokableRun +// for the LLM-facing argument shape that produces this struct. +type SpawnRequest struct { + // AgentName is the persona name (frontmatter `name`). Must be + // resolvable through the Runtime's AgentRegistry. + AgentName string + // Prompt is the initial user message for the worker. + Prompt string + // Background = true: fire-and-forget; SpawnWorker returns immediately + // with status=running. Background = false: SpawnWorker blocks until + // the worker reaches a terminal state. + Background bool + // ParentSession is the coordinator's chat session id. + ParentSession string + // ParentEmit is the SSE emitter the coordinator runtime threads + // through. Used to deliver the task_notification frame when the + // worker (background=true) finishes. nil = no notification (sync + // callers don't need it). + ParentEmit Emit + // SessionKind overrides the persisted chat_sessions.kind for the + // worker session. Empty = "user" (default). The investigator usecase + // sets "investigation" so auto-spawned RCA transcripts stay out of + // the /chat list. + SessionKind string + // OwnerUserID overrides the persisted chat_sessions.user_id when + // the worker has no ParentSession to inherit from. The investigator + // usecase passes 0 to mark these rows as "system-owned" — they + // don't belong to any operator and the /chat filter (which checks + // user_id) won't surface them. + OwnerUserID uint64 +} + +// TaskNotification is the payload of a task_notification streaming +// frame. Mirrors the schema laid out in — coordinator's +// SSE listener fans this back through the legacy SSE envelope so the +// SPA only needs to learn one new event type. +type TaskNotification struct { + TaskID string `json:"task_id"` + Status WorkerStatus `json:"status"` + Summary string `json:"summary"` + Result string `json:"result,omitempty"` + Err string `json:"error,omitempty"` + Usage map[string]any `json:"usage,omitempty"` +} + +// EventTaskNotification is the new streaming event type the runtime +// emits when a background worker reaches a terminal state. The SPA +// listens for this in addition to the existing assistant / tool / +// done frames. +const EventTaskNotification EventType = "task_notification" + +// emitCtxKey is the unexported context key that threads the active +// per-request Emit through the graph layer down into AgentTool's +// InvokableRun via the WorkerSpawner shim. The shim reads it through +// EmitFromContext to populate SpawnRequest.ParentEmit so a background +// worker can fire its task_notification through the SSE channel that +// owns the user's chat session. +type emitCtxKeyT struct{} + +var emitCtxKey = emitCtxKeyT{} + +// withEmit returns ctx augmented with emit. Internal — Handle threads +// the request emitter in before invoking the graph. +func withEmit(ctx context.Context, emit Emit) context.Context { + if emit == nil { + return ctx + } + return context.WithValue(ctx, emitCtxKey, emit) +} + +// EmitFromContext retrieves the active streaming Emit from ctx, if any. +// Returns nil when no emitter was attached (e.g. blocking call without +// SSE). The wiring shim at the cmd/main.go boundary uses this to thread +// task_notification frames through the same SSE channel that owns the +// coordinator's chat session. +func EmitFromContext(ctx context.Context) Emit { + v, _ := ctx.Value(emitCtxKey).(Emit) + return v +} + +// SpawnWorker starts a worker. See package header for state machine. +// +// Background=false: blocks until the worker reaches completed / failed / +// killed; the returned *Worker has Result + Err filled. +// +// Background=true: returns immediately with a Worker whose Status = +// running; the graph runs in a detached goroutine and the terminal +// transition emits a task_notification through req.ParentEmit. +func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, error) { + if rt == nil { + return nil, errors.New("chatruntime: nil runtime") + } + if rt.cfg.ChatModel == nil { + return nil, errors.New("chatruntime: ChatModel is required") + } + if rt.cfg.AgentRegistry == nil { + return nil, fmt.Errorf("chatruntime: agent %q: registry not wired", req.AgentName) + } + agentDef, ok := rt.cfg.AgentRegistry.ByName(req.AgentName) + if !ok || agentDef == nil { + return nil, fmt.Errorf("chatruntime: agent %q not found", req.AgentName) + } + + id := newWorkerID() + sessID := newWorkerSessionID(id) + + // Persist the worker session row up-front so audit + worker-tree + // queries can resolve the (parent → worker) relationship without + // waiting for the worker to finish. We need the parent's UserID for + // row-level scoping; missing parent → worker session is orphan-able + // (still persisted, but parent_session_id=nil) so the LLM-driven + // flow remains usable in tests / sync calls without a parent row. + var ownerUserID uint64 + var parentRefForRow *string + if req.ParentSession != "" && rt.cfg.Sessions != nil { + parentSess, err := rt.cfg.Sessions.GetSession(ctx, req.ParentSession) + if err == nil && parentSess != nil { + ownerUserID = parentSess.UserID + pid := req.ParentSession + parentRefForRow = &pid + } + // On lookup failure we proceed without a parent ref so the + // worker can still run; the audit row simply has parent_session_id + // = nil. Logged at info — not fatal. + if err != nil && rt.log != nil { + rt.log.Info("chatruntime: worker spawn — parent session lookup failed", + "parent_session_id", req.ParentSession, + "err", err.Error()) + } + } + if rt.cfg.Sessions != nil { + agentName := agentDef.Name + // Optional caller-supplied overrides: SessionKind labels the + // transcript so /chat list can filter; OwnerUserID is used + // when there's no ParentSession to inherit from (e.g. the + // alert investigator spawns system-owned workers). + kind := aiopsmodel.SessionKindUser + if req.SessionKind != "" { + kind = req.SessionKind + } + effectiveOwner := ownerUserID + if effectiveOwner == 0 && req.OwnerUserID != 0 { + effectiveOwner = req.OwnerUserID + } + row := &aiopsmodel.Session{ + ID: sessID, + UserID: effectiveOwner, + Title: fmt.Sprintf("Worker: %s", agentDef.Name), + AgentID: &agentName, + ParentSessionID: parentRefForRow, + Background: req.Background, + Kind: kind, + CreatedAt: time.Now().UTC(), + UpdatedAt: time.Now().UTC(), + } + if err := rt.cfg.Sessions.CreateSession(ctx, row); err != nil { + return nil, fmt.Errorf("chatruntime: worker session create: %w", err) + } + } + + // Background spawn always derives from the long-lived runtime context + // so a finishing HTTP request can't tear down the worker mid-run. + // Sync spawn inherits the caller's ctx so a cancel propagates. + parent := ctx + if req.Background { + parent = context.Background() + } + workerCtx, cancel := context.WithCancel(parent) + + w := &Worker{ + ID: id, + AgentName: agentDef.Name, + SessionID: sessID, + ParentSessionID: req.ParentSession, + Prompt: req.Prompt, + Status: WorkerStatusPending, + Background: req.Background, + cancel: cancel, + } + + rt.workersMu.Lock() + if rt.workers == nil { + rt.workers = map[string]*Worker{} + } + rt.workers[id] = w + rt.workersMu.Unlock() + + run := func() { + // Close the session row when the goroutine terminates by any + // path (normal completion, error, kill, panic). Without this + // the chat_sessions row stays closed_at=NULL forever and + // every SpawnWorker call leaks one "looks-active" row — see + // the orphan accumulation that hit 161 rows on the test env + // before this fix landed. We use context.Background so the + // close fires even when the caller's ctx is already cancelled + // (which is the common case for the cancel-killed branch). + defer func() { + if rt.cfg.Sessions != nil && sessID != "" { + if err := rt.cfg.Sessions.CloseSession(context.Background(), sessID); err != nil && rt.log != nil { + rt.log.Warn("chatruntime: worker session close failed", + "session_id", sessID, "err", err.Error()) + } + } + }() + + w.mu.Lock() + w.Status = WorkerStatusRunning + w.StartedAt = time.Now().UTC() + w.mu.Unlock() + + result, err := rt.runWorker(workerCtx, agentDef, sessID, req.Prompt) + + w.mu.Lock() + end := time.Now().UTC() + w.EndedAt = &end + switch { + case workerCtx.Err() != nil && w.Status == WorkerStatusRunning && err != nil: + // Killed via cancel(); preserve the cancellation classification + // over a stray invoke error. + w.Status = WorkerStatusKilled + w.Err = workerCtx.Err().Error() + case err != nil: + w.Status = WorkerStatusFailed + w.Err = err.Error() + default: + w.Status = WorkerStatusCompleted + w.Result = result + } + w.mu.Unlock() + + if req.Background && req.ParentEmit != nil { + req.ParentEmit(rt.notificationFor(w)) + } + } + + if req.Background { + go run() + return rt.snapshotWorker(w), nil + } + + run() + return rt.snapshotWorker(w), nil +} + +// CountWorkersByStatus returns the live worker counts split by status. +// Used by the ADR-026 self-obs sampler ticker (cmd/ongrid/main.go) to +// keep ongrid_chatruntime_worker_sessions{status="running|pending"} +// fresh. Holds the runtime mutex briefly; safe at any interval. +func (rt *Runtime) CountWorkersByStatus() (running, pending int) { + if rt == nil { + return 0, 0 + } + rt.workersMu.Lock() + defer rt.workersMu.Unlock() + for _, w := range rt.workers { + if w == nil { + continue + } + w.mu.Lock() + switch w.Status { + case WorkerStatusRunning: + running++ + case WorkerStatusPending: + pending++ + } + w.mu.Unlock() + } + return +} + +// SendToWorker continues a worker by appending a follow-up user message. +// The worker must be running or already in a terminal state — claude-code +// allows continuing a completed worker so a coordinator can refine the +// task without a fresh spawn (figure: SendMessageTool +// targets either state). For a running worker the message is queued +// until the worker yields; for a completed worker we re-run with the +// follow-up prompt. +// +// First version: we run the follow-up synchronously in the current +// goroutine (no queueing), and disallow continuing a killed worker — +// the LLM should spawn a new one in that case. +func (rt *Runtime) SendToWorker(ctx context.Context, workerID, message string) error { + if rt == nil { + return errors.New("chatruntime: nil runtime") + } + rt.workersMu.Lock() + w := rt.workers[workerID] + rt.workersMu.Unlock() + if w == nil { + return fmt.Errorf("chatruntime: worker %q not found", workerID) + } + w.mu.Lock() + status := w.Status + agentName := w.AgentName + w.mu.Unlock() + switch status { + case WorkerStatusKilled: + return fmt.Errorf("chatruntime: worker %q is killed; spawn a fresh one", workerID) + case WorkerStatusFailed: + return fmt.Errorf("chatruntime: worker %q is failed; spawn a fresh one", workerID) + } + if rt.cfg.AgentRegistry == nil { + return errors.New("chatruntime: agent registry not wired") + } + agentDef, ok := rt.cfg.AgentRegistry.ByName(agentName) + if !ok || agentDef == nil { + return fmt.Errorf("chatruntime: agent %q vanished from registry", agentName) + } + + w.mu.Lock() + w.Status = WorkerStatusRunning + sessID := w.SessionID + w.mu.Unlock() + + result, err := rt.runWorker(ctx, agentDef, sessID, message) + + w.mu.Lock() + defer w.mu.Unlock() + end := time.Now().UTC() + w.EndedAt = &end + if err != nil { + w.Status = WorkerStatusFailed + w.Err = err.Error() + return err + } + w.Status = WorkerStatusCompleted + w.Result = result + return nil +} + +// StopWorker cancels a running worker. Returns nil even when the worker +// is already terminal — stopping is idempotent. +func (rt *Runtime) StopWorker(_ context.Context, workerID string) error { + if rt == nil { + return errors.New("chatruntime: nil runtime") + } + rt.workersMu.Lock() + w := rt.workers[workerID] + rt.workersMu.Unlock() + if w == nil { + return fmt.Errorf("chatruntime: worker %q not found", workerID) + } + w.mu.Lock() + defer w.mu.Unlock() + if w.Status == WorkerStatusRunning || w.Status == WorkerStatusPending { + if w.cancel != nil { + w.cancel() + } + // The goroutine flips Status to killed when it observes ctx.Done(). + // For pending workers there's no goroutine yet; stamp here. + if w.Status == WorkerStatusPending { + w.Status = WorkerStatusKilled + end := time.Now().UTC() + w.EndedAt = &end + } + } + return nil +} + +// GetWorker returns a snapshot copy of the worker by id. The returned +// *Worker is safe to read without locks; callers must NOT mutate it. +// (false, nil) when not found. +func (rt *Runtime) GetWorker(workerID string) (*Worker, bool) { + if rt == nil { + return nil, false + } + rt.workersMu.Lock() + w, ok := rt.workers[workerID] + rt.workersMu.Unlock() + if !ok { + return nil, false + } + return rt.snapshotWorker(w), true +} + +// runWorker is the inner spawn-and-invoke routine shared by SpawnWorker +// and SendToWorker. It builds a per-worker tool bag (whitelist + black- +// list filtered against the runtime's tool bag), composes the worker's +// system prompt, and drives one graph.Invoke against the agent's +// configured Model + MaxTurns. +// +// sessID is the worker's own chat_sessions row id. It is threaded into +// the persistence callback so worker assistant + tool messages are +// written into chat_messages under the worker's session (not the +// coordinator's). The user-role prompt is persisted up-front here, +// matching Handle()'s "user message lands on disk before the LLM call" +// invariant — same survival semantics on a graph crash. +func (rt *Runtime) runWorker(ctx context.Context, agentDef *Agent, sessID, userText string) (string, error) { + workerTools := filterToolsForAgent(rt.cfg.ToolBag, agentDef, false) + + systemPrompt := ComposeSystemPrompt(rt.cfg.BasePrompt, nil, agentDef) + + // KB-first prologue. Weak coordinator models (GLM-4 etc) don't + // reliably follow "rule 0 — query_knowledge before any other + // tool" written in the persona prompt; we observed specialists + // jumping straight to host_du_summary / query_promql. So when + // the persona declares query_knowledge in its Tools whitelist, + // we call it ONCE here with the user task as the query and + // prepend the top match (if score ≥ 0.6) to userText as + // explicit context. The worker LLM then sees the KB material as + // part of its input — no choice required. + // + // Cost: one extra qdrant query per worker spawn (~50ms typical), + // saves an entire ReAct turn when the rule would have been + // followed, and forces RAG-first behavior when it wouldn't have. + if hasTool(workerTools, "query_knowledge") { + if kbCtx := rt.prologueKBLookup(ctx, workerTools, userText); kbCtx != "" { + userText = kbCtx + "\n\n---\n\n用户原始任务：\n" + userText + } + } + + cfg := rt.cfg.GraphCfg + if agentDef.MaxTurns > 0 { + cfg.MaxIterations = agentDef.MaxTurns + } + if agentDef.Model != "" { + cfg.Model = agentDef.Model + } + + g, err := graph.BuildReActGraph(rt.cfg.ChatModel, workerTools, cfg) + if err != nil { + return "", fmt.Errorf("chatruntime: build worker graph: %w", err) + } + + // Persist the user-role prompt under the worker's session id. Same + // invariant Handle() honours on the coordinator path — the user turn + // lives on disk before the graph runs so a crash mid-invoke leaves + // an auditable trace. + var handlers []einocallbacks.Handler + if rt.cfg.Sessions != nil && sessID != "" { + content := userText + userMsg := &aiopsmodel.Message{ + SessionID: sessID, + Role: aiopsmodel.RoleUser, + Content: &content, + CreatedAt: time.Now().UTC(), + } + if err := rt.cfg.Sessions.AppendMessage(ctx, userMsg); err != nil { + return "", fmt.Errorf("chatruntime: persist worker user msg: %w", err) + } + + // Wire the per-worker persistence callback. Reuse the runtime's + // CallbackDeps for cross-cutting context (logger, registerer) + // but override Persistence.SessionID so writes land on the + // worker row. SSE / Audit / Metrics inherit from the runtime + // config (Audit + Metrics are session-id-aware via their own + // deps; we keep the worker's session id flowing into them too). + deps := rt.cfg.CallbackDeps + deps.Persistence.SessionID = sessID + if deps.Persistence.Repo == nil { + deps.Persistence.Repo = rt.cfg.Sessions + } + // Workers don't stream to the parent SSE channel directly; their + // terminal status fans back through req.ParentEmit's task_notification. + deps.SSE = nil + handlers = callbacks.NewDefaultHandlers(deps) + } + + out, err := g.Invoke(ctx, &graph.Input{ + SystemPrompt: systemPrompt, + History: nil, + UserText: userText, + }, compose.WithCallbacks(handlers...)) + if err != nil { + return "", err + } + if out == nil || out.AssistantMessage == nil { + return "", nil + } + return out.AssistantMessage.Content, nil +} + +// notificationFor produces a task_notification Event for the given +// terminal worker. Field shape mirrors +func (rt *Runtime) notificationFor(w *Worker) Event { + w.mu.Lock() + defer w.mu.Unlock() + summary := fmt.Sprintf("Agent %s %s", w.AgentName, w.Status) + usage := map[string]any{} + if !w.StartedAt.IsZero() && w.EndedAt != nil { + usage["duration_ms"] = w.EndedAt.Sub(w.StartedAt).Milliseconds() + } + notif := &TaskNotification{ + TaskID: w.ID, + Status: w.Status, + Summary: summary, + Result: w.Result, + Err: w.Err, + Usage: usage, + } + return Event{Type: EventTaskNotification, Notification: notif} +} + +// coordinatorOnlyTools is the implicit blacklist applied to every worker +// in this first-version implementation: the control tools live on +// the coordinator only. Even if a worker persona accidentally whitelists +// "AgentTool" we still strip it here, so workers cannot spawn nested +// workers / send messages / stop tasks. +var coordinatorOnlyTools = map[string]bool{ + "AgentTool": true, + "SendMessage": true, + "TaskStop": true, +} + +// filterToolsForAgent applies the agent persona's whitelist + blacklist +// against the runtime tool bag. Black wins : +// +// - if Tools is empty, every tool is initially allowed (inherit from +// policy); otherwise only tools whose Info().Name appears in the +// whitelist are allowed. +// - DisallowedTools is then subtracted. Pattern "*_skill" is a +// suffix wildcard supported for ergonomic blacklists ( +// example). +// - The control tools (AgentTool / SendMessage / TaskStop) are +// stripped only for workers (isCoordinator=false) — workers cannot +// spawn nested workers in this first version. Coordinator passes +// isCoordinator=true so AgentTool stays visible even when the +// coordinator persona's Tools whitelist is non-empty. +// - viewerOnly==true drops every tool whose Info().Class is +// not "read". Viewer chat is degraded to single-agent read-only, +// which is fine — the coordinator can still answer with read tools +// directly (no need to dispatch since there's nothing mutating to do). +func filterToolsForAgent(bag []basetool.BaseTool, agentDef *Agent, isCoordinator bool) []basetool.BaseTool { + return filterToolsForAgentRole(bag, agentDef, isCoordinator, false) +} + +func filterToolsForAgentRole(bag []basetool.BaseTool, agentDef *Agent, isCoordinator bool, viewerOnly bool) []basetool.BaseTool { + whitelist := map[string]bool{} + blacklist := []string(nil) + if agentDef != nil { + for _, n := range agentDef.Tools { + whitelist[strings.TrimSpace(n)] = true + } + blacklist = agentDef.DisallowedTools + } + + out := make([]basetool.BaseTool, 0, len(bag)) + for _, t := range bag { + if t == nil { + continue + } + info, err := t.Info(context.Background()) + if err != nil || info == nil { + continue + } + // Viewer downgrade: drop any tool that is not Class="read". + // Applies BEFORE persona / control-tool checks so that even the + // coordinator chat for a viewer cannot dispatch (AgentTool is + // Class="write") or mutate (host_restart_service, etc.). + if viewerOnly && info.Class != "read" { + continue + } + // Coordinator-only tools survive the strip when we ARE the + // coordinator. They're always in scope for the coordinator + // regardless of any persona whitelist — control plane. + if coordinatorOnlyTools[info.Name] { + if !isCoordinator { + continue + } + } else if agentDef != nil && len(whitelist) > 0 && !whitelist[info.Name] { + continue + } + if matchesAny(info.Name, blacklist) { + continue + } + out = append(out, t) + } + return out +} + +// matchesAny returns true when name matches any pattern in patterns. +// Patterns: exact name, or "*" / "*" wildcard. +func matchesAny(name string, patterns []string) bool { + for _, p := range patterns { + p = strings.TrimSpace(p) + if p == "" { + continue + } + switch { + case p == name: + return true + case strings.HasPrefix(p, "*") && strings.HasSuffix(name, p[1:]): + return true + case strings.HasSuffix(p, "*") && strings.HasPrefix(name, p[:len(p)-1]): + return true + } + } + return false +} + +// snapshotWorker returns a copy of w safe to publish to a caller. The +// internal lock + cancel func are not copied — the returned *Worker is +// strictly read-only. +func (rt *Runtime) snapshotWorker(w *Worker) *Worker { + if w == nil { + return nil + } + w.mu.Lock() + defer w.mu.Unlock() + cp := &Worker{ + ID: w.ID, + AgentName: w.AgentName, + SessionID: w.SessionID, + ParentSessionID: w.ParentSessionID, + Prompt: w.Prompt, + Status: w.Status, + Background: w.Background, + StartedAt: w.StartedAt, + Result: w.Result, + Err: w.Err, + } + if w.EndedAt != nil { + t := *w.EndedAt + cp.EndedAt = &t + } + return cp +} + +// newWorkerID returns a fresh "agent-<8hex>" identifier. crypto/rand +// keeps collisions astronomically unlikely; callers don't need to retry. +func newWorkerID() string { + var b [4]byte + if _, err := rand.Read(b[:]); err != nil { + // crypto/rand failure is treated as fatal in the Go stdlib + // design; fall back to a timestamp-derived id to avoid panic. + return fmt.Sprintf("agent-%08x", time.Now().UnixNano()&0xffffffff) + } + return "agent-" + hex.EncodeToString(b[:]) +} + +// newWorkerSessionID derives a session id from the worker id. Distinct +// from the parent session so worker chat history can be persisted +// independently. The "worker-<8hex>" shape mirrors the worker id's +// hex tail and stays under chat_sessions.id (char(36)) — short, prefix- +// searchable, and visually distinct from coordinator UUIDs. +func newWorkerSessionID(workerID string) string { + // workerID is "agent-<8hex>"; rebrand the prefix to "worker-" so + // the chat_sessions row id is self-describing in audit dumps. + if strings.HasPrefix(workerID, "agent-") { + return "worker-" + workerID[len("agent-"):] + } + return "worker-" + workerID +} + +// hasTool reports whether `name` is present in the BaseTool slice. Used +// by runWorker to decide whether to run the KB prologue. +func hasTool(bag []basetool.BaseTool, name string) bool { + for _, t := range bag { + if t == nil { + continue + } + info, err := t.Info(context.Background()) + if err != nil || info == nil { + continue + } + if info.Name == name { + return true + } + } + return false +} + +// prologueKBLookup runs `query_knowledge` once with the user's task as +// the natural-language query and, when the top hit has score ≥ 0.6, +// returns a formatted KB context block ready to prepend to userText. +// +// All failure modes (tool missing, decode error, no hits, low score) +// return "" — the caller falls back to the bare user prompt. This is +// deliberately silent: the LLM should never see a "KB lookup failed" +// message, just sometimes-richer context. +func (rt *Runtime) prologueKBLookup(ctx context.Context, bag []basetool.BaseTool, userText string) string { + var kb basetool.BaseTool + for _, t := range bag { + if t == nil { + continue + } + info, err := t.Info(context.Background()) + if err != nil || info == nil || info.Name != "query_knowledge" { + continue + } + kb = t + break + } + if kb == nil { + return "" + } + // Schema across versions has used both "query" and "q"; send + // "query" and let the tool ignore extras. + args, _ := json.Marshal(map[string]any{ + "query": userText, + "top_k": 3, + "min_score": 0.6, + }) + out, err := kb.InvokableRun(ctx, string(args)) + if err != nil || out == "" { + return "" + } + // Parse loosely — the response shape is {"items":[{"title":..,"preview":..,"score":..}, ...]} + var parsed struct { + Items []struct { + Title string `json:"title"` + Preview string `json:"preview"` + Path string `json:"path"` + Score float64 `json:"score"` + URL string `json:"url"` + } `json:"items"` + } + if jerr := json.Unmarshal([]byte(out), &parsed); jerr != nil { + return "" + } + if len(parsed.Items) == 0 { + return "" + } + top := parsed.Items[0] + if top.Score < 0.6 { + return "" + } + var b strings.Builder + b.WriteString("[KB 自动注入 · 协调员认为这是相关 playbook]\n") + b.WriteString("Title: ") + b.WriteString(top.Title) + b.WriteString(fmt.Sprintf(" (score=%.2f)\n", top.Score)) + if top.Path != "" { + b.WriteString("Path: ") + b.WriteString(top.Path) + b.WriteString("\n") + } + b.WriteString("\n") + b.WriteString(top.Preview) + if top.URL != "" { + b.WriteString("\n\nSource: ") + b.WriteString(top.URL) + } + b.WriteString("\n\n请基于这份 playbook 推进诊断；末尾在答案里标注 `（参考 KB: " + top.Title + "）`。如果 playbook 不适用，可以忽略并自由发挥。") + return b.String() +} diff --git a/internal/manager/biz/aiops/graph/callbacks/audit.go b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/audit.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/audit.go rename to internal/manager/biz/aiops/runtime/v1/graph/callbacks/audit.go diff --git a/internal/manager/biz/aiops/graph/callbacks/audit_test.go b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/audit_test.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/audit_test.go rename to internal/manager/biz/aiops/runtime/v1/graph/callbacks/audit_test.go diff --git a/internal/manager/biz/aiops/runtime/v1/graph/callbacks/chain.go b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/chain.go new file mode 100644 index 000000000..2c3478990 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/chain.go @@ -0,0 +1,134 @@ +package callbacks + +import ( + "errors" + "sync/atomic" + + "github.com/cloudwego/eino/callbacks" + "github.com/prometheus/client_golang/prometheus" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// assistantIDRelay is the shared slot the PersistenceHandler writes +// the freshly-persisted chat_messages.id into; SSEHandler reads it +// inside OnEnd to attach to the assistant_end SSE frame. Both handlers +// run on the same goroutine for a given graph iteration (persistence +// is registered before SSE in NewDefaultHandlers and eino preserves +// registration order on OnEnd), so a plain string with atomic Load/ +// Store is sufficient — no race between writer and reader within an +// iteration. +// +// Frontend ChatThread used to dedupe by a synthetic assistant-iter-N +// id (v0.7.63 hotfix) because this field was empty. Now it can use +// the real DB id. +type assistantIDRelay struct { + id atomic.Pointer[string] +} + +func (r *assistantIDRelay) store(id string) { + if r == nil { + return + } + r.id.Store(&id) +} + +func (r *assistantIDRelay) load() string { + if r == nil { + return "" + } + if p := r.id.Load(); p != nil { + return *p + } + return "" +} + +// Deps bundles every dependency the default callback chain needs. +// Each handler treats nil dependencies as "skip me" — pass partial +// Deps to wire only some handlers (e.g. tests that only want metrics). +// +// Reference: Callback 链 + 主参考图 callback 链. +type Deps struct { + // Persistence wiring (chat_messages / chat_tool_calls writes). + Persistence PersistenceDeps + + // SSE emitter (when nil, no SSE handler is wired). Cutover layer + // builds an emitter that pipes into the existing http.go writeSSE. + SSE SSEEmitter + + // Audit (slog INFO records). Logger required. + Audit AuditDeps + + // Metrics (Prom counters). Registerer required. + Metrics MetricsDeps + + // Budget gate (PR-1). Both fields required to wire it. + BudgetChecker llm.BudgetChecker + BudgetUserID uint64 +} + +// NewDefaultHandlers builds the default ordered callback chain for an +// agent graph run. Order is incidental — eino does not document a +// guaranteed handler ordering — but the slice is stable across calls +// so tests can assert positional indices. +// +// Handlers with nil dependencies are skipped, so passing a partial +// Deps yields a sparser chain. The PR-1 BudgetCallbackHandler is +// included whenever Deps.BudgetChecker is non-nil. +// +// Cutover layer (NEXT PR) calls this once per request, threads the +// returned slice into compose.WithCallbacks at Invoke time, and +// discards the handlers when the request finishes (so per-call state +// is bounded by the request lifetime). +func NewDefaultHandlers(deps Deps) []callbacks.Handler { + out := make([]callbacks.Handler, 0, 5) + + // Shared relay so Persistence (runs first) can hand the freshly- + // written assistant row id to SSE (runs second) for the + // assistant_end frame. + relay := &assistantIDRelay{} + + if h := NewPersistenceHandler(deps.Persistence); h != nil { + h.assistantIDRelay = relay + out = append(out, h) + } + if h := NewSSEHandler(deps.SSE); h != nil { + h.assistantIDRelay = relay + out = append(out, h) + } + if h := NewAuditHandler(deps.Audit); h != nil { + out = append(out, h) + } + if h := NewMetricsHandler(deps.Metrics); h != nil { + out = append(out, h) + } + if deps.BudgetChecker != nil { + out = append(out, llm.NewBudgetCallbackHandler(deps.BudgetChecker, deps.BudgetUserID)) + } + return out +} + +// ErrNoHandlers is returned by helpers that require at least one +// handler in the chain. Reserved for future use; PR-6 callers tolerate +// an empty slice. +var ErrNoHandlers = errors.New("graph/callbacks: no handlers configured") + +// registerOrExisting is the package-private register helper. Mirrors +// llm/metrics.go's registerOrExisting + tools/decorators/metric.go's +// regOrExist; lives here so persistence.go can use it without +// triggering an import cycle (both packages are siblings of llm but +// llm/metrics.go's helper is unexported). +func registerOrExisting(reg prometheus.Registerer, c prometheus.Collector) prometheus.Collector { + if reg == nil { + reg = prometheus.DefaultRegisterer + } + err := reg.Register(c) + if err == nil { + return c + } + var are prometheus.AlreadyRegisteredError + if errors.As(err, &are) { + return are.ExistingCollector + } + panic(err) +} diff --git a/internal/manager/biz/aiops/graph/callbacks/chain_test.go b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/chain_test.go similarity index 90% rename from internal/manager/biz/aiops/graph/callbacks/chain_test.go rename to internal/manager/biz/aiops/runtime/v1/graph/callbacks/chain_test.go index 0b6f2ab7b..7aadee069 100644 --- a/internal/manager/biz/aiops/graph/callbacks/chain_test.go +++ b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/chain_test.go @@ -16,10 +16,10 @@ func TestNewDefaultHandlers_AllWiredEmitsFive(t *testing.T) { repo := newFakeSessionRepo() reg := prometheus.NewRegistry() deps := Deps{ - Persistence: PersistenceDeps{SessionID: "s", Repo: repo, Registerer: reg}, - SSE: func(SSEEvent) {}, - Audit: AuditDeps{Logger: slog.Default(), SessionID: "s"}, - Metrics: MetricsDeps{Registerer: reg}, + Persistence: PersistenceDeps{SessionID: "s", Repo: repo, Registerer: reg}, + SSE: func(SSEEvent) {}, + Audit: AuditDeps{Logger: slog.Default(), SessionID: "s"}, + Metrics: MetricsDeps{Registerer: reg}, BudgetChecker: stubBudgetChecker{}, BudgetUserID: 1, } diff --git a/internal/manager/biz/aiops/graph/callbacks/metrics.go b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/metrics.go similarity index 99% rename from internal/manager/biz/aiops/graph/callbacks/metrics.go rename to internal/manager/biz/aiops/runtime/v1/graph/callbacks/metrics.go index 01736ea52..738f093d7 100644 --- a/internal/manager/biz/aiops/graph/callbacks/metrics.go +++ b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/metrics.go @@ -120,7 +120,7 @@ func regOrExist(reg prometheus.Registerer, c prometheus.Collector) prometheus.Co type MetricsHandler struct { collectors *metricsCollectors - chatTurns atomic.Int64 + chatTurns atomic.Int64 toolStartsMu sync.Mutex toolStarts map[string]time.Time diff --git a/internal/manager/biz/aiops/graph/callbacks/metrics_test.go b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/metrics_test.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/metrics_test.go rename to internal/manager/biz/aiops/runtime/v1/graph/callbacks/metrics_test.go diff --git a/internal/manager/biz/aiops/runtime/v1/graph/callbacks/persistence.go b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/persistence.go new file mode 100644 index 000000000..575fc8245 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/persistence.go @@ -0,0 +1,435 @@ +// Package callbacks bundles the eino callbacks.Handler implementations +// that wire ongrid cross-cutting concerns into the new compose.Graph +// agent kernel: persistence, SSE streaming, audit, metrics, and +// (carried over from PR-1) budget gating. The handler set is documented +// in (Callback 链) + (主参考图 — callback chain +// 横切区块). +// +// Each file in this package owns exactly one handler; chain.go combines +// them into a default list. Handlers are wired at Invoke time via +// compose.WithCallbacks (see graph.BuildReActGraph header comment) — no +// global registration is performed by these constructors so a graph +// run cannot accidentally pick up handlers from another tenant or +// session. PR-6 of scaffolding only — the cutover layer (NEXT +// PR) will assemble Deps and pass the chain through compose.Invoke. +package callbacks + +import ( + "context" + "errors" + "log/slog" + "sync" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" + "github.com/prometheus/client_golang/prometheus" + + biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" +) + +// PersistenceDeps bundles the persistence handler's collaborators. +// SessionID identifies which chat row to write into. Repo is the +// SessionRepo binding (— write chat_messages and +// chat_tool_calls). Logger is optional — used to record best-effort +// persistence failures so they don't fail user-visible requests but +// stay observable. Registerer is optional — when set, the handler +// increments ongrid_persist_errors_total{kind=...} for any persist +// failure (错误处理 — persist 失败不阻断 graph). +type PersistenceDeps struct { + SessionID string + Repo biz.SessionRepo + Logger *slog.Logger + Registerer prometheus.Registerer +} + +// PersistenceHandler writes chat_messages + chat_tool_calls rows as +// the graph executes. Mirrors the persistence side-effects the legacy +// agent.go for-loop performs synchronously; spec: +// +// OnChatModelEnd → INSERT chat_messages (role=assistant) +// OnToolStart → INSERT chat_tool_calls (status=pending) +// OnToolEnd → UPDATE chat_tool_calls + INSERT chat_messages (role=tool) +// +// User-message persistence stays at the chatruntime entry point (as +// agent.go does today) — by the time the graph starts, the user's +// turn is already on disk. This handler is only responsible for the +// agent's own writes. +// +// Concurrency: handler instances are designed for ONE graph run each. +// The cutover layer constructs a fresh PersistenceHandler per request +// so the SessionID + per-call state stay scoped. Tool start times are +// stashed on context (per-call) which is goroutine-safe. +// +// Errors: persist failures NEVER abort the graph — the handler logs + +// counts and returns ctx unchanged. 可观测性: audit/ +// persist best-effort. +type PersistenceHandler struct { + deps PersistenceDeps + + // errCounter is the lazy collector for persist failures. Resolved + // at construction; nil when Registerer is nil. + errCounter *prometheus.CounterVec + + // toolCalls maps eino tool_call_id → the chat_tool_calls row id we + // inserted on OnStart, so OnEnd can update the same row. eino + // guarantees the tool_call_id is unique within a graph run; we + // scope the map to this handler instance, not globally. + toolCalls map[string]toolCallEntry + toolCallsMu sync.Mutex + + // asstMu protects iteration-counting state used by OnEnd to + // distinguish the terminal assistant turn from intermediate ones. + asstMu sync.Mutex + asstWrites int + + // assistantIDRelay (optional) is the cross-handler share with + // SSEHandler — see chain.go. Set by NewDefaultHandlers; nil when + // tests use NewPersistenceHandler directly. + assistantIDRelay *assistantIDRelay +} + +// toolCallEntry tracks per-call state across OnStart → OnEnd. +type toolCallEntry struct { + rowID string + startedAt time.Time + toolName string + argsJSON string + toolCallID string // eino's tool_call_id (== ChatGPT-shape call id) +} + +// NewPersistenceHandler builds the handler. Returns nil if SessionID +// or Repo is missing — the cutover layer treats nil as "persistence +// disabled" (matching the existing optional-handler pattern). +func NewPersistenceHandler(deps PersistenceDeps) *PersistenceHandler { + if deps.SessionID == "" || deps.Repo == nil { + return nil + } + h := &PersistenceHandler{ + deps: deps, + toolCalls: make(map[string]toolCallEntry), + } + if deps.Registerer != nil { + h.errCounter = registerOrExisting(deps.Registerer, prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_persist_errors_total", + Help: "Total ongrid AIOps persistence failures (chat_messages / chat_tool_calls writes).", + }, + []string{"kind"}, + )).(*prometheus.CounterVec) + } + return h +} + +// Needed signals which timings to wire so eino can short-circuit the +// expensive ones we don't use. +func (h *PersistenceHandler) Needed(_ context.Context, info *callbacks.RunInfo, timing callbacks.CallbackTiming) bool { + if h == nil { + return false + } + if info == nil { + return false + } + switch info.Component { + case components.ComponentOfChatModel: + return timing == callbacks.TimingOnEnd + case components.ComponentOfTool: + return timing == callbacks.TimingOnStart || timing == callbacks.TimingOnEnd || timing == callbacks.TimingOnError + default: + return false + } +} + +// OnStart is the hook fired before a component runs. For PersistenceHandler: +// - ChatModel start: no-op (user message is persisted by chatruntime +// entry, before the graph starts; spec). +// - Tool start: write a chat_tool_calls row in pending state and +// stash the row id on the handler so OnEnd can find it. +func (h *PersistenceHandler) OnStart(ctx context.Context, info *callbacks.RunInfo, input callbacks.CallbackInput) context.Context { + if h == nil || info == nil { + return ctx + } + if info.Component != components.ComponentOfTool { + return ctx + } + tin := einotool.ConvCallbackInput(input) + if tin == nil { + return ctx + } + startedAt := time.Now().UTC() + row := &model.ToolCall{ + // MessageID is filled by chatruntime when it appends the assistant + // row; for PR-6 we leave it empty and rely on the cutover layer + // to provide a per-turn message id via context. The legacy + // agent.go also stores tool_calls under the assistant message id; + // spec preserves that. + MessageID: messageIDFromCtx(ctx), + ToolName: info.Name, + ArgumentsJSON: tin.ArgumentsInJSON, + Status: model.StatusPending, + StartedAt: startedAt, + CreatedAt: startedAt, + } + if err := h.deps.Repo.CreateToolCall(ctx, row); err != nil { + h.recordErr("tool_call_insert", err) + return ctx + } + h.toolCallsMu.Lock() + h.toolCalls[toolCallIDFromCtx(ctx, info)] = toolCallEntry{ + rowID: row.ID, + startedAt: startedAt, + toolName: info.Name, + argsJSON: tin.ArgumentsInJSON, + toolCallID: toolCallIDFromCtx(ctx, info), + } + h.toolCallsMu.Unlock() + return ctx +} + +// OnEnd is the hook fired after a component succeeds. +// - ChatModel end: append a chat_messages row (role=assistant) with +// the model's content + token usage. +// - Tool end: update the chat_tool_calls row with status=success + +// the result JSON, AND insert a chat_messages row (role=tool) so +// history replay sees the tool result on the next turn. +func (h *PersistenceHandler) OnEnd(ctx context.Context, info *callbacks.RunInfo, output callbacks.CallbackOutput) context.Context { + if h == nil || info == nil { + return ctx + } + switch info.Component { + case components.ComponentOfChatModel: + mo := einomodel.ConvCallbackOutput(output) + if mo == nil || mo.Message == nil { + return ctx + } + h.persistAssistant(ctx, mo) + case components.ComponentOfTool: + tout := einotool.ConvCallbackOutput(output) + if tout == nil { + return ctx + } + h.persistToolEnd(ctx, info, tout, nil) + } + return ctx +} + +// OnError fires when a component returns a non-nil error. For tool +// errors we still update the chat_tool_calls row to status=error so +// the audit trail captures the failure; ChatModel errors are recorded +// by the audit handler (this layer doesn't produce a chat row for +// them — there's no message to persist). +func (h *PersistenceHandler) OnError(ctx context.Context, info *callbacks.RunInfo, err error) context.Context { + if h == nil || info == nil || err == nil { + return ctx + } + if info.Component != components.ComponentOfTool { + return ctx + } + h.persistToolEnd(ctx, info, nil, err) + return ctx +} + +// OnStartWithStreamInput is a no-op (we only consume the eventual end value). +func (h *PersistenceHandler) OnStartWithStreamInput(ctx context.Context, _ *callbacks.RunInfo, in *schema.StreamReader[callbacks.CallbackInput]) context.Context { + if in != nil { + in.Close() + } + return ctx +} + +// OnEndWithStreamOutput is a no-op for PR-6. token-level streaming +// persistence (writing the final assembled message at stream-end) is +// owned by the cutover layer in PR-7; for now we drain + close so we +// don't leak goroutines. +func (h *PersistenceHandler) OnEndWithStreamOutput(ctx context.Context, _ *callbacks.RunInfo, out *schema.StreamReader[callbacks.CallbackOutput]) context.Context { + if out != nil { + out.Close() + } + return ctx +} + +func (h *PersistenceHandler) persistAssistant(ctx context.Context, mo *einomodel.CallbackOutput) { + msg := mo.Message + row := &model.Message{ + SessionID: h.deps.SessionID, + Role: string(msg.Role), + CreatedAt: time.Now().UTC(), + } + if msg.Content != "" { + c := msg.Content + row.Content = &c + } + if mo.TokenUsage != nil { + pt := mo.TokenUsage.PromptTokens + ct := mo.TokenUsage.CompletionTokens + row.PromptTokens = &pt + row.CompletionTokens = &ct + } else if msg.ResponseMeta != nil && msg.ResponseMeta.Usage != nil { + pt := msg.ResponseMeta.Usage.PromptTokens + ct := msg.ResponseMeta.Usage.CompletionTokens + row.PromptTokens = &pt + row.CompletionTokens = &ct + } + if err := h.deps.Repo.AppendMessage(ctx, row); err != nil { + h.recordErr("assistant_insert", err) + return + } + // Hand the persisted row id to SSEHandler so the assistant_end + // frame can ship it. AppendMessage stamps row.ID via gorm's auto- + // generated string id (see model.Message ID field). + h.assistantIDRelay.store(row.ID) + h.asstMu.Lock() + h.asstWrites++ + h.asstMu.Unlock() +} + +func (h *PersistenceHandler) persistToolEnd(ctx context.Context, info *callbacks.RunInfo, tout *einotool.CallbackOutput, execErr error) { + tcID := toolCallIDFromCtx(ctx, info) + h.toolCallsMu.Lock() + entry, ok := h.toolCalls[tcID] + delete(h.toolCalls, tcID) + h.toolCallsMu.Unlock() + if !ok { + // No matching OnStart — chatruntime didn't fire it (e.g. + // graph injection edge case). Skip the update so we don't + // orphan a tool_calls row. + return + } + + endedAt := time.Now().UTC() + status := model.StatusSuccess + var errStr *string + var resultPtr *string + if execErr != nil { + s := execErr.Error() + errStr = &s + status = model.StatusError + if errors.Is(execErr, context.DeadlineExceeded) { + status = model.StatusTimeout + } + } else if tout != nil { + s := tout.Response + resultPtr = &s + } + + if err := h.deps.Repo.UpdateToolCallResult(ctx, entry.rowID, status, resultPtr, errStr, endedAt); err != nil { + h.recordErr("tool_call_update", err) + } + + // Append a role=tool message so history replay sees the tool + // result. agent.go's existing behaviour — preserved by + // 数据层 invariants. + tname := entry.toolName + tcallID := entry.toolCallID + body := "" + if resultPtr != nil { + body = *resultPtr + } else if errStr != nil { + body = `{"error":"` + *errStr + `"}` + } else { + body = `{}` + } + row := &model.Message{ + SessionID: h.deps.SessionID, + Role: model.RoleTool, + Content: &body, + ToolCallID: &tcallID, + ToolName: &tname, + CreatedAt: endedAt, + } + if err := h.deps.Repo.AppendMessage(ctx, row); err != nil { + h.recordErr("tool_msg_insert", err) + } +} + +func (h *PersistenceHandler) recordErr(kind string, err error) { + if h.deps.Logger != nil { + h.deps.Logger.Warn("persistence handler write failed", + slog.String("kind", kind), + slog.String("session_id", h.deps.SessionID), + slog.String("error", err.Error())) + } + if h.errCounter != nil { + h.errCounter.WithLabelValues(kind).Inc() + } +} + +// AssistantWriteCount reports how many assistant rows this handler +// has persisted. Exposed for tests; production callers should not +// depend on it. +func (h *PersistenceHandler) AssistantWriteCount() int { + if h == nil { + return 0 + } + h.asstMu.Lock() + defer h.asstMu.Unlock() + return h.asstWrites +} + +// Compile-time assertion. +var ( + _ callbacks.Handler = (*PersistenceHandler)(nil) + _ callbacks.TimingChecker = (*PersistenceHandler)(nil) +) + +// --- ctx helpers --------------------------------------------------------- + +// messageIDCtxKey is the context key carrying the assistant chat_messages +// row id under which tool calls should be parented. The cutover layer +// will set this; PR-6 leaves the field zero-valued when the key is +// absent (the SQL schema permits NULL message_id during the migration +// window — chatruntime will backfill). +type messageIDCtxKey struct{} + +// toolCallIDCtxKey lets the chatruntime + assembler pass eino's +// tool_call_id (the ChatGPT-style call id) through to the persistence +// handler so OnStart and OnEnd correlate via the same key. +type toolCallIDCtxKey struct{} + +// WithMessageID returns ctx with mid stashed as the parent assistant +// row id. Used by the cutover layer to link chat_tool_calls rows back +// to their owning assistant turn. +func WithMessageID(ctx context.Context, mid string) context.Context { + if mid == "" { + return ctx + } + return context.WithValue(ctx, messageIDCtxKey{}, mid) +} + +func messageIDFromCtx(ctx context.Context) string { + if v, ok := ctx.Value(messageIDCtxKey{}).(string); ok { + return v + } + return "" +} + +// WithToolCallID returns ctx with the eino tool_call_id stashed so +// PersistenceHandler.OnStart / OnEnd can correlate using a stable key. +// In production wiring eino sets the per-call id internally; for tests +// we expose this seam so handlers can be exercised in isolation. +func WithToolCallID(ctx context.Context, id string) context.Context { + if id == "" { + return ctx + } + return context.WithValue(ctx, toolCallIDCtxKey{}, id) +} + +// toolCallIDFromCtx prefers an explicit context value; otherwise it +// falls back to a tuple of (run-info name, run-info type) which is +// stable for the duration of a single tool invocation. eino guarantees +// the same RunInfo pointer is reused across OnStart/OnEnd of one call, +// but inspecting it via Type+Name keeps us decoupled from internal +// pointer identity. +func toolCallIDFromCtx(ctx context.Context, info *callbacks.RunInfo) string { + if v, ok := ctx.Value(toolCallIDCtxKey{}).(string); ok && v != "" { + return v + } + if info == nil { + return "" + } + return info.Name + "|" + info.Type +} diff --git a/internal/manager/biz/aiops/runtime/v1/graph/callbacks/persistence_test.go b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/persistence_test.go new file mode 100644 index 000000000..bf9fa7cd0 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/persistence_test.go @@ -0,0 +1,300 @@ +package callbacks + +import ( + "context" + "errors" + "strconv" + "sync" + "testing" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" + "github.com/prometheus/client_golang/prometheus" + + biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +// fakeSessionRepo is a goroutine-safe in-memory SessionRepo used by the +// persistence handler tests. Only the methods PersistenceHandler calls +// are exercised; the rest are no-ops to satisfy the interface. +type fakeSessionRepo struct { + mu sync.Mutex + messages []*model.Message + toolCalls map[string]*model.ToolCall + nextID int + + failAppend bool + failCreate bool + failUpdate bool +} + +func newFakeSessionRepo() *fakeSessionRepo { + return &fakeSessionRepo{toolCalls: map[string]*model.ToolCall{}} +} + +func (r *fakeSessionRepo) CreateSession(context.Context, *model.Session) error { + return errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) GetSession(context.Context, string) (*model.Session, error) { + return nil, errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) ListSessions(context.Context, uint64, int, int, *uint64) ([]*model.Session, error) { + return nil, errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) ListByParent(context.Context, string) ([]*model.Session, error) { + return nil, errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) RenameSession(context.Context, string, string) error { return nil } +func (r *fakeSessionRepo) CloseSession(context.Context, string) error { + return errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) DeleteSession(context.Context, string) error { + return errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) AppendMessage(_ context.Context, m *model.Message) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.failAppend { + return errors.New("inject: append") + } + r.nextID++ + if m.ID == "" { + m.ID = "msg-" + strconv.Itoa(r.nextID) + } + cp := *m + r.messages = append(r.messages, &cp) + return nil +} +func (r *fakeSessionRepo) ListMessages(context.Context, string, int) ([]*model.Message, error) { + return nil, errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) CreateToolCall(_ context.Context, tc *model.ToolCall) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.failCreate { + return errors.New("inject: create") + } + r.nextID++ + if tc.ID == "" { + tc.ID = "tc-" + strconv.Itoa(r.nextID) + } + cp := *tc + r.toolCalls[tc.ID] = &cp + return nil +} +func (r *fakeSessionRepo) UpdateToolCallResult(_ context.Context, id, status string, resultJSON, errStr *string, endedAt time.Time) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.failUpdate { + return errors.New("inject: update") + } + tc, ok := r.toolCalls[id] + if !ok { + return errs.ErrNotFound + } + tc.Status = status + tc.ResultJSON = resultJSON + tc.Error = errStr + tc.EndedAt = &endedAt + return nil +} +func (r *fakeSessionRepo) SumTokensSince(context.Context, time.Time) (biz.TokenSums, error) { + return biz.TokenSums{}, nil +} + +var _ biz.SessionRepo = (*fakeSessionRepo)(nil) + +func chatModelInfo() *callbacks.RunInfo { + return &callbacks.RunInfo{Name: "ChatModel", Type: "Test", Component: components.ComponentOfChatModel} +} + +func toolInfo(name string) *callbacks.RunInfo { + return &callbacks.RunInfo{Name: name, Type: "Test", Component: components.ComponentOfTool} +} + +func TestPersistenceHandler_NewNilDeps(t *testing.T) { + t.Parallel() + if NewPersistenceHandler(PersistenceDeps{}) != nil { + t.Fatalf("nil session id should yield nil handler") + } + if NewPersistenceHandler(PersistenceDeps{SessionID: "s"}) != nil { + t.Fatalf("nil repo should yield nil handler") + } +} + +func TestPersistenceHandler_AssistantWriteOnEnd(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "sess-1", Repo: repo}) + if h == nil { + t.Fatalf("handler is nil") + } + out := &einomodel.CallbackOutput{ + Message: &schema.Message{Role: schema.Assistant, Content: "hello"}, + TokenUsage: &einomodel.TokenUsage{ + PromptTokens: 5, + CompletionTokens: 3, + TotalTokens: 8, + }, + } + h.OnEnd(context.Background(), chatModelInfo(), out) + if got := h.AssistantWriteCount(); got != 1 { + t.Errorf("assistant writes = %d, want 1", got) + } + repo.mu.Lock() + defer repo.mu.Unlock() + if len(repo.messages) != 1 { + t.Fatalf("messages persisted = %d, want 1", len(repo.messages)) + } + row := repo.messages[0] + if row.SessionID != "sess-1" { + t.Errorf("session_id = %q, want sess-1", row.SessionID) + } + if row.Role != string(schema.Assistant) { + t.Errorf("role = %q, want assistant", row.Role) + } + if row.Content == nil || *row.Content != "hello" { + t.Errorf("content = %v, want hello", row.Content) + } + if row.PromptTokens == nil || *row.PromptTokens != 5 { + t.Errorf("prompt tokens = %v, want 5", row.PromptTokens) + } +} + +func TestPersistenceHandler_ToolStartEndCycle(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "sess-1", Repo: repo}) + ctx := WithToolCallID(WithMessageID(context.Background(), "msg-1"), "call-1") + + // Start + h.OnStart(ctx, toolInfo("query_promql"), &einotool.CallbackInput{ArgumentsInJSON: `{"query":"up"}`}) + + repo.mu.Lock() + if len(repo.toolCalls) != 1 { + repo.mu.Unlock() + t.Fatalf("expected 1 pending tool_call after OnStart, got %d", len(repo.toolCalls)) + } + var tcID string + for k, tc := range repo.toolCalls { + tcID = k + if tc.Status != model.StatusPending { + t.Errorf("tool_call.status = %q, want pending", tc.Status) + } + if tc.ToolName != "query_promql" { + t.Errorf("tool_name = %q, want query_promql", tc.ToolName) + } + if tc.ArgumentsJSON != `{"query":"up"}` { + t.Errorf("arguments_json = %q", tc.ArgumentsJSON) + } + if tc.MessageID != "msg-1" { + t.Errorf("message_id = %q, want msg-1", tc.MessageID) + } + } + repo.mu.Unlock() + _ = tcID + + // End (success) + h.OnEnd(ctx, toolInfo("query_promql"), &einotool.CallbackOutput{Response: `{"ok":true}`}) + + repo.mu.Lock() + defer repo.mu.Unlock() + tc := repo.toolCalls[tcID] + if tc.Status != model.StatusSuccess { + t.Errorf("status after success = %q, want success", tc.Status) + } + if tc.ResultJSON == nil || *tc.ResultJSON != `{"ok":true}` { + t.Errorf("result_json = %v", tc.ResultJSON) + } + // chat_messages role=tool was appended + if len(repo.messages) != 1 { + t.Fatalf("expected 1 message (role=tool) after success, got %d", len(repo.messages)) + } + if repo.messages[0].Role != model.RoleTool { + t.Errorf("appended role = %q, want tool", repo.messages[0].Role) + } +} + +func TestPersistenceHandler_ToolErrorMarksError(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "sess-1", Repo: repo}) + ctx := WithToolCallID(context.Background(), "call-2") + h.OnStart(ctx, toolInfo("flaky"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("flaky"), errors.New("boom")) + + repo.mu.Lock() + defer repo.mu.Unlock() + if len(repo.toolCalls) != 1 { + t.Fatalf("toolCalls count = %d", len(repo.toolCalls)) + } + for _, tc := range repo.toolCalls { + if tc.Status != model.StatusError { + t.Errorf("status = %q, want error", tc.Status) + } + if tc.Error == nil || *tc.Error != "boom" { + t.Errorf("error = %v", tc.Error) + } + } +} + +func TestPersistenceHandler_ToolTimeoutClassified(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s1", Repo: repo}) + ctx := WithToolCallID(context.Background(), "call-3") + h.OnStart(ctx, toolInfo("slow"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("slow"), context.DeadlineExceeded) + repo.mu.Lock() + defer repo.mu.Unlock() + for _, tc := range repo.toolCalls { + if tc.Status != model.StatusTimeout { + t.Errorf("status = %q, want timeout", tc.Status) + } + } +} + +func TestPersistenceHandler_PersistFailureNonFatal(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + repo.failAppend = true + reg := prometheus.NewRegistry() + h := NewPersistenceHandler(PersistenceDeps{ + SessionID: "s1", + Repo: repo, + Registerer: reg, + }) + out := &einomodel.CallbackOutput{Message: &schema.Message{Role: schema.Assistant, Content: "x"}} + // No panic expected; failure is just logged + counted. + h.OnEnd(context.Background(), chatModelInfo(), out) +} + +func TestPersistenceHandler_NeededFiltersComponent(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s", Repo: repo}) + if !h.Needed(context.Background(), chatModelInfo(), callbacks.TimingOnEnd) { + t.Errorf("ChatModel OnEnd should be needed") + } + if h.Needed(context.Background(), chatModelInfo(), callbacks.TimingOnStart) { + t.Errorf("ChatModel OnStart should NOT be needed (user msg is persisted upstream)") + } + if !h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnStart) { + t.Errorf("Tool OnStart should be needed") + } + other := &callbacks.RunInfo{Component: components.Component("Embedding")} + if h.Needed(context.Background(), other, callbacks.TimingOnEnd) { + t.Errorf("non-tool/non-chat component should be skipped") + } +} + +// Compatibility with the current shared repository contract; v1 never calls this method. +func (r *fakeSessionRepo) UpdateSessionModel(context.Context, string, string, string) error { + return errs.ErrNotWiredYet +} diff --git a/internal/manager/biz/aiops/graph/callbacks/sse.go b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/sse.go similarity index 96% rename from internal/manager/biz/aiops/graph/callbacks/sse.go rename to internal/manager/biz/aiops/runtime/v1/graph/callbacks/sse.go index 99432f8be..07d1b0251 100644 --- a/internal/manager/biz/aiops/graph/callbacks/sse.go +++ b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/sse.go @@ -57,12 +57,12 @@ const ( type SSEEvent struct { Type SSEEventType Iteration int - Assistant *AssistantPayload // for assistant_start / assistant_end - Delta *AssistantDelta // for assistant_delta - Tool *ToolPayload // for tool_start / tool_end - Done *DonePayload // for done - Error *ErrorPayload // for error - Notification *TaskNotificationPayload // for task_notification + Assistant *AssistantPayload // for assistant_start / assistant_end + Delta *AssistantDelta // for assistant_delta + Tool *ToolPayload // for tool_start / tool_end + Done *DonePayload // for done + Error *ErrorPayload // for error + Notification *TaskNotificationPayload // for task_notification } // AssistantPayload is the start/end frame body. ContentSoFar is empty @@ -114,7 +114,7 @@ type ErrorPayload struct { } // TaskNotificationPayload is the body of a task_notification frame -//Emitted by the coordinator runtime when a background +// Emitted by the coordinator runtime when a background // worker spawned via AgentTool reaches a terminal state. The shape is // stable wire because the SPA reads each field directly: // diff --git a/internal/manager/biz/aiops/graph/callbacks/sse_test.go b/internal/manager/biz/aiops/runtime/v1/graph/callbacks/sse_test.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/sse_test.go rename to internal/manager/biz/aiops/runtime/v1/graph/callbacks/sse_test.go diff --git a/internal/manager/biz/aiops/runtime/v1/graph/react.go b/internal/manager/biz/aiops/runtime/v1/graph/react.go new file mode 100644 index 000000000..007154076 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/graph/react.go @@ -0,0 +1,305 @@ +package graph + +import ( + "context" + "errors" + "fmt" + "strings" + + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/flow/agent/react" + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// Node names exposed by the wrapper graph. Stable so audit / SSE +// callbacks can filter by RunInfo.Name. The inner ReAct subgraph names +// are owned by eino (react.GraphName / react.ModelNodeName / +// react.ToolsNodeName) and reused so persistence / SSE handlers see +// "ChatModel" / "Tools" exactly like the canonical eino layout. +const ( + // NodeAssembler is the lambda that turns *Input into []*schema.Message. + NodeAssembler = "MessageAssembler" + + // NodeReact is the wrapped eino react.Agent subgraph. + NodeReact = "ReActSubgraph" + + // NodeProjector is the lambda that turns *schema.Message into *Output. + NodeProjector = "OutputProjector" +) + +// SystemReminderTag wraps the per-turn anti-drift block injected ahead +// of the user message. System-Reminder 周期注入 — kept as +// a constant so audit / red-team grep can locate every injected block. +// +// PR-6: the block content is the bare hardcoded constraints (web-search +// gate, "do not loop on the same failing tool"); the configurable +// criticalReminder field on agent persona lands in a +// later PR (TODO marked in injectSystemReminder below). +const SystemReminderTag = "system-reminder" + +// BuildReActGraph constructs the wrapper graph + inner ReAct subgraph +// The returned Runnable accepts an *Input, +// drives the ReAct loop until either a final assistant message lands +// or MaxIterations is reached, and returns an *Output. +// +// Topology: +// +// START +// ↓ +// MessageAssembler (lambda) -- *Input → []*schema.Message +// ↓ +// ReActSubgraph (compose.AnyGraph) -- []*schema.Message → *schema.Message +// ↓ +// OutputProjector (lambda) -- *schema.Message → *Output +// ↓ +// END +// +// The ReActSubgraph is eino's stock react.Agent.ExportGraph() output. +// Internally it expands to ChatModel ↔ Branch(tool_calls?) ↔ ToolsNode +// per the diagram in — we are NOT re-implementing that +// inner shape because eino's flow/agent/react package is already the +// canonical maintained one. Callbacks (PR-6) registered via +// compose.WithCallbacks see the inner ChatModel + ToolsNode invocations +// just as if we had wired them by hand. +// +// Callbacks: handlers are NOT registered at build time. eino's graph +// compile API does not accept a Handler list — handlers ride on the +// Invoke call instead via compose.WithCallbacks(handlers...). The +// cutover layer (NEXT PR) builds the handler chain via +// callbacks.NewDefaultHandlers and threads it into Invoke; this +// function stays callback-agnostic so the same compiled graph can be +// re-used across requests with different per-call handler sets (e.g. +// per-tenant audit sinks). +func BuildReActGraph( + model einomodel.ToolCallingChatModel, + tools []basetool.BaseTool, + cfg Config, +) (compose.Runnable[*Input, *Output], error) { + if model == nil { + return nil, errors.New("graph: BuildReActGraph: model is required") + } + cfg = cfg.applyDefaults() + + // 1. Build the inner eino react agent. We pass our adapted tools + // in via ToolsConfig.Tools — eino infers the schema by calling + // each tool's Info(ctx). + einoTools := WrapBaseTools(tools) + baseTools := make([]einotool.BaseTool, 0, len(einoTools)) + for _, t := range einoTools { + baseTools = append(baseTools, t) + } + reactCfg := &react.AgentConfig{ + ToolCallingModel: model, + ToolsConfig: compose.ToolsNodeConfig{ + Tools: baseTools, + }, + // eino's "step" counts every graph node visit — one ReAct + // iteration is ChatModel + ToolsNode = 2 steps. So to give the + // LLM cfg.MaxIterations actual ChatModel turns we double it. + // We also leave headroom (+2) for the framing nodes + // (MessageAssembler / OutputProjector) eino counts toward the + // outer graph budget. Without this fix the LLM was capped at + // ~15 ChatModel turns when MaxIterations=30, surfacing as a + // graph ErrExceededMaxSteps mid-conversation (stream error). + MaxStep: cfg.MaxIterations*2 + 2, + GraphName: "ReActAgent", + ModelNodeName: "ChatModel", + ToolsNodeName: "Tools", + } + reactAgent, err := react.NewAgent(context.Background(), reactCfg) + if err != nil { + // Most common cause: a tool's Info(ctx) returned an unparseable + // JSON Schema (see tool_adapter.go). Wrap so the build site + // gets a clear hint without losing the underlying error. + return nil, fmt.Errorf("graph: build inner ReAct agent: %w", err) + } + + innerGraph, innerNodeOpts := reactAgent.ExportGraph() + + // 2. Build the outer wrapper graph: MessageAssembler → ReAct → OutputProjector. + g := compose.NewGraph[*Input, *Output]() + + assembler := compose.InvokableLambda(func(ctx context.Context, in *Input) ([]*schema.Message, error) { + return assembleMessages(in) + }) + if err := g.AddLambdaNode(NodeAssembler, assembler); err != nil { + return nil, fmt.Errorf("graph: add assembler node: %w", err) + } + + if err := g.AddGraphNode(NodeReact, innerGraph, innerNodeOpts...); err != nil { + return nil, fmt.Errorf("graph: add ReAct subgraph: %w", err) + } + + projector := compose.InvokableLambda(func(ctx context.Context, msg *schema.Message) (*Output, error) { + out := &Output{AssistantMessage: msg} + if msg != nil && msg.ResponseMeta != nil && msg.ResponseMeta.Usage != nil { + out.Usage.PromptTokens = msg.ResponseMeta.Usage.PromptTokens + out.Usage.CompletionTokens = msg.ResponseMeta.Usage.CompletionTokens + out.Usage.TotalTokens = msg.ResponseMeta.Usage.TotalTokens + } + // Iterations is filled in by the metrics / audit handler on the + // caller side (it has the per-graph counter); we leave it 0 here + // because the projector lambda only sees the terminal message. + // — MetricsHandler observes the ChatModel turns. + return out, nil + }) + if err := g.AddLambdaNode(NodeProjector, projector); err != nil { + return nil, fmt.Errorf("graph: add projector node: %w", err) + } + + if err := g.AddEdge(compose.START, NodeAssembler); err != nil { + return nil, err + } + if err := g.AddEdge(NodeAssembler, NodeReact); err != nil { + return nil, err + } + if err := g.AddEdge(NodeReact, NodeProjector); err != nil { + return nil, err + } + if err := g.AddEdge(NodeProjector, compose.END); err != nil { + return nil, err + } + + runnable, err := g.Compile(context.Background(), + compose.WithGraphName("OngridReActAgent"), + // Outer steps = assembler + react-subgraph + projector. The + // inner ReAct's own MaxStep was already applied in reactCfg. + compose.WithMaxRunSteps(cfg.MaxIterations+10), + ) + if err != nil { + return nil, fmt.Errorf("graph: compile wrapper: %w", err) + } + return runnable, nil +} + +// assembleMessages fans Input out into the canonical eino message +// shape: system → history → user-role → user text. +// +// Order matches agent.go's existing for-loop with the per-turn +// system-reminder block extracted into its own user-role message so +// the LLM sees it as a fresh injection on every turn (— +// claude-code's anti-drift mechanism). Specifically: +// +// - system message goes first (skipped if SystemPrompt is empty) +// - history is replayed verbatim (chatruntime strips the trailing +// user row before passing History so the model doesn't see the +// same turn twice) +// - block is its own user-role message just before +// the user text — re-injected per turn so long sessions cannot +// drift past the rules even after the system message scrolls out +// of an attention budget +// - the user text (with mentions inlined as a markdown preamble) +// follows +// +// When UserText is empty the user-role messages are skipped entirely +// (matches the legacy code path where the caller already appended the +// user turn into History). +func assembleMessages(in *Input) ([]*schema.Message, error) { + if in == nil { + return nil, errors.New("graph: assembler: nil input") + } + out := make([]*schema.Message, 0, len(in.History)+3) + // Append the response-language directive to the system prompt (one + // system message — strongest signal). The personas are Chinese, so + // without this the model answers in Chinese even in English mode. + sp := in.SystemPrompt + if dir := languageDirective(in.Locale); dir != "" { + if sp != "" { + sp += "\n\n" + } + sp += dir + } + if sp != "" { + out = append(out, schema.SystemMessage(sp)) + } + out = append(out, in.History...) + + if in.UserText != "" { + // 1. Per-turn system-reminder, as its own user-role message. + // — re-injected on every turn so it survives + // long-session attention drift. + if reminder := buildSystemReminder(in); reminder != "" { + out = append(out, schema.UserMessage(reminder)) + } + // 2. The actual user turn with @-mentions inlined as a + // markdown preamble (legacy agent.go format). + userBody := in.UserText + if in.MentionsRendered != "" { + userBody = in.MentionsRendered + "\n\n" + userBody + } + out = append(out, schema.UserMessage(userBody)) + } + return out, nil +} + +// buildSystemReminder returns the per-turn anti-drift reminder block +// the MessageAssembler injects ahead of the user message. +// — claude-code's per-turn `` injection adapted for +// ongrid: +// +// - hardcoded baseline rules (always present): device_id digit-id, +// no-loop-on-failing-tool, tools-are-facts, web_search gating +// - persona critical_reminder (Input.AgentReminder) — one line, when +// a worker persona is active for this turn +// - runtime-computed dynamic hints (Input.DynamicHints) — N lines, +// e.g. "tool X failed 2 times in a row", "iteration > 20, summarize +// and respond" +// +// Returns "" when there is literally nothing to inject (currently the +// hardcoded baseline guarantees at least three bullets, so this only +// happens if a future caller drops the baseline; we keep the empty +// branch so the assembler can skip the inline prepend cleanly). +// languageDirective maps a UI locale to an explicit "answer in this +// language" instruction. Empty locale → "" (no directive, back-compat). +// It covers tool-call narration explicitly because that's what drifts +// back to Chinese first when the persona is Chinese. +func languageDirective(locale string) string { + switch locale { + case "en-US": + return "Respond in English. Everything you write to the user — prose, explanations, headings, and the narration around every tool call — must be in English. Tool descriptions, knowledge-base snippets, persona text, and logs may be in Chinese; render their MEANING in English and never echo raw Chinese to the user. Translate domain terms to their English equivalents (e.g. \"0号病人\" → \"patient zero\", \"根因\" → \"root cause\", \"告警\" → \"alert\", \"巡检\" → \"inspection\"). Leave only proper nouns, identifiers, hostnames, file paths, code, and raw command output verbatim." + case "zh-CN": + return "用中文回复：你的所有叙述、解释、标题，以及每次工具调用前后的说明都必须用中文。工具描述、知识库片段、日志可能是英文，把含义用中文表达即可；标识符、主机名、文件路径、代码、命令原始输出保持原样。" + } + return "" +} + +func buildSystemReminder(in *Input) string { + if in == nil { + return "" + } + lines := []string{ + "- 同一工具失败两次后请换思路，不要重复调用", + "- device_id / alert_id 必须是数字 ID（@-mention 已经为你解析）", + "- 工具结果是事实，不要在没有数据时编造", + } + // Re-assert the response language every turn (system prompt scrolls + // out of attention in long sessions). Prepended so it leads the block. + if dir := languageDirective(in.Locale); dir != "" { + lines = append([]string{"- " + dir}, lines...) + } + if !in.WebSearchEnabled { + lines = append(lines, "- web_search 已被关闭，本轮不要调用") + } + if r := strings.TrimSpace(in.AgentReminder); r != "" { + lines = append(lines, "- "+r) + } + for _, h := range in.DynamicHints { + h = strings.TrimSpace(h) + if h == "" { + continue + } + lines = append(lines, "- "+h) + } + if len(lines) == 0 { + return "" + } + parts := make([]string, 0, len(lines)+2) + parts = append(parts, "<"+SystemReminderTag+">") + parts = append(parts, lines...) + parts = append(parts, "") + return strings.Join(parts, "\n") +} diff --git a/internal/manager/biz/aiops/runtime/v1/graph/react_test.go b/internal/manager/biz/aiops/runtime/v1/graph/react_test.go new file mode 100644 index 000000000..dc6ca9541 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/graph/react_test.go @@ -0,0 +1,398 @@ +package graph + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// scriptedState is the shared mutable bookkeeping for scriptedChatModel +// instances. eino's ToolCallingChatModel.WithTools returns a *new* +// instance with tools bound, which means the test must share counters / +// reply pointers between the original and any derived instance — the +// inner ReAct agent calls .WithTools then dispatches to the new copy. +type scriptedState struct { + mu sync.Mutex + replies []*schema.Message + idx int + generateCalls atomic.Int32 + genErr error + inputsLog [][]*schema.Message +} + +// scriptedChatModel is a tool-calling ChatModel test double used by +// the react_test integration tests. It returns one *schema.Message per +// Generate call from the `replies` list, in order; once exhausted, it +// returns the last reply (so loops with MaxIterations work as expected). +// +// Multiple scriptedChatModel instances may share a single *scriptedState +// (eino's WithTools copies the receiver) — counters must therefore be +// stored on the shared state, not the wrapping instance. +type scriptedChatModel struct { + st *scriptedState + + // boundTools is captured by WithTools so the test can assert the + // tool list shape was forwarded. Lives on the per-instance struct + // because each derivation has its own bound tool list. + boundTools []*schema.ToolInfo +} + +func newScriptedChatModel(replies ...*schema.Message) *scriptedChatModel { + return &scriptedChatModel{st: &scriptedState{replies: replies}} +} + +func (s *scriptedChatModel) Generate(_ context.Context, input []*schema.Message, _ ...model.Option) (*schema.Message, error) { + s.st.generateCalls.Add(1) + s.st.mu.Lock() + cp := make([]*schema.Message, len(input)) + copy(cp, input) + s.st.inputsLog = append(s.st.inputsLog, cp) + if s.st.genErr != nil { + err := s.st.genErr + s.st.mu.Unlock() + return nil, err + } + if len(s.st.replies) == 0 { + s.st.mu.Unlock() + return &schema.Message{Role: schema.Assistant, Content: "ok"}, nil + } + if s.st.idx < len(s.st.replies) { + out := s.st.replies[s.st.idx] + s.st.idx++ + s.st.mu.Unlock() + return out, nil + } + out := s.st.replies[len(s.st.replies)-1] + s.st.mu.Unlock() + return out, nil +} + +func (s *scriptedChatModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { + msg, err := s.Generate(ctx, input, opts...) + if err != nil { + return nil, err + } + return schema.StreamReaderFromArray([]*schema.Message{msg}), nil +} + +func (s *scriptedChatModel) BindTools(tools []*schema.ToolInfo) error { + s.boundTools = tools + return nil +} + +// WithTools returns a NEW instance that shares the underlying state +// pointer so Generate counters / reply cursor are observable from the +// original receiver after the inner ReAct agent dispatches through the +// derived copy. +func (s *scriptedChatModel) WithTools(tools []*schema.ToolInfo) (model.ToolCallingChatModel, error) { + return &scriptedChatModel{st: s.st, boundTools: tools}, nil +} + +func (s *scriptedChatModel) generateCalls() int32 { return s.st.generateCalls.Load() } + +func makeAssistantNoTools(content string) *schema.Message { + return &schema.Message{Role: schema.Assistant, Content: content} +} + +func makeAssistantToolCall(content, callID, toolName, args string) *schema.Message { + return &schema.Message{ + Role: schema.Assistant, + Content: content, + ToolCalls: []schema.ToolCall{ + { + ID: callID, + Type: "function", + Function: schema.FunctionCall{Name: toolName, Arguments: args}, + }, + }, + } +} + +func TestBuildReActGraph_FinalReplyOneTurn(t *testing.T) { + t.Parallel() + scripted := newScriptedChatModel(makeAssistantNoTools("hello world")) + tool := &fakeBaseTool{name: "noop", parameters: `{"type":"object","properties":{}}`} + g, err := BuildReActGraph(scripted, []basetool.BaseTool{tool}, Config{}) + if err != nil { + t.Fatalf("BuildReActGraph: %v", err) + } + out, err := g.Invoke(context.Background(), &Input{ + SystemPrompt: "be helpful", + UserText: "hi", + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if out == nil || out.AssistantMessage == nil { + t.Fatalf("expected non-nil output") + } + if out.AssistantMessage.Content != "hello world" { + t.Errorf("content = %q, want hello world", out.AssistantMessage.Content) + } + if scripted.generateCalls() != 1 { + t.Errorf("Generate calls = %d, want 1", scripted.generateCalls()) + } +} + +func TestBuildReActGraph_ToolCallThenFinal(t *testing.T) { + t.Parallel() + scripted := newScriptedChatModel( + makeAssistantToolCall("", "call_1", "echo", `{"a":1}`), + makeAssistantNoTools("done"), + ) + echo := &fakeBaseTool{ + name: "echo", + parameters: `{"type":"object","properties":{"a":{"type":"integer"}}}`, + runResp: `{"echoed":1}`, + } + g, err := BuildReActGraph(scripted, []basetool.BaseTool{echo}, Config{MaxIterations: 5}) + if err != nil { + t.Fatalf("BuildReActGraph: %v", err) + } + out, err := g.Invoke(context.Background(), &Input{UserText: "do it"}) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if out.AssistantMessage.Content != "done" { + t.Errorf("content = %q, want done", out.AssistantMessage.Content) + } + if echo.calls.Load() != 1 { + t.Errorf("tool calls = %d, want 1", echo.calls.Load()) + } + if scripted.generateCalls() != 2 { + t.Errorf("Generate calls = %d, want 2", scripted.generateCalls()) + } +} + +func TestBuildReActGraph_RecoversFromToolError(t *testing.T) { + t.Parallel() + scripted := newScriptedChatModel( + makeAssistantToolCall("", "call_1", "flaky", `{}`), + makeAssistantNoTools("recovered"), + ) + flaky := &fakeBaseTool{ + name: "flaky", + parameters: `{"type":"object","properties":{}}`, + runErr: errors.New("temporary"), + } + g, err := BuildReActGraph(scripted, []basetool.BaseTool{flaky}, Config{MaxIterations: 5}) + if err != nil { + t.Fatalf("BuildReActGraph: %v", err) + } + out, err := g.Invoke(context.Background(), &Input{UserText: "do it"}) + // einoToolAdapter wraps tool errors as JSON envelopes ( + // invariant: tool failures are facts the LLM consumes, not graph- + // fatal events). ToolsNode therefore returns successfully with the + // envelope in the tool message; ChatModel then runs the second + // scripted reply ("recovered") and the graph completes cleanly. + if err != nil { + t.Fatalf("Invoke should recover from tool error, got %v", err) + } + if out == nil || out.AssistantMessage == nil || !strings.Contains(out.AssistantMessage.Content, "recovered") { + t.Errorf("expected recovered final reply, got %+v", out) + } + if flaky.calls.Load() != 1 { + t.Errorf("tool calls = %d, want 1 (one attempt then ChatModel recovers)", flaky.calls.Load()) + } + if scripted.generateCalls() != 2 { + t.Errorf("ChatModel generate calls = %d, want 2 (initial tool_call turn + recovery turn)", scripted.generateCalls()) + } +} + +func TestBuildReActGraph_NilModelFails(t *testing.T) { + t.Parallel() + if _, err := BuildReActGraph(nil, nil, Config{}); err == nil { + t.Fatalf("expected error for nil model") + } +} + +func TestBuildReActGraph_BadToolSchemaFailsBuild(t *testing.T) { + t.Parallel() + scripted := newScriptedChatModel() + bad := &fakeBaseTool{name: "bad", parameters: `{not json`} + if _, err := BuildReActGraph(scripted, []basetool.BaseTool{bad}, Config{}); err == nil { + t.Fatalf("expected build to fail when tool schema parsing fails") + } +} + +func TestAssembleMessages_MentionsAndReminderInline(t *testing.T) { + t.Parallel() + in := &Input{ + SystemPrompt: "agent rules", + UserText: "find big files on edge-1", + MentionsRendered: "- edge-1: device_id=42 (online)", + WebSearchEnabled: false, + } + out, err := assembleMessages(in) + if err != nil { + t.Fatalf("assembleMessages: %v", err) + } + // system + reminder (user) + user (mentions+text) = 3 + if len(out) != 3 { + t.Fatalf("expected 3 messages, got %d (%+v)", len(out), out) + } + if out[0].Role != schema.System || out[0].Content != "agent rules" { + t.Errorf("first message should be system: got role=%s content=%q", out[0].Role, out[0].Content) + } + reminder := out[1] + if reminder.Role != schema.User { + t.Errorf("reminder role = %s, want user", reminder.Role) + } + if !strings.HasPrefix(reminder.Content, "") || !strings.HasSuffix(reminder.Content, "") { + t.Errorf("reminder message is not a bare block: %q", reminder.Content) + } + if !strings.Contains(reminder.Content, "web_search 已被关闭") { + t.Errorf("reminder missing web_search disabled note: %q", reminder.Content) + } + user := out[2] + if user.Role != schema.User { + t.Errorf("last role should be user, got %s", user.Role) + } + if strings.Contains(user.Content, "") { + t.Errorf("user message must NOT inline the reminder block (it is now a separate message): %q", user.Content) + } + if !strings.Contains(user.Content, "edge-1: device_id=42") { + t.Errorf("user content missing mention block: %q", user.Content) + } + if !strings.Contains(user.Content, "find big files on edge-1") { + t.Errorf("user content missing the actual user text: %q", user.Content) + } +} + +func TestAssembleMessages_AgentReminderAndDynamicHints(t *testing.T) { + t.Parallel() + in := &Input{ + SystemPrompt: "agent rules", + UserText: "继续诊断", + WebSearchEnabled: true, + AgentReminder: "结论先于过程，最多列 3 条假设", + DynamicHints: []string{ + "注意: query_logql 已连续失败 2 次，请换工具或要求用户澄清", + "已经跑了 22 轮，如不能在下一轮给出最终回答应当总结当前已知信息回答用户", + }, + } + out, err := assembleMessages(in) + if err != nil { + t.Fatalf("assembleMessages: %v", err) + } + if len(out) != 3 { + t.Fatalf("expected 3 messages got %d", len(out)) + } + rem := out[1].Content + if !strings.Contains(rem, "结论先于过程") { + t.Errorf("reminder missing AgentReminder bullet: %q", rem) + } + if !strings.Contains(rem, "query_logql 已连续失败 2 次") { + t.Errorf("reminder missing dynamic-hint failure bullet: %q", rem) + } + if !strings.Contains(rem, "已经跑了 22 轮") { + t.Errorf("reminder missing dynamic-hint iteration bullet: %q", rem) + } + // web_search ON -> the disabled-note bullet must NOT be present. + if strings.Contains(rem, "web_search 已被关闭") { + t.Errorf("reminder should not mention web_search disabled when WebSearchEnabled=true: %q", rem) + } +} + +func TestAssembleMessages_LocaleDirective(t *testing.T) { + t.Parallel() + // en-US: both the system message and the per-turn reminder must carry + // an explicit English directive (personas are Chinese). + en, err := assembleMessages(&Input{SystemPrompt: "agent rules", UserText: "status?", Locale: "en-US"}) + if err != nil { + t.Fatalf("assembleMessages: %v", err) + } + if !strings.Contains(en[0].Content, "agent rules") || !strings.Contains(en[0].Content, "Respond in English") { + t.Errorf("en system message should append the English directive: %q", en[0].Content) + } + if !strings.Contains(en[1].Content, "Respond in English") { + t.Errorf("en reminder should carry the English directive: %q", en[1].Content) + } + + // zh-CN: Chinese directive instead. + zh, _ := assembleMessages(&Input{SystemPrompt: "agent rules", UserText: "状态？", Locale: "zh-CN"}) + if !strings.Contains(zh[0].Content, "用中文回复") { + t.Errorf("zh system message should append the Chinese directive: %q", zh[0].Content) + } + + // empty locale: no directive at all (back-compat for non-SPA callers + // like the IM bridge — system prompt is left exactly as-is). + none, _ := assembleMessages(&Input{SystemPrompt: "agent rules", UserText: "x"}) + if none[0].Content != "agent rules" { + t.Errorf("empty locale must not alter the system prompt: %q", none[0].Content) + } + if strings.Contains(none[1].Content, "Respond in English") || strings.Contains(none[1].Content, "用中文回复") { + t.Errorf("empty locale reminder must carry no language directive: %q", none[1].Content) + } +} + +func TestBuildSystemReminder_EmptyHintsTrimmed(t *testing.T) { + t.Parallel() + got := buildSystemReminder(&Input{ + WebSearchEnabled: true, + AgentReminder: " ", // all whitespace -> dropped + DynamicHints: []string{"", " ", "real hint"}, + }) + if !strings.Contains(got, "real hint") { + t.Errorf("real hint missing from reminder: %q", got) + } + // AgentReminder was whitespace-only -> no extra empty bullet. + if strings.Contains(got, "- \n") || strings.Contains(got, "- \n") { + t.Errorf("empty bullet leaked: %q", got) + } +} + +func TestAssembleMessages_HistoryReplay(t *testing.T) { + t.Parallel() + in := &Input{ + SystemPrompt: "be cool", + History: []*schema.Message{ + {Role: schema.User, Content: "earlier"}, + {Role: schema.Assistant, Content: "earlier reply"}, + }, + UserText: "follow up", + } + out, err := assembleMessages(in) + if err != nil { + t.Fatalf("assembleMessages: %v", err) + } + // system + 2 history + 1 reminder (user) + 1 user = 5 + if len(out) != 5 { + t.Fatalf("expected 5 messages got %d", len(out)) + } + if out[1].Content != "earlier" { + t.Errorf("history user content lost: %+v", out[1]) + } + if !strings.HasPrefix(out[3].Content, "") { + t.Errorf("expected reminder at index 3: %q", out[3].Content) + } + if out[4].Content != "follow up" { + t.Errorf("user text lost: %q", out[4].Content) + } +} + +func TestAssembleMessages_NilInputFails(t *testing.T) { + t.Parallel() + if _, err := assembleMessages(nil); err == nil { + t.Fatalf("expected error for nil input") + } +} + +func TestConfig_Defaults(t *testing.T) { + t.Parallel() + c := Config{}.applyDefaults() + if c.MaxIterations != 30 { + t.Errorf("MaxIterations default = %d, want 30", c.MaxIterations) + } + if c.ToolTimeout.Seconds() != 15 { + t.Errorf("ToolTimeout default = %v, want 15s", c.ToolTimeout) + } +} diff --git a/internal/manager/biz/aiops/runtime/v1/graph/tool_adapter.go b/internal/manager/biz/aiops/runtime/v1/graph/tool_adapter.go new file mode 100644 index 000000000..4c093680f --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/graph/tool_adapter.go @@ -0,0 +1,299 @@ +package graph + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" + "github.com/eino-contrib/jsonschema" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// toolMemo is a per-run cache of identical (read-tool, args) calls. The +// graph — and therefore the wrapped tool set — is rebuilt per request, so +// one memo per WrapBaseTools call is naturally scoped to a single ReAct +// run. Within that run an identical call (same tool, byte-identical args, +// seconds apart) cannot yield new information and is almost always a ReAct +// loop artifact; returning the prior result skips re-executing an expensive +// tool (SSH probe, PromQL, LLM-backed query_translate) and keeps an +// identical-call loop from burning the iteration budget on real work. Only +// Class=="read" tools are memoized — write/destructive tools never touch +// this path, so the review/mutation flow is unaffected. +type toolMemo struct { + mu sync.Mutex + m map[string]string // (tool\x00args) -> result, identical-call cache + counts map[string]int // tool name -> distinct executions this run +} + +func newToolMemo() *toolMemo { + return &toolMemo{m: make(map[string]string), counts: make(map[string]int)} +} + +func (t *toolMemo) get(k string) (string, bool) { + t.mu.Lock() + defer t.mu.Unlock() + v, ok := t.m[k] + return v, ok +} + +func (t *toolMemo) put(k, v string) { + t.mu.Lock() + defer t.mu.Unlock() + t.m[k] = v +} + +// count returns how many real executions of the named tool have happened +// this run; bump records one more. +func (t *toolMemo) count(name string) int { + t.mu.Lock() + defer t.mu.Unlock() + return t.counts[name] +} + +func (t *toolMemo) bump(name string) { + t.mu.Lock() + defer t.mu.Unlock() + t.counts[name]++ +} + +// maxToolCallsPerRun caps how many times any one tool may EXECUTE within a +// single agent run. Identical-arg repeats are served from the memo and don't +// count; this catches the other failure mode — the model calling the same +// tool many times with slightly different args (e.g. query_promql across a +// dozen metrics, query_alert_rules over and over) without converging. Past +// the cap the tool returns a "synthesize now" directive instead of running, +// which forces the agent to answer from what it already gathered. Generous +// enough that normal multi-step investigation isn't clipped. +const maxToolCallsPerRun = 8 + +// toolBudgetExceeded is the synthetic tool result returned once a tool hits +// maxToolCallsPerRun. Shaped like a normal JSON tool result so the LLM reads +// it as data and (re)directs to answering. +func toolBudgetExceeded(name string, n int) string { + b, _ := json.Marshal(map[string]any{ + "status": "call_budget_exceeded", + "tool": name, + "calls": n, + "instruction": fmt.Sprintf("You have already called %q %d times this turn — that is the per-tool limit. Do NOT call it again. Answer the user from the results you already gathered; if they're insufficient, state specifically what is missing and ask the user.", name, n), + }) + return string(b) +} + +// WrapBaseTool adapts an ongrid basetool.BaseTool to eino's +// components/tool.BaseTool + InvokableTool surface so the eino ToolsNode +// can dispatch to it. PR-3's basetool was deliberately mirror-shaped +// against eino (see basetool.go header comment), so this adapter is +// thin: Info is a 1-1 field copy and InvokableRun forwards the args +// JSON verbatim. +// +// graph 执行层 ToolsNode 接收的是 eino +// tool.BaseTool；本 adapter 是仓库自家 BaseTool 与 eino 之间唯一胶水点。 +// +// Per-call options (tenant / user / device id) ride on +// `basetool.InvokeOption` slots; eino's `tool.Option` system carries an +// impl-specific bag for them — see WithInvokeOpts. If the caller does +// not pass any impl-specific options the inner tool runs with its +// decorator-resolved defaults (the typical path). +func WrapBaseTool(t basetool.BaseTool) einotool.InvokableTool { + if t == nil { + return nil + } + return &einoToolAdapter{inner: t} +} + +// einoInvokeOptKey is the internal carrier for ongrid InvokeOptions +// passed through eino's `tool.Option` slot. Unexported so callers +// route through WithInvokeOpts. +type einoInvokeOptKey struct { + opts []basetool.InvokeOption +} + +// WithInvokeOpts is the eino-side option helper that carries +// basetool.InvokeOption into a ToolsNode call. The graph wiring layer +// (PR-N chatruntime) will use this to thread per-request tenant / user +// id through the graph runtime down to each tool's InvokableRun call. +// +// Usage from a graph client: +// +// runnable.Invoke(ctx, in, compose.WithToolsNodeOption( +// compose.WithToolOption(graph.WithInvokeOpts( +// basetool.WithUserID(uid), +// basetool.WithTenant(tenantID), +// ), +// ) +func WithInvokeOpts(opts ...basetool.InvokeOption) einotool.Option { + return einotool.WrapImplSpecificOptFn(func(k *einoInvokeOptKey) { + k.opts = append(k.opts, opts...) + }) +} + +// einoToolAdapter wraps a basetool.BaseTool to satisfy eino's +// InvokableTool interface. The struct is intentionally trivial — all +// real behaviour (tenant/audit/timeout/ratelimit/metric) lives in the +// PR-3 decorator chain wrapped *around* the inner tool *before* it +// reaches this adapter. +type einoToolAdapter struct { + inner basetool.BaseTool + // memo is the per-run identical-call cache (nil = memoization off, e.g. + // the single-tool WrapBaseTool path used by tests). Shared across all + // adapters built by one WrapBaseTools call. + memo *toolMemo + // Info() is resolved once (name + read-ness) for the memo key + gate. + infoOnce sync.Once + cacheName string + cacheable bool // Class == "read" +} + +// resolveInfo lazily caches the tool's name + whether it's a pure-read tool +// (the only class we memoize). Info() is otherwise called by eino at build +// time; caching here avoids a call per dispatch. +func (a *einoToolAdapter) resolveInfo(ctx context.Context) { + a.infoOnce.Do(func() { + if info, err := a.inner.Info(ctx); err == nil && info != nil { + a.cacheName = info.Name + a.cacheable = info.Class == "read" + } + }) +} + +// Info returns the eino schema.ToolInfo for this tool. WhenToUse from +// our extended ToolInfo is appended to the description (with a +// "When to use:" prefix) so the LLM sees both halves through the +// standard schema field. — Tool 层 description vs +// when_to_use 拆分。 +func (a *einoToolAdapter) Info(ctx context.Context) (*schema.ToolInfo, error) { + if a == nil || a.inner == nil { + return nil, fmt.Errorf("graph: tool adapter has nil inner tool") + } + info, err := a.inner.Info(ctx) + if err != nil { + return nil, err + } + if info == nil { + return nil, fmt.Errorf("graph: tool returned nil ToolInfo") + } + desc := info.Description + if info.WhenToUse != "" { + if desc != "" { + desc = desc + "\n\nWhen to use: " + info.WhenToUse + } else { + desc = "When to use: " + info.WhenToUse + } + } + out := &schema.ToolInfo{ + Name: info.Name, + Desc: desc, + } + if len(info.Parameters) > 0 { + // Preserve the existing JSON-Schema bytes verbatim by re-parsing + // into eino's jsonschema.Schema. PR-3's basetool.ToolInfo carries + // the schema as raw JSON; eino's ParamsOneOf wants a typed + // *jsonschema.Schema, so we deserialize. A failure here means the + // upstream tool produced invalid JSON Schema — bubble it as an + // error so the graph build refuses to compile. + js := &jsonschema.Schema{} + if err := json.Unmarshal(info.Parameters, js); err != nil { + return nil, fmt.Errorf("graph: tool %q: parse parameters JSON Schema: %w", info.Name, err) + } + out.ParamsOneOf = schema.NewParamsOneOfByJSONSchema(js) + } + return out, nil +} + +// InvokableRun forwards to the inner basetool.BaseTool. Per-call +// InvokeOptions are extracted from the eino tool.Option bag if the +// caller used WithInvokeOpts. +// +// **Tool errors are converted to a JSON envelope, never returned as a +// Go error.** Eino's ToolsNode treats Go-level errors as graph-fatal +// (terminates the whole invoke + SSE stream); ongrid's invariant is +// "tool failures are facts the LLM can recover from" — the LLM should +// see the error text as a tool result and decide to retry / switch / +// ask the user, NOT have the conversation aborted. We mirror what the +// legacy agent.go for-loop did: marshal err into a result-shaped JSON +// like {"error": "..."} so the LLM consumes it as data. +// +// True nil-receiver / unrecoverable bugs (we wrote the wrong inner) +// still surface as Go error so eino can panic-loud, since those are +// not user-fixable. +func (a *einoToolAdapter) InvokableRun(ctx context.Context, argumentsInJSON string, opts ...einotool.Option) (string, error) { + if a == nil || a.inner == nil { + return "", fmt.Errorf("graph: tool adapter has nil inner tool") + } + var memoKey string + if a.memo != nil { + a.resolveInfo(ctx) + // 1. Identical-call memo (read tools only): a byte-identical repeat + // returns the prior result without re-executing. + if a.cacheable && a.cacheName != "" { + memoKey = a.cacheName + "\x00" + argumentsInJSON + if cached, ok := a.memo.get(memoKey); ok { + return cached, nil + } + } + // 2. Per-tool execution cap (all tools): once a tool has run + // maxToolCallsPerRun times this run, stop executing it and hand + // back a "synthesize now" directive. Catches the distinct-args + // repeat loop (query_promql/query_alert_rules called over and over) + // that the memo can't. + if a.cacheName != "" && a.memo.count(a.cacheName) >= maxToolCallsPerRun { + return toolBudgetExceeded(a.cacheName, a.memo.count(a.cacheName)), nil + } + } + resolved := einotool.GetImplSpecificOptions(&einoInvokeOptKey{}, opts...) + out, err := a.inner.InvokableRun(ctx, argumentsInJSON, resolved.opts...) + // Count this real execution toward the per-tool cap (success or failure; + // a failing tool that's hammered is also waste the cap should bound). + if a.memo != nil && a.cacheName != "" { + a.memo.bump(a.cacheName) + } + if err != nil { + // Re-shape as a tool-result-style JSON so the LLM gets it as a + // message instead of having the graph terminate. Truncate long + // errors so we don't blow the context window with stack traces. + msg := err.Error() + const cap = 2048 + if len(msg) > cap { + msg = msg[:cap] + "...(truncated)" + } + envelope, mErr := json.Marshal(map[string]any{ + "error": msg, + "status": "failed", + }) + if mErr != nil { + // Marshal of a string + status into a 2-key map should be + // infallible; if it isn't, fall back to the original error. + return "", err + } + return string(envelope), nil + } + // Cache successful read-tool results only — a failed call stays + // retryable (a transient error shouldn't be pinned for the whole run). + if memoKey != "" { + a.memo.put(memoKey, out) + } + return out, nil +} + +// WrapBaseTools is the slice-flavoured WrapBaseTool. Returns a slice of +// eino tool.BaseTool ready to feed into compose.ToolsNodeConfig.Tools. +// Nil entries in the input are skipped so callers can pass a sparse +// list (e.g. from a skill activation filter). +func WrapBaseTools(tools []basetool.BaseTool) []einotool.BaseTool { + // One memo shared by every tool in this build = scoped to one run + // (the graph is rebuilt per request). The single-tool WrapBaseTool + // path deliberately leaves memo nil (tests / non-graph callers). + memo := newToolMemo() + out := make([]einotool.BaseTool, 0, len(tools)) + for _, t := range tools { + if t == nil { + continue + } + out = append(out, &einoToolAdapter{inner: t, memo: memo}) + } + return out +} diff --git a/internal/manager/biz/aiops/runtime/v1/graph/tool_adapter_test.go b/internal/manager/biz/aiops/runtime/v1/graph/tool_adapter_test.go new file mode 100644 index 000000000..f3dbfd178 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/graph/tool_adapter_test.go @@ -0,0 +1,290 @@ +package graph + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync/atomic" + "testing" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// fakeBaseTool is the test double used by tool_adapter + react tests. +// It records every invocation so assertions can inspect args + opts. +type fakeBaseTool struct { + name string + desc string + whenToUse string + parameters string // raw JSON Schema + class string + infoErr error + runErr error + runResp string + calls atomic.Int32 + lastArgs string + lastOptsLen int +} + +func (f *fakeBaseTool) Info(_ context.Context) (*basetool.ToolInfo, error) { + if f.infoErr != nil { + return nil, f.infoErr + } + return &basetool.ToolInfo{ + Name: f.name, + Description: f.desc, + WhenToUse: f.whenToUse, + Parameters: json.RawMessage(f.parameters), + Class: f.class, + }, nil +} + +func (f *fakeBaseTool) InvokableRun(_ context.Context, argsJSON string, opts ...basetool.InvokeOption) (string, error) { + f.calls.Add(1) + f.lastArgs = argsJSON + f.lastOptsLen = len(opts) + if f.runErr != nil { + return "", f.runErr + } + return f.runResp, nil +} + +func TestWrapBaseTool_NilInner(t *testing.T) { + t.Parallel() + if got := WrapBaseTool(nil); got != nil { + t.Fatalf("WrapBaseTool(nil) = %v, want nil", got) + } +} + +func TestWrapBaseTool_InfoMergesWhenToUse(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{ + name: "search", + desc: "search the web", + whenToUse: "user explicitly asks the public web", + parameters: `{"type":"object","properties":{"q":{"type":"string"}},"required":["q"]}`, + } + wrapped := WrapBaseTool(inner) + info, err := wrapped.Info(context.Background()) + if err != nil { + t.Fatalf("Info: %v", err) + } + if info.Name != "search" { + t.Errorf("Name = %q, want search", info.Name) + } + if !strings.Contains(info.Desc, "search the web") { + t.Errorf("Desc missing description body: %q", info.Desc) + } + if !strings.Contains(info.Desc, "When to use") { + t.Errorf("Desc missing when-to-use header: %q", info.Desc) + } + if !strings.Contains(info.Desc, "user explicitly asks the public web") { + t.Errorf("Desc missing when-to-use body: %q", info.Desc) + } + if info.ParamsOneOf == nil { + t.Errorf("ParamsOneOf is nil; expected populated") + } +} + +func TestWrapBaseTool_InfoNoWhenToUse(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "ping", desc: "ping a host"} + wrapped := WrapBaseTool(inner) + info, err := wrapped.Info(context.Background()) + if err != nil { + t.Fatalf("Info: %v", err) + } + if info.Desc != "ping a host" { + t.Errorf("Desc = %q, want %q", info.Desc, "ping a host") + } +} + +func TestWrapBaseTool_InfoEmptyParameters(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "noargs"} + wrapped := WrapBaseTool(inner) + info, err := wrapped.Info(context.Background()) + if err != nil { + t.Fatalf("Info: %v", err) + } + if info.ParamsOneOf != nil { + t.Errorf("ParamsOneOf should be nil when no parameters declared") + } +} + +func TestWrapBaseTool_InfoBadJSONSchemaFails(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "bad", parameters: `{not valid json`} + wrapped := WrapBaseTool(inner) + if _, err := wrapped.Info(context.Background()); err == nil { + t.Fatalf("expected Info to fail on invalid JSON Schema") + } +} + +func TestWrapBaseTool_InfoBubbleInnerErr(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "x", infoErr: errors.New("boom")} + wrapped := WrapBaseTool(inner) + if _, err := wrapped.Info(context.Background()); err == nil { + t.Fatalf("expected Info to bubble inner error") + } +} + +func TestWrapBaseTool_InvokableRunForwardsArgs(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "echo", runResp: `{"ok":true}`} + wrapped := WrapBaseTool(inner) + out, err := wrapped.InvokableRun(context.Background(), `{"a":1}`) + if err != nil { + t.Fatalf("InvokableRun: %v", err) + } + if out != `{"ok":true}` { + t.Errorf("response = %q, want %q", out, `{"ok":true}`) + } + if inner.calls.Load() != 1 { + t.Errorf("inner call count = %d, want 1", inner.calls.Load()) + } + if inner.lastArgs != `{"a":1}` { + t.Errorf("lastArgs = %q, want %q", inner.lastArgs, `{"a":1}`) + } +} + +func TestWrapBaseTool_WithInvokeOpts(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "echo", runResp: `{}`} + wrapped := WrapBaseTool(inner) + _, err := wrapped.InvokableRun(context.Background(), `{}`, + WithInvokeOpts(basetool.WithUserID(42), basetool.WithTenant("acme")), + ) + if err != nil { + t.Fatalf("InvokableRun: %v", err) + } + if inner.lastOptsLen != 2 { + t.Errorf("inner saw %d opts, want 2", inner.lastOptsLen) + } +} + +func TestWrapBaseTool_RunErrIntoEnvelope(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "bad", runErr: errors.New("boom: sandbox path /etc not allowed")} + wrapped := WrapBaseTool(inner) + out, err := wrapped.InvokableRun(context.Background(), `{}`) + // Adapter folds tool errors into a JSON envelope so eino's ToolsNode + // doesn't treat them as graph-fatal. The LLM consumes the envelope + // as a tool result and decides to retry / switch / ask the user. + if err != nil { + t.Fatalf("err should be nil (folded into envelope), got %v", err) + } + if !strings.Contains(out, `"error"`) || !strings.Contains(out, "boom") { + t.Errorf("envelope should carry error text, got %q", out) + } + if !strings.Contains(out, `"status":"failed"`) { + t.Errorf("envelope should mark status=failed, got %q", out) + } +} + +func TestWrapBaseTools_SkipsNil(t *testing.T) { + t.Parallel() + tools := []basetool.BaseTool{nil, &fakeBaseTool{name: "a"}, nil, &fakeBaseTool{name: "b"}} + out := WrapBaseTools(tools) + if len(out) != 2 { + t.Fatalf("WrapBaseTools dropped wrong count: got %d entries, want 2", len(out)) + } +} + +// Per-run memo: an identical read-tool call returns the cached result +// without re-executing; distinct args re-execute. +func TestEinoToolAdapter_MemoizesIdenticalReadCalls(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "query_promql", class: "read", runResp: `{"v":1}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + r1, err1 := a.InvokableRun(ctx, `{"q":"up"}`) + r2, err2 := a.InvokableRun(ctx, `{"q":"up"}`) + if err1 != nil || err2 != nil { + t.Fatalf("errs: %v %v", err1, err2) + } + if r1 != `{"v":1}` || r2 != `{"v":1}` { + t.Fatalf("results = %q, %q; want cached identical", r1, r2) + } + if got := inner.calls.Load(); got != 1 { + t.Errorf("identical read calls should execute once, got %d", got) + } + if _, err := a.InvokableRun(ctx, `{"q":"down"}`); err != nil { + t.Fatalf("distinct call err: %v", err) + } + if got := inner.calls.Load(); got != 2 { + t.Errorf("distinct args should re-execute, got %d", got) + } +} + +// Write/destructive tools are never memoized — the review/mutation flow +// must see every call. +func TestEinoToolAdapter_DoesNotMemoizeWriteTool(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "host_restart_service", class: "destructive", runResp: `{"ok":true}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + _, _ = a.InvokableRun(ctx, `{"svc":"nginx"}`) + _, _ = a.InvokableRun(ctx, `{"svc":"nginx"}`) + if got := inner.calls.Load(); got != 2 { + t.Errorf("destructive tool must NOT be memoized; want 2 executions, got %d", got) + } +} + +// A failed call stays retryable — the error envelope is not cached. +func TestEinoToolAdapter_DoesNotMemoizeErrors(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "query_x", class: "read", runErr: errors.New("boom")} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + _, _ = a.InvokableRun(ctx, `{"q":"a"}`) + _, _ = a.InvokableRun(ctx, `{"q":"a"}`) + if got := inner.calls.Load(); got != 2 { + t.Errorf("errored read calls must stay retryable; want 2 executions, got %d", got) + } +} + +// After maxToolCallsPerRun distinct executions, the tool stops running and +// returns a "synthesize now" directive (catches the varying-args repeat loop +// the memo can't). +func TestEinoToolAdapter_PerToolCallCap(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "query_promql", class: "read", runResp: `{"v":1}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + // Distinct args each time so the identical-call memo doesn't short-circuit; + // the cap counts executions. + for i := 0; i < maxToolCallsPerRun; i++ { + out, _ := a.InvokableRun(ctx, fmt.Sprintf(`{"q":"m%d"}`, i)) + if strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("call %d should execute, got budget directive: %s", i, out) + } + } + if got := inner.calls.Load(); got != int32(maxToolCallsPerRun) { + t.Fatalf("expected %d executions, got %d", maxToolCallsPerRun, got) + } + // One past the cap → directive, no execution. + out, _ := a.InvokableRun(ctx, `{"q":"over"}`) + if !strings.Contains(out, "call_budget_exceeded") { + t.Errorf("past the cap should return the budget directive, got %q", out) + } + if got := inner.calls.Load(); got != int32(maxToolCallsPerRun) { + t.Errorf("over-cap call must NOT execute; still want %d, got %d", maxToolCallsPerRun, got) + } +} + +// The single-tool WrapBaseTool path leaves memo nil — no caching. +func TestEinoToolAdapter_NoMemoByDefault(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "query_x", class: "read", runResp: "ok"} + a := &einoToolAdapter{inner: inner} // memo nil + ctx := context.Background() + _, _ = a.InvokableRun(ctx, `{"q":"a"}`) + _, _ = a.InvokableRun(ctx, `{"q":"a"}`) + if got := inner.calls.Load(); got != 2 { + t.Errorf("memo-less adapter must execute each call; want 2, got %d", got) + } +} diff --git a/internal/manager/biz/aiops/runtime/v1/graph/types.go b/internal/manager/biz/aiops/runtime/v1/graph/types.go new file mode 100644 index 000000000..62cd1b5bf --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/graph/types.go @@ -0,0 +1,172 @@ +// Package graph is the eino-based agent execution kernel introduced by +// / (Agent compose.Graph 拼 ReAct) + (主参考图 +// — Graph 执行层). It builds a `compose.Runnable[*Input, *Output]` whose +// internal shape is the standard ReAct loop: +// +// START → MessageAssembler → ChatModel → Branch(tool_calls?) → +// yes → ToolsNode → MsgAppend → 回 ChatModel +// no → END +// +// Implementation note (architectural deviation ): the +// inner ReAct subgraph is built via cloudwego/eino's stock +// `flow/agent/react.NewAgent`, which already produces the exact graph +// the spec calls out (ChatModel ↔ Branch ↔ ToolsNode), bound at compile +// time with the user-supplied `model.ToolCallingChatModel` + tools. Our +// own wrapper graph (this package) adds the **MessageAssembler** lambda +// in front so callers pass `*Input` (system prompt + history + user +// text + system-reminder cfg) instead of pre-flattened `[]*schema.Message`, +// and an **OutputProjector** lambda in the back so callers receive +// `*Output` (assistant message + iteration count + usage) instead of +// the bare assistant `*schema.Message`. We are NOT re-implementing the +// branch / tools-node wiring by hand because eino's ReAct implementation +// is the canonical maintained one — re-rolling it would be churn against +// no benefit. See react.go header comment for the layered topology. +// +// PR-6 of scaffolding only. The cutover (replacing agent.go's +// 660-line for-loop with this graph) is the NEXT PR — main.go is not +// touched here. Callers in this PR are tests + internal wiring only. +package graph + +import ( + "time" + + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// Input is the structured request for one ReAct-loop run. It mirrors the +// information the legacy agent.go for-loop assembles from the chat +// session before dispatching to the LLM. The MessageAssembler lambda +// (react.go) flattens this into `[]*schema.Message` for the eino graph. +// +// ASCII shows MessageAssembler taking +// `Input{Messages, UserText, SystemPrompt}`; we add WebSearchEnabled and +// MentionsRendered as separate fields so the assembler can decide where +// to inline them (mentions go above the user text; web_search gating is +// enforced by the toolBag, not by message content). +type Input struct { + // SystemPrompt is the base agent instructions. Empty = no system + // message. PR-6: the caller (chatruntime, NEXT PR) stitches together + // base prompt + active skill prompts + agent persona; this layer + // just consumes the final string. + SystemPrompt string + + // History is the prior conversation in eino message form. Includes + // user / assistant / tool messages; ordered chronologically. + // HistoryLimit gating happens upstream — this layer takes the slice + // as-is. + History []*schema.Message + + // UserText is the new user turn (verbatim, post-mention-inlining). + // MentionsRendered, when non-empty, is prepended as a markdown + // preamble per agent.go's existing behaviour. + UserText string + + // WebSearchEnabled is the per-call gate the SPA's globe toggle + // flips. PR-6 carries this through but enforcement remains at the + // toolBag layer (chatruntime wires the right tool list); the field + // is here so the assembler can echo it into a system-reminder block + // for belt-and-braces (a hijacked tool_call from history replay + // won't slip through). + WebSearchEnabled bool + + // MentionsRendered is the pre-rendered markdown bullet list of + // @-mention context blocks. Empty = no mentions on this turn. + // agent.go's exact format ("用户在消息中引用了以下平台对象 ...") is + // reproduced here for replay parity. + MentionsRendered string + + // AgentReminder is the persona-level critical_reminder string the + // coordinator/runtime resolved for this turn. When + // non-empty it is appended verbatim — without further wrapping — as + // one bullet inside the per-turn block the + // MessageAssembler injects. Empty = no persona reminder for this + // turn (coordinator persona without a critical_reminder, or no + // persona at all). + // + // Static persona prose still lives inside SystemPrompt via + // chatruntime.ComposeSystemPrompt — this field carries only the + // short reminder line that gets re-injected on every turn. + AgentReminder string + // DynamicHints is the runtime-computed list of per-turn hint lines + // (each rendered as one bullet inside the block). + // — examples: "tool X failed N times in a row", "we + // already ran Y iterations". Computed by chatruntime from the + // session's recent message + tool history; the graph is not the + // source of truth and never inspects history to derive its own + // hints. + DynamicHints []string + + // Locale is the UI language the answer should be written in + // ("en-US" / "zh-CN"). The personas are Chinese, so without this the + // model defaults to Chinese even when the SPA is in English mode. The + // assembler turns it into an explicit "respond in " + // directive (system prompt + per-turn reminder). Empty = no directive + // (back-compat; e.g. the IM bridge, which doesn't carry a UI locale). + Locale string +} + +// Output is the terminal result of a graph run. Mirrors the parts of +// agent.Reply this layer can populate without reaching into the +// session repo (which is the persistence callback's job). Iterations +// is best-effort — eino's react agent doesn't expose the step count +// directly, so the OutputProjector counts ChatModel turns on the +// callback channel and stamps the value here. +type Output struct { + // AssistantMessage is the final assistant turn the model produced + // (no tool_calls). Always non-nil on success. + AssistantMessage *schema.Message + + // Iterations is the number of ChatModel calls that ran during this + // turn. Counted via the metrics/audit handler chain; 0 if no + // counter handler was wired. + Iterations int + + // Usage is the aggregated token usage across every ChatModel call + // in this run. Populated by the MetricsHandler (callbacks/metrics.go). + // Zero values when no counter was wired. + Usage llm.Usage +} + +// Config tunes graph behaviour. Defaults match agent.go's existing +// for-loop so the cutover PR sees zero behavioural drift. +// +// 改进点 #5: MaxIterations 内嵌 — eino's ReAct accepts +// MaxStep at construction; we expose it here. +type Config struct { + // Model is the LLM model id (e.g. "gpt-4o", "claude-sonnet-4-6"). + // Empty = use the underlying ChatModel's default. + Model string + + // Provider is the routing key recognised by RoutingChatModel + // (— provider 路由保留). Empty = use the routing + // model's defaultProvider. + Provider string + + // Temperature controls sampling randomness. 0 retains the legacy + // agent default of 0.1. + Temperature float32 + + // MaxIterations caps the outer ReAct loop. 0 -> 30 (agent.go default). + MaxIterations int + + // ToolTimeout is the per-tool wall clock ceiling. 0 -> 15s + // (agent.go default). Enforced by the BaseTool decorator chain + // (PR-3); this field is kept here so the cutover PR can pass the + // value through to chatruntime when it builds the toolBag. + ToolTimeout time.Duration +} + +// applyDefaults fills in zero-valued Config fields with the same +// defaults agent.go uses. Returned by value so callers don't see +// mutation of their Config struct. +func (c Config) applyDefaults() Config { + if c.MaxIterations <= 0 { + c.MaxIterations = 30 + } + if c.ToolTimeout <= 0 { + c.ToolTimeout = 15 * time.Second + } + return c +} diff --git a/internal/manager/biz/aiops/runtime/v1/source.json b/internal/manager/biz/aiops/runtime/v1/source.json new file mode 100644 index 000000000..1c420261b --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v1/source.json @@ -0,0 +1,376 @@ +{ + "release": "v0.7.158", + "commit": "04b2796319785df840d7a472569c380e0871730b", + "files": [ + { + "path": "chatruntime/agent_parser.go", + "source_blob": "059f2a40f3d8da243290d8b5b46457b24321e4bc", + "mode": "100644" + }, + { + "path": "chatruntime/agent_parser_test.go", + "source_blob": "d683e5a7a74900f87828cdb6ce2cab63bd9e19d0", + "mode": "100644" + }, + { + "path": "chatruntime/agent_registry.go", + "source_blob": "fa1e4474d67b9b300fb5958a1e6369414e3f4c3d", + "mode": "100644" + }, + { + "path": "chatruntime/agent_registry_test.go", + "source_blob": "d6935c46368ae1152d3fd0d9ffe1e61947a4b5e5", + "mode": "100644" + }, + { + "path": "chatruntime/bare_skills_test.go", + "source_blob": "390c0df30112b32a2987d1d08565fb99f7b3602d", + "mode": "100644" + }, + { + "path": "chatruntime/chatmodelopts_test.go", + "source_blob": "dcfd314dc03f42faeb82b2a29d03c45c7cebf63f", + "mode": "100644" + }, + { + "path": "chatruntime/command_parser.go", + "source_blob": "a294aa1283d3dd5c7897495e4a7052fd1baa9e85", + "mode": "100644" + }, + { + "path": "chatruntime/command_parser_test.go", + "source_blob": "19534a03fb3160d0dad4d5694f7548626d0a2f51", + "mode": "100644" + }, + { + "path": "chatruntime/load_all.go", + "source_blob": "304ef393320a1f290576f691c572208d0d0b394f", + "mode": "100644" + }, + { + "path": "chatruntime/load_all_test.go", + "source_blob": "bebe1581af2035f6073b51ed70409543ead883a7", + "mode": "100644" + }, + { + "path": "chatruntime/plugin_container.go", + "source_blob": "55bd09118921dedfd903cce3e917a85f354f2dcb", + "mode": "100644" + }, + { + "path": "chatruntime/plugin_container_test.go", + "source_blob": "93fc67b3e2d885b47ed2fbf2e3b3074dfa5aaae5", + "mode": "100644" + }, + { + "path": "chatruntime/runtime.go", + "source_blob": "21234123143c3b3cc1caaf016657728a5ec0d267", + "mode": "100644" + }, + { + "path": "chatruntime/runtime_test.go", + "source_blob": "9b15e96cdfdb6021b54b7022361c374bdacf4ec7", + "mode": "100644" + }, + { + "path": "chatruntime/runtime_worker_test.go", + "source_blob": "9fa63969e477c9a193d23c4b93a9331715dda94a", + "mode": "100644" + }, + { + "path": "chatruntime/skill_parser.go", + "source_blob": "0aa1f698962db68ce06356cd611a72c22ec93c32", + "mode": "100644" + }, + { + "path": "chatruntime/skill_parser_test.go", + "source_blob": "abe27da3b74fa13750101a645d5f881fd5288290", + "mode": "100644" + }, + { + "path": "chatruntime/skill_registry.go", + "source_blob": "d7ffc79c986c7575c4e8f6bfc08982864a3a6290", + "mode": "100644" + }, + { + "path": "chatruntime/skill_registry_test.go", + "source_blob": "20496349467411cd59082d37002efa4a616600e1", + "mode": "100644" + }, + { + "path": "chatruntime/system_prompt.go", + "source_blob": "421f9c2f4c5a9b15c83658538c39b989d9306a72", + "mode": "100644" + }, + { + "path": "chatruntime/system_prompt_test.go", + "source_blob": "7c0c606aeedc9492470e77f424e263106d3ddb45", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_parser/minimal/incident-investigator.md", + "source_blob": "ab3c0adf5103ac1c15d3c00f274934bced64ef46", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_parser/missing_when_to_use/foo.md", + "source_blob": "a8a7d82cf5e3186519c0b1600f3fc45fcbec350a", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md", + "source_blob": "ac8e534216a81e1281ae586bf3faf297152977d1", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md", + "source_blob": "6126be6350121eaa920551219a8512a075e25dac", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_registry/multi/README.md", + "source_blob": "fe1b6dbad2476cb6ca521af1e51416325f348eec", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_registry/multi/incident-investigator.md", + "source_blob": "3e9cd127eb07d1b576e6f702a72b29a4a42bf7f3", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_registry/multi/reviewer.md", + "source_blob": "2b5b7571f4e6a37ae953da9f821dbb6df3736334", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json", + "source_blob": "35a928395bf934cb1ea3c5da3d66708cc37b1cae", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json", + "source_blob": "7c92e55c3a2fefdde62ef3a49151e99158da9b25", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json", + "source_blob": "da39e4ffafe816be90259a3f68b763a3f71b93ed", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md", + "source_blob": "2f5cd59517e9ac7c24259f30dcbb2d386af164b4", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json", + "source_blob": "90566062121c9cc4af73a1fc2b48aa8e193c443f", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json", + "source_blob": "90b0c506aefd8bc4ba2f65a9fd67ddb3dfc3e379", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/path_traversal/skills/pwned", + "source_blob": "6998f20a07b1d03c62a19b010d26e3844be24112", + "mode": "120000" + }, + { + "path": "chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md", + "source_blob": "dba27c4eb35eb7bb75233992c014a4cec75bbba5", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json", + "source_blob": "b4853c2c112529ca01548882e3ace065c7c4978e", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/.mcp.json", + "source_blob": "5392ca626579dde6e2b956e632f557d162fc9b83", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md", + "source_blob": "0265b69d1e3f28d934912f51041404b1ef569d07", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md", + "source_blob": "a38e3afedc44973b49226263f88571122bf64345", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md", + "source_blob": "2f5cd59517e9ac7c24259f30dcbb2d386af164b4", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md", + "source_blob": "14870376331ae58968a2ade178201029ae684cdb", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json", + "source_blob": "72b2c845b9b1b346754d47cbb17c0e3dce936d35", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md", + "source_blob": "b820c41153888824658ad32a49d1d65b3e81f513", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md", + "source_blob": "baf6620bd2f182e8fd49223c4c2e1aaecc8c419d", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/minimal/SKILL.md", + "source_blob": "14870376331ae58968a2ade178201029ae684cdb", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/missing_description/SKILL.md", + "source_blob": "a517a9e42c235a311ef34f9a0798f30f19bfaa65", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/name_with_dash/SKILL.md", + "source_blob": "7b9cb73116176c3cad3ade313ef3f84105b7b73b", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/typical/SKILL.md", + "source_blob": "60ce5c8422705cf8b27c2446f09cc326e204addf", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md", + "source_blob": "b5143a6a10f9b10ad2769863c918d9f8ebc5e22d", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md", + "source_blob": "c428a8e0cddf8e21e89c5c74ab874dce223f0d86", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md", + "source_blob": "e935d3aaac26cbf723acf72bde3bd1f39a3e2e5a", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md", + "source_blob": "1b5ea8da6600914beecb051517fd998da5c1711b", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md", + "source_blob": "82023343213f597789836b8de5263a4a2912d7b0", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md", + "source_blob": "a1f04f7194c7505bf096285e9f5fc6cdcfa2c81e", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md", + "source_blob": "e038d9d44835a98ca2300d95326efd1baa1b8a0f", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md", + "source_blob": "9412ba7d741aa3052fd82d77b83dc060d1a24f1b", + "mode": "100644" + }, + { + "path": "chatruntime/types.go", + "source_blob": "cb3b52762aebcd766ebcbae156a12e2c92e9db83", + "mode": "100644" + }, + { + "path": "chatruntime/worker.go", + "source_blob": "d95c653d3fd739eb86c8b595b7044caa4c4cc99d", + "mode": "100644" + }, + { + "path": "graph/callbacks/audit.go", + "source_blob": "fd990b8c22a1d60a418c0e5c5be5d0ae549335d3", + "mode": "100644" + }, + { + "path": "graph/callbacks/audit_test.go", + "source_blob": "10d0689b2d8696541d1d0d5f07db2318a0ca0e78", + "mode": "100644" + }, + { + "path": "graph/callbacks/chain.go", + "source_blob": "9f81cd80e8447c68cba6a68098454dcb7b695b55", + "mode": "100644" + }, + { + "path": "graph/callbacks/chain_test.go", + "source_blob": "0b6f2ab7b1e189b2cafb75f71c3bad14a98368aa", + "mode": "100644" + }, + { + "path": "graph/callbacks/metrics.go", + "source_blob": "01736ea520bf373678ad3aba0ffc00ff7eb9d9ef", + "mode": "100644" + }, + { + "path": "graph/callbacks/metrics_test.go", + "source_blob": "23369d42cf58d06f9c76cb071c2a742885aca218", + "mode": "100644" + }, + { + "path": "graph/callbacks/persistence.go", + "source_blob": "d2c872c27965340b6ff5b7ccf25bef79cb0085db", + "mode": "100644" + }, + { + "path": "graph/callbacks/persistence_test.go", + "source_blob": "a923ff867922e13e314db36ffe89e3508d062ab9", + "mode": "100644" + }, + { + "path": "graph/callbacks/sse.go", + "source_blob": "99432f8be298de0134c337e618ec174f88d8cbc7", + "mode": "100644" + }, + { + "path": "graph/callbacks/sse_test.go", + "source_blob": "39779c1a0dee9b4f6cfaec5b110347b080275a30", + "mode": "100644" + }, + { + "path": "graph/react.go", + "source_blob": "007154076e8f3ea591e21641aa7303958c6310f6", + "mode": "100644" + }, + { + "path": "graph/react_test.go", + "source_blob": "2d123f1a4c7781298a09d375b0674b9c2309b1d8", + "mode": "100644" + }, + { + "path": "graph/tool_adapter.go", + "source_blob": "aad18b6061553c4adbeff6ffa44404d72a3f0ca5", + "mode": "100644" + }, + { + "path": "graph/tool_adapter_test.go", + "source_blob": "f3dbfd178a044bd7e6f2c2faa948acc3adef6637", + "mode": "100644" + }, + { + "path": "graph/types.go", + "source_blob": "62cd1b5bf8779727fd35486486f5f4d2cb64d2ee", + "mode": "100644" + } + ] +} diff --git a/internal/manager/biz/aiops/runtime/v2/README.md b/internal/manager/biz/aiops/runtime/v2/README.md new file mode 100644 index 000000000..28edf2dae --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/README.md @@ -0,0 +1,9 @@ +# Agent Runtime v2 + +来源：[Ongrid v0.18.2](https://github.com/ongridio/ongrid/releases/tag/v0.18.2)，提交 `ad9e52f670aa8a23976942954acbf103e1a6ef2d`。 + +提取该 release 的 `biz/aiops/chatruntime/` 与 `biz/aiops/graph/`，包括测试、testdata、prompt、Worker 和回调；不提取 legacy `agent/`。原文件清单与 Git blob 记录在 [source.json](source.json)。 + +适配：两个包的内部 import 改为本版本路径。无其他生产逻辑修改。 + +当前与主模块共享 Eino/LLM 依赖、领域模型、业务工具和存储接口，因此不是旧 release 的完整可执行镜像。此目录参与编译和 race 测试，但尚未接入生产路由或管理员切换。接入前必须验证工具装配、权限、审批及后台任务生命周期，不能用历史策略绕过当前安全约束。 diff --git a/internal/manager/biz/aiops/chatruntime/agent_parser.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_parser.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/agent_parser.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/agent_parser.go diff --git a/internal/manager/biz/aiops/chatruntime/agent_parser_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_parser_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/agent_parser_test.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/agent_parser_test.go diff --git a/internal/manager/biz/aiops/chatruntime/agent_registry.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/agent_registry.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry.go diff --git a/internal/manager/biz/aiops/chatruntime/agent_registry_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/agent_registry_test.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry_test.go diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/bare_skills_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/bare_skills_test.go new file mode 100644 index 000000000..390c0df30 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/bare_skills_test.go @@ -0,0 +1,138 @@ +package chatruntime + +import ( + "os" + "path/filepath" + "testing" +) + +// TestDetectContainer_BareSkillsDir — a skills.sh-style drop is a +// directory with skills//SKILL.md and NO plugin manifest. Should +// be recognized as ContainerBareSkills. +func TestDetectContainer_BareSkillsDir(t *testing.T) { + root := t.TempDir() + writeSkillMd(t, filepath.Join(root, "skills", "find-skills", "SKILL.md"), `--- +name: find-skills +description: Find and install agent skills when asked +--- + +Body of the skill. +`) + kind, manifest, err := DetectContainer(root) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerBareSkills { + t.Errorf("want ContainerBareSkills, got %q", kind) + } + if manifest != "" { + t.Errorf("bare-skills has no manifest path, got %q", manifest) + } +} + +// TestDetectContainer_BareSkillsRootSkillMd — a single-skill pack drops +// SKILL.md at the root of the cloned directory. +func TestDetectContainer_BareSkillsRootSkillMd(t *testing.T) { + root := t.TempDir() + writeSkillMd(t, filepath.Join(root, "SKILL.md"), `--- +name: solo-skill +description: A standalone skill at the root +--- +body +`) + kind, _, err := DetectContainer(root) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerBareSkills { + t.Errorf("want ContainerBareSkills, got %q", kind) + } +} + +// TestDetectContainer_ClaudeWins — when both .claude-plugin/plugin.json +// AND skills/ exist, the claude marker wins (existing behaviour). +func TestDetectContainer_ClaudeWins(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, ".claude-plugin"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(root, ".claude-plugin", "plugin.json"), + []byte(`{"id":"x","name":"X","version":"1.0.0"}`), 0o644); err != nil { + t.Fatalf("write manifest: %v", err) + } + writeSkillMd(t, filepath.Join(root, "skills", "s", "SKILL.md"), `--- +name: s +description: d +---`) + + kind, _, err := DetectContainer(root) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerClaude { + t.Errorf("want ContainerClaude, got %q", kind) + } +} + +// TestLoadPluginContainer_BareSkillsSynthesizesPack — bare-skill load +// produces a Pack with id derived from the directory basename. +func TestLoadPluginContainer_BareSkillsSynthesizesPack(t *testing.T) { + parent := t.TempDir() + root := filepath.Join(parent, "vercel-labs__skills") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + writeSkillMd(t, filepath.Join(root, "skills", "alpha", "SKILL.md"), `--- +name: alpha +description: First skill +--- +body alpha +`) + writeSkillMd(t, filepath.Join(root, "skills", "beta", "SKILL.md"), `--- +name: beta +description: Second skill +--- +body beta +`) + + res, err := LoadPluginContainer(root) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + if res.Pack == nil { + t.Fatalf("Pack nil") + } + if res.Pack.ID != "vercel-labs-skills" { + t.Errorf("Pack.ID = %q, want vercel-labs-skills", res.Pack.ID) + } + if len(res.Skills) != 2 { + t.Errorf("want 2 skills loaded, got %d", len(res.Skills)) + } +} + +// TestBareSkillsPackID — id normalisation rules. +func TestBareSkillsPackID(t *testing.T) { + cases := []struct{ in, want string }{ + {"vercel-labs__skills", "vercel-labs-skills"}, + {"My Awesome Pack v2", "my-awesome-pack-v2"}, + {"____", "untitled-skill-pack"}, + {"", "untitled-skill-pack"}, + {"snake_case_repo", "snake-case-repo"}, + {"already-clean", "already-clean"}, + } + for _, tc := range cases { + if got := bareSkillsPackID(tc.in); got != tc.want { + t.Errorf("bareSkillsPackID(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func writeSkillMd(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write: %v", err) + } +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/chatmodelopts_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/chatmodelopts_test.go new file mode 100644 index 000000000..dcfd314dc --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/chatmodelopts_test.go @@ -0,0 +1,23 @@ +package chatruntime + +import "testing" + +// chatModelOpts must add an option only for a set field — the empty/default +// path adds nothing so the routing default is preserved. +func TestChatModelOpts(t *testing.T) { + if got := chatModelOpts(nil); got != nil { + t.Errorf("nil req -> nil, got %v", got) + } + if got := chatModelOpts(&Request{}); len(got) != 0 { + t.Errorf("empty req -> no options, got %d", len(got)) + } + if got := chatModelOpts(&Request{Provider: " "}); len(got) != 0 { + t.Errorf("whitespace provider -> no options, got %d", len(got)) + } + if got := chatModelOpts(&Request{Provider: "anthropic"}); len(got) != 1 { + t.Errorf("provider only -> 1 option, got %d", len(got)) + } + if got := chatModelOpts(&Request{Provider: "anthropic", Model: "claude-3-opus-latest"}); len(got) != 2 { + t.Errorf("provider+model -> 2 options, got %d", len(got)) + } +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/command_parser.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/command_parser.go new file mode 100644 index 000000000..a294aa128 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/command_parser.go @@ -0,0 +1,151 @@ +package chatruntime + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// commandFrontmatter is the parsed YAML frontmatter of a claude/cursor +// `commands/.md` file. We model only the fields +// asks for; everything else lands in the prompt body unchanged. +// +// `allowed-tools` (claude-specific, kebab-cased — claude-code uses +// kebab-case in commands frontmatter, unlike SKILL.md / agents) is +// captured purely so we can echo it into the prompt body as a soft +// hint. ongrid does NOT enforce it as a hard restriction, since claude +// command tools (Bash / Edit / ...) don't map 1:1 to ongrid tool +// classes. +type commandFrontmatter struct { + Description string `yaml:"description"` + AllowedTools []string `yaml:"allowed-tools"` +} + +// ConvertCommandFile parses a claude/cursor `commands/.md` file +// at path and returns a Skill with mode=keyword. : +// +// - keyword = ["/", ""] (file basename, optional snake-cased +// alias when the basename contains dashes) +// - frontmatter description → Skill.Description +// - body → Skill.PromptBody (with `[能力: cmd_]` header) +// - tools: [] (commands are pure prompt injection — no new tools wired +// by ongrid; the claude-side tool restriction is informational) +// - name prefixed with `cmd_` to avoid collision with regular skills +// - frontmatter `allowed-tools: [Bash, Edit]` (claude-specific) → appended +// to the prompt body as a soft hint, NOT enforced. +// +// The basename rule mirrors how claude-code resolves slash commands: +// `commands/foo.md` → `/foo`. Nested subdirs are flattened +// (`commands/git/commit.md` → `/commit`); the loader (LoadPluginContainer) +// is responsible for walking commands/ recursively. +// +// Returns the parsed Skill, any non-fatal warnings, and a parse error +// when the file is unreadable / not markdown / missing required fields. +func ConvertCommandFile(path string) (*Skill, []LoadWarning, error) { + if !strings.EqualFold(filepath.Ext(path), ".md") { + return nil, nil, fmt.Errorf("chatruntime: %s: command file must end in .md", path) + } + + raw, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: read %s: %w", path, err) + } + + frontmatter, body, err := splitFrontmatter(raw) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: %s: %w", path, err) + } + + var fm commandFrontmatter + if len(frontmatter) > 0 { + if err := yaml.Unmarshal(frontmatter, &fm); err != nil { + return nil, nil, fmt.Errorf("chatruntime: parse %s frontmatter: %w", path, err) + } + } + + base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + base = strings.TrimSpace(base) + if base == "" { + return nil, nil, fmt.Errorf("chatruntime: %s: command file has empty basename", path) + } + + // Build keyword list + // Always include "/" and ""; if the basename contains + // dashes, also include the snake_cased alias so users typing either + // "review-pr" or "review_pr" trigger activation. + keywords := []string{"/" + base, base} + if strings.ContainsAny(base, "-. ") { + alias := normalizeSnakeName(base) + if alias != "" && alias != base { + keywords = append(keywords, alias) + } + } + + // Skill name uses cmd_ prefix to avoid colliding with regular skills. + // If the basename isn't snake_case, normalize after the prefix so the + // final name is always valid. + nameSuffix := base + if !skillNameRe.MatchString(nameSuffix) { + nameSuffix = normalizeSnakeName(nameSuffix) + } + skillName := "cmd_" + nameSuffix + + description := strings.TrimSpace(fm.Description) + if description == "" { + description = fmt.Sprintf("Slash-command equivalent of `/%s` (claude commands import).", base) + } + + // Compose the prompt body. The H1 is the canonical [能力: cmd_] + // tag so system_prompt assembly picks it up as-is. + bodyText := strings.TrimRight(string(body), "\n") + header := "[能力: " + skillName + "]" + + var b strings.Builder + b.WriteString(header) + b.WriteString("\n\n") + if bodyText != "" { + b.WriteString(bodyText) + b.WriteString("\n") + } + if len(fm.AllowedTools) > 0 { + // Soft hint, not a hard restriction. We surface the upstream + // allowed-tools list verbatim so the LLM can take it as a clue + // about the author's intent. + b.WriteString("\n") + b.WriteString(fmt.Sprintf("> 上游 (claude) 允许使用的工具: %s。ongrid 不强制此清单，仅作提示。\n", + strings.Join(fm.AllowedTools, ", "))) + } + + sk := &Skill{ + Name: skillName, + Description: description, + Activation: Activation{ + Mode: "keyword", + Keywords: keywords, + }, + PromptBody: strings.TrimRight(b.String(), "\n"), + Metadata: SkillMetadata{ + Ongrid: OngridExt{ + Activation: Activation{ + Mode: "keyword", + Keywords: keywords, + }, + }, + }, + Dir: filepath.Dir(path), + } + + var warnings []LoadWarning + if strings.TrimSpace(fm.Description) == "" { + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "command_missing_description", + Reason: fmt.Sprintf("command %q has no frontmatter description; using a generated default", base), + }) + } + + return sk, warnings, nil +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/command_parser_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/command_parser_test.go new file mode 100644 index 000000000..19534a03f --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/command_parser_test.go @@ -0,0 +1,159 @@ +package chatruntime + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// writeTemp writes content to a fresh tempdir/file and returns the path. +func writeTemp(t *testing.T, name, content string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + return path +} + +func TestConvertCommandFile_Minimal(t *testing.T) { + path := writeTemp(t, "commit.md", `--- +description: Commit the staged changes. +--- + +Run git commit with a conventional message. +`) + sk, warns, err := ConvertCommandFile(path) + if err != nil { + t.Fatalf("ConvertCommandFile: %v", err) + } + if len(warns) != 0 { + t.Errorf("unexpected warnings: %+v", warns) + } + if sk.Name != "cmd_commit" { + t.Errorf("Name = %q, want cmd_commit", sk.Name) + } + if sk.Description != "Commit the staged changes." { + t.Errorf("Description = %q", sk.Description) + } + if sk.Activation.Mode != "keyword" { + t.Errorf("Activation.Mode = %q, want keyword", sk.Activation.Mode) + } + if !hasName(sk.Activation.Keywords, "/commit") { + t.Errorf("missing /commit keyword; got %v", sk.Activation.Keywords) + } + if !hasName(sk.Activation.Keywords, "commit") { + t.Errorf("missing commit keyword; got %v", sk.Activation.Keywords) + } + // metadata.ongrid.activation should mirror the top-level activation + // so downstream consumers reading either key see keyword mode. + if sk.Metadata.Ongrid.Activation.Mode != "keyword" { + t.Errorf("metadata.ongrid.activation.mode = %q, want keyword", sk.Metadata.Ongrid.Activation.Mode) + } + if len(sk.Tools) != 0 { + t.Errorf("Tools should be empty; got %d", len(sk.Tools)) + } + if !strings.HasPrefix(strings.TrimSpace(sk.PromptBody), "[能力: cmd_commit]") { + t.Errorf("body should start with [能力: cmd_commit]; got:\n%s", sk.PromptBody) + } +} + +func TestConvertCommandFile_AllowedToolsAppendedToBody(t *testing.T) { + path := writeTemp(t, "deploy.md", `--- +description: Deploy to staging. +allowed-tools: [Bash, Edit, Read] +--- + +Run the deploy script. +`) + sk, _, err := ConvertCommandFile(path) + if err != nil { + t.Fatalf("ConvertCommandFile: %v", err) + } + for _, tool := range []string{"Bash", "Edit", "Read"} { + if !strings.Contains(sk.PromptBody, tool) { + t.Errorf("body should mention allowed tool %q; got:\n%s", tool, sk.PromptBody) + } + } + // Soft-hint, not enforcement: Tools slice is still empty. + if len(sk.Tools) != 0 { + t.Errorf("Tools should remain empty (soft hint, not hard restriction); got %d", len(sk.Tools)) + } +} + +func TestConvertCommandFile_DashedBasename_SnakeAlias(t *testing.T) { + path := writeTemp(t, "review-pr.md", `--- +description: Review the PR diff. +--- + +Walk through the diff and call out issues. +`) + sk, _, err := ConvertCommandFile(path) + if err != nil { + t.Fatalf("ConvertCommandFile: %v", err) + } + if sk.Name != "cmd_review_pr" { + t.Errorf("Name = %q, want cmd_review_pr", sk.Name) + } + for _, kw := range []string{"/review-pr", "review-pr", "review_pr"} { + if !hasName(sk.Activation.Keywords, kw) { + t.Errorf("missing keyword %q; got %v", kw, sk.Activation.Keywords) + } + } +} + +func TestConvertCommandFile_NonMarkdown_Error(t *testing.T) { + path := writeTemp(t, "weird.txt", `not markdown`) + if _, _, err := ConvertCommandFile(path); err == nil { + t.Fatalf("expected error for .txt file") + } +} + +func TestConvertCommandFile_NoFrontmatter_GeneratesDescription(t *testing.T) { + path := writeTemp(t, "raw.md", "Just a plain markdown body, no frontmatter.\n") + sk, warns, err := ConvertCommandFile(path) + if err != nil { + t.Fatalf("ConvertCommandFile: %v", err) + } + if sk.Name != "cmd_raw" { + t.Errorf("Name = %q, want cmd_raw", sk.Name) + } + if sk.Description == "" { + t.Errorf("Description should be generated when missing") + } + hasMissingDesc := false + for _, w := range warns { + if w.Code == "command_missing_description" { + hasMissingDesc = true + } + } + if !hasMissingDesc { + t.Errorf("expected command_missing_description warning; got %+v", warns) + } +} + +func TestConvertCommandFile_NestedFlattens(t *testing.T) { + // A file at commands/git/commit.md should flatten to cmd_commit by + // basename — confirms the loader's flatten policy at unit level. + dir := t.TempDir() + nested := filepath.Join(dir, "git") + if err := os.MkdirAll(nested, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + path := filepath.Join(nested, "commit.md") + if err := os.WriteFile(path, []byte(`--- +description: nested commit cmd +--- +body`), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + sk, _, err := ConvertCommandFile(path) + if err != nil { + t.Fatalf("ConvertCommandFile: %v", err) + } + if sk.Name != "cmd_commit" { + t.Errorf("Name = %q, want cmd_commit (basename flatten)", sk.Name) + } +} diff --git a/internal/manager/biz/aiops/chatruntime/config_confirm.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/config_confirm.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/config_confirm.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/config_confirm.go diff --git a/internal/manager/biz/aiops/chatruntime/decision.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/decision.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/decision.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/decision.go diff --git a/internal/manager/biz/aiops/chatruntime/decision_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/decision_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/decision_test.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/decision_test.go diff --git a/internal/manager/biz/aiops/chatruntime/k8s_action_e2e_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/k8s_action_e2e_test.go similarity index 99% rename from internal/manager/biz/aiops/chatruntime/k8s_action_e2e_test.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/k8s_action_e2e_test.go index 203e91bb7..4f775e215 100644 --- a/internal/manager/biz/aiops/chatruntime/k8s_action_e2e_test.go +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/k8s_action_e2e_test.go @@ -11,7 +11,7 @@ import ( "github.com/cloudwego/eino/schema" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/load_all.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/load_all.go new file mode 100644 index 000000000..304ef3933 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/load_all.go @@ -0,0 +1,245 @@ +package chatruntime + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// LoadAllConfig is the input to LoadAll — the unified entry point for +// populating SkillRegistry + AgentRegistry from disk. PR-skill-load +// (simplified loader) collapses what used to be two +// separate walks (SkillRegistry.Load + AgentRegistry.Load) into one. +// +// Both roots are optional. A non-existent root is not an error (a +// fresh install with no skills/agents boots fine). +type LoadAllConfig struct { + // SkillsRoot is the directory walked for SKILL.md files and plugin + // containers. Plugin containers found here contribute their skills, + // agents, and command-converted skills to the result. + SkillsRoot string + + // AgentsRoot is the directory walked for *.md agent personas. + // Plugin containers found here contribute their agents (and their + // skills + command-skills, since a pack drop in either root is + // equally valid). + AgentsRoot string + + // ExtraSkillsRoots is a list of additional directories to walk + // alongside SkillsRoot. Used to keep image-baked built-in skills + // in one root and marketplace-installed packs in another, so + // marketplace Reload() doesn't have to drop the built-ins to refresh + // the user-installed set. Empty / non-existent entries are skipped + // silently. + ExtraSkillsRoots []string + + // ExtraAgentsRoots mirrors ExtraSkillsRoots for the agents side — + // each entry is walked with the "loose *.md = agent persona" rule + // so additional persona drops pile on top of AgentsRoot without + // the loose .md files being silently filtered out as non-SKILL.md. + ExtraAgentsRoots []string +} + +// LoadAll walks SkillsRoot + AgentsRoot once, dispatching to the right +// parser per file / directory: +// +// - /.../SKILL.md → ParseSkillMd +// - /...//.claude-plugin/plugin.json +// - /...//openclaw.plugin.json → LoadPluginContainer (recursive) +// - /.../*.md (not under a pack) → ParseAgentMd +// +// When a plugin container is detected, the walk SkipDir's into it and +// hands off to LoadPluginContainer; this prevents double-loading the +// same SKILL.md once from the outer walk and again from the recursive +// container load. +// +// The LoadResult.Pack field is left nil — LoadAll aggregates packs +// across multiple roots and the field is reserved for the +// single-container LoadPluginContainer flow. +func LoadAll(cfg LoadAllConfig) (*LoadResult, error) { + res := &LoadResult{} + + if cfg.SkillsRoot != "" { + more, err := walkLoadRoot(cfg.SkillsRoot, true) + if err != nil { + return nil, fmt.Errorf("chatruntime: load skills root %q: %w", cfg.SkillsRoot, err) + } + res.Skills = append(res.Skills, more.Skills...) + res.Agents = append(res.Agents, more.Agents...) + res.Warnings = append(res.Warnings, more.Warnings...) + } + + for _, extra := range cfg.ExtraSkillsRoots { + if extra == "" { + continue + } + more, err := walkLoadRoot(extra, true) + if err != nil { + return nil, fmt.Errorf("chatruntime: load extra skills root %q: %w", extra, err) + } + res.Skills = append(res.Skills, more.Skills...) + res.Agents = append(res.Agents, more.Agents...) + res.Warnings = append(res.Warnings, more.Warnings...) + } + + for _, extra := range cfg.ExtraAgentsRoots { + if extra == "" { + continue + } + more, err := walkLoadRoot(extra, false) + if err != nil { + return nil, fmt.Errorf("chatruntime: load extra agents root %q: %w", extra, err) + } + res.Skills = append(res.Skills, more.Skills...) + res.Agents = append(res.Agents, more.Agents...) + res.Warnings = append(res.Warnings, more.Warnings...) + } + + if cfg.AgentsRoot != "" { + more, err := walkLoadRoot(cfg.AgentsRoot, false) + if err != nil { + return nil, fmt.Errorf("chatruntime: load agents root %q: %w", cfg.AgentsRoot, err) + } + res.Skills = append(res.Skills, more.Skills...) + res.Agents = append(res.Agents, more.Agents...) + res.Warnings = append(res.Warnings, more.Warnings...) + } + return res, nil +} + +// walkLoadRoot walks root once. wantSkills selects whether we treat +// loose SKILL.md (true) or loose *.md (false → agent personas) as the +// primary content type — i.e. the root's "purpose". Plugin containers +// found in either root contribute everything they ship. +func walkLoadRoot(root string, wantSkills bool) (*LoadResult, error) { + res := &LoadResult{} + + info, err := os.Stat(root) + if err != nil { + if os.IsNotExist(err) { + return res, nil + } + return nil, fmt.Errorf("stat root %q: %w", root, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("root %q is not a directory", root) + } + + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return nil, fmt.Errorf("eval symlinks %q: %w", root, err) + } + + walkErr := filepath.Walk(root, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + res.Warnings = append(res.Warnings, LoadWarning{ + Path: path, Code: "walk_error", Reason: walkErr.Error(), + }) + return nil + } + if info == nil { + return nil + } + + // A symlink at the top of a tree is a common drop pattern (e.g. + // /var/lib/ongrid/system/skills/ → real-bundle). filepath.Walk + // won't follow it on its own, so we Stat-resolve it: if it lands + // inside resolvedRoot AND points to a directory that hosts a + // plugin container, treat it as a container. + if info.Mode()&os.ModeSymlink != 0 { + if !pathSafeUnderRoot(path, resolvedRoot, &res.Warnings) { + return nil + } + target, err := os.Stat(path) + if err != nil || !target.IsDir() { + return nil + } + kind, _, _ := DetectContainer(path) + if kind != ContainerNone { + lr, err := LoadPluginContainer(path) + if err != nil { + res.Warnings = append(res.Warnings, LoadWarning{ + Path: path, Code: "container_load_failed", Reason: err.Error(), + }) + return nil + } + res.Skills = append(res.Skills, lr.Skills...) + res.Agents = append(res.Agents, lr.Agents...) + res.Warnings = append(res.Warnings, lr.Warnings...) + } + return nil + } + + if info.IsDir() { + // Plugin container marker: hand off to LoadPluginContainer + // and SkipDir so the outer walk doesn't double-process the + // same SKILL.md / agent .md. + kind, _, _ := DetectContainer(path) + if kind != ContainerNone { + lr, err := LoadPluginContainer(path) + if err != nil { + res.Warnings = append(res.Warnings, LoadWarning{ + Path: path, Code: "container_load_failed", Reason: err.Error(), + }) + return filepath.SkipDir + } + res.Skills = append(res.Skills, lr.Skills...) + res.Agents = append(res.Agents, lr.Agents...) + res.Warnings = append(res.Warnings, lr.Warnings...) + return filepath.SkipDir + } + return nil + } + + // File handlers — only the root's "primary" content type is + // loaded loose. Skills root walks SKILL.md; agents root walks + // loose *.md (one persona per file). + base := info.Name() + if wantSkills { + if base != "SKILL.md" { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &res.Warnings) { + return nil + } + sk, ws, err := ParseSkillMd(path) + res.Warnings = append(res.Warnings, ws...) + if err != nil { + res.Warnings = append(res.Warnings, LoadWarning{ + Path: path, Code: "parse_failed", Reason: err.Error(), + }) + return nil + } + sk.Dir = filepath.Dir(path) + res.Skills = append(res.Skills, sk) + return nil + } + + // Agent root — *.md files are personas; README.md skipped. + if !strings.EqualFold(filepath.Ext(base), ".md") { + return nil + } + if strings.EqualFold(base, "README.md") { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &res.Warnings) { + return nil + } + ag, ws, err := ParseAgentMd(path) + res.Warnings = append(res.Warnings, ws...) + if err != nil { + res.Warnings = append(res.Warnings, LoadWarning{ + Path: path, Code: "parse_failed", Reason: err.Error(), + }) + return nil + } + ag.Dir = filepath.Dir(path) + res.Agents = append(res.Agents, ag) + return nil + }) + if walkErr != nil { + return nil, walkErr + } + return res, nil +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/load_all_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/load_all_test.go new file mode 100644 index 000000000..bebe1581a --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/load_all_test.go @@ -0,0 +1,146 @@ +package chatruntime + +import ( + "os" + "path/filepath" + "testing" +) + +// TestLoadAll_WiresPluginContainersFromSkillsRoot drops a recursive_claude +// pack under skillsRoot and asserts both its skills AND its agents reach +// the LoadResult. +func TestLoadAll_WiresPluginContainersFromSkillsRoot(t *testing.T) { + root := t.TempDir() + // Mirror the recursive_claude fixture into root/recursive so that + // SkillsRoot is the conventional "parent of packs" layout. + src, err := filepath.Abs(filepath.Join("testdata", "plugin_container", "recursive_claude")) + if err != nil { + t.Fatalf("abs: %v", err) + } + dst := filepath.Join(root, "recursive") + if err := copyTree(src, dst); err != nil { + t.Fatalf("copyTree: %v", err) + } + + res, err := LoadAll(LoadAllConfig{SkillsRoot: root}) + if err != nil { + t.Fatalf("LoadAll: %v", err) + } + if len(res.Skills) == 0 { + t.Fatalf("LoadAll returned no skills; warns=%+v", res.Warnings) + } + hasHello := false + hasCmd := false + for _, sk := range res.Skills { + if sk.Name == "hello_world" { + hasHello = true + } + if sk.Name == "cmd_commit" { + hasCmd = true + } + } + if !hasHello { + t.Errorf("hello_world not loaded from pack under skills root; got %v", skillNames(res.Skills)) + } + if !hasCmd { + t.Errorf("cmd_commit not loaded from pack under skills root; got %v", skillNames(res.Skills)) + } + // Agents from the pack should also surface. + hasAgent := false + for _, ag := range res.Agents { + if ag.Name == "pr_summary" { + hasAgent = true + } + } + if !hasAgent { + t.Errorf("pr_summary agent should reach LoadAll result; got %d agents", len(res.Agents)) + } +} + +func TestLoadAll_NonExistentRoots_NoError(t *testing.T) { + res, err := LoadAll(LoadAllConfig{ + SkillsRoot: filepath.Join(t.TempDir(), "missing-skills"), + AgentsRoot: filepath.Join(t.TempDir(), "missing-agents"), + }) + if err != nil { + t.Fatalf("LoadAll on missing roots should be nil-error, got %v", err) + } + if len(res.Skills) != 0 || len(res.Agents) != 0 { + t.Errorf("expected empty result; got skills=%d agents=%d", len(res.Skills), len(res.Agents)) + } +} + +func TestLoadAll_AgentsRootLoadsLoosePersonas(t *testing.T) { + src, err := filepath.Abs(filepath.Join("testdata", "agent_registry", "multi")) + if err != nil { + t.Fatalf("abs: %v", err) + } + res, err := LoadAll(LoadAllConfig{AgentsRoot: src}) + if err != nil { + t.Fatalf("LoadAll: %v", err) + } + if len(res.Agents) != 2 { + t.Errorf("expected 2 agents, got %d", len(res.Agents)) + } +} + +// TestLoadAll_ExtraAgentsRoots — regression for the marketplace reload +// path. When the AgentRegistry reloads after a pack install, the +// marketplace passes the builtin agents directory as an extra. The +// previous implementation routed extras through ExtraSkillsRoots which +// silently dropped loose *.md agent personas (only SKILL.md matches +// the wantSkills=true filter). Result: every marketplace install +// emptied the assistant page until manager restart. +func TestLoadAll_ExtraAgentsRoots(t *testing.T) { + personas, err := filepath.Abs(filepath.Join("testdata", "agent_registry", "multi")) + if err != nil { + t.Fatalf("abs: %v", err) + } + // Treat the persona dir as an EXTRA — primary is empty. Should + // still load both .md personas. + res, err := LoadAll(LoadAllConfig{ExtraAgentsRoots: []string{personas}}) + if err != nil { + t.Fatalf("LoadAll: %v", err) + } + if len(res.Agents) != 2 { + t.Errorf("ExtraAgentsRoots: want 2 agents, got %d", len(res.Agents)) + } + // Sanity: ExtraSkillsRoots on the same dir would NOT yield the + // agents (only SKILL.md matches the skills filter). This guards + // against future refactors collapsing the two extra-lists back + // into one without restoring the semantic split. + res2, err := LoadAll(LoadAllConfig{ExtraSkillsRoots: []string{personas}}) + if err != nil { + t.Fatalf("LoadAll (skills extras): %v", err) + } + if len(res2.Agents) != 0 { + t.Errorf("ExtraSkillsRoots: agent personas leaked into skills walk, got %d", len(res2.Agents)) + } +} + +// copyTree recursively copies the contents of src to dst. Used by +// LoadAll tests so the test's tempdir is a self-contained skills root +// (no symlink escape). +func copyTree(src, dst string) error { + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + if info.IsDir() { + return os.MkdirAll(target, 0o755) + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + return os.WriteFile(target, raw, 0o600) + }) +} diff --git a/internal/manager/biz/aiops/chatruntime/plugin_container.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/plugin_container.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/plugin_container.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/plugin_container.go diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/plugin_container_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/plugin_container_test.go new file mode 100644 index 000000000..93fc67b3e --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/plugin_container_test.go @@ -0,0 +1,302 @@ +package chatruntime + +import ( + "path/filepath" + "strings" + "testing" +) + +func fixturePack(name string) string { + return filepath.Join("testdata", "plugin_container", name) +} + +func TestDetectContainer_Claude(t *testing.T) { + kind, path, err := DetectContainer(fixturePack("claude_pack")) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerClaude { + t.Errorf("kind = %q, want claude", kind) + } + if filepath.Base(path) != "plugin.json" { + t.Errorf("path = %q", path) + } +} + +func TestDetectContainer_Openclaw(t *testing.T) { + kind, path, err := DetectContainer(fixturePack("openclaw_pack")) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerOpenclaw { + t.Errorf("kind = %q, want openclaw", kind) + } + if filepath.Base(path) != "openclaw.plugin.json" { + t.Errorf("path = %q", path) + } +} + +func TestDetectContainer_None(t *testing.T) { + kind, _, err := DetectContainer(t.TempDir()) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerNone { + t.Errorf("kind = %q, want none", kind) + } +} + +func TestLoadPluginContainer_Claude(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("claude_pack")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + if res.Pack == nil { + t.Fatal("Pack is nil") + } + if res.Pack.ID != "acme-tools" { + t.Errorf("ID = %q", res.Pack.ID) + } + if res.Pack.Version != "1.0.0" { + t.Errorf("Version = %q", res.Pack.Version) + } + if res.Pack.DisplayName != "Acme Tools" { + t.Errorf("DisplayName = %q", res.Pack.DisplayName) + } + if len(res.Pack.ConfigSchema) == 0 { + t.Error("ConfigSchema empty") + } + if res.Pack.ManifestSHA256 == "" { + t.Error("ManifestSHA256 empty") + } + if res.Pack.SignatureState != "unsigned" { + t.Errorf("SignatureState = %q", res.Pack.SignatureState) + } +} + +func TestLoadPluginContainer_OpenclawLegacyPreserved(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("openclaw_pack")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + if res.Pack.ID != "openclaw-bundle" { + t.Errorf("ID = %q", res.Pack.ID) + } + legacy, ok := res.Pack.UIMetadata["openclaw_legacy"].(map[string]any) + if !ok { + t.Fatalf("openclaw_legacy missing or wrong type: %#v", res.Pack.UIMetadata) + } + for _, k := range []string{"providers", "channels", "cliBackends", "providerAuthChoices"} { + if _, ok := legacy[k]; !ok { + t.Errorf("openclaw_legacy missing key %q; got %v", k, mapKeys(legacy)) + } + } + hasLegacyWarn := false + for _, w := range res.Warnings { + if w.Code == "openclaw_legacy_preserved" { + hasLegacyWarn = true + } + } + if !hasLegacyWarn { + t.Errorf("expected openclaw_legacy_preserved warning; got %+v", res.Warnings) + } +} + +func TestLoadPluginContainer_HooksAndMcpWarnings(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("claude_pack_with_extras")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + codes := map[string]bool{} + for _, w := range res.Warnings { + codes[w.Code] = true + } + if !codes["hooks_unsupported"] { + t.Errorf("expected hooks_unsupported warning; got %+v", res.Warnings) + } + if !codes["mcp_unsupported"] { + t.Errorf("expected mcp_unsupported warning; got %+v", res.Warnings) + } +} + +func TestLoadPluginContainer_NoMarker(t *testing.T) { + _, err := LoadPluginContainer(t.TempDir()) + if err == nil { + t.Fatal("expected error when no marker present") + } +} + +// --- recursive load (PR-skill-load) --- + +func TestLoadPluginContainer_ClaudeRecursive_Skills(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_claude")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + // Expect: 1 SKILL.md (hello_world) + 1 command-converted skill (cmd_commit). + var helloFound, cmdCommitFound bool + for _, sk := range res.Skills { + if sk.Name == "hello_world" { + helloFound = true + if sk.Dir == "" { + t.Errorf("hello_world.Dir empty") + } + if sk.Activation.Mode != "always" { + t.Errorf("hello_world activation = %q, want always (default)", sk.Activation.Mode) + } + } + if sk.Name == "cmd_commit" { + cmdCommitFound = true + } + } + if !helloFound { + t.Errorf("hello_world skill not loaded; got %v", skillNames(res.Skills)) + } + if !cmdCommitFound { + t.Errorf("cmd_commit (from commands/) not loaded; got %v", skillNames(res.Skills)) + } +} + +func TestLoadPluginContainer_ClaudeRecursive_Agents(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_claude")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + if len(res.Agents) != 1 { + t.Fatalf("Agents len = %d, want 1", len(res.Agents)) + } + if res.Agents[0].Name != "pr_summary" { + t.Errorf("agent name = %q, want pr_summary", res.Agents[0].Name) + } + if res.Agents[0].Dir == "" { + t.Errorf("agent.Dir empty") + } +} + +func TestLoadPluginContainer_ClaudeRecursive_Commands(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_claude")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + var cmd *Skill + for _, sk := range res.Skills { + if sk.Name == "cmd_commit" { + cmd = sk + break + } + } + if cmd == nil { + t.Fatalf("cmd_commit not in skills; got %v", skillNames(res.Skills)) + } + if cmd.Activation.Mode != "keyword" { + t.Errorf("cmd_commit activation = %q, want keyword", cmd.Activation.Mode) + } + wantKw := map[string]bool{"/commit": false, "commit": false} + for _, kw := range cmd.Activation.Keywords { + if _, ok := wantKw[kw]; ok { + wantKw[kw] = true + } + } + for kw, found := range wantKw { + if !found { + t.Errorf("cmd_commit missing keyword %q; got %v", kw, cmd.Activation.Keywords) + } + } + // Body should carry the soft-hint about claude allowed-tools. + if !strings.Contains(cmd.PromptBody, "Bash") || !strings.Contains(cmd.PromptBody, "Edit") { + t.Errorf("cmd_commit prompt body should include allowed-tools hint; got:\n%s", cmd.PromptBody) + } + if !strings.HasPrefix(strings.TrimSpace(cmd.PromptBody), "[能力: cmd_commit]") { + t.Errorf("cmd_commit body should start with [能力: cmd_commit] header; got:\n%s", cmd.PromptBody) + } +} + +func TestLoadPluginContainer_HookSubdir_Warning(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_claude")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + codes := map[string]int{} + for _, w := range res.Warnings { + codes[w.Code]++ + } + if codes["hooks_unsupported"] == 0 { + t.Errorf("expected hooks_unsupported warning; got %v", codes) + } + if codes["hooks_dropped"] == 0 { + t.Errorf("expected hooks_dropped warning per hook file; got %v", codes) + } +} + +func TestLoadPluginContainer_McpFile_Warning(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_claude")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + hasMcp := false + for _, w := range res.Warnings { + if w.Code == "mcp_unsupported" { + hasMcp = true + } + } + if !hasMcp { + t.Errorf("expected mcp_unsupported warning; got %+v", res.Warnings) + } +} + +func TestLoadPluginContainer_OpenclawRecursive_RelativeSkills(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_openclaw")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + if len(res.Skills) != 2 { + t.Fatalf("Skills len = %d, want 2 (foo, bar); got %v", len(res.Skills), skillNames(res.Skills)) + } + names := map[string]bool{} + for _, sk := range res.Skills { + names[sk.Name] = true + } + for _, expected := range []string{"foo", "bar"} { + if !names[expected] { + t.Errorf("skill %q not loaded; got %v", expected, skillNames(res.Skills)) + } + } +} + +func TestLoadPluginContainer_PathTraversal_Rejected(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("path_traversal")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + // The pwned skill (under a symlink that escapes the pack root) MUST + // NOT make it into res.Skills. + for _, sk := range res.Skills { + if sk.Name == "pwned" { + t.Fatalf("pwned skill loaded despite path-traversal symlink: %+v", sk) + } + } + // Expect an escapes_root warning. + hasEscape := false + for _, w := range res.Warnings { + if w.Code == "escapes_root" { + hasEscape = true + if !strings.Contains(strings.ToLower(w.Reason), "path traversal") && + !strings.Contains(strings.ToLower(w.Reason), "escapes") { + t.Errorf("escapes_root reason should mention traversal/escapes; got %q", w.Reason) + } + } + } + if !hasEscape { + t.Errorf("expected escapes_root warning; got %+v", res.Warnings) + } +} + +// --- helpers --- + +func mapKeys(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/manager/biz/aiops/chatruntime/reasoning_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/reasoning_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/reasoning_test.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/reasoning_test.go diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime.go new file mode 100644 index 000000000..b9e894370 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime.go @@ -0,0 +1,1801 @@ +// runtime.go is the in-process orchestration entry the new graph-based +// agent path goes through. PR-9 of — the cutover layer that +// pre-PR-9 callers (legacy agent.go) bypass entirely. After this PR +// the manager's HTTP service can pick which path runs based on the +// ONGRID_AGENT_KERNEL feature flag (default = legacy). +// +// Responsibilities: +// +// 1. Ownership check (caller user_id == session.user_id; admin bypass +// happens upstream — by the time Handle is called the service has +// already resolved the owning user_id). +// 2. Skill resolution against the user query (SkillRegistry.Resolve) +// — the active skills shape the system prompt. +// 3. System prompt assembly via ComposeSystemPrompt — base prompt + +// active skill prompts + (optional) agent persona. +// 4. @-mention inlining — same markdown-bullet preamble the legacy +// agent.go produces, so chat history replay is byte-identical +// across kernels (legacy mention text persists into the new path +// unchanged). +// 5. User-message persistence (chat_messages role=user). Mirrors the +// legacy "persist before LLM call" invariant from agent.go so a +// downstream crash leaves the user turn on disk. +// 6. Graph invoke with the default callback chain +// (callbacks.NewDefaultHandlers) wired via compose.WithCallbacks. +// The callback chain handles persistence (assistant + tool rows), +// SSE streaming, audit, metrics, and budget gating. +// 7. Reply translation — a chatruntime.Reply that the service layer +// translates back to agent.Reply for HTTP response shape parity. +// +// Behaviour parity with legacy agent.go is the explicit goal — the +// SPA, the persistence schema, and downstream reports must not see +// the kernel switch. Where exact parity isn't reachable (e.g. eino's +// graph runs the loop internally so we don't see per-iteration +// AssistantEvent counts), the value is best-effort and documented +// inline. +package chatruntime + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strconv" + "strings" + "sync" + "time" + + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/schema" + "go.opentelemetry.io/otel/attribute" + + biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph/callbacks" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/toolreplay" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/errs" + "github.com/ongridio/ongrid/internal/pkg/llm" + "github.com/ongridio/ongrid/internal/pkg/tracing" +) + +// Mention mirrors the legacy agent.Mention shape so chatruntime can +// be called without dragging the manager/biz/aiops/agent package as +// a dependency. The service-layer adapter does the type translation. +type Mention struct { + Type string + ID string + Label string +} + +// MentionResolver hydrates a list of @-mentions into one bullet line +// each. Mirrors agent.MentionResolver. nil = no inlining. +type MentionResolver interface { + Resolve(ctx context.Context, mentions []Mention) []string +} + +// EventType enumerates streaming events emitted by the runtime. The +// values are byte-equal to the legacy agent.EventType strings so the +// SSE adapter at the service layer can route both kernels through +// the same wire format. New event names introduced in PR-6 +// (assistant_start / assistant_delta / assistant_end) are NOT +// surfaced through this enum yet — the cutover keeps the legacy +// frame names so the SPA round-trips without changes. +type EventType string + +const ( + // EventAssistant fires after every assistant turn is persisted. + EventAssistant EventType = "assistant" + // EventToolStart fires after a tool_call row is persisted as + // pending, before the tool actually runs. + EventToolStart EventType = "tool_start" + // EventToolEnd fires once the tool finishes (success/error/timeout). + EventToolEnd EventType = "tool_end" + // EventDone fires once at terminal success with the final Reply. + EventDone EventType = "done" + // EventError fires on terminal failure. + EventError EventType = "error" +) + +// AssistantEvent describes one assistant turn's persisted row. Same +// shape as legacy agent.AssistantEvent. +type AssistantEvent struct { + Iteration int + MessageID string + Content string + CreatedAt time.Time + PendingToolCalls int +} + +// ToolEvent describes one tool_call lifecycle event. Same shape as +// legacy agent.ToolEvent. +type ToolEvent struct { + ToolCallID string + Name string + DeviceID *uint64 + Status string + StartedAt time.Time + EndedAt *time.Time + DurationMs int64 + Error string + ArgsJSON string + ResultJSON string +} + +// Event is the streaming envelope. Mirrors legacy agent.Event so the +// service-layer adapter can copy fields one-to-one without a switch. +// +// Notification carries the task_notification payload — set when +// Type == EventTaskNotification and a background worker reaches a +// terminal state. The SPA / SSE adapter renders it as a separate frame +// type next to assistant / tool / done. +type Event struct { + Type EventType + Assistant *AssistantEvent + Tool *ToolEvent + Done *Reply + Error string + Notification *TaskNotification + // Approval carries the approval_pending payload — set when + // Type == EventApprovalPending and a blocking tool (cloud_bash) has + // queued a proposal and is awaiting the human decision (HLD-021). + Approval *ApprovalPending +} + +// Emit is the streaming callback. nil-safe. +type Emit func(Event) + +// Request bundles every per-call input the runtime needs. +type Request struct { + SessionID string + UserID uint64 + // Role is the caller's system role (admin | user | viewer). + // gates the toolbag against this — viewer drops Class!=read so the + // LLM cannot reach mutating tools no matter what the persona allows. + Role string + UserText string + Attachments []aiopsmodel.Attachment + Mentions []Mention + Provider string + Model string + WebSearchEnabled bool + // Locale is the UI language ("en-US"/"zh-CN") the reply should use; + // threaded into graph.Input so the assembler adds a language directive. + Locale string + // Emit is the streaming sink. nil = no streaming (blocking call). + Emit Emit +} + +// Reply is the terminal result returned by Handle. Mirrors the +// legacy agent.Reply shape so the service layer can fan it back out +// through the same DTO without translation losses. +type Reply struct { + Message *aiopsmodel.Message + Usage llm.Usage + Iterations int + ToolCalls []*aiopsmodel.ToolCall +} + +// Config tunes runtime behaviour. ToolBag is the wrapped (decorated) +// tool list; the runtime does NOT wrap tools itself — that's the +// caller's job at construction time so the decorator deps (audit +// sink, limiter, registerer) live exactly once per process. +type Config struct { + // SkillRegistry holds the parsed SKILL.md descriptors. May be + // empty (nil-safe ComposeSystemPrompt). + SkillRegistry *SkillRegistry + + // AgentRegistry holds the parsed agent personas. May be empty. + AgentRegistry *AgentRegistry + + // Sessions is the persistence binding (chat_sessions / + // chat_messages / chat_tool_calls). Required. + Sessions biz.SessionRepo + + // ChatModel is the eino model.ToolCallingChatModel. Required. + // Production wiring passes RoutingChatModel from PR-1; tests + // inject a scriptedChatModel. + ChatModel model.ToolCallingChatModel + + // ToolBag is the pre-decorated BaseTool list the graph exposes + // to the LLM. cmd/ongrid/main.go assembles this once via + // Registry.BuildBaseTools + AppendHostFilesTools + Wrap. + ToolBag []basetool.BaseTool + + // MentionResolver hydrates @-mentions into markdown bullets. + // Optional. + MentionResolver MentionResolver + + // CredentialBinder (optional) maps the session's active skill names to + // the vault credential names their installed packs are bound to + // (HLD-017 design-time binding). The runtime attaches the result to ctx + // (basetool.WithBoundCredentials) so cloud_bash queues those credentials + // for injection at approve time — no LLM/user run-time choice. Wired + // post-construction via SetCredentialBinder (marketplace usecase is + // built after the runtime). + CredentialBinder CredentialBinder + + // BasePrompt is the universal preamble prepended to every + // system prompt. Empty = none. + BasePrompt string + + // HistoryLimit caps how many past messages we replay. 0 → 50 + // (legacy default). + HistoryLimit int + + // GraphCfg tunes the graph engine (max iterations, temperature, + // per-tool timeout). + GraphCfg graph.Config + + // CallbackDeps are the cross-cutting deps wired into every graph + // run. Persistence.SessionID is filled per-request from + // Request.SessionID; the rest of Persistence + Audit + Metrics + // + Budget are filled at construction. + CallbackDeps callbacks.Deps + + // AgentWriteEnabled, when set, is consulted live (per request) to decide + // whether the agent may use write/mutating tools. Returning false forces a + // read-only toolbag (same filter as a viewer): every Class!="read" tool is + // stripped before the LLM sees it. nil = always enabled (no gate), so an + // unwired runtime keeps full behaviour. + AgentWriteEnabled func(ctx context.Context) bool + + // Logger may be nil. + Logger *slog.Logger +} + +// ToolBagProvider is the local mirror of tools.DeferredToolBagProvider — +// chatruntime cannot import the tools package (cyclic: tools imports +// nothing from chatruntime, but cmd-side wiring goes the other +// direction and would close the loop). The shape is intentionally +// identical so a *tools.ToolBag value satisfies this interface +// directly without an adapter. +// +// — the runtime needs a handle on the unredacted tool +// universe so future code paths (e.g. logging the active toolBag at +// boot, exposing a /metrics list of registered tools) can introspect +// what the LLM has access to even when deferral is on. Today the +// handle is read-only — set once via SetToolBag, never mutated. +type ToolBagProvider interface { + DeferredTools() []basetool.BaseTool + AllTools() []basetool.BaseTool +} + +// mutableToolBagProvider is intentionally optional so lightweight test +// providers remain read-only. The concrete tools.ToolBag implements it and +// keeps ToolSearch aligned with runtime-discovered MCP tools. +type mutableToolBagProvider interface { + ToolBagProvider + ReplaceToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) +} + +// CredentialBinder resolves the vault credential NAMES bound to whichever +// installed packs ship the given active skills (HLD-017). The marketplace +// usecase satisfies this structurally. Returns nil when nothing is bound. +type CredentialBinder interface { + BoundCredentialNamesForSkills(ctx context.Context, skillNames []string) []string +} + +// Runtime is the cutover entry. Construct once at boot via +// NewRuntime; call Handle per HTTP request. +// +// workers holds the in-process map of spawned sub-agents keyed by +// worker id. Lifetime: bounded by the Runtime's lifetime — there's no +// auto-eviction yet (a follow-up PR adds a TTL sweeper once we move +// chat_sessions to track parent_session_id; until then workers persist +// in memory until process restart, which matches a fresh sandbox per +// chat in dev/test). +type Runtime struct { + cfg Config + log *slog.Logger + + workersMu sync.Mutex + workers map[string]*Worker + + // toolMu protects cfg.ToolBag, which is updated when an MCP server is + // verified, edited, disabled, or deleted while chats are in flight. + toolMu sync.RWMutex + + // bag is the unredacted ToolBag handle wired by SetToolBag. nil + // when the caller wires the runtime without calling SetToolBag — + // not all code paths need it (legacy callers only feed + // cfg.ToolBag through the graph). + bag ToolBagProvider +} + +// NewRuntime builds a Runtime. Returns an error when required deps +// are missing — Sessions and ChatModel. +func NewRuntime(cfg Config) (*Runtime, error) { + if cfg.Sessions == nil { + return nil, errors.New("chatruntime: Sessions is required") + } + if cfg.ChatModel == nil { + return nil, errors.New("chatruntime: ChatModel is required") + } + if cfg.HistoryLimit <= 0 { + cfg.HistoryLimit = 50 + } + log := cfg.Logger + if log == nil { + log = slog.Default() + } + return &Runtime{cfg: cfg, log: log}, nil +} + +// SetMentionResolver wires the @-mention hydrator post-construction. +// Optional — when unset the runtime runs with no mention inlining +// (matches the legacy agent's nil-resolver behaviour). +func (rt *Runtime) SetMentionResolver(r MentionResolver) { + if rt == nil { + return + } + rt.cfg.MentionResolver = r +} + +// SetCredentialBinder wires the active-skill→bound-credentials resolver onto +// the Runtime after construction (the marketplace usecase is built later than +// the runtime — same chicken-and-egg as the cloud_bash proposer). nil-safe. +func (rt *Runtime) SetCredentialBinder(b CredentialBinder) { + if rt == nil { + return + } + rt.cfg.CredentialBinder = b +} + +// SetAgentWriteEnabledProvider wires the live write-action gate (an admin +// system_settings toggle). fn is consulted per request; returning false forces +// a read-only toolbag. nil clears the gate (always enabled). Post-construction +// setter so cmd/ongrid can hand in a closure over the setting service after the +// runtime is built. +func (rt *Runtime) SetAgentWriteEnabledProvider(fn func(ctx context.Context) bool) { + if rt == nil { + return + } + rt.cfg.AgentWriteEnabled = fn +} + +// SetToolBag wires the unredacted ToolBag handle onto the Runtime so +// downstream introspection (ToolSearch, future /metrics +// surfaces) can query the full tool universe even when the LLM-facing +// view is the deferred / redacted one. Idempotent; nil is a no-op so +// tests can opt out without a panic. +// +// Note: the bag is INFORMATIONAL only. The graph still feeds the +// LLM via cfg.ToolBag, which the caller assembles from +// bag.SchemasForLLM() before calling NewRuntime — see +// cmd/ongrid/main.go's buildAIOpsRuntime for the wiring. +func (rt *Runtime) SetToolBag(bag ToolBagProvider) { + if rt == nil { + return + } + rt.bag = bag +} + +// AppendToolBag appends extra BaseTools to the coordinator tool bag. +// — used by main.go to bolt the coordinator-only AgentTool +// / SendMessageTool / TaskStopTool onto the Runtime AFTER construction +// (the chicken-and-egg: those tools need the Runtime as their spawner, +// the Runtime needs the tool bag at NewRuntime time). Workers do NOT +// observe these added tools — filterToolsForAgent strips them +// unconditionally via coordinatorOnlyTools (see worker.go). +// +// This is intentionally distinct from a generic "AddTool" method: it +// signals "these are coordinator-only" so a future reader of main.go +// understands why three more tools appear after NewRuntime returned. +func (rt *Runtime) AppendToolBag(tools []basetool.BaseTool) { + if rt == nil || len(tools) == 0 { + return + } + rt.toolMu.Lock() + defer rt.toolMu.Unlock() + index := map[string]int{} + for i, t := range rt.cfg.ToolBag { + if t == nil { + continue + } + if info, err := t.Info(context.Background()); err == nil && info != nil { + index[info.Name] = i + } + } + for _, t := range tools { + if t == nil { + continue + } + info, err := t.Info(context.Background()) + if err == nil && info != nil { + if i, ok := index[info.Name]; ok { + rt.cfg.ToolBag[i] = t + continue + } + index[info.Name] = len(rt.cfg.ToolBag) + } + rt.cfg.ToolBag = append(rt.cfg.ToolBag, t) + } +} + +// ReplaceToolsByNamePrefix replaces tools in the graph-facing bag whose wire +// names share prefix. MCP uses one prefix per server, preventing a refresh of +// one registration from removing tools offered by other servers. +func (rt *Runtime) ReplaceToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) { + if rt == nil || strings.TrimSpace(prefix) == "" { + return + } + rt.toolMu.Lock() + defer rt.toolMu.Unlock() + next := make([]basetool.BaseTool, 0, len(rt.cfg.ToolBag)+len(replacements)) + for _, tool := range rt.cfg.ToolBag { + if toolNameHasPrefix(tool, prefix) { + continue + } + next = append(next, tool) + } + rt.cfg.ToolBag = append(next, replacements...) +} + +// ReplaceCatalogToolsByNamePrefix updates the unredacted catalog used by +// ToolSearch. It is separate from ReplaceToolsByNamePrefix because the graph +// sees decorated tools while ToolSearch must retain raw schemas. +func (rt *Runtime) ReplaceCatalogToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) { + if rt == nil || strings.TrimSpace(prefix) == "" { + return + } + if bag, ok := rt.bag.(mutableToolBagProvider); ok { + bag.ReplaceToolsByNamePrefix(prefix, replacements) + } +} + +func (rt *Runtime) toolBagSnapshot() []basetool.BaseTool { + if rt == nil { + return nil + } + rt.toolMu.RLock() + defer rt.toolMu.RUnlock() + return append([]basetool.BaseTool(nil), rt.cfg.ToolBag...) +} + +func toolNameHasPrefix(tool basetool.BaseTool, prefix string) bool { + if tool == nil { + return false + } + info, err := tool.Info(context.Background()) + return err == nil && info != nil && strings.HasPrefix(info.Name, prefix) +} + +// AgentRegistry exposes the registry for narrow read-only callers +// (cmd/main.go uses this when building AgentTool's subagent registry +// shim). nil-safe. +func (rt *Runtime) AgentRegistry() *AgentRegistry { + if rt == nil { + return nil + } + return rt.cfg.AgentRegistry +} + +// SkillRegistry exposes the skill registry for narrow read-only +// callers — today the marketplace usecase wires Reload calls +// against it on install/uninstall. nil-safe. +func (rt *Runtime) SkillRegistry() *SkillRegistry { + if rt == nil { + return nil + } + return rt.cfg.SkillRegistry +} + +// ToolCount returns how many tools are bound into the toolBag. Used +// at boot time + tests for visibility (the spec asks us to log this +// when ONGRID_AGENT_KERNEL=graph). +func (rt *Runtime) ToolCount() int { + if rt == nil { + return 0 + } + return len(rt.toolBagSnapshot()) +} + +// ToolNames returns the resolved tool names (best-effort — Info() +// errors are skipped). Production wiring logs this at boot so the +// operator sees what the LLM is exposed to. +func (rt *Runtime) ToolNames(ctx context.Context) []string { + if rt == nil { + return nil + } + tools := rt.toolBagSnapshot() + out := make([]string, 0, len(tools)) + for _, t := range tools { + if t == nil { + continue + } + info, err := t.Info(ctx) + if err != nil || info == nil { + continue + } + out = append(out, info.Name) + } + return out +} + +// Handle runs one user turn end-to-end. Mirrors legacy +// agent.runInternal: ownership check → mention inline → user message +// persistence → graph invoke with callback chain → reply. +func (rt *Runtime) Handle(ctx context.Context, req *Request) (result *Reply, retErr error) { + if rt == nil || rt.cfg.Sessions == nil || rt.cfg.ChatModel == nil { + return nil, errs.ErrNotWiredYet + } + if req == nil { + return nil, fmt.Errorf("%w: request required", errs.ErrInvalid) + } + + emit := req.Emit + if emit == nil { + emit = func(Event) {} + } + // Thread the active emitter into ctx so AgentTool's InvokableRun + // (deep inside the graph) can fire a task_notification SSE frame + // when a background worker terminates. / + ctx = withEmit(ctx, req.Emit) + + // 1. Ownership check. + sess, err := rt.cfg.Sessions.GetSession(ctx, req.SessionID) + if err != nil { + return nil, err + } + if sess.UserID != req.UserID { + // Match legacy behaviour: surface ErrNotFound (not + // ErrForbidden) so non-owners can't fingerprint sessions. + return nil, errs.ErrNotFound + } + ctx, span := startAgentSpan(ctx, sess, "default") + tracing.RecordAgentContent(span, "input.value", req.UserText) + var runErr error + defer func() { + if result != nil && result.Message != nil && result.Message.Content != nil { + tracing.RecordAgentContent(span, "output.value", *result.Message.Content) + } + if runErr == nil { + runErr = retErr + } + tracing.EndSpan(span, runErr) + }() + span.SetAttributes(attribute.String("ongrid.agent.kernel", "graph")) + + // 2. Render @-mention bullets. + mentionsRendered := "" + augmentedUserText := req.UserText + if len(req.Mentions) > 0 && rt.cfg.MentionResolver != nil { + bullets := rt.cfg.MentionResolver.Resolve(ctx, req.Mentions) + if len(bullets) > 0 { + // Same prefix the legacy agent.go uses — keeps history + // replay byte-identical across kernels. + mentionsRendered = "用户在消息中引用了以下平台对象，请在回答时考虑：\n" + + strings.Join(bullets, "\n") + augmentedUserText = mentionsRendered + "\n\n" + req.UserText + } + } + + // 3. Persist the user message first (legacy invariant — survives + // a graph crash). + userContent := augmentedUserText + userMsg := &aiopsmodel.Message{ + SessionID: sess.ID, + Role: aiopsmodel.RoleUser, + Content: &userContent, + Attachments: req.Attachments, + AttachmentIDs: attachmentIDs(req.Attachments), + CreatedAt: time.Now().UTC(), + } + if err := rt.cfg.Sessions.AppendMessage(ctx, userMsg); err != nil { + return nil, fmt.Errorf("chatruntime: persist user msg: %w", err) + } + + // 4. Load history (includes the just-appended user message). + history, err := rt.cfg.Sessions.ListMessages(ctx, sess.ID, rt.cfg.HistoryLimit) + if err != nil { + return nil, fmt.Errorf("chatruntime: load history: %w", err) + } + if reply, handled := rt.tryApplyConfirmedConfigDraft(ctx, req, sess, history, emit); handled { + return reply, nil + } + // Resolve and Decide run before the LLM tool loop. This prevents a model + // from turning a candidate discovered during reasoning into an executable + // capture target. The reply is persisted and streamed like any other turn. + turnPlan, clarification := resolveTurn(req) + if turnPlan.Decision == DecisionClarify { + message := &aiopsmodel.Message{SessionID: sess.ID, Role: aiopsmodel.RoleAssistant, Content: &clarification, CreatedAt: time.Now().UTC()} + if err := rt.cfg.Sessions.AppendMessage(ctx, message); err != nil { + return nil, fmt.Errorf("chatruntime: persist clarification: %w", err) + } + emit(Event{Type: EventAssistant, Assistant: &AssistantEvent{MessageID: message.ID, Content: clarification, CreatedAt: message.CreatedAt}}) + reply := &Reply{Message: message} + emit(Event{Type: EventDone, Done: reply}) + return reply, nil + } + + // 5. Resolve active skills + compose the system prompt. + policy := Policy{AllowedClasses: []string{"*"}} + var activeSkills []*Skill + if rt.cfg.SkillRegistry != nil { + activeSkills = rt.cfg.SkillRegistry.Resolve(req.UserText, policy) + } + // 5·HLD-017: attach the active skills' design-time bound credentials to + // ctx so cloud_bash queues them for injection at approve time. The + // binding was chosen at install time; nothing is decided at run time. + if rt.cfg.CredentialBinder != nil && len(activeSkills) > 0 { + names := make([]string, 0, len(activeSkills)) + for _, s := range activeSkills { + names = append(names, s.Name) + } + if creds := rt.cfg.CredentialBinder.BoundCredentialNamesForSkills(ctx, names); len(creds) > 0 { + ctx = basetool.WithBoundCredentials(ctx, creds) + } + } + // 5a. Top-level persona (Phase 2 of the user-Agent initiative): when + // the session was created with a chosen agent (sess.AgentID set), use + // that persona's SystemPrompt as the base instead of the global + // BasePrompt and filter the ToolBag through its allowed/disallowed + // list. Worker spawn already does this for sub-agents (see worker.go); + // the coordinator path now honors it too. Stale agent ids fall back + // to the global default with a log line — never crash a session. + basePrompt := rt.cfg.BasePrompt + runtimeToolBag := rt.toolBagSnapshot() + sessionToolBag := runtimeToolBag + agentReminderForPersona := "" + // Coordinator iff session is pinned to "default" or has no + // AgentID at all. Specialist personas (= anything else in the + // registry) are workers, not coordinators. + isCoordinator := sess.AgentID == nil || *sess.AgentID == "" || *sess.AgentID == "default" + // viewer downgrade — drop every Class!=read tool out of the + // toolbag before persona / control filtering. This applies even when + // no persona is loaded, so the LLM never sees host_bash / AgentTool / + // restart_service via the viewer's chat session. + viewerOnly := req.Role == "viewer" + // Global write-action gate (admin setting, consulted live per request): + // when disabled, force the same read-only toolbag a viewer gets — strip + // every Class!="read" tool before the LLM sees it, so even the + // propose-confirm writes (cloud_bash, apply_config_change, serve_page, + // AgentTool dispatch, host_restart_service, …) are unavailable. A nil + // provider means "no gate" → enabled, preserving full behaviour. + writeEnabled := true + if rt.cfg.AgentWriteEnabled != nil { + writeEnabled = rt.cfg.AgentWriteEnabled(ctx) + } + readOnly := viewerOnly || !writeEnabled + if readOnly { + sessionToolBag = filterToolsForAgentRole(runtimeToolBag, nil, isCoordinator, true) + } + // Resolve the active persona. Sessions with no AgentID still + // route through the "default" persona — that's where the curated + // coordinator toolbag lives. Without this lookup, a CreateSession + // without an explicit agent_id would see the full toolbag and + // the rule-1 dispatch policy becomes advisory again. + personaName := "default" + if sess.AgentID != nil && *sess.AgentID != "" { + personaName = *sess.AgentID + } + if rt.cfg.AgentRegistry != nil { + if persona, ok := rt.cfg.AgentRegistry.ByName(personaName); ok { + if strings.TrimSpace(persona.SystemPrompt) != "" { + basePrompt = persona.SystemPrompt + } + // Apply the persona's Tools whitelist even for the + // "default" coordinator — so a curated coordinator + // toolbag (AgentTool + a handful of triage tools) can + // force dispatch instead of letting the LLM see and + // reach for every deep-dive tool itself. The + // coordinatorOnlyTools (AgentTool/SendMessage/TaskStop) + // survive the strip via isCoordinator=true. + sessionToolBag = filterToolsForAgentRole(runtimeToolBag, persona, isCoordinator, readOnly) + agentReminderForPersona = persona.CriticalReminder + } else if rt.log != nil && personaName != "default" { + rt.log.Info("chatruntime: session agent_id not in registry — using default", + slog.String("session_id", sess.ID), + slog.String("agent_id", personaName)) + } + } + // Tool exposure is governed by persona and permission policy above. Do + // not further prune it with request-keyword heuristics: the legacy kernel + // exposed its permitted registry schemas and relied on each tool's own + // semantic contract. Keyword pruning hid valid tools (for example, + // host_du_summary for a disk-directory request) before the model could + // make that choice. + // AgentID="default" is the virtual top-level persona. It owns the + // complete policy-permitted tool surface and may ask a specialist for + // additional evidence, but it never loses ownership of this session. + // 5b. Multi-agent catalog: when the session is the coordinator + // (no specific persona pinned), append a markdown list of available + // specialist personas so the LLM knows what subagent_type values + // AgentTool will accept. Workers don't get the catalog (they can't + // spawn nested workers — see coordinatorOnlyTools). + if isCoordinator && rt.cfg.AgentRegistry != nil { + if catalog := buildAgentCatalog(rt.cfg.AgentRegistry); catalog != "" { + basePrompt = basePrompt + "\n\n" + catalog + } + } + if digest := buildToolCapabilityDigest(sessionToolBag); digest != "" { + basePrompt = basePrompt + "\n\n" + digest + } + systemPrompt := ComposeSystemPrompt(basePrompt, activeSkills, nil) + + // 6. Build the eino history slice. Convert persisted rows into + // schema.Message so the graph assembler can replay them. + einoHistory := buildEinoHistory(history) + + // 7. Build the graph (per-request — eino graphs are cheap to + // compose and the inner ReAct agent is rebuilt on every call + // today; an in-memory cache keyed on (toolBag identity, cfg) + // is a future optimisation). + graphCfg := rt.cfg.GraphCfg + // Persona-level MaxTurns override — already honoured for workers + // via worker.go runWorker; mirror it here for the coordinator + // path so the default persona can have a tighter cap. Without + // this, the global 30-iteration cap lets a misbehaving + // coordinator chain 120+ tool calls (we observed exactly this + // pattern in the E2E eval before adding the override). + if rt.cfg.AgentRegistry != nil { + if persona, ok := rt.cfg.AgentRegistry.ByName(personaName); ok && persona.MaxTurns > 0 { + graphCfg.MaxIterations = persona.MaxTurns + } + } + // 8. Wire the per-request callback chain. Persistence's + // SessionID is filled in here; everything else lives on + // rt.cfg.CallbackDeps. The SSE handler is appended via the + // cutover layer's emitter when req.Emit != nil. + deps := rt.cfg.CallbackDeps + deps.AlertDraftGuard.UserText = req.UserText + deps.Persistence.SessionID = sess.ID + deps.Persistence.Model = strings.TrimSpace(req.Model) + if deps.Persistence.Repo == nil { + deps.Persistence.Repo = rt.cfg.Sessions + } + if req.Emit != nil { + deps.SSE = rt.toCallbackEmitter(req.Emit, sess.ID) + } else { + deps.SSE = nil + } + handlers := callbacks.NewDefaultHandlers(deps) + toolPersistence := callbacks.EnableSynchronousToolPersistence(handlers) + graphCfg.ToolPersistence = toolPersistence + g, err := graph.BuildReActGraph(rt.cfg.ChatModel, sessionToolBag, graphCfg) + if err != nil { + return nil, fmt.Errorf("chatruntime: build graph: %w", err) + } + + // 9. Compute per-turn dynamic hints from history + persona's + // critical_reminder (if a worker persona is active for this turn). + // — these get inlined into the per-turn + // block by graph.assembleMessages. + dynamicHints := rt.calcDynamicHints(history) + if turnPlan.Phase == PhaseAct || turnPlan.Phase == PhaseOperate || turnPlan.Phase == PhasePropose { + dynamicHints = append(dynamicHints, turnPlan.ModelBoundary()) + } + // AgentReminder is populated when the session is pinned to a + // persona (Phase 2). — anti-drift reminder gets + // inlined into the per-turn block. Worker spawns + // have always passed CriticalReminder; the coordinator path now + // does too when sess.AgentID is set. + agentReminder := agentReminderForPersona + + // 10. Invoke. The graph's outer Output carries the assistant + // message + usage; the persistence callback already wrote the + // assistant + tool rows by the time Invoke returns. + // + // Per-request model selection (SPA model picker): thread + // req.Provider/req.Model into the ChatModel node as eino model + // options. RoutingChatModel.pick consumes WithProvider; the inner + // clientChatModel honours WithModel. Without this the graph kernel + // silently ignored the picker and always used the routing default + // — so a user who chose e.g. claude-opus still got the default + // model. When both are empty no option is added (default path + // unchanged). + invokeOpts := []compose.Option{compose.WithCallbacks(handlers...)} + if mopts := chatModelOpts(req); len(mopts) > 0 { + invokeOpts = append(invokeOpts, compose.WithChatModelOption(mopts...)) + } + invokeOpts = append(invokeOpts, compose.WithToolsNodeOption( + compose.WithToolOption( + graph.WithInvokeOpts( + basetool.WithUserText(req.UserText), + basetool.WithConfirmedDeviceIDs(confirmedDeviceIDs(req.Mentions)), + basetool.WithHostWritePermission(writeEnabled && !viewerOnly), + ), + graph.WithToolInvocationPersistence(toolPersistence), + ), + )) + // The tool adapter receives these invoke options, but coordinator-only + // tools can synchronously spawn a worker from the graph context. Stamp the + // same gate on the request context so that worker inherits the caller's + // permission instead of accidentally gaining a broader toolbag. + ctx = basetool.WithAgentWriteAllowed(ctx, writeEnabled && !viewerOnly) + // Thread the final per-turn tool view onto ctx so ToolSearch only + // returns tools callable in this graph. Coordinator routing stubs are + // deliberately included: a deferred schema lookup must return a + // callable handoff, never an unreachable direct tool schema. + ctx = basetool.WithFilteredTools(ctx, sessionToolBag) + // Thread the UI locale onto ctx so AgentTool can pick it up and + // forward it into the sub-agent's SpawnRequest. Without this, a + // coordinator that handles an English question hands off to a + // specialist that answers in zh (GLM default) — see + // feedback_ai_output_locale.md regression 2026-06-02. + ctx = basetool.WithLocale(ctx, req.Locale) + // Same idea for the LLM choice: stamp the coordinator's resolved + // provider+model so AgentTool can plumb them into the sub-agent's + // SpawnRequest. runWorker reads them back as chatModelOpts. + // Without this, sub-agents fall through to the routing default + // ("openai"), and installs without an OpenAI key see specialists + // fail with `provider "openai" not configured`. + ctx = basetool.WithLLMChoice(ctx, req.Provider, req.Model) + // HLD-019: attach the session id so cloud_bash can resolve a per-session + // agent workspace at exec time (files persist across commands in a session + // instead of running in a throwaway temp dir). + ctx = basetool.WithSessionID(ctx, sess.ID) + ctx = basetool.WithAPMSource(ctx, sess.APMSource) + // Tag artifacts produced in this turn (serve_page) as chat-sourced so the + // operations UI's 生成来源 column can tell assistant pages from workflow pages. + ctx = basetool.WithArtifactSource(ctx, basetool.ArtifactSourceChat) + // Forward the admin write gate to host_bash: when ON, the tool tells the + // edge to bypass cmdpolicy and run the raw command. `writeEnabled` was + // resolved above (same value that gated the toolbag); reuse it so a single + // setting read drives both tool exposure and host-command authority. + ctx = basetool.WithHostWriteAllowed(ctx, writeEnabled && !viewerOnly) + // Always autoheal any in-flight tool batch on the way out — covers + // the "user closed browser mid-tool-batch" case the in-session + // ChatModel.OnStart flush can't reach. Defer with a background-rooted + // ctx so a cancelled request ctx doesn't block the stub inserts. + defer func() { + flushCtx := context.WithoutCancel(ctx) + callbacks.FinalizeBatches(flushCtx, handlers) + }() + out, invokeErr := g.Invoke(ctx, &graph.Input{ + SystemPrompt: systemPrompt, + History: einoHistory, + UserText: req.UserText, + UserImages: imageInputParts(req.Attachments), + WebSearchEnabled: req.WebSearchEnabled, + MentionsRendered: mentionsRendered, + AgentReminder: agentReminder, + DynamicHints: dynamicHints, + Locale: req.Locale, + }, invokeOpts...) + if invokeErr != nil { + runErr = invokeErr + // Soft-fail graph-level errors: write an apology assistant + // message, emit it like a normal turn, and emit Done. Keeps + // SPA experience consistent with legacy agent.go's + // MaxIterations apology path. Without this, errors like + // eino's ErrExceededMaxSteps surface as "stream error" in + // the user's chat instead of an actionable message. + apology := buildGraphErrorApology(invokeErr) + fallbackMsg := &aiopsmodel.Message{ + SessionID: sess.ID, + Role: aiopsmodel.RoleAssistant, + Content: &apology, + CreatedAt: time.Now().UTC(), + } + // Best-effort persist; if it fails we still emit so the user + // sees something instead of a stream error. + if persistErr := rt.cfg.Sessions.AppendMessage(ctx, fallbackMsg); persistErr != nil && rt.log != nil { + rt.log.Warn("chatruntime: persist apology failed", + slog.String("err", persistErr.Error())) + } + emit(Event{Type: EventAssistant, Assistant: &AssistantEvent{ + MessageID: fallbackMsg.ID, + Content: apology, + CreatedAt: fallbackMsg.CreatedAt, + }}) + reply := &Reply{ + Message: fallbackMsg, + } + emit(Event{Type: EventDone, Done: reply}) + if rt.log != nil { + rt.log.Warn("chatruntime: graph invoke failed (apology emitted)", + slog.String("err", invokeErr.Error())) + } + return reply, nil + } + + // 10. Translate Output → Reply. The graph's Output owns the + // assistant message and best-effort iterations + usage; the + // persistence handler has already written the chat_messages + // row, so we re-fetch the recently-persisted assistant message + // ID for handoff to the SSE adapter when needed. + reply := &Reply{Usage: out.Usage, Iterations: out.Iterations} + if toolPersistence != nil { + reply.ToolCalls = toolPersistence.ToolCalls() + } + if out.AssistantMessage != nil { + // Build a synthetic *aiopsmodel.Message so the upper layer + // can re-render the legacy postMessageResp shape. The + // canonical row written by PersistenceHandler lives in + // chat_messages; we mirror its fields here for the wire DTO. + content := out.AssistantMessage.Content + if guard := callbacks.AlertDraftGuardFromHandlers(handlers); guard != nil { + content = guard.SanitizeAssistantContent(content) + } + var contentPtr *string + if content != "" { + contentPtr = &content + } + reply.Message = &aiopsmodel.Message{ + SessionID: sess.ID, + Role: aiopsmodel.RoleAssistant, + Content: contentPtr, + CreatedAt: time.Now().UTC(), + } + } + + // EventDone — keep parity with legacy agent.go's terminal + // emission. PersistenceHandler may have already written the + // assistant row; the legacy contract is "Done frame fires + // exactly once at terminal success". + emit(Event{Type: EventDone, Done: reply}) + return reply, nil +} + +func confirmedDeviceIDs(mentions []Mention) []uint64 { + ids := make([]uint64, 0, len(mentions)) + for _, mention := range mentions { + if !strings.EqualFold(strings.TrimSpace(mention.Type), "device") { + continue + } + id, err := strconv.ParseUint(strings.TrimSpace(mention.ID), 10, 64) + if err == nil && id != 0 { + ids = append(ids, id) + } + } + return ids +} + +func resolveTurn(req *Request) (TurnPlan, string) { + if req == nil { + return PlanTurn(ResolvedFacts{Permitted: false, Reason: "nil request"}), "" + } + if needsPacketCaptureSessionID(req.UserText) { + plan := PlanTurn(ResolvedFacts{Missing: true, Permitted: req.Role != "viewer"}) + return plan, "当前没有可枚举“最近抓包会话”的助理工具；`get_packet_capture_session` 需要明确的 `pcap-session-*`。请提供抓包会话 ID，或先发起一次新的抓包任务后再分析。" + } + if needsPacketPathDetails(req.UserText) { + plan := PlanTurn(ResolvedFacts{Missing: true, Permitted: req.Role != "viewer"}) + return plan, "不能确定 NAT 或网关路径。要判断这个包是否经过 NAT 或网关，需要先明确具体包或会话：请提供 `pcap-session-*`、源/目的地址、端口、协议、时间窗，或说明从哪台 edge 到哪台设备/服务。没有这些信息我不能推断包走向。" + } + captureIntent := isStartCaptureIntent(req.UserText) + // Semantic target resolution stays in Understand with full history. A + // sensitive candidate such as capture transitions to Propose at the shared + // tool gate, which freezes tool+args and emits the existing confirmation + // card before execution. + plan := PlanTurn(ResolvedFacts{Permitted: req.Role != "viewer", NeedsConfirmation: captureIntent, LongRunning: captureIntent}) + return plan, "" +} + +func needsPacketCaptureSessionID(userText string) bool { + text := strings.ToLower(userText) + if strings.Contains(text, "pcap-session-") { + return false + } + if strings.Contains(userText, "页面") || + strings.Contains(userText, "产物关联") || + strings.Contains(text, "page not found") || + strings.Contains(text, "why") { + return false + } + hasCapture := strings.Contains(userText, "抓包") || + strings.Contains(userText, "数据包") || + strings.Contains(text, "packet capture") || + strings.Contains(text, "pcap") + if !hasCapture { + return false + } + for _, phrase := range []string{"最近", "最新", "有哪些", "列出", "告诉我", "分析最近", "recent", "latest", "list"} { + if strings.Contains(userText, phrase) || strings.Contains(text, phrase) { + return true + } + } + return false +} + +func needsPacketPathDetails(userText string) bool { + text := strings.ToLower(userText) + if strings.Contains(text, "pcap-session-") || strings.Contains(text, "device_id") || strings.Contains(userText, "@") { + return false + } + hasPacket := strings.Contains(userText, "这个包") || strings.Contains(userText, "该包") || strings.Contains(text, "this packet") + if !hasPacket { + return false + } + return strings.Contains(userText, "NAT") || + strings.Contains(strings.ToLower(userText), "nat") || + strings.Contains(userText, "网关") || + strings.Contains(userText, "链路") || + strings.Contains(text, "gateway") || + strings.Contains(text, "path") +} + +func isStartCaptureIntent(userText string) bool { + if isCancelIntent(userText) { + return false + } + text := strings.ToLower(userText) + if hasReadOnlyCaptureIntent(userText) { + return false + } + startMarkers := []string{"开始", "发起", "启动", "抓包", "抓取", "抓一下", "抓 ", "抓\t", "capture ", "start capture", "run capture"} + hasStartMarker := false + for _, marker := range startMarkers { + if strings.Contains(text, marker) || strings.Contains(userText, marker) { + hasStartMarker = true + break + } + } + if !hasStartMarker { + return false + } + return strings.Contains(text, "pcap") || + strings.Contains(text, "packet capture") || + strings.Contains(text, "capture packet") || + strings.Contains(text, "tcp port") || + strings.Contains(text, "udp port") || + strings.Contains(userText, "抓包") || + strings.Contains(userText, "流量") || + strings.Contains(userText, "数据包") || + strings.Contains(userText, "的包") +} + +func hasReadOnlyCaptureIntent(userText string) bool { + text := strings.ToLower(userText) + for _, phrase := range []string{ + "分析", "查看", "列出", "最近", "有哪些", "告诉我", "入口", "源码", "实现", "页面", "产物", "会话", "能不能", "是否", + "analyze", "inspect", "show", "list", "recent", "artifact", "session", "source", "code", "can ", + } { + if strings.Contains(text, phrase) || strings.Contains(userText, phrase) { + return true + } + } + return false +} + +func isCancelIntent(userText string) bool { + text := strings.ToLower(userText) + for _, phrase := range []string{"停止", "取消", "终止", "stop", "cancel", "abort"} { + if strings.Contains(text, phrase) || strings.Contains(userText, phrase) { + return true + } + } + return false +} + +// toCallbackEmitter adapts a chatruntime.Emit into the +// callbacks.SSEEmitter the persistence/SSE chain consumes. The +// adapter normalises the new event names back to the legacy ones +// (SSEEventAssistantStart / SSEEventAssistantEnd → "assistant"; +// SSEEventAssistantDelta dropped — see PR-9 spec note about +// IncludeDelta=false until the SPA catches up). +func (rt *Runtime) toCallbackEmitter(emit Emit, sessionID string) callbacks.SSEEmitter { + return func(ev callbacks.SSEEvent) { + switch ev.Type { + case callbacks.SSEEventAssistantEnd: + if ev.Assistant == nil { + return + } + emit(Event{Type: EventAssistant, Assistant: &AssistantEvent{ + Iteration: ev.Assistant.Iteration, + MessageID: ev.Assistant.MessageID, + Content: ev.Assistant.Content, + CreatedAt: ev.Assistant.CreatedAt, + PendingToolCalls: ev.Assistant.PendingToolCalls, + }}) + case callbacks.SSEEventAssistantStart: + // Legacy frontend doesn't have an "assistant_start" + // frame; suppress it so the SPA doesn't see a phantom + // empty assistant bubble. The assistant_end frame + // (above) carries the full content + pending tool + // count which matches the legacy "assistant" frame. + return + case callbacks.SSEEventAssistantDelta: + // Token-level streaming is gated behind a feature flag + // pending SPA support. Drop for now — see PR-9 spec. + return + case callbacks.SSEEventToolStart: + if ev.Tool == nil { + return + } + emit(Event{Type: EventToolStart, Tool: &ToolEvent{ + ToolCallID: ev.Tool.ToolCallID, + Name: ev.Tool.Name, + ArgsJSON: ev.Tool.ArgsJSON, + Status: ev.Tool.Status, + StartedAt: ev.Tool.StartedAt, + }}) + case callbacks.SSEEventToolEnd: + if ev.Tool == nil { + return + } + emit(Event{Type: EventToolEnd, Tool: &ToolEvent{ + ToolCallID: ev.Tool.ToolCallID, + Name: ev.Tool.Name, + ArgsJSON: ev.Tool.ArgsJSON, + Status: ev.Tool.Status, + StartedAt: ev.Tool.StartedAt, + EndedAt: ev.Tool.EndedAt, + DurationMs: ev.Tool.DurationMs, + Error: ev.Tool.Error, + ResultJSON: ev.Tool.ResultJSON, + }}) + case callbacks.SSEEventDone: + // Done is emitted by Handle directly so the chatruntime + // caller sees a single, well-typed Done event with the + // full Reply. The graph's terminal "done" callback fires + // without a Reply payload, which is unhelpful for the + // adapter. Drop the empty version to avoid double-fire. + return + case callbacks.SSEEventError: + msg := "" + if ev.Error != nil { + msg = ev.Error.Message + } + emit(Event{Type: EventError, Error: msg}) + } + } +} + +// buildEinoHistory translates persisted aiopsmodel.Message rows into +// schema.Message. +// +// Tool-call replay: tool-only assistant rows (Content=NULL, with hydrated +// ToolCalls) are emitted as {role:assistant, content:"", tool_calls:[...]} +// so the following role=tool messages remain paired. Strict providers +// (DeepSeek v4+) reject orphan tool messages with HTTP 400; OpenAI silently +// tolerated them. See toolreplay.Resolve for the LLM-call-id recovery +// rules (post-fix llm_call_id column + back-compat pair-by-order). +// +// The graph assembler appends the user turn separately (from Input.UserText); +// to avoid the LLM seeing the same turn twice we strip a trailing role=user +// row from the persisted history. +func imageInputParts(images []aiopsmodel.Attachment) []schema.MessageInputPart { + parts := make([]schema.MessageInputPart, 0, len(images)) + for _, image := range images { + data := base64.StdEncoding.EncodeToString(image.Data) + parts = append(parts, schema.MessageInputPart{ + Type: schema.ChatMessagePartTypeImageURL, + Image: &schema.MessageInputImage{MessagePartCommon: schema.MessagePartCommon{Base64Data: &data, MIMEType: image.MIMEType}, Detail: schema.ImageURLDetailAuto}, + }) + } + return parts +} + +func attachmentIDs(items []aiopsmodel.Attachment) []string { + ids := make([]string, 0, len(items)) + for _, item := range items { + ids = append(ids, item.ID) + } + return ids +} + +func buildEinoHistory(rows []*aiopsmodel.Message) []*schema.Message { + if len(rows) == 0 { + return nil + } + end := len(rows) + if end > 0 && rows[end-1].Role == aiopsmodel.RoleUser { + end-- + } + // Resolve against the WHOLE history (not just rows[:end]) so the + // trailing-user trim doesn't accidentally break pair-by-order + // resolution for an assistant just before the trimmed user. + callIDs := toolreplay.Resolve(rows) + // Index role=tool rows by tool_call_id so an assistant with tool_calls + // can hoist its responses to the immediate-next position regardless + // of the persisted created_at order. Without this, long-running + // AgentTool sub-agent spawns leave their parent assistant's tool_call + // slot dangling — strict providers (DeepSeek v4+) reject with + // HTTP 400 "insufficient tool messages following tool_calls message". + toolIdx := toolreplay.IndexToolMessagesByCallID(rows) + skipTool := make(map[int]bool) + out := make([]*schema.Message, 0, end) + + // emitToolByCallID looks up the tool row matching callID in toolIdx, + // appends it to out (as schema.Message), and marks it skipped so the + // natural-order iterator below doesn't re-emit it. + emitToolByCallID := func(callID string) { + j, ok := toolIdx[callID] + if !ok || j >= end { + // Tool result wasn't persisted (in flight) or sits in the + // trimmed trailing user range. Skip silently — the assistant + // turn that emitted this tool_call may need to be dropped + // upstream; here we simply omit this slot. + return + } + tm := rows[j] + content := "" + if tm.Content != nil { + content = sanitizeToolReplayContent(*tm.Content) + } + tcID := "" + if tm.ToolCallID != nil { + tcID = *tm.ToolCallID + } + tname := "" + if tm.ToolName != nil { + tname = *tm.ToolName + } + out = append(out, &schema.Message{ + Role: schema.RoleType(aiopsmodel.RoleTool), + Content: content, + ToolCallID: tcID, + ToolName: tname, + }) + skipTool[j] = true + } + for i := 0; i < end; i++ { + m := rows[i] + switch m.Role { + case aiopsmodel.RoleUser, aiopsmodel.RoleSystem: + if m.Content == nil { + continue + } + msg := &schema.Message{ + Role: schema.RoleType(m.Role), + Content: *m.Content, + UserInputMultiContent: imageInputParts(m.Attachments), + } + out = append(out, msg) + case aiopsmodel.RoleAssistant: + calls, ok := callIDs[m.ID] + if len(m.ToolCalls) > 0 && !ok { + toolreplay.MarkDependentToolsForSkip(rows, i, len(m.ToolCalls), skipTool) + continue + } + // Precheck: every tool_call we'd emit must have a hoistable + // response inside the current window. If any is missing + // (parallel ToolsNode dropped an OnEnd → chat_tool_calls + // row written but no role=tool chat_messages row), drop the + // whole assistant turn + its dependent tools rather than + // send an envelope strict providers reject with HTTP 400 + // "insufficient tool messages following tool_calls". + if ok && len(calls) > 0 { + complete := true + for _, tc := range calls { + if j, found := toolIdx[tc.ID]; !found || j >= end { + complete = false + break + } + } + if !complete { + toolreplay.MarkDependentToolsForSkip(rows, i, len(calls), skipTool) + continue + } + } + content := "" + if m.Content != nil { + content = *m.Content + } + msg := &schema.Message{ + Role: schema.RoleType(m.Role), + Content: content, + } + if m.ReasoningContent != nil { + msg.ReasoningContent = *m.ReasoningContent + } + if ok { + msg.ToolCalls = make([]schema.ToolCall, 0, len(calls)) + for _, tc := range calls { + msg.ToolCalls = append(msg.ToolCalls, schema.ToolCall{ + ID: tc.ID, + Type: "function", + Function: schema.FunctionCall{ + Name: tc.Name, + Arguments: tc.ArgsJSON, + }, + }) + } + } + if content == "" && len(msg.ToolCalls) == 0 { + // Polluted-data case: assistant with no replayable + // signal AND no hydrated tool_calls. Drop assistant + // + dependent tool rows so the LLM never sees orphans. + toolreplay.MarkAllFollowingToolsForSkip(rows, i, skipTool) + continue + } + out = append(out, msg) + // HOIST: for every tool_call in this assistant's slot, + // look up its response by tool_call_id and emit it RIGHT + // HERE so the LLM API's "tool messages must immediately + // follow tool_calls" invariant holds — even when the + // natural created_at order interleaves another assistant + // (e.g. long-running AgentTool finishing after later + // short tools). + for _, tc := range msg.ToolCalls { + emitToolByCallID(tc.ID) + } + case aiopsmodel.RoleTool: + // Tool messages are emitted ONLY via hoisting (emitToolByCallID, + // right after their assistant's tool_calls — which sets + // skipTool for that row). A role=tool row reaching this branch + // un-hoisted is an ORPHAN: its tool_call_id matches no preceding + // assistant tool_call. That happens when an out-of-order + // parallel completion persisted the response under a synthetic + // "|einoToolAdapter" id (the autoheal stub then filled the + // real id slot, so the assistant turn survived the precheck and + // this real response is left dangling). Emitting it bare yields + // the provider 400 "Messages with role 'tool' must be a response + // to a preceding message with 'tool_calls'". Drop it — the + // assistant's slot is already satisfied by the hoisted row. + continue + } + } + return out +} + +const ( + toolBudgetExceededStatus = "call_budget_exceeded" + toolBudgetExpiredPreviousTurn = "expired_previous_turn" + toolBudgetExpiredReplayMessage = "This per-turn tool budget result belonged to a previous user turn and has expired. The tool may be called again in the current user turn if needed." +) + +type toolBudgetReplayEnvelope struct { + Status string `json:"status"` + Scope string `json:"scope,omitempty"` + Tool string `json:"tool,omitempty"` + Calls int `json:"calls,omitempty"` + Instruction string `json:"instruction"` +} + +func sanitizeToolReplayContent(content string) string { + var env toolBudgetReplayEnvelope + if err := json.Unmarshal([]byte(content), &env); err != nil { + return content + } + if env.Status != toolBudgetExceededStatus { + return content + } + env.Scope = toolBudgetExpiredPreviousTurn + env.Instruction = toolBudgetExpiredReplayMessage + b, err := json.Marshal(env) + if err != nil { + return content + } + return string(b) +} + +// chatModelOpts turns the per-request model selection (SPA picker) into eino +// model options for the graph's ChatModel node. Empty fields add no option, +// so the routing default applies (unchanged default path). Provider routes +// via RoutingChatModel.pick (WithProvider); model name is honoured by the +// inner clientChatModel (WithModel). +func chatModelOpts(req *Request) []model.Option { + if req == nil { + return nil + } + var opts []model.Option + if p := strings.TrimSpace(req.Provider); p != "" { + opts = append(opts, llm.WithProvider(p)) + } + if m := strings.TrimSpace(req.Model); m != "" { + opts = append(opts, model.WithModel(m)) + } + return opts +} + +// calcDynamicHints produces the per-turn hint bullets that get injected +// into the block. Pure: depends only +// on the persisted history rows; no LLM, no I/O. Returns nil when no +// hint applies for this turn. +// +// Current heuristics (kept intentionally narrow — false positives on a +// reminder we re-show every turn corrode the LLM's trust in the block): +// +// (a) consecutive same-tool failures ≥ 2 → "switch tools or ask user" +// (b) repeated (tool, args) call ≥ 3 times → "stop repeating the same call" +func (rt *Runtime) calcDynamicHints(history []*aiopsmodel.Message) []string { + if rt == nil { + return nil + } + var hints []string + _, currentUser := latestUserMessage(history) + // Historical loop evidence is advisory only. It must not be injected + // into an unrelated new question: the graph's real tool memo is rebuilt + // per request, so carrying this hint across questions makes the model + // believe a previous turn exhausted this turn's budget. + if looksLikeToolLoopContinuation(currentUser) { + if name, n := consecutiveFailedTool(history, 2); n >= 2 { + hints = append(hints, fmt.Sprintf("%s 已连续失败 %d 次：换工具，或问用户澄清", name, n)) + } + // Repeat-call detection — same tool with similar args ≥ 3 times in + // the trailing window. This guards a requested continuation without + // turning a prior investigation into a session-wide tool budget. + if name, args, n := repeatedToolCall(history, 3); n >= 3 { + hints = append(hints, fmt.Sprintf("%s 已重复调用 %d 次（args: %s）：从你已有的数据下结论，不要再调用同款工具", name, n, args)) + } + } + if alertDraftGuardNeedsDraftRetry(history) { + hints = append(hints, "上一轮告警草案被安全闸门拦截：当前用户消息仍在继续创建告警时，直接调用 draft_config_change 生成 config_draft/draft_hash；metric 类规则需要先用 list_metric_catalog 获取当前指标。不要要求用户重新发送，不要输出文字草案，不要仅解释流程") + } + // "Promise-without-execution" detection — see ongridBasePrompt() + // 中等档位的 LLM (e.g. glm-4-plus) 偶尔写 "让我..." / "我先..." 这种 + // 计划句但没真发出 tool_call，导致 graph 提前 END，用户体验是消息 + // 戛然而止。这条 hint 在下一轮把 LLM 重新拽回来 — 但只在用户给了 + // 续聊指令（如 "继续"）的时候触发，避免在用户主动终止对话时打扰。 + if excerpt, found := detectUnfollowedPromise(history); found { + hints = append(hints, fmt.Sprintf(`上一轮你说"%s"但没真发 tool_call。如要继续探索请直接发 tool_call，不要再写计划句；如已有结论请直接给用户答复`, excerpt)) + } + return hints +} + +// looksLikeToolLoopContinuation limits historical loop hints to an explicit +// request to continue the preceding investigation. A new question in the same +// session always receives a fresh per-tool execution budget. +func looksLikeToolLoopContinuation(text string) bool { + trimmed := strings.TrimSpace(text) + if trimmed == "" { + return false + } + lower := strings.ToLower(trimmed) + return containsAnyString(trimmed, lower, []string{ + "继续", "接着", "重试", "再试", "上一条", "上一步", "刚才那个", + "continue", "retry", "try again", "previous step", "same investigation", + }) +} + +// repeatedToolCall walks the previous user turn's tool messages looking for the +// most frequent tool. Returns (name, argsExcerpt, n) where n is occurrence +// count. Args excerpt = first 80 bytes of the latest call's args. n < minN is +// reported as 0. +func repeatedToolCall(history []*aiopsmodel.Message, minN int) (string, string, int) { + if minN <= 0 { + minN = 3 + } + start, end := previousUserTurnBounds(history) + counts := map[string]int{} + args := map[string]string{} + for i := end - 1; i >= start; i-- { + m := history[i] + if m == nil || m.Role != aiopsmodel.RoleTool { + continue + } + name := "" + if m.ToolName != nil { + name = *m.ToolName + } + if name == "" { + continue + } + counts[name]++ + // snapshot args for highest-count latest occurrence + if _, ok := args[name]; !ok && m.Content != nil { + c := *m.Content + if len(c) > 80 { + c = c[:80] + } + args[name] = c + } + } + var topName string + var topN int + for name, n := range counts { + if n > topN { + topName = name + topN = n + } + } + if topN < minN { + return "", "", 0 + } + return topName, args[topName], topN +} + +func previousUserTurnBounds(history []*aiopsmodel.Message) (int, int) { + if len(history) == 0 { + return 0, 0 + } + currentUserIdx := -1 + for i := len(history) - 1; i >= 0; i-- { + m := history[i] + if m != nil && m.Role == aiopsmodel.RoleUser { + currentUserIdx = i + break + } + } + if currentUserIdx < 0 { + return 0, len(history) + } + prevUserIdx := -1 + for i := currentUserIdx - 1; i >= 0; i-- { + m := history[i] + if m != nil && m.Role == aiopsmodel.RoleUser { + prevUserIdx = i + break + } + } + if prevUserIdx < 0 { + return 0, currentUserIdx + } + return prevUserIdx + 1, currentUserIdx +} + +// consecutiveFailedTool walks history backwards looking at the trailing +// run of tool messages. Returns (toolName, n) where n is how many of the +// most-recent tool turns were the same tool AND each one's persisted +// content carries a failure marker. n < minN is reported as 0. +// +// Failure marker (deliberately simple): the persisted Content JSON +// contains the substring `"error"` — every tool decorator (PR-3) writes +// `{"error":"..."}` on tool failure (timeout, panic, returned error), +// and successful payloads do not happen to ship an `"error"` field. We +// keep the parse loose so a future tool that reports partial failures +// inside a richer JSON still gets caught. +// +// Boundary cases this honours: +// - the trailing tool block must be unbroken — interleaved +// user/assistant turns reset the counter (n = 0) +// - if the trailing tool block has mixed tool names, only the +// most-recent tool's run is counted (we walk backwards; on a +// name-mismatch we stop) +// - a single failure (n == 1) returns ("", 0) when minN == 2 — we do +// NOT want to nag on every transient error +func consecutiveFailedTool(history []*aiopsmodel.Message, minN int) (string, int) { + if minN <= 0 { + minN = 2 + } + name := "" + n := 0 + for i := len(history) - 1; i >= 0; i-- { + m := history[i] + if m == nil { + continue + } + if m.Role != aiopsmodel.RoleTool { + break + } + // Tool message — extract name + check failure. + curName := "" + if m.ToolName != nil { + curName = *m.ToolName + } + if curName == "" { + break + } + if name == "" { + name = curName + } else if curName != name { + // different tool — stop counting; we only flag a single + // stuck-loop tool, not a pattern across multiple tools. + break + } + content := "" + if m.Content != nil { + content = *m.Content + } + if !looksLikeToolFailure(content) { + break + } + n++ + } + if n < minN { + return "", 0 + } + return name, n +} + +func alertDraftGuardNeedsDraftRetry(history []*aiopsmodel.Message) bool { + currentIdx, currentUser := latestUserMessage(history) + if currentIdx < 0 || strings.TrimSpace(currentUser) == "" { + return false + } + if !callbacks.LooksLikeAlertRuleCreationText(currentUser) && !looksLikeAlertDraftContinuation(currentUser) { + return false + } + for i := currentIdx - 1; i >= 0; i-- { + m := history[i] + if m == nil { + continue + } + switch m.Role { + case aiopsmodel.RoleAssistant: + if m.Content == nil { + return false + } + return callbacks.LooksLikeAlertDraftGuardBlockedMessage(*m.Content) + case aiopsmodel.RoleUser: + return false + } + } + return false +} + +func latestUserMessage(history []*aiopsmodel.Message) (int, string) { + for i := len(history) - 1; i >= 0; i-- { + m := history[i] + if m == nil || m.Role != aiopsmodel.RoleUser || m.Content == nil { + continue + } + return i, *m.Content + } + return -1, "" +} + +func looksLikeAlertDraftContinuation(text string) bool { + trimmed := strings.TrimSpace(text) + if trimmed == "" { + return false + } + lower := strings.ToLower(trimmed) + if strings.Contains(trimmed, "为什么") || + strings.Contains(trimmed, "原因") || + strings.Contains(lower, "why") { + return false + } + return containsAnyString(trimmed, lower, []string{ + "继续", "继续创建", "直接创建", "生成草案", "生成 draft", "创建 draft", "draft_config_change", + "continue", "create it", "draft it", "generate draft", + }) +} + +func containsAnyString(text, lowerText string, needles []string) bool { + for _, needle := range needles { + if strings.Contains(lowerText, needle) || strings.Contains(text, needle) { + return true + } + } + return false +} + +// looksLikeToolFailure is the heuristic used by consecutiveFailedTool. +// True when the persisted tool message Content carries an "error" JSON +// key — the decorator chain's standard failure shape. +func looksLikeToolFailure(content string) bool { + if content == "" { + return false + } + // Cheap substring scan — JSON validity is enforced by the + // decorator path, and we only need a yes/no signal here. + return strings.Contains(content, `"error"`) +} + +// promiseMarkers are the substrings that indicate the assistant wrote a +// plan句 ("let me...") but did not actually emit a tool_call. Detected +// in the most recent assistant message; if matched + no tool message +// follows before the next user, calcDynamicHints injects a nudge for +// the next turn (see ongridBasePrompt). +var promiseMarkers = []string{ + "让我", "我先", "我来", "我会", "我将", "接下来", "首先", "先让我", "让我先", +} + +// detectUnfollowedPromise walks history backwards looking for the last +// assistant message. Returns (excerpt, true) when: +// +// 1. that message contains any promiseMarker token, AND +// 2. no role=tool message follows it before the next role=user message +// (or before end of history). +// +// Returns ("", false) otherwise. The excerpt is a 30-rune prefix of the +// matched assistant content for inclusion in the hint. +func detectUnfollowedPromise(history []*aiopsmodel.Message) (string, bool) { + if len(history) == 0 { + return "", false + } + // Find last assistant message. + asstIdx := -1 + for i := len(history) - 1; i >= 0; i-- { + if history[i].Role == aiopsmodel.RoleAssistant { + asstIdx = i + break + } + } + if asstIdx < 0 || history[asstIdx].Content == nil { + return "", false + } + content := strings.TrimSpace(*history[asstIdx].Content) + if callbacks.LooksLikeAlertDraftGuardBlockedMessage(content) { + return "", false + } + if content == "" || !containsAnyPromiseMarker(content) { + return "", false + } + // Look forward from asstIdx+1: did a tool message follow before the + // next user message? + for j := asstIdx + 1; j < len(history); j++ { + switch history[j].Role { + case aiopsmodel.RoleTool: + return "", false // promise WAS executed + case aiopsmodel.RoleUser: + // User came in before any tool ran — promise unfollowed. + break + } + } + // Build a clean excerpt: strip leading bullet/whitespace, take 30 runes. + excerpt := content + if r := []rune(excerpt); len(r) > 30 { + excerpt = string(r[:30]) + "..." + } + return excerpt, true +} + +func containsAnyPromiseMarker(content string) bool { + for _, m := range promiseMarkers { + if strings.Contains(content, m) { + return true + } + } + return false +} + +// buildGraphErrorApology turns a graph-level error into a user-facing +// markdown apology, picking different wording per error class so the user +// (and downstream Ant audit) can tell what happened. Mirrors the apology +// style of legacy agent.go's MaxIterations fallback. +func buildGraphErrorApology(err error) string { + if err == nil { + return "抱歉，处理消息时遇到未知错误。请换个问法再问一次。" + } + msg := err.Error() + low := strings.ToLower(msg) + switch { + // LLM hallucinated a tool name not in its actual schema (e.g. + // model "remembers" `get_host_load` from training even though + // only `query_devices / query_knowledge / ...` were presented). + // Eino's tool router can't dispatch → whole graph aborts. Map + // to a friendly retry message that nudges toward the right + // flow (AgentTool dispatch). + case strings.Contains(low, "not found in toolsnode"), + strings.Contains(low, "tool") && strings.Contains(low, "not found"): + return "这个问题需要的深度查询能力不在我（协调员）的工具范围内 — 那些工具都在专家手里。请直接告诉我你的具体诉求（例如「看 CPU/内存」「检查磁盘容量」「网络连通性」），我会派对应的 specialist 处理。" + // Match both eino's wording shapes: "exceeds max steps" (current + // runtime) and the legacy "exceeded max iterations" / "maxsteps" + // variants. The bug we ate before: matcher said "exceeded" (past + // tense) but eino emits "exceeds" (present tense), so every + // max-steps abort dumped raw "[GraphRunError]…" to the user. + case strings.Contains(low, "exceeds max"), + strings.Contains(low, "exceeded max"), + strings.Contains(low, "max steps"), + strings.Contains(low, "maxstep"), + strings.Contains(low, "max iterations"): + return "我跑了很多轮工具调用还是没收敛 — 大概率是搜索路径走偏了。上面工具调用区已经有阶段性数据，可以直接看；也可以这样收敛：\n• 把问题缩小成「看 X 指标的 Y 现象」——给具体 metric / device 让我聚焦\n• 或者告诉我「已经看到 A / B / C，下一步从 D 这条线索查」\n• 也可以直接 @ 一个 specialist（如 @specialist-compute 看 CPU/内存）让专家接手" + case strings.Contains(low, "context canceled"), strings.Contains(low, "context deadline"): + return "本次请求超时或被取消。一般是上游 LLM / 设备响应慢。请稍后重试，或换简单的问法。" + case strings.Contains(low, "budget"): + return "今日 LLM 预算已用完。请联系 admin 调整配额或明天再试。" + // History replay produced a tool_calls envelope that the upstream + // provider rejected because one or more tool responses are missing + // from chat_messages (eino ToolsNode dropped an OnEnd in parallel + // execution; the chat_tool_calls row is there but the role=tool + // chat_messages row is not). Distinct user message — this is a + // dirty-session issue, not a provider or quota problem; new session + // is the clean way out. + case strings.Contains(low, "insufficient tool messages"), + strings.Contains(low, "tool_calls must be followed"), + strings.Contains(low, "tool messages responding"): + return "本会话历史里有一轮工具调用的响应没落库（并行调用写入丢失），后续每条消息送上游都会被拒。请新开一个会话继续，这个会话先放一放——后端会再修。" + // LLM provider quota / rate-limit / insufficient balance. Substring + // covers OpenAI ("insufficient_quota"), Anthropic ("credit balance"), + // Zhipu ("余额不足"), generic 429 wording. Operator-side issues, not + // user-side. The `insufficient` substring is intentionally narrow — + // the bare word also appears in protocol error strings like + // "insufficient tool messages following tool_calls", which used to + // land here and mislead the user into chasing a quota that was fine. + case strings.Contains(low, "429"), + strings.Contains(low, "too many requests"), + strings.Contains(low, "余额不足"), + strings.Contains(low, "insufficient_quota"), + strings.Contains(low, "insufficient balance"), + strings.Contains(low, "insufficient credit"), + strings.Contains(low, "insufficient funds"), + strings.Contains(low, "credit balance is too low"), + strings.Contains(low, "rate limit"), + strings.Contains(low, "quota"): + return "LLM provider 当前不可用——可能是配额用尽 / 限流 / 余额不足。请到「设置 → 模型」检查 provider 的状态和余额；问题在配置层面，重试不会解决。" + // Other LLM provider errors (auth failure, model not found, etc.) + // land here so the user gets a hint to check provider config + // instead of a 200-char raw API error dump. + case strings.Contains(low, "llm: chat completion"), + strings.Contains(low, "openai api"), + strings.Contains(low, "api error"): + return "LLM provider 报错（可能是 API key / 模型名 / 网络）。请到「设置 → 模型」检查 provider 配置；详细 raw 错误请看 manager 日志。" + default: + // 非已知 class：给一段保守的原文摘要，让用户能截图反馈。 + short := msg + if len(short) > 200 { + short = short[:200] + "..." + } + return "抱歉，处理消息时遇到错误：\n```\n" + short + "\n```\n请换个问法再试，或截图反馈给我们。" + } +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_test.go new file mode 100644 index 000000000..11228338f --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_test.go @@ -0,0 +1,1096 @@ +package chatruntime + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + einomodel "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + + biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph/callbacks" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +// scriptedChatModel returns one *schema.Message per Generate call, +// tracking generateCalls so tests can assert how many turns ran. +// Mirrors the pattern in graph/react_test.go. +type scriptedChatModel struct { + mu sync.Mutex + replies []*schema.Message + idx int + calls atomic.Int32 + inputs [][]*schema.Message +} + +func newScriptedChatModel(replies ...*schema.Message) *scriptedChatModel { + return &scriptedChatModel{replies: replies} +} + +func (s *scriptedChatModel) Generate(_ context.Context, input []*schema.Message, _ ...einomodel.Option) (*schema.Message, error) { + s.calls.Add(1) + s.mu.Lock() + defer s.mu.Unlock() + inputCopy := make([]*schema.Message, len(input)) + copy(inputCopy, input) + s.inputs = append(s.inputs, inputCopy) + if len(s.replies) == 0 { + return &schema.Message{Role: schema.Assistant, Content: "ok"}, nil + } + if s.idx < len(s.replies) { + out := s.replies[s.idx] + s.idx++ + return out, nil + } + return s.replies[len(s.replies)-1], nil +} + +func (s *scriptedChatModel) Stream(ctx context.Context, input []*schema.Message, opts ...einomodel.Option) (*schema.StreamReader[*schema.Message], error) { + msg, err := s.Generate(ctx, input, opts...) + if err != nil { + return nil, err + } + return schema.StreamReaderFromArray([]*schema.Message{msg}), nil +} + +func (s *scriptedChatModel) BindTools(_ []*schema.ToolInfo) error { return nil } + +func (s *scriptedChatModel) WithTools(_ []*schema.ToolInfo) (einomodel.ToolCallingChatModel, error) { + return s, nil +} + +// memSessions is an in-memory SessionRepo for runtime tests. Only the +// methods runtime.Handle exercises are implemented; the rest panic on +// purpose so a future refactor can't silently slip past coverage. +type memSessions struct { + mu sync.Mutex + sessions map[string]*model.Session + messages []*model.Message + toolCalls []*model.ToolCall +} + +func newMemSessions(seed *model.Session) *memSessions { + m := &memSessions{sessions: map[string]*model.Session{}} + if seed != nil { + m.sessions[seed.ID] = seed + } + return m +} + +func (m *memSessions) CreateSession(_ context.Context, s *model.Session) error { + m.mu.Lock() + defer m.mu.Unlock() + m.sessions[s.ID] = s + return nil +} +func (m *memSessions) GetSession(_ context.Context, id string) (*model.Session, error) { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.sessions[id] + if !ok { + return nil, errs.ErrNotFound + } + cp := *s + return &cp, nil +} +func (m *memSessions) ListSessions(_ context.Context, _ uint64, _, _ int, _ *uint64) ([]*model.Session, error) { + return nil, nil +} +func (m *memSessions) ListByParent(_ context.Context, parentID string) ([]*model.Session, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]*model.Session, 0) + for _, s := range m.sessions { + if s.ParentSessionID != nil && *s.ParentSessionID == parentID { + cp := *s + out = append(out, &cp) + } + } + return out, nil +} +func (m *memSessions) CloseSession(_ context.Context, id string) error { + m.mu.Lock() + defer m.mu.Unlock() + if s, ok := m.sessions[id]; ok { + now := time.Now().UTC() + s.ClosedAt = &now + } + return nil +} +func (m *memSessions) RenameSession(_ context.Context, _, _ string) error { return nil } +func (m *memSessions) UpdateSessionModel(_ context.Context, _, _, _ string) error { return nil } +func (m *memSessions) DeleteSession(_ context.Context, _ string) error { return nil } +func (m *memSessions) AppendMessage(_ context.Context, msg *model.Message) error { + m.mu.Lock() + defer m.mu.Unlock() + if msg.ID == "" { + msg.ID = "m" + idgen() + } + m.messages = append(m.messages, msg) + return nil +} +func (m *memSessions) ListMessages(_ context.Context, sessionID string, _ int) ([]*model.Message, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]*model.Message, 0, len(m.messages)) + for _, mg := range m.messages { + if mg.SessionID == sessionID { + out = append(out, mg) + } + } + return out, nil +} +func (m *memSessions) CreateToolCall(_ context.Context, tc *model.ToolCall) error { + m.mu.Lock() + defer m.mu.Unlock() + if tc.ID == "" { + tc.ID = "tc" + idgen() + } + m.toolCalls = append(m.toolCalls, tc) + return nil +} +func (m *memSessions) UpdateToolCallResult(_ context.Context, id string, status string, resultJSON, errStr *string, endedAt time.Time) error { + m.mu.Lock() + defer m.mu.Unlock() + for _, tc := range m.toolCalls { + if tc.ID != id { + continue + } + tc.Status = status + tc.ResultJSON = resultJSON + tc.Error = errStr + tc.EndedAt = &endedAt + return nil + } + return nil +} +func (m *memSessions) SumTokensSince(_ context.Context, _ time.Time) (biz.TokenSums, error) { + return biz.TokenSums{}, nil +} + +func TestMemSessions_GetSessionReturnsSnapshot(t *testing.T) { + store := newMemSessions(&model.Session{ID: "session-1"}) + + row, err := store.GetSession(context.Background(), "session-1") + if err != nil { + t.Fatalf("GetSession: %v", err) + } + now := time.Now().UTC() + row.ClosedAt = &now + + stored, err := store.GetSession(context.Background(), "session-1") + if err != nil { + t.Fatalf("GetSession after mutating snapshot: %v", err) + } + if stored.ClosedAt != nil { + t.Fatalf("stored ClosedAt = %v after mutating snapshot, want nil", stored.ClosedAt) + } +} + +var idCounter atomic.Int64 + +func idgen() string { + n := idCounter.Add(1) + return time.Now().UTC().Format("20060102") + "-" + atoi(int(n)) +} + +func atoi(n int) string { + if n == 0 { + return "0" + } + out := []byte{} + for n > 0 { + out = append([]byte{byte('0' + n%10)}, out...) + n /= 10 + } + return string(out) +} + +// TestRuntime_NewRuntime_RequiresDeps confirms NewRuntime fails fast +// when ChatModel or Sessions is missing — production wiring must not +// silently drop a key dep. +func TestRuntime_NewRuntime_RequiresDeps(t *testing.T) { + if _, err := NewRuntime(Config{}); err == nil { + t.Errorf("NewRuntime{} returned nil error — expected dep check") + } + if _, err := NewRuntime(Config{Sessions: newMemSessions(nil)}); err == nil { + t.Errorf("NewRuntime sans ChatModel should error") + } + if _, err := NewRuntime(Config{ChatModel: newScriptedChatModel()}); err == nil { + t.Errorf("NewRuntime sans Sessions should error") + } +} + +// TestRuntime_Handle_OwnershipCheck enforces the "non-owner gets +// ErrNotFound" invariant. Mirrors the legacy agent's behaviour +// . +func TestRuntime_Handle_OwnershipCheck(t *testing.T) { + sess := &model.Session{ID: "s1", UserID: 7} + rt, err := NewRuntime(Config{ + Sessions: newMemSessions(sess), + ChatModel: newScriptedChatModel(), + ToolBag: nil, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + _, err = rt.Handle(context.Background(), &Request{ + SessionID: "s1", + UserID: 99, // not the owner + UserText: "hi", + }) + if !errors.Is(err, errs.ErrNotFound) { + t.Errorf("non-owner err = %v, want ErrNotFound", err) + } +} + +// TestRuntime_Handle_HappyPath_FinalReply runs the graph once +// against a scriptedChatModel that returns a no-tools assistant +// message. Asserts: +// - user message persisted +// - Reply.Message non-nil with the model's content +// - terminal Done event fires once +func TestRuntime_Handle_HappyPath_FinalReply(t *testing.T) { + sess := &model.Session{ID: "s1", UserID: 7} + store := newMemSessions(sess) + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "all good", + }) + rt, err := NewRuntime(Config{ + Sessions: store, + ChatModel: scripted, + ToolBag: nil, + GraphCfg: graph.Config{MaxIterations: 5}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + var ( + mu sync.Mutex + events []Event + ) + emit := func(ev Event) { + mu.Lock() + defer mu.Unlock() + events = append(events, ev) + } + + reply, err := rt.Handle(context.Background(), &Request{ + SessionID: "s1", + UserID: 7, + UserText: "what's up", + Emit: emit, + }) + if err != nil { + t.Fatalf("Handle: %v", err) + } + if reply == nil || reply.Message == nil { + t.Fatalf("expected non-nil reply.Message") + } + if reply.Message.Content == nil || *reply.Message.Content != "all good" { + t.Errorf("reply content = %v, want \"all good\"", reply.Message.Content) + } + + // User message must be persisted. + store.mu.Lock() + defer store.mu.Unlock() + if len(store.messages) == 0 { + t.Errorf("expected at least one persisted message (user turn)") + } + if store.messages[0].Role != model.RoleUser { + t.Errorf("first persisted message role = %q, want user", store.messages[0].Role) + } + + // Done event fires once at terminal success. + mu.Lock() + defer mu.Unlock() + if len(events) == 0 { + t.Fatalf("expected at least one event") + } + if events[len(events)-1].Type != EventDone { + t.Errorf("last event type = %q, want done", events[len(events)-1].Type) + } +} + +func TestRuntime_Handle_HostFollowUpFromHistory_ReachesModel(t *testing.T) { + tests := []struct { + name string + question string + answer string + }{ + {name: "chinese", question: "哪个磁盘占比比较大？", answer: "设备 1 的根分区占用最高"}, + {name: "english", question: "Which disk is using the most space on this device?", answer: "The root filesystem on device 1 has the highest usage."}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sess := &model.Session{ID: "s-follow-up-" + tt.name, UserID: 7} + store := newMemSessions(sess) + callID := "call_previous_" + tt.name + store.messages = append(store.messages, + &model.Message{ID: "u-prev-" + tt.name, SessionID: sess.ID, Role: model.RoleUser, Content: strPtr("device_id = 1")}, + &model.Message{ID: "a-tool-" + tt.name, SessionID: sess.ID, Role: model.RoleAssistant, ToolCalls: []model.ToolCall{{ + ToolName: "host_bash", ArgumentsJSON: `{"device_ids":[1],"cmd":"df -h"}`, + Status: model.StatusSuccess, LLMCallID: &callID, + }}}, + &model.Message{ID: "t-prev-" + tt.name, SessionID: sess.ID, Role: model.RoleTool, Content: strPtr(`{"success_count":1}`), ToolName: strPtr("host_bash"), ToolCallID: &callID}, + &model.Message{ID: "a-prev-" + tt.name, SessionID: sess.ID, Role: model.RoleAssistant, Content: strPtr("device 1 root filesystem usage is 77%")}, + ) + scripted := newScriptedChatModel(&schema.Message{Role: schema.Assistant, Content: tt.answer}) + rt, err := NewRuntime(Config{Sessions: store, ChatModel: scripted, GraphCfg: graph.Config{MaxIterations: 5}}) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + reply, err := rt.Handle(context.Background(), &Request{SessionID: sess.ID, UserID: sess.UserID, Role: "admin", UserText: tt.question}) + if err != nil { + t.Fatalf("Handle: %v", err) + } + if scripted.calls.Load() != 1 { + t.Fatalf("LLM calls = %d, want 1; request was likely intercepted before history replay", scripted.calls.Load()) + } + if reply == nil || reply.Message == nil || reply.Message.Content == nil || *reply.Message.Content != tt.answer { + t.Fatalf("reply = %+v, want %q", reply, tt.answer) + } + scripted.mu.Lock() + inputs := append([][]*schema.Message(nil), scripted.inputs...) + scripted.mu.Unlock() + if len(inputs) != 1 || !schemaMessagesContain(inputs[0], "device_id = 1") || !schemaMessagesContain(inputs[0], tt.question) { + t.Fatalf("model input did not contain prior target and current question: %+v", inputs) + } + }) + } +} + +func schemaMessagesContain(messages []*schema.Message, needle string) bool { + for _, msg := range messages { + if msg != nil && contains(msg.Content, needle) { + return true + } + } + return false +} + +func TestRuntime_Handle_ConfirmedConfigDraft_AppliesWithoutLLM(t *testing.T) { + sess := &model.Session{ID: "s1", UserID: 7} + store := newMemSessions(sess) + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "should not run", + }) + apply := &captureApplyTool{ + resp: `{"kind":"config_apply_result","domain":"alert_rule","action":"create","status":"applied","resource_id":42,"resource":{"name":"MySQL 连接使用率过高预警","type":"alert_rule"}}`, + } + rt, err := NewRuntime(Config{ + Sessions: store, + ChatModel: scripted, + ToolBag: []basetool.BaseTool{apply}, + GraphCfg: graph.Config{MaxIterations: 5}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + payload := `{"action":"create","draft_id":"draft-1","rule":{"rule_key":"mysql_connection_usage_high","name":"MySQL 连接使用率过高预警","kind":"metric_raw","severity":"warning","spec":{"expr":"up > 0","for":"5m"}}}` + userText := "确认创建这条告警规则\n" + + "domain: alert_rule\n" + + "action: create\n" + + "apply_tool: apply_config_change\n" + + "draft_hash: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" + + "payload:\n```json\n" + payload + "\n```" + + var events []Event + reply, err := rt.Handle(context.Background(), &Request{ + SessionID: sess.ID, + UserID: sess.UserID, + Role: "admin", + UserText: userText, + Emit: func(ev Event) { + events = append(events, ev) + }, + }) + if err != nil { + t.Fatalf("Handle: %v", err) + } + if scripted.calls.Load() != 0 { + t.Fatalf("LLM calls = %d, want 0 for deterministic confirm apply", scripted.calls.Load()) + } + if apply.calls.Load() != 1 { + t.Fatalf("apply_config_change calls = %d, want 1", apply.calls.Load()) + } + argsValue := apply.args.Load() + if argsValue == nil { + t.Fatalf("captured apply args is nil") + } + argsJSON, ok := argsValue.(string) + if !ok { + t.Fatalf("captured apply args type = %T, want string", argsValue) + } + for _, want := range []string{ + `"domain":"alert_rule"`, + `"action":"create"`, + `"confirmed":true`, + `"draft_hash":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`, + `"payload":{`, + `"rule_key":"mysql_connection_usage_high"`, + } { + if !contains(argsJSON, want) { + t.Fatalf("apply args missing %q: %s", want, argsJSON) + } + } + if reply == nil || reply.Message == nil || reply.Message.Content == nil { + t.Fatalf("expected persisted assistant reply") + } + if !contains(*reply.Message.Content, "已确认并创建告警规则") || !contains(*reply.Message.Content, "ID: 42") { + t.Fatalf("reply content = %q", *reply.Message.Content) + } + + var sawStart, sawEnd bool + for _, ev := range events { + if ev.Tool == nil || ev.Tool.Name != applyConfigChangeToolName { + continue + } + switch ev.Type { + case EventToolStart: + sawStart = true + if ev.Tool.Status != "running" { + t.Fatalf("tool_start status = %q, want running", ev.Tool.Status) + } + case EventToolEnd: + sawEnd = true + if ev.Tool.Status != "success" { + t.Fatalf("tool_end status = %q, want success", ev.Tool.Status) + } + } + } + if !sawStart || !sawEnd { + t.Fatalf("missing apply tool events: sawStart=%v sawEnd=%v events=%v", sawStart, sawEnd, events) + } + if len(store.messages) != 2 { + t.Fatalf("persisted messages = %d, want user + assistant", len(store.messages)) + } +} + +func TestRuntime_Handle_PlainOKAppliesLatestConfigDraftWithoutLLM(t *testing.T) { + sess := &model.Session{ID: "s1", UserID: 7} + store := newMemSessions(sess) + payload := `{"action":"create","draft_id":"draft-plain-ok","rule":{"rule_key":"mysql_slow_queries_surge","name":"MySQL 慢查询异常增多","kind":"metric_raw","severity":"warning","spec":{"expr":"rate(mysql_global_status_slow_queries[5m]) > 0.5","for":"5m"}}}` + draftResult := `{"kind":"config_draft","domain":"alert_rule","action":"create","draft_hash":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","payload":` + payload + `}` + store.messages = append(store.messages, + &model.Message{ID: "m-user", SessionID: sess.ID, Role: model.RoleUser, Content: strPtr("创建 MySQL 慢查询告警"), CreatedAt: time.Now().Add(-3 * time.Minute)}, + &model.Message{ID: "m-tool", SessionID: sess.ID, Role: model.RoleTool, ToolName: strPtr("draft_config_change"), Content: strPtr(draftResult), CreatedAt: time.Now().Add(-2 * time.Minute)}, + &model.Message{ID: "m-assistant", SessionID: sess.ID, Role: model.RoleAssistant, Content: strPtr("草稿已生成，确认后创建。"), CreatedAt: time.Now().Add(-time.Minute)}, + ) + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "should not run", + }) + apply := &captureApplyTool{ + resp: `{"kind":"config_apply_result","domain":"alert_rule","action":"create","status":"applied","resource_id":95,"resource":{"name":"MySQL 慢查询异常增多","type":"alert_rule"}}`, + } + rt, err := NewRuntime(Config{ + Sessions: store, + ChatModel: scripted, + ToolBag: []basetool.BaseTool{apply}, + GraphCfg: graph.Config{MaxIterations: 5}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + reply, err := rt.Handle(context.Background(), &Request{ + SessionID: sess.ID, + UserID: sess.UserID, + Role: "admin", + UserText: "ok", + }) + if err != nil { + t.Fatalf("Handle: %v", err) + } + if scripted.calls.Load() != 0 { + t.Fatalf("LLM calls = %d, want 0 for plain confirmation apply", scripted.calls.Load()) + } + if apply.calls.Load() != 1 { + t.Fatalf("apply_config_change calls = %d, want 1", apply.calls.Load()) + } + argsValue := apply.args.Load() + if argsValue == nil { + t.Fatalf("captured apply args is nil") + } + argsJSON, ok := argsValue.(string) + if !ok { + t.Fatalf("captured apply args type = %T, want string", argsValue) + } + for _, want := range []string{ + `"confirmed":true`, + `"draft_hash":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"`, + `"draft_id":"draft-plain-ok"`, + `"confirmation_text":"ok"`, + } { + if !contains(argsJSON, want) { + t.Fatalf("apply args missing %q: %s", want, argsJSON) + } + } + if reply == nil || reply.Message == nil || reply.Message.Content == nil { + t.Fatalf("expected persisted assistant reply") + } + if !contains(*reply.Message.Content, "已确认并创建告警规则") || !contains(*reply.Message.Content, "ID: 95") { + t.Fatalf("reply content = %q", *reply.Message.Content) + } +} + +func TestLatestConfigDraftApplyArgs_StopsAtAppliedResult(t *testing.T) { + payload := `{"action":"create","draft_id":"draft-old","rule":{"rule_key":"old","name":"Old","kind":"metric_raw","severity":"warning","spec":{"expr":"up > 0","for":"5m"}}}` + draftResult := `{"kind":"config_draft","domain":"alert_rule","action":"create","draft_hash":"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","payload":` + payload + `}` + applyResult := `{"kind":"config_apply_result","domain":"alert_rule","action":"create","status":"applied","resource_id":95}` + + args, err, ok := latestConfigDraftApplyArgs([]*model.Message{ + {Role: model.RoleUser, Content: strPtr("创建旧告警")}, + {Role: model.RoleTool, Content: strPtr(draftResult)}, + {Role: model.RoleUser, Content: strPtr("ok")}, + {Role: model.RoleTool, Content: strPtr(applyResult)}, + {Role: model.RoleUser, Content: strPtr("ok")}, + }, "ok") + if err != nil { + t.Fatalf("latestConfigDraftApplyArgs: %v", err) + } + if ok || args != "" { + t.Fatalf("latestConfigDraftApplyArgs returned (%q, %v), want no pending draft", args, ok) + } +} + +func TestLatestConfigDraftApplyArgs_DoesNotCrossPreviousUserTurn(t *testing.T) { + payload := `{"action":"create","draft_id":"draft-stale","rule":{"rule_key":"stale","name":"Stale","kind":"metric_raw","severity":"warning","spec":{"expr":"up > 0","for":"5m"}}}` + draftResult := `{"kind":"config_draft","domain":"alert_rule","action":"create","draft_hash":"sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","payload":` + payload + `}` + + args, err, ok := latestConfigDraftApplyArgs([]*model.Message{ + {Role: model.RoleUser, Content: strPtr("创建旧告警")}, + {Role: model.RoleTool, Content: strPtr(draftResult)}, + {Role: model.RoleAssistant, Content: strPtr("草稿已生成，确认后创建。")}, + {Role: model.RoleUser, Content: strPtr("先问另一个问题")}, + {Role: model.RoleAssistant, Content: strPtr("好的。")}, + {Role: model.RoleUser, Content: strPtr("ok")}, + }, "ok") + if err != nil { + t.Fatalf("latestConfigDraftApplyArgs: %v", err) + } + if ok || args != "" { + t.Fatalf("latestConfigDraftApplyArgs returned (%q, %v), want no stale draft across user turn", args, ok) + } +} + +// TestRuntime_ToolCount + ToolNames provides the per-spec visibility. +// (a) ONGRID_AGENT_KERNEL=graph startup logs how many BaseTools are +// bound — this is the seam main.go logs against. +func TestRuntime_ToolCountAndNames(t *testing.T) { + sess := &model.Session{ID: "s1", UserID: 7} + rt, err := NewRuntime(Config{ + Sessions: newMemSessions(sess), + ChatModel: newScriptedChatModel(), + ToolBag: []basetool.BaseTool{ + &fakeTool{name: "echo", schema: `{"type":"object","properties":{}}`}, + &fakeTool{name: "ping", schema: `{"type":"object","properties":{}}`}, + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + if rt.ToolCount() != 2 { + t.Errorf("ToolCount = %d, want 2", rt.ToolCount()) + } + names := rt.ToolNames(context.Background()) + t.Logf("toolBag names = %v (count=%d)", names, len(names)) + if len(names) != 2 { + t.Errorf("ToolNames len = %d, want 2", len(names)) + } +} + +func TestRuntime_ReplaceToolsByNamePrefix(t *testing.T) { + sess := &model.Session{ID: "s1", UserID: 7} + rt, err := NewRuntime(Config{ + Sessions: newMemSessions(sess), + ChatModel: newScriptedChatModel(), + ToolBag: []basetool.BaseTool{ + &fakeTool{name: "query_promql", schema: `{"type":"object"}`}, + &fakeTool{name: "mcp__github__old_search", schema: `{"type":"object"}`}, + &fakeTool{name: "mcp__grafana__query", schema: `{"type":"object"}`}, + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + rt.ReplaceToolsByNamePrefix("mcp__github__", []basetool.BaseTool{ + &fakeTool{name: "mcp__github__code_search", schema: `{"type":"object"}`}, + }) + names := rt.ToolNames(context.Background()) + got := map[string]bool{} + for _, name := range names { + got[name] = true + } + for _, want := range []string{"query_promql", "mcp__github__code_search", "mcp__grafana__query"} { + if !got[want] { + t.Errorf("ToolNames missing %q: %v", want, names) + } + } + if got["mcp__github__old_search"] { + t.Errorf("stale MCP tool remained in runtime: %v", names) + } +} + +// strPtr is a small helper for tool-message content/name pointer +// fields in the test fixtures below. Inline helpers everywhere bloats +// the test fixture noise and obscures the assertions. +func strPtr(s string) *string { return &s } + +// makeToolMsg builds a chat_messages row in the persisted shape the +// runtime sees from ListMessages. role=tool, ToolName + Content are +// non-nil; failure flag is "is the JSON content carrying an error +// field" (looksLikeToolFailure). +func makeToolMsg(toolName string, fail bool) *model.Message { + body := `{"ok":true}` + if fail { + body = `{"error":"timeout"}` + } + return &model.Message{ + Role: model.RoleTool, + ToolName: strPtr(toolName), + Content: strPtr(body), + } +} + +// TestConsecutiveFailedTool covers the boundary cases the spec calls +// out: below threshold / different tool / mixed success+fail. +func TestConsecutiveFailedTool(t *testing.T) { + t.Parallel() + + t.Run("two_in_a_row_same_tool", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + } + name, n := consecutiveFailedTool(hist, 2) + if name != "query_logql" || n != 2 { + t.Errorf("got (%q, %d), want (query_logql, 2)", name, n) + } + }) + + t.Run("three_in_a_row_same_tool", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + } + name, n := consecutiveFailedTool(hist, 2) + if name != "query_logql" || n != 3 { + t.Errorf("got (%q, %d), want (query_logql, 3)", name, n) + } + }) + + t.Run("below_minN", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + } + name, n := consecutiveFailedTool(hist, 2) + if name != "" || n != 0 { + t.Errorf("got (%q, %d), want zero result below minN", name, n) + } + }) + + t.Run("different_tool_resets", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + makeToolMsg("query_promql", true), + } + name, n := consecutiveFailedTool(hist, 2) + // trailing tool is query_promql with one fail — below minN. + if name != "" || n != 0 { + t.Errorf("got (%q, %d), want zero (different tool breaks the run)", name, n) + } + }) + + t.Run("success_in_middle_breaks_run", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", false), // success -> resets + makeToolMsg("query_logql", true), + } + name, n := consecutiveFailedTool(hist, 2) + // only the trailing single fail counts. + if name != "" || n != 0 { + t.Errorf("got (%q, %d), want zero (success in middle breaks)", name, n) + } + }) + + t.Run("non_tool_role_at_tail", func(t *testing.T) { + hist := []*model.Message{ + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + {Role: model.RoleAssistant, Content: strPtr("ok then")}, + } + // The trailing message is assistant, so the tool block isn't + // the most-recent thing — we should NOT report a stuck loop. + name, n := consecutiveFailedTool(hist, 2) + if name != "" || n != 0 { + t.Errorf("got (%q, %d), want zero (non-tool role at tail)", name, n) + } + }) + + t.Run("empty_history", func(t *testing.T) { + name, n := consecutiveFailedTool(nil, 2) + if name != "" || n != 0 { + t.Errorf("got (%q, %d), want zero", name, n) + } + }) +} + +// TestCalcDynamicHints checks the high-level wiring of both heuristics +// — consecutive-failed-tool + iteration-cap. +func TestCalcDynamicHints(t *testing.T) { + t.Parallel() + rt, err := NewRuntime(Config{ + Sessions: newMemSessions(&model.Session{ID: "s", UserID: 1}), + ChatModel: newScriptedChatModel(), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + t.Run("trailing_3_failed_logql_emits_failure_hint", func(t *testing.T) { + hist := []*model.Message{ + {Role: model.RoleAssistant, Content: strPtr("trying logql")}, + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + makeToolMsg("query_logql", true), + {Role: model.RoleUser, Content: strPtr("继续排查")}, + } + hints := rt.calcDynamicHints(hist) + if len(hints) == 0 { + t.Fatalf("expected at least one hint, got none") + } + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if !contains(joined, "query_logql") || (!contains(joined, "连续失败") && !contains(joined, "已重复调用")) { + t.Errorf("missing loop-protection hint: %q", joined) + } + }) + + t.Run("new_question_does_not_inherit_previous_turn_tool_loop_hint", func(t *testing.T) { + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("检查主机磁盘")}, + makeToolMsg("host_bash", false), + makeToolMsg("host_bash", false), + makeToolMsg("host_bash", false), + {Role: model.RoleAssistant, Content: strPtr("已完成磁盘检查")}, + {Role: model.RoleUser, Content: strPtr("看看 docker images 有没有能够清理的？")}, + } + hints := rt.calcDynamicHints(hist) + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if contains(joined, "host_bash 已重复调用") || contains(joined, "host_bash 已连续失败") { + t.Errorf("independent question inherited prior tool loop hint: %q", joined) + } + }) + + t.Run("clean_history_no_hints", func(t *testing.T) { + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("hi")}, + {Role: model.RoleAssistant, Content: strPtr("hi back")}, + } + hints := rt.calcDynamicHints(hist) + if len(hints) != 0 { + t.Errorf("expected no hints, got %v", hints) + } + }) + + t.Run("unfollowed_promise_emits_nudge", func(t *testing.T) { + // Last assistant said "让我..." but no tool message follows. + // Reproduces the d9fa4f42 session 17:00:36 trail-off. + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("看一下磁盘")}, + {Role: model.RoleAssistant, Content: strPtr("让我先查看 /data 目录的磁盘使用情况")}, + } + hints := rt.calcDynamicHints(hist) + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if !contains(joined, "没真发 tool_call") { + t.Errorf("missing unfollowed-promise hint: %q", joined) + } + }) + + t.Run("promise_followed_by_tool_no_hint", func(t *testing.T) { + // Same promise but tool ran — no hint expected. + toolName := "host_du_summary" + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("看一下磁盘")}, + {Role: model.RoleAssistant, Content: strPtr("让我先查看 /data")}, + {Role: model.RoleTool, ToolName: &toolName, Content: strPtr(`{"subpaths":[]}`)}, + } + hints := rt.calcDynamicHints(hist) + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if contains(joined, "没真发 tool_call") { + t.Errorf("hint should not fire when tool followed promise: %q", joined) + } + }) + + t.Run("new_alert_request_after_many_created_rules_no_repeat_tool_hint", func(t *testing.T) { + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("创建 CPU 告警")}, + makeToolMsg("list_metric_catalog", false), + makeToolMsg("draft_config_change", false), + {Role: model.RoleAssistant, Content: strPtr("草稿已生成")}, + {Role: model.RoleUser, Content: strPtr("ok")}, + {Role: model.RoleAssistant, Content: strPtr("已确认并创建告警规则")}, + {Role: model.RoleUser, Content: strPtr("创建 PostgreSQL 告警")}, + makeToolMsg("list_metric_catalog", false), + makeToolMsg("draft_config_change", false), + {Role: model.RoleAssistant, Content: strPtr("草稿已生成")}, + {Role: model.RoleUser, Content: strPtr("ok")}, + {Role: model.RoleAssistant, Content: strPtr("已确认并创建告警规则")}, + {Role: model.RoleUser, Content: strPtr("创建 Redis 告警：缓存命中率偏低时提醒我")}, + } + hints := rt.calcDynamicHints(hist) + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if contains(joined, "已重复调用") || contains(joined, "不要再调用同款工具") { + t.Errorf("new alert request should not inherit old repeat-tool hint: %q", joined) + } + }) + + t.Run("previous_turn_repeated_tool_still_emits_hint", func(t *testing.T) { + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("继续排查日志")}, + makeToolMsg("query_logql", false), + makeToolMsg("query_logql", false), + makeToolMsg("query_logql", false), + {Role: model.RoleAssistant, Content: strPtr("还是没有更多结论")}, + {Role: model.RoleUser, Content: strPtr("继续")}, + } + hints := rt.calcDynamicHints(hist) + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if !contains(joined, "query_logql") || !contains(joined, "已重复调用") { + t.Errorf("missing previous-turn repeat-tool hint: %q", joined) + } + }) + + t.Run("alert_draft_guard_block_then_create_request_emits_draft_retry_hint", func(t *testing.T) { + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("为 MySQL 创建连接数超过 85% 的告警")}, + {Role: model.RoleAssistant, Content: strPtr(callbacks.AlertDraftGuardBlockedMessage)}, + {Role: model.RoleUser, Content: strPtr("为 MySQL 创建一条告警，连接数达到 max_connections 的 85% 且持续 5 分钟触发 Warning")}, + } + hints := rt.calcDynamicHints(hist) + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if !contains(joined, "draft_config_change") || !contains(joined, "不要要求用户重新发送") { + t.Errorf("missing alert draft retry hint: %q", joined) + } + }) + + t.Run("alert_draft_guard_block_then_why_question_no_draft_retry_hint", func(t *testing.T) { + hist := []*model.Message{ + {Role: model.RoleUser, Content: strPtr("为 MySQL 创建连接数超过 85% 的告警")}, + {Role: model.RoleAssistant, Content: strPtr(callbacks.AlertDraftGuardBlockedMessage)}, + {Role: model.RoleUser, Content: strPtr("为什么要这样提示，不能直接创建 draft 吗")}, + } + hints := rt.calcDynamicHints(hist) + joined := "" + for _, h := range hints { + joined += h + "\n" + } + if contains(joined, "draft_config_change") { + t.Errorf("why-question should not trigger draft retry hint: %q", joined) + } + }) +} + +// contains is a tiny test helper so we don't import strings in this +// file just for one substring check. +func contains(s, sub string) bool { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false +} + +// fakeTool is a minimal BaseTool used by TestRuntime_ToolCountAndNames. +type fakeTool struct { + name string + schema string +} + +func (f *fakeTool) Info(_ context.Context) (*basetool.ToolInfo, error) { + return &basetool.ToolInfo{ + Name: f.name, + Description: "fake", + Parameters: []byte(f.schema), + Class: "read", + }, nil +} + +func (f *fakeTool) InvokableRun(_ context.Context, _ string, _ ...basetool.InvokeOption) (string, error) { + return `{"ok":true}`, nil +} + +type captureApplyTool struct { + resp string + calls atomic.Int32 + args atomic.Value +} + +func (t *captureApplyTool) Info(_ context.Context) (*basetool.ToolInfo, error) { + return &basetool.ToolInfo{ + Name: applyConfigChangeToolName, + Description: "fake apply", + Parameters: []byte(`{"type":"object","properties":{}}`), + Class: "write", + }, nil +} + +func (t *captureApplyTool) InvokableRun(_ context.Context, argsJSON string, _ ...basetool.InvokeOption) (string, error) { + t.calls.Add(1) + t.args.Store(argsJSON) + return t.resp, nil +} + +// TestBuildEinoHistory_DropsOrphanToolMessage reproduces the .91 session +// eff73a55 corruption: parallel tools (get_host_processes, query_promql) +// completed out of issue-order, so query_promql's real response was +// persisted under the synthetic id "query_promql|einoToolAdapter" while +// an autoheal stub filled the real call_B slot. The assistant turn then +// survives the completeness precheck (both real ids have a row), but the +// orphan synthetic-id row used to be emitted bare in natural order → +// provider 400 "Messages with role 'tool' must be a response to a +// preceding message with 'tool_calls'". buildEinoHistory must drop it. +func TestBuildEinoHistory_DropsOrphanToolMessage(t *testing.T) { + callA, callB := "call_00_aaa", "call_01_bbb" + asst := &model.Message{ + ID: "asst-1", + Role: model.RoleAssistant, + Content: strPtr("checking host + metrics"), + ToolCalls: []model.ToolCall{ + {ToolName: "get_host_processes", LLMCallID: strPtr(callA), ArgumentsJSON: "{}"}, + {ToolName: "query_promql", LLMCallID: strPtr(callB), ArgumentsJSON: "{}"}, + }, + } + rows := []*model.Message{ + {ID: "u0", Role: model.RoleUser, Content: strPtr("load + cpu?")}, + asst, + // real get_host_processes response (correct id) + {ID: "t-a", Role: model.RoleTool, ToolCallID: strPtr(callA), ToolName: strPtr("get_host_processes"), Content: strPtr(`{"procs":[]}`)}, + // ORPHAN: query_promql's real response stamped with the synthetic + // adapter id after the out-of-order completion. + {ID: "t-orphan", Role: model.RoleTool, ToolCallID: strPtr("query_promql|einoToolAdapter"), ToolName: strPtr("query_promql"), Content: strPtr(`{"resultType":"matrix"}`)}, + // autoheal stub filling the real call_B slot. + {ID: "t-b", Role: model.RoleTool, ToolCallID: strPtr(callB), ToolName: strPtr("query_promql"), Content: strPtr(`{"error":"tool response was not persisted","autoheal":true}`)}, + {ID: "u1", Role: model.RoleUser, Content: strPtr("1+2")}, + } + + out := buildEinoHistory(rows) + + // Every tool message must carry an id that the assistant actually + // emitted — no orphan synthetic-id row survives. + valid := map[string]bool{callA: true, callB: true} + toolCount := 0 + for k, msg := range out { + if msg.Role != schema.RoleType(model.RoleTool) { + continue + } + toolCount++ + if !valid[msg.ToolCallID] { + t.Errorf("orphan tool message survived: id=%q at %d", msg.ToolCallID, k) + } + // A tool message must be preceded (somewhere before) by an + // assistant carrying a matching tool_call id. + if k == 0 || out[k-1].Role == schema.RoleType(model.RoleUser) { + t.Errorf("tool message at %d not preceded by an assistant/tool", k) + } + } + if toolCount != 2 { + t.Errorf("emitted %d tool messages, want 2 (callA + callB stub)", toolCount) + } + // The assistant slot must be present with both tool_calls. + var sawAsst bool + for _, msg := range out { + if msg.Role == schema.RoleType(model.RoleAssistant) && len(msg.ToolCalls) == 2 { + sawAsst = true + } + } + if !sawAsst { + t.Error("assistant turn with 2 tool_calls missing from replay") + } +} + +func TestBuildEinoHistory_SanitizesExpiredToolBudgetResult(t *testing.T) { + callID := "call_budget_1" + rows := []*model.Message{ + {ID: "u0", Role: model.RoleUser, Content: strPtr("创建数据库连接告警")}, + { + ID: "asst-1", + Role: model.RoleAssistant, + Content: strPtr("checking metric catalog"), + ToolCalls: []model.ToolCall{ + {ToolName: "list_metric_catalog", LLMCallID: strPtr(callID), ArgumentsJSON: `{"query":"mysql connection usage"}`}, + }, + }, + { + ID: "t-budget", + Role: model.RoleTool, + ToolCallID: strPtr(callID), + ToolName: strPtr("list_metric_catalog"), + Content: strPtr(`{"status":"call_budget_exceeded","tool":"list_metric_catalog","calls":1,"instruction":"You have already called \"list_metric_catalog\" 1 times this turn. Do NOT call it again."}`), + }, + {ID: "u1", Role: model.RoleUser, Content: strPtr("继续创建")}, + } + + out := buildEinoHistory(rows) + + var toolMsg *schema.Message + for _, msg := range out { + if msg.Role == schema.RoleType(model.RoleTool) { + toolMsg = msg + break + } + } + if toolMsg == nil { + t.Fatalf("expected replayed tool message") + } + if toolMsg.ToolCallID != callID { + t.Fatalf("tool_call_id = %q, want %q", toolMsg.ToolCallID, callID) + } + if contains(toolMsg.Content, "Do NOT call it again") { + t.Fatalf("stale current-turn directive survived replay: %q", toolMsg.Content) + } + if !contains(toolMsg.Content, `"scope":"expired_previous_turn"`) { + t.Fatalf("expired scope missing from replay content: %q", toolMsg.Content) + } + if !contains(toolMsg.Content, "may be called again in the current user turn") { + t.Fatalf("replay content should tell the model the budget expired: %q", toolMsg.Content) + } +} diff --git a/internal/manager/biz/aiops/chatruntime/runtime_worker_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_worker_test.go similarity index 99% rename from internal/manager/biz/aiops/chatruntime/runtime_worker_test.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_worker_test.go index c26286dea..b36b64ad9 100644 --- a/internal/manager/biz/aiops/chatruntime/runtime_worker_test.go +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_worker_test.go @@ -10,7 +10,7 @@ import ( "github.com/cloudwego/eino/schema" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" model "github.com/ongridio/ongrid/internal/manager/model/aiops" ) diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_parser.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_parser.go new file mode 100644 index 000000000..0aa1f6989 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_parser.go @@ -0,0 +1,248 @@ +package chatruntime + +import ( + "bufio" + "bytes" + "fmt" + "os" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +// frontmatterDelimiter is the YAML frontmatter fence used by SKILL.md / +// agents/*.md and every other Markdown frontmatter dialect. +const frontmatterDelimiter = "---" + +// skillNameRe is the snake_case validator. says the name is +// lowercase + digits + underscore; this matches what openclaw enforces too. +var skillNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9_]*$`) + +// nonSnakeRe matches characters that should be normalized to underscore +// in a non-conforming name (dashes, dots, spaces). +var nonSnakeRe = regexp.MustCompile(`[^a-z0-9_]+`) + +// h1Re matches the leading `#`-style markdown H1 header. +var h1Re = regexp.MustCompile(`^#\s+.+\s*$`) + +// ParseSkillMd reads a SKILL.md file at path and returns the parsed +// Skill plus any non-fatal warnings. Required fields +// are name + description; missing them produces an error rather than a +// warning (a SKILL.md without a description is unusable). +// +// Behavioral notes: +// - Frontmatter is preserved into Skill.UnknownFields for any keys we +// don't model — explicitly requires this so upstream +// schema additions don't break ongrid loading. +// - The body H1 is auto-normalized to `[能力: ]` +// - name is snake_case validated; non-conforming names are normalized +// and a warning with code "name_normalized" is emitted. +// +// The returned Skill.Dir is the directory containing path. PromptBody is +// the markdown body verbatim (after H1 normalization). Tag-less fields +// (Provenance, Dir) stay zero-valued — the install path fills them. +func ParseSkillMd(path string) (*Skill, []LoadWarning, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: read %s: %w", path, err) + } + frontmatter, body, err := splitFrontmatter(raw) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: %s: %w", path, err) + } + + var warnings []LoadWarning + + // Decode into the typed struct first, then again into a map so we + // can compute "unknown" keys for forward compatibility. + var sk Skill + if len(frontmatter) > 0 { + if err := yaml.Unmarshal(frontmatter, &sk); err != nil { + return nil, nil, fmt.Errorf("chatruntime: parse %s frontmatter: %w", path, err) + } + } + + rawMap := map[string]any{} + if len(frontmatter) > 0 { + if err := yaml.Unmarshal(frontmatter, &rawMap); err != nil { + return nil, nil, fmt.Errorf("chatruntime: parse %s frontmatter (raw): %w", path, err) + } + } + sk.UnknownFields = retainUnknownSkillFields(rawMap) + + // Required fields — + if strings.TrimSpace(sk.Name) == "" { + return nil, warnings, fmt.Errorf("chatruntime: %s: frontmatter missing required field 'name'", path) + } + if strings.TrimSpace(sk.Description) == "" { + return nil, warnings, fmt.Errorf("chatruntime: %s: frontmatter missing required field 'description'", path) + } + + // Snake-case enforcement with auto-normalization. + if !skillNameRe.MatchString(sk.Name) { + original := sk.Name + sk.Name = normalizeSnakeName(sk.Name) + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "name_normalized", + Reason: fmt.Sprintf("name %q is not snake_case; normalized to %q", original, sk.Name), + }) + } + + // Body: normalize the leading H1 to [能力: ] + sk.PromptBody = normalizeSkillBodyH1(body, sk.Name) + + // Reconcile metadata.ongrid.activation vs top-level activation. + // — both are accepted; ongrid one wins on conflict, with + // a warning so the author knows. + if sk.Metadata.Ongrid.Activation.Mode != "" { + if sk.Activation.Mode != "" && sk.Metadata.Ongrid.Activation.Mode != sk.Activation.Mode { + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "activation_conflict", + Reason: "both top-level activation and metadata.ongrid.activation are set; using metadata.ongrid.activation", + }) + } + sk.Activation = sk.Metadata.Ongrid.Activation + } + + return &sk, warnings, nil +} + +// splitFrontmatter splits a SKILL.md / agent .md byte slice into +// (frontmatterYAML, bodyMarkdown). When no frontmatter is present (no +// leading `---` line) it returns frontmatter=nil and body=full input. +// +// A malformed frontmatter (opening `---` but no closing `---`) is an error. +func splitFrontmatter(raw []byte) ([]byte, []byte, error) { + scanner := bufio.NewScanner(bytes.NewReader(raw)) + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) + + // Peek the first non-empty line. If it isn't `---`, treat the whole + // thing as body (no frontmatter). + var firstLine string + var leading []string + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "" { + leading = append(leading, line) + continue + } + firstLine = line + break + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("scan: %w", err) + } + if strings.TrimRight(firstLine, " \t") != frontmatterDelimiter { + // No frontmatter — return the entire raw as body. + return nil, raw, nil + } + + var fmLines []string + closed := false + for scanner.Scan() { + line := scanner.Text() + if strings.TrimRight(line, " \t") == frontmatterDelimiter { + closed = true + break + } + fmLines = append(fmLines, line) + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("scan: %w", err) + } + if !closed { + return nil, nil, fmt.Errorf("frontmatter started with --- but never closed") + } + + var bodyLines []string + for scanner.Scan() { + bodyLines = append(bodyLines, scanner.Text()) + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("scan: %w", err) + } + + frontmatter := []byte(strings.Join(fmLines, "\n")) + body := []byte(strings.Join(bodyLines, "\n")) + _ = leading + return frontmatter, body, nil +} + +// normalizeSkillBodyH1 rewrites the leading H1 (`# Title`) to +// `[能力: ]` If there's no H1, prepends one. +func normalizeSkillBodyH1(body []byte, name string) string { + const tagPrefix = "[能力: " + header := tagPrefix + name + "]" + + text := string(body) + if strings.TrimSpace(text) == "" { + return header + } + + lines := strings.Split(text, "\n") + // Skip leading blanks to find the first content line; we drop those + // leading blanks so the canonical header is always the first byte. + idx := 0 + for idx < len(lines) && strings.TrimSpace(lines[idx]) == "" { + idx++ + } + if idx >= len(lines) { + return header + } + lines = lines[idx:] + first := lines[0] + if h1Re.MatchString(strings.TrimSpace(first)) { + // Replace the H1 with the canonical tag. + lines[0] = header + } else if strings.HasPrefix(strings.TrimSpace(first), tagPrefix) { + // Already normalized — leave as is. + } else { + // Prepend the canonical tag as a new first line. + newLines := []string{header, ""} + newLines = append(newLines, lines...) + lines = newLines + } + return strings.TrimRight(strings.Join(lines, "\n"), "\n") +} + +// normalizeSnakeName converts dashes / dots / spaces / etc. to underscores +// and lowercases the result. Used when the author wrote `hello-world` etc. +func normalizeSnakeName(in string) string { + out := strings.ToLower(strings.TrimSpace(in)) + out = nonSnakeRe.ReplaceAllString(out, "_") + out = strings.Trim(out, "_") + if out == "" { + out = "unnamed" + } + return out +} + +// retainUnknownSkillFields returns a map of frontmatter keys that we +// don't model on the Skill struct. wants these preserved so +// upstream openclaw / claude-code schema additions don't break loading. +func retainUnknownSkillFields(raw map[string]any) map[string]any { + known := map[string]struct{}{ + "name": {}, + "version": {}, + "description": {}, + "when_to_use": {}, + "activation": {}, + "config_section": {}, + "tools": {}, + "metadata": {}, + } + out := map[string]any{} + for k, v := range raw { + if _, ok := known[k]; ok { + continue + } + out[k] = v + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_parser_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_parser_test.go new file mode 100644 index 000000000..abe27da3b --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_parser_test.go @@ -0,0 +1,230 @@ +package chatruntime + +import ( + "path/filepath" + "strings" + "testing" +) + +// fixtureSkill returns the absolute path to a SKILL.md fixture under +// testdata/skill_parser//SKILL.md. +func fixtureSkill(scenario string) string { + return filepath.Join("testdata", "skill_parser", scenario, "SKILL.md") +} + +func TestParseSkillMd_Minimal(t *testing.T) { + sk, warns, err := ParseSkillMd(fixtureSkill("minimal")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if len(warns) != 0 { + t.Fatalf("expected no warnings, got %+v", warns) + } + if sk.Name != "hello_world" { + t.Errorf("Name = %q, want hello_world", sk.Name) + } + if sk.Description == "" { + t.Error("Description is empty") + } + // Body's H1 should be normalized to [能力: hello_world]. + if !strings.HasPrefix(sk.PromptBody, "[能力: hello_world]") { + t.Errorf("PromptBody = %q, want H1 normalized", sk.PromptBody) + } + if !strings.Contains(sk.PromptBody, "Hello") { + t.Errorf("PromptBody lost the body content: %q", sk.PromptBody) + } +} + +func TestParseSkillMd_Typical(t *testing.T) { + sk, warns, err := ParseSkillMd(fixtureSkill("typical")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if len(warns) != 0 { + t.Fatalf("expected no warnings, got %+v", warns) + } + if sk.Version != "1.0.0" { + t.Errorf("Version = %q, want 1.0.0", sk.Version) + } + wantOS := []string{"darwin", "linux"} + if got := sk.Metadata.OS; !strSliceEq(got, wantOS) { + t.Errorf("Metadata.OS = %v, want %v", got, wantOS) + } + if got := sk.Metadata.Requires.Bins; !strSliceEq(got, []string{"find", "du"}) { + t.Errorf("Metadata.Requires.Bins = %v", got) + } + if got := sk.Metadata.Requires.Config; !strSliceEq(got, []string{"accountsPath"}) { + t.Errorf("Metadata.Requires.Config = %v", got) + } + if len(sk.Tools) != 2 { + t.Fatalf("Tools len = %d, want 2", len(sk.Tools)) + } + if sk.Tools[0].Name != "find_files" || sk.Tools[0].Class != ClassRead { + t.Errorf("Tools[0] = %+v", sk.Tools[0]) + } + if sk.Tools[0].WhenToUse == "" { + t.Error("Tools[0].WhenToUse is empty — snake_case decode failed") + } +} + +func TestParseSkillMd_WithWhenToUse(t *testing.T) { + sk, _, err := ParseSkillMd(fixtureSkill("with_when_to_use")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if sk.WhenToUse == "" { + t.Fatal("WhenToUse is empty") + } + if !strings.Contains(sk.WhenToUse, "log content") { + t.Errorf("WhenToUse content unexpected: %q", sk.WhenToUse) + } +} + +func TestParseSkillMd_WithOngridExt(t *testing.T) { + sk, warns, err := ParseSkillMd(fixtureSkill("with_ongrid_ext")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if len(warns) != 0 { + t.Fatalf("expected no warnings, got %+v", warns) + } + if sk.Metadata.Ongrid.Scope != "edge" { + t.Errorf("Ongrid.Scope = %q, want edge", sk.Metadata.Ongrid.Scope) + } + if sk.Metadata.Ongrid.EdgeRuntime != "subprocess" { + t.Errorf("Ongrid.EdgeRuntime = %q", sk.Metadata.Ongrid.EdgeRuntime) + } + if len(sk.Metadata.Ongrid.EdgeCapabilities) != 2 { + t.Errorf("EdgeCapabilities len = %d, want 2", len(sk.Metadata.Ongrid.EdgeCapabilities)) + } + if sk.Metadata.Ongrid.MinOngridVersion != ">=0.7.30" { + t.Errorf("MinOngridVersion = %q", sk.Metadata.Ongrid.MinOngridVersion) + } + // metadata.ongrid.activation should win and be reflected in + // top-level Activation per the parser's reconciliation rule. + if sk.Activation.Mode != "keyword" { + t.Errorf("Activation.Mode = %q, want keyword (from ongrid ext)", sk.Activation.Mode) + } + if !strSliceEq(sk.Activation.Keywords, []string{"disk", "磁盘", "du"}) { + t.Errorf("Activation.Keywords = %v", sk.Activation.Keywords) + } +} + +func TestParseSkillMd_WithUnknownFields(t *testing.T) { + sk, _, err := ParseSkillMd(fixtureSkill("with_unknown_fields")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if sk.UnknownFields == nil { + t.Fatal("UnknownFields is nil; expected upstream-only keys preserved") + } + if _, ok := sk.UnknownFields["custom_field_one"]; !ok { + t.Errorf("custom_field_one not preserved; got keys %v", keys(sk.UnknownFields)) + } + if _, ok := sk.UnknownFields["custom_field_two"]; !ok { + t.Errorf("custom_field_two not preserved") + } + if _, ok := sk.UnknownFields["upstream_only_array"]; !ok { + t.Errorf("upstream_only_array not preserved") + } + // Known fields should NOT leak into UnknownFields. + if _, ok := sk.UnknownFields["name"]; ok { + t.Errorf("'name' should not appear in UnknownFields") + } +} + +func TestParseSkillMd_NameWithDash(t *testing.T) { + sk, warns, err := ParseSkillMd(fixtureSkill("name_with_dash")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if sk.Name != "hello_world" { + t.Errorf("Name = %q, want normalized hello_world", sk.Name) + } + found := false + for _, w := range warns { + if w.Code == "name_normalized" { + found = true + break + } + } + if !found { + t.Errorf("expected name_normalized warning; got %+v", warns) + } +} + +func TestParseSkillMd_MissingDescription(t *testing.T) { + _, _, err := ParseSkillMd(fixtureSkill("missing_description")) + if err == nil { + t.Fatal("expected error for missing description, got nil") + } + if !strings.Contains(err.Error(), "description") { + t.Errorf("error should mention 'description'; got %v", err) + } +} + +func TestParseSkillMd_FileMissing(t *testing.T) { + _, _, err := ParseSkillMd(filepath.Join("testdata", "skill_parser", "does_not_exist", "SKILL.md")) + if err == nil { + t.Fatal("expected error for missing file") + } +} + +func TestSplitFrontmatter_NoFrontmatter(t *testing.T) { + body := []byte("# Just a body\n\nNo frontmatter here.\n") + fm, b, err := splitFrontmatter(body) + if err != nil { + t.Fatalf("splitFrontmatter: %v", err) + } + if fm != nil { + t.Errorf("expected nil frontmatter, got %q", fm) + } + if string(b) != string(body) { + t.Errorf("body should be untouched") + } +} + +func TestSplitFrontmatter_MalformedUnclosed(t *testing.T) { + body := []byte("---\nname: x\n\nno closing fence\n") + _, _, err := splitFrontmatter(body) + if err == nil { + t.Fatal("expected error for unclosed frontmatter") + } +} + +func TestNormalizeSkillBodyH1_NoH1Adds(t *testing.T) { + out := normalizeSkillBodyH1([]byte("Just a paragraph.\n"), "abc") + if !strings.HasPrefix(out, "[能力: abc]") { + t.Errorf("expected canonical header prepended; got %q", out) + } +} + +func TestNormalizeSkillBodyH1_AlreadyTagged(t *testing.T) { + body := []byte("[能力: abc]\n\nbody\n") + out := normalizeSkillBodyH1(body, "abc") + if strings.Count(out, "[能力: abc]") != 1 { + t.Errorf("expected single tag header; got %q", out) + } +} + +// --- helpers --- + +func strSliceEq(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func keys(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_registry.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_registry.go new file mode 100644 index 000000000..d7ffc79c9 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_registry.go @@ -0,0 +1,239 @@ +package chatruntime + +import ( + "fmt" + "path/filepath" + "strings" + "sync" +) + +// SkillRegistry holds every Skill discovered under a skills root plus the +// non-fatal warnings accumulated during the walk. Mirrors the shape of +// sealsuite-agent's SkillRegistry but adapted to SKILL.md. +// +// PR-2 scope: load + Resolve. Tool-class filter logic happens here at +// the skill level (we drop tools whose Class isn't allowed by the +// policy); the actual factory binding will land in PR-3. +// +// Concurrency: a sync.RWMutex protects the slices. marketplace +// Install / Uninstall calls Reload while in-flight chats may be calling +// Resolve / All — Reload swaps a freshly-built slice atomically under +// the write lock and returning slices are *value copies* so callers +// already holding a result don't observe mutations. +type SkillRegistry struct { + mu sync.RWMutex + skills []*Skill + warnings []LoadWarning +} + +// NewSkillRegistry returns an empty registry. +func NewSkillRegistry() *SkillRegistry { return &SkillRegistry{} } + +// Load walks skillsRoot looking for SKILL.md files and plugin containers +// and parses each one. Walks recursively; any depth under skillsRoot is OK. +// A non-existent skillsRoot is not an error (a fresh install with no skills +// boots fine). +// +// As of PR-skill-load (simplified loader) Load delegates to +// LoadAll, which also recurses into `.claude-plugin/` and +// `openclaw.plugin.json` containers and pulls their skills + commands +// (agents from packs in the skills root are also kept on the result, but +// they are ignored by this method since SkillRegistry only owns skills — +// callers wanting agents from packs should use LoadAll directly). +// +// Path safety: symlinks are resolved and verified to land back inside +// skillsRoot before parse. Per-file parse errors land as warnings rather +// than aborting; one bad SKILL.md should not take down boot. +func (r *SkillRegistry) Load(skillsRoot string) error { + res, err := LoadAll(LoadAllConfig{SkillsRoot: skillsRoot}) + if err != nil { + return fmt.Errorf("chatruntime: skill registry load: %w", err) + } + r.mu.Lock() + r.skills = append([]*Skill(nil), res.Skills...) + r.warnings = append([]LoadWarning(nil), res.Warnings...) + r.mu.Unlock() + return nil +} + +// Reload replaces the registry contents with a fresh scan of skillsRoot +// (delegating through LoadAll so packs / plugin containers / warnings +// behave identically to the boot-time path). Used by marketplace +// Install/Uninstall to hot-reload without restart. +// +// Atomicity: builds the new slice outside the lock, then swaps under +// r.mu.Lock() in O(1). All previously-returned slices from All() / +// Warnings() / Resolve() are value copies and remain unchanged — an +// in-flight chat that already grabbed a Resolve() result keeps running +// against the snapshot it was given. +// +// Empty skillsRoot wipes the registry (i.e. behaves like Load("") which +// is a no-op, but explicitly: skills slice goes to nil so a subsequent +// All() observes "nothing"). A non-existent skillsRoot is not an error +// — same convention as Load. +// +// Variadic extras lets the marketplace pass (builtinRoot, userRoot) so +// the image-baked built-in skills survive a hot-reload triggered by a +// pack install — a single-root Reload would otherwise drop them. +func (r *SkillRegistry) Reload(skillsRoot string, extras ...string) error { + res, err := LoadAll(LoadAllConfig{SkillsRoot: skillsRoot, ExtraSkillsRoots: extras}) + if err != nil { + return fmt.Errorf("chatruntime: skill registry reload: %w", err) + } + newSkills := append([]*Skill(nil), res.Skills...) + newWarnings := append([]LoadWarning(nil), res.Warnings...) + r.mu.Lock() + r.skills = newSkills + r.warnings = newWarnings + r.mu.Unlock() + return nil +} + +// All returns a copy of every loaded skill (so callers can't mutate +// the registry's internal slice). +func (r *SkillRegistry) All() []*Skill { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]*Skill, len(r.skills)) + copy(out, r.skills) + return out +} + +// Warnings returns a copy of every warning recorded during Load. +func (r *SkillRegistry) Warnings() []LoadWarning { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]LoadWarning, len(r.warnings)) + copy(out, r.warnings) + return out +} + +// Add inserts a programmatically-constructed skill. Used by feature-flag +// gated skills that don't exist as SKILL.md on disk (e.g. a runtime +// proposal skill). +func (r *SkillRegistry) Add(sk *Skill) { + if sk == nil { + return + } + r.mu.Lock() + r.skills = append(r.skills, sk) + r.mu.Unlock() +} + +// AddAll inserts a batch of skills. Nil entries are skipped. Used by +// LoadAll to merge plugin-container output into the registry. +func (r *SkillRegistry) AddAll(skills []*Skill) { + for _, sk := range skills { + r.Add(sk) + } +} + +// AddWarnings appends loader warnings (e.g. from LoadAll) so +// SkillRegistry.Warnings() exposes them alongside per-skill parse +// warnings. +func (r *SkillRegistry) AddWarnings(ws []LoadWarning) { + if len(ws) == 0 { + return + } + r.mu.Lock() + r.warnings = append(r.warnings, ws...) + r.mu.Unlock() +} + +// Resolve filters loaded skills against (query, policy): +// +// 1. Activation filter: mode "always" → include; mode "keyword" → +// include only when query (lowercased) contains any keyword. +// 2. Policy filter: drop tools whose Class is not in policy.AllowedClasses. +// The skill survives as long as at least one tool survives. A skill +// declaring zero tools is always kept after the activation filter +// (it may be a pure-prompt skill). +// +// Returned skills are *copies* with their Tools slice already filtered. +// The original registry skills are not mutated. +func (r *SkillRegistry) Resolve(query string, policy Policy) []*Skill { + queryLower := strings.ToLower(strings.TrimSpace(query)) + r.mu.RLock() + skills := make([]*Skill, len(r.skills)) + copy(skills, r.skills) + r.mu.RUnlock() + var out []*Skill + for _, sk := range skills { + if !activationMatches(sk.Activation, queryLower) { + continue + } + filtered := filterToolsByPolicy(sk.Tools, policy) + if len(sk.Tools) > 0 && len(filtered) == 0 { + // Skill had tools but none survived policy — drop the skill. + continue + } + clone := *sk + clone.Tools = filtered + out = append(out, &clone) + } + return out +} + +// activationMatches reports whether a skill's Activation matches the +// (already lowercased) query. +func activationMatches(a Activation, queryLower string) bool { + mode := strings.ToLower(strings.TrimSpace(a.Mode)) + switch mode { + case "", "always": + return true + case "keyword": + for _, kw := range a.Keywords { + kw = strings.ToLower(strings.TrimSpace(kw)) + if kw == "" { + continue + } + if strings.Contains(queryLower, kw) { + return true + } + } + return false + default: + // Unknown mode — fail open; the parse-time warning is the place + // to surface this instead of silently disabling the skill. + return true + } +} + +// filterToolsByPolicy drops tools whose Class is not allowed. +func filterToolsByPolicy(tools []ToolDecl, policy Policy) []ToolDecl { + if len(tools) == 0 { + return nil + } + out := make([]ToolDecl, 0, len(tools)) + for _, t := range tools { + // Empty class → assume read-only (safe default). + class := t.Class + if class == "" { + class = ClassRead + } + if !policy.Allows(class) { + continue + } + out = append(out, t) + } + return out +} + +// pathHasPrefix returns true when child sits inside parent (or is the +// parent itself). Cleans both paths first so trailing slashes don't +// cause false negatives. Mirrors internal/skill/loader.go for consistency. +func pathHasPrefix(child, parent string) bool { + child = filepath.Clean(child) + parent = filepath.Clean(parent) + if child == parent { + return true + } + rel, err := filepath.Rel(parent, child) + if err != nil { + return false + } + if strings.HasPrefix(rel, "..") { + return false + } + return true +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_registry_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_registry_test.go new file mode 100644 index 000000000..204963494 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/skill_registry_test.go @@ -0,0 +1,166 @@ +package chatruntime + +import ( + "path/filepath" + "testing" +) + +func fixtureSkillRoot(scenario string) string { + return filepath.Join("testdata", "skill_registry", scenario) +} + +func TestSkillRegistry_LoadMulti(t *testing.T) { + r := NewSkillRegistry() + if err := r.Load(fixtureSkillRoot("multi")); err != nil { + t.Fatalf("Load: %v", err) + } + all := r.All() + if len(all) != 3 { + t.Fatalf("All() len = %d, want 3 (always_skill, keyword_skill, buried_skill)", len(all)) + } + names := map[string]bool{} + for _, sk := range all { + names[sk.Name] = true + if sk.Dir == "" { + t.Errorf("skill %q has empty Dir", sk.Name) + } + } + for _, expected := range []string{"always_skill", "keyword_skill", "buried_skill"} { + if !names[expected] { + t.Errorf("skill %q not loaded", expected) + } + } +} + +func TestSkillRegistry_LoadNonExistent(t *testing.T) { + r := NewSkillRegistry() + if err := r.Load(fixtureSkillRoot("does_not_exist")); err != nil { + t.Fatalf("Load on missing dir should be nil-error, got %v", err) + } + if len(r.All()) != 0 { + t.Errorf("expected zero skills") + } +} + +func TestSkillRegistry_BadSkillIsWarning(t *testing.T) { + r := NewSkillRegistry() + if err := r.Load(fixtureSkillRoot("with_bad_skill")); err != nil { + t.Fatalf("Load: %v", err) + } + if len(r.All()) != 1 { + t.Errorf("All() len = %d, want 1 (only 'good' parses)", len(r.All())) + } + hasParseFailed := false + for _, w := range r.Warnings() { + if w.Code == "parse_failed" { + hasParseFailed = true + } + } + if !hasParseFailed { + t.Errorf("expected parse_failed warning for the bad skill; got %+v", r.Warnings()) + } +} + +func TestSkillRegistry_ResolveAlwaysVsKeyword(t *testing.T) { + r := NewSkillRegistry() + if err := r.Load(fixtureSkillRoot("multi")); err != nil { + t.Fatalf("Load: %v", err) + } + policy := Policy{AllowedClasses: []string{"*"}} + + // Without any keyword in the query, only always-mode + buried (default + // always) should resolve. keyword_skill should be excluded. + got := r.Resolve("hello world", policy) + names := skillNames(got) + if hasName(names, "keyword_skill") { + t.Errorf("keyword_skill should NOT resolve without keyword; got %v", names) + } + if !hasName(names, "always_skill") { + t.Errorf("always_skill should resolve; got %v", names) + } + if !hasName(names, "buried_skill") { + t.Errorf("buried_skill (default-always) should resolve; got %v", names) + } + + // With "logs" in the query, keyword_skill should activate. + got = r.Resolve("show me the logs from edge X", policy) + if !hasName(skillNames(got), "keyword_skill") { + t.Errorf("keyword_skill should activate on 'logs'; got %v", skillNames(got)) + } +} + +func TestSkillRegistry_ResolvePolicyDropsTools(t *testing.T) { + r := NewSkillRegistry() + if err := r.Load(fixtureSkillRoot("multi")); err != nil { + t.Fatalf("Load: %v", err) + } + + // Read-only policy on a 'logs' query: keyword_skill activates, but its + // destructive tool (delete_logs) should be filtered out. + policy := Policy{AllowedClasses: []string{"read"}} + got := r.Resolve("logs please", policy) + for _, sk := range got { + if sk.Name != "keyword_skill" { + continue + } + if len(sk.Tools) != 1 { + t.Fatalf("keyword_skill Tools len = %d, want 1 (delete_logs filtered)", len(sk.Tools)) + } + if sk.Tools[0].Name != "query_logs" { + t.Errorf("expected query_logs to survive; got %q", sk.Tools[0].Name) + } + } +} + +func TestSkillRegistry_ResolveDropsSkillIfAllToolsFiltered(t *testing.T) { + r := NewSkillRegistry() + // Load a synthetic skill with only destructive tools; under a + // read-only policy the skill should be dropped entirely. + r.Add(&Skill{ + Name: "destructive_only", + Description: "Has only destructive tools.", + Activation: Activation{Mode: "always"}, + Tools: []ToolDecl{ + {Name: "wipe", Impl: "x", Class: ClassDestructive, Description: "Wipes the disk."}, + }, + }) + policy := Policy{AllowedClasses: []string{"read"}} + got := r.Resolve("hi", policy) + if hasName(skillNames(got), "destructive_only") { + t.Errorf("destructive_only should be dropped; got %v", skillNames(got)) + } +} + +func TestSkillRegistry_ResolvePromptOnlySkillSurvives(t *testing.T) { + r := NewSkillRegistry() + r.Add(&Skill{ + Name: "pure_prompt", + Description: "Prompt-only skill, no tools.", + Activation: Activation{Mode: "always"}, + PromptBody: "guidance text", + }) + policy := Policy{AllowedClasses: []string{"read"}} + got := r.Resolve("anything", policy) + if !hasName(skillNames(got), "pure_prompt") { + t.Errorf("pure_prompt skill should survive policy with no tools; got %v", skillNames(got)) + } +} + +// --- helpers --- + +func skillNames(skills []*Skill) []string { + out := make([]string, 0, len(skills)) + for _, sk := range skills { + out = append(out, sk.Name) + } + return out +} + +func hasName(list []string, name string) bool { + for _, n := range list { + if n == name { + return true + } + } + return false +} diff --git a/internal/manager/biz/aiops/chatruntime/system_prompt.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/system_prompt.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/system_prompt.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/system_prompt.go diff --git a/internal/manager/biz/aiops/chatruntime/system_prompt_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/system_prompt_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/system_prompt_test.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/system_prompt_test.go diff --git a/internal/manager/biz/aiops/chatruntime/testdata/agent_parser/minimal/incident-investigator.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/minimal/incident-investigator.md similarity index 100% rename from internal/manager/biz/aiops/chatruntime/testdata/agent_parser/minimal/incident-investigator.md rename to internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/minimal/incident-investigator.md diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md new file mode 100644 index 000000000..a8a7d82cf --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md @@ -0,0 +1,6 @@ +--- +name: foo +description: Missing when_to_use to test required-field error. +--- + +This file is intentionally missing the when_to_use field. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md new file mode 100644 index 000000000..ac8e53421 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md @@ -0,0 +1,23 @@ +--- +name: auditor +description: Read-only auditor that watches mutating decisions for compliance breaches. +when_to_use: Spawn whenever a mutating proposal lands in audit mode. +tools: + - get_audit_log + - query_promql +permission_mode: read-only +max_turns: 8 +critical_reminder: | + - Do not approve or reject; only flag. + - Quote SOP IDs verbatim; never paraphrase. + - If unsure, say so explicitly. +background: false +omit_claude_md: false +--- + +You are the ongrid compliance auditor agent. + +Workflow: +1. Pull recent audit_log entries for the incident in scope. +2. Cross-check each mutating action against the SOP catalogue. +3. Output a flag list (no approve/reject). diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md new file mode 100644 index 000000000..6126be635 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md @@ -0,0 +1,23 @@ +--- +name: reviewer +description: SOP dual-sign reviewer for mutating proposals. +when_to_use: Spawn only when the user proposes a mutating / destructive operation. +tools: + - get_incident_detail + - query_promql + - query_logql + - get_sop_text +disallowed_tools: + - "*_skill" + - run_shell +permission_mode: read-only +max_turns: 5 +model: anthropic/claude-opus-4-7 +--- + +You are the ongrid high-risk-operation second-review agent. + +After receiving a proposal {action, target, reason, blast_radius}: +1. Verify a relevant SOP exists. +2. Verify target device's current state. +3. Return approve / reject with one-line rationale. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/README.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/README.md new file mode 100644 index 000000000..fe1b6dbad --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/README.md @@ -0,0 +1,4 @@ +# agents/ + +Test fixture root for AgentRegistry.Load. README.md is intentionally +present to verify the loader skips it. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/incident-investigator.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/incident-investigator.md new file mode 100644 index 000000000..3e9cd127e --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/incident-investigator.md @@ -0,0 +1,7 @@ +--- +name: incident_investigator +description: Alert root-cause investigator. +when_to_use: When the user asks about an alert's root cause or blast radius. +--- + +You are the ongrid alert root-cause investigation agent. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/reviewer.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/reviewer.md new file mode 100644 index 000000000..2b5b7571f --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/agent_registry/multi/reviewer.md @@ -0,0 +1,9 @@ +--- +name: reviewer +description: SOP dual-sign reviewer. +when_to_use: Spawn for any mutating proposal that needs HLD-003 dual sign. +permission_mode: read-only +max_turns: 5 +--- + +You are the ongrid SOP reviewer. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json new file mode 100644 index 000000000..35a928395 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json @@ -0,0 +1,13 @@ +{ + "id": "acme-tools", + "name": "Acme Tools", + "version": "1.0.0", + "description": "A bundle of acme tools.", + "configSchema": { + "type": "object", + "properties": { + "endpoint": { "type": "string" } + }, + "required": ["endpoint"] + } +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json new file mode 100644 index 000000000..7c92e55c3 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json @@ -0,0 +1,7 @@ +{ + "id": "acme-extras", + "name": "Acme Extras", + "version": "1.0.0", + "description": "Bundle that ships hooks and an MCP config.", + "configSchema": {} +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json new file mode 100644 index 000000000..da39e4ffa --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json @@ -0,0 +1,3 @@ +{ + "mcpServers": {} +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md new file mode 100644 index 000000000..2f5cd5951 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md @@ -0,0 +1,3 @@ +# pre-prompt hook + +(unused, present only to test that the hooks/ subdir is detected and warned about) diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json new file mode 100644 index 000000000..905660621 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json @@ -0,0 +1,18 @@ +{ + "id": "openclaw-bundle", + "name": "OpenClaw Bundle", + "version": "2.3.4", + "description": "OpenClaw plugin with provider-specific extras.", + "providers": { + "openai": { "baseUrl": "https://api.openai.com/v1" }, + "anthropic": { "baseUrl": "https://api.anthropic.com" } + }, + "channels": ["feishu", "cli"], + "cliBackends": ["claude", "codex"], + "providerAuthChoices": { + "preferred": "cliFlag" + }, + "configSchema": { + "type": "object" + } +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json new file mode 100644 index 000000000..90b0c506a --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json @@ -0,0 +1,6 @@ +{ + "id": "evil", + "name": "Evil Plugin", + "version": "0.0.1", + "description": "A plugin whose skills/ directory contains a symlink that escapes the pack root." +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal/skills/pwned b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal/skills/pwned new file mode 120000 index 000000000..6998f20a0 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal/skills/pwned @@ -0,0 +1 @@ +../../path_traversal_target \ No newline at end of file diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md new file mode 100644 index 000000000..dba27c4eb --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md @@ -0,0 +1,8 @@ +--- +name: pwned +description: I am outside the pack root and should not be loaded. +--- + +# Pwned + +This SKILL.md is outside the path_traversal pack — loader must reject access via symlink. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json new file mode 100644 index 000000000..b4853c2c1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json @@ -0,0 +1,6 @@ +{ + "id": "recursive-test", + "name": "Recursive Test", + "version": "1.0.0", + "description": "Bundle that ships skills, agents, commands, hooks and an MCP config; used to test recursive sub-tree load." +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json new file mode 100644 index 000000000..5392ca626 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "example": { + "command": "node", + "args": ["./mcp-server.js"] + } + } +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md new file mode 100644 index 000000000..0265b69d1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md @@ -0,0 +1,7 @@ +--- +name: pr_summary +description: Summarizes the diff of a pull request. +when_to_use: When the user asks for a summary of the current PR or branch diff. +--- + +You are the PR summary agent. Read the diff, then output a 3-bullet summary. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md new file mode 100644 index 000000000..a38e3afed --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md @@ -0,0 +1,7 @@ +--- +description: Commit the staged changes with a conventional commit message. +allowed-tools: [Bash, Edit] +--- + +Commit the currently staged changes. Generate a conventional commit message +based on the diff. Then run `git status` to show the result. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md new file mode 100644 index 000000000..2f5cd5951 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md @@ -0,0 +1,3 @@ +# pre-prompt hook + +(unused, present only to test that the hooks/ subdir is detected and warned about) diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md new file mode 100644 index 000000000..148703763 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md @@ -0,0 +1,8 @@ +--- +name: hello_world +description: A simple skill that says hello. +--- + +# Hello World + +When the user asks for a greeting, respond with "Hello". diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json new file mode 100644 index 000000000..72b2c845b --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json @@ -0,0 +1,7 @@ +{ + "id": "openclaw-recursive", + "name": "OpenClaw Recursive", + "version": "0.1.0", + "description": "OpenClaw bundle declaring its skills via the manifest skills[] array.", + "skills": ["./skills/foo", "./skills/bar"] +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md new file mode 100644 index 000000000..b820c4115 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md @@ -0,0 +1,8 @@ +--- +name: bar +description: The bar skill. +--- + +# Bar + +Do bar things. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md new file mode 100644 index 000000000..baf6620bd --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md @@ -0,0 +1,8 @@ +--- +name: foo +description: The foo skill. +--- + +# Foo + +Do foo things. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/minimal/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/minimal/SKILL.md new file mode 100644 index 000000000..148703763 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/minimal/SKILL.md @@ -0,0 +1,8 @@ +--- +name: hello_world +description: A simple skill that says hello. +--- + +# Hello World + +When the user asks for a greeting, respond with "Hello". diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/missing_description/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/missing_description/SKILL.md new file mode 100644 index 000000000..a517a9e42 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/missing_description/SKILL.md @@ -0,0 +1,7 @@ +--- +name: nope +--- + +# No Description + +Frontmatter is missing description; this should be a parse error. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md new file mode 100644 index 000000000..7b9cb7311 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md @@ -0,0 +1,9 @@ +--- +name: hello-world +description: Skill whose name uses a dash, expected to be normalized. +--- + +# Hello World Dashed + +This skill name is `hello-world` which is not snake_case. The parser +should normalize it to `hello_world` and emit a warning. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/typical/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/typical/SKILL.md new file mode 100644 index 000000000..60ce5c842 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/typical/SKILL.md @@ -0,0 +1,29 @@ +--- +name: file_inspector +version: 1.0.0 +description: Inspect files on the host filesystem. +metadata: + os: ["darwin", "linux"] + requires: + bins: [find, du] + config: [accountsPath] +tools: + - name: find_files + impl: builtin:filetools.Find + class: read + description: Search files by name pattern. + when_to_use: When the user asks to find files. + - name: du_summary + impl: builtin:filetools.DuSummary + class: read + description: Disk usage summary. +--- + +# File Inspector + +Use this skill to find files on disk and report disk usage. + +## Workflow + +1. Use `find_files` to locate candidate files. +2. Use `du_summary` to size up the matches. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md new file mode 100644 index 000000000..b5143a6a1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md @@ -0,0 +1,27 @@ +--- +name: edge_disk_audit +description: Audit disk usage on edge devices. +metadata: + os: [linux] + requires: + bins: [find, du] + ongrid: + scope: edge + edge_runtime: subprocess + edge_capabilities: + - filesystem.read.path: ["/data", "/var/log"] + - process.exec: [find, du] + activation: + mode: keyword + keywords: [disk, 磁盘, du] + min_ongrid_version: ">=0.7.30" +tools: + - name: edge_du + impl: edge:disk.DuSummary + class: read + description: Disk usage on edge. +--- + +# Edge Disk Audit + +Inspect disk consumption on connected edges via the plugin runtime. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md new file mode 100644 index 000000000..c428a8e0c --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md @@ -0,0 +1,16 @@ +--- +name: future_skill +description: A skill that uses fields ongrid does not yet recognize. +custom_field_one: hello +custom_field_two: + nested: + value: 42 +upstream_only_array: + - a + - b +--- + +# Future Skill + +Newer SKILL.md spec adds fields ongrid hasn't modeled yet — they should +survive into UnknownFields. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md new file mode 100644 index 000000000..e935d3aaa --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md @@ -0,0 +1,16 @@ +--- +name: log_search +description: Search Loki logs. +when_to_use: | + When the user asks about log content, error patterns, or log volume per device. + NOT for: file-system search (use file_inspector), metric queries (use metric_query). +tools: + - name: query_logql + impl: builtin:loki.Query + class: read + description: Run LogQL. +--- + +# Log Search + +Use LogQL to search distributed logs. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md new file mode 100644 index 000000000..1b5ea8da6 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md @@ -0,0 +1,15 @@ +--- +name: always_skill +description: Always active. +activation: + mode: always +tools: + - name: ping + impl: builtin:ping.Ping + class: read + description: Ping a host. +--- + +# Always Skill + +Always loaded into the toolBag. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md new file mode 100644 index 000000000..820233432 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md @@ -0,0 +1,20 @@ +--- +name: keyword_skill +description: Activated by the word "logs". +activation: + mode: keyword + keywords: [logs, 日志] +tools: + - name: query_logs + impl: builtin:loki.Query + class: read + description: Search logs. + - name: delete_logs + impl: builtin:loki.Delete + class: destructive + description: Wipe a log range. +--- + +# Keyword Skill + +Only loaded when the user mentions logs / 日志. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md new file mode 100644 index 000000000..a1f04f719 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md @@ -0,0 +1,8 @@ +--- +name: buried_skill +description: A skill nested several directories deep, to verify recursive walk. +--- + +# Buried Skill + +Recursive walk should still find this. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md new file mode 100644 index 000000000..e038d9d44 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md @@ -0,0 +1,7 @@ +--- +name: bad +--- + +# No Description + +Missing description — should produce a parse_failed warning, not crash the walk. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md new file mode 100644 index 000000000..9412ba7d7 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md @@ -0,0 +1,8 @@ +--- +name: good +description: A well-formed skill. +--- + +# Good + +This one parses fine. diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/tracing.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/tracing.go new file mode 100644 index 000000000..372bcce7d --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/tracing.go @@ -0,0 +1,31 @@ +package chatruntime + +import ( + "context" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/tracing" +) + +func startAgentSpan(ctx context.Context, session *aiopsmodel.Session, name string) (context.Context, trace.Span) { + conversationID := session.ID + if session.RootSessionID != nil && *session.RootSessionID != "" { + conversationID = *session.RootSessionID + } + if session.AgentID != nil && *session.AgentID != "" { + name = *session.AgentID + } + return otel.Tracer("github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/chatruntime").Start(ctx, "invoke_agent "+name, + trace.WithAttributes( + attribute.String("gen_ai.operation.name", "invoke_agent"), + attribute.String("gen_ai.agent.name", name), + attribute.String("gen_ai.conversation.id", conversationID), + attribute.String("ongrid.agent.session.id", session.ID), + attribute.Bool("ongrid.agent.content.capture", tracing.AgentContentEnabled()), + ), + ) +} diff --git a/internal/manager/biz/aiops/chatruntime/tracing_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/tracing_test.go similarity index 98% rename from internal/manager/biz/aiops/chatruntime/tracing_test.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/tracing_test.go index a3dcb67f6..f2c0b5a05 100644 --- a/internal/manager/biz/aiops/chatruntime/tracing_test.go +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/tracing_test.go @@ -25,8 +25,8 @@ import ( "go.opentelemetry.io/otel/sdk/trace/tracetest" "github.com/ongridio/ongrid/internal/manager/biz/agenttrace" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph/callbacks" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph/callbacks" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" "github.com/ongridio/ongrid/internal/pkg/llm" diff --git a/internal/manager/biz/aiops/chatruntime/types.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/types.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/types.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/types.go diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/worker.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/worker.go new file mode 100644 index 000000000..5c7f527ad --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/worker.go @@ -0,0 +1,1080 @@ +// worker.go implements — the coordinator/worker multi-agent +// path. The Runtime owns a small in-memory map of spawned workers; each +// worker is an independent graph.Invoke against a tool-bag filtered by +// the worker agent's persona (whitelist + disallowed_tools — black wins). +// +// Each worker owns an internal work session. The root session remains owned +// by the default assistant; parent_session_id records only the immediate +// collaboration edge, while root_session_id preserves the user task chain. +// Worker-to-worker delegation is introduced only through an explicit budgeted +// policy at Resolve time, never by giving every worker an unrestricted spawn +// tool. +// +// State machine (figure): +// +// pending — set transiently between SpawnRequest accept and goroutine +// start (effectively never observable for sync spawns) +// ↓ +// running — goroutine has begun graph.Invoke; status flips here +// BEFORE the child graph emits its first event +// ↓ +// completed — graph.Invoke returned a non-nil AssistantMessage with +// no error; Result holds the final assistant content +// failed — graph.Invoke returned err != nil OR the agent name was +// unknown / agent registry was nil +// killed — StopWorker called while status was running; cancel() +// fires and the goroutine observes ctx.Done() then sets +// EndedAt + status=killed (without overwriting err) +// +// Background semantics — when SpawnRequest.Background is true SpawnWorker +// returns immediately with status=running and the graph.Invoke runs in a +// detached goroutine; the goroutine emits a "task_notification" SSE +// envelope back through req.ParentEmit upon terminal status. Callers +// (AgentTool) get the worker_id from the synchronous return and can +// answer the LLM with `{task_id, status: "pending"}` while the SPA +// renders a live tile that gets updated when the notification fires. +package chatruntime + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + einocallbacks "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/compose" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph/callbacks" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/llm" + "github.com/ongridio/ongrid/internal/pkg/tracing" +) + +// WorkerStatus enumerates the worker state machine. See package header +// for the transition diagram. +type WorkerStatus string + +const ( + // WorkerStatusPending — accepted but not yet started. Observable + // only in tests that intercept between SpawnWorker accept and goroutine + // scheduling; in production this state is essentially transient. + WorkerStatusPending WorkerStatus = "pending" + // WorkerStatusRunning — graph.Invoke is in flight. + WorkerStatusRunning WorkerStatus = "running" + // WorkerStatusCompleted — graph.Invoke returned a final assistant + // message with no error. + WorkerStatusCompleted WorkerStatus = "completed" + // WorkerStatusFailed — graph.Invoke returned an error, or the agent + // name was unknown. + WorkerStatusFailed WorkerStatus = "failed" + // WorkerStatusKilled — explicitly stopped via StopWorker. + WorkerStatusKilled WorkerStatus = "killed" +) + +// Worker tracks a spawned sub-agent. Unique by ID within a Runtime +// instance. +type Worker struct { + // ID is "agent-<8 hex>" — unique per Runtime, stable for the + // worker's lifetime. + ID string + // AgentName is the persona name (frontmatter `name` field). + AgentName string + // SessionID is the worker's own session id (distinct from the + // coordinator's session). For PR-1 of this is a synthetic + // in-memory id; the chat_sessions row + parent_session_id column + // migration is follow-up. + SessionID string + // ParentSessionID is the coordinator session that requested the + // spawn. Tracked here so a future audit query can rebuild the parent + // → worker tree without a schema change. + ParentSessionID string + // Prompt is the initial user message handed to the worker. + Prompt string + // Status is the lifecycle state. Read-only outside the runtime; + // callers should always pull through GetWorker for a snapshot. + Status WorkerStatus + // Background indicates whether this is a fire-and-forget spawn. + Background bool + // StartedAt is the wall time the worker entered "running". + StartedAt time.Time + // EndedAt is the wall time of any terminal transition. nil while + // running. + EndedAt *time.Time + // Result is the worker's final assistant content (when terminal = + // completed). Empty for failed / killed. + Result string + // Err is the failure reason (when terminal = failed / killed). + Err string + + // Internal fields — not surfaced to callers via copies. + cancel context.CancelFunc + mu sync.Mutex +} + +// SpawnRequest describes a worker to spawn. See AgentTool.InvokableRun +// for the LLM-facing argument shape that produces this struct. +type SpawnRequest struct { + // AgentName is the persona name (frontmatter `name`). Must be + // resolvable through the Runtime's AgentRegistry. + AgentName string + // Prompt is the initial user message for the worker. + Prompt string + // Background = true: fire-and-forget; SpawnWorker returns immediately + // with status=running. Background = false: SpawnWorker blocks until + // the worker reaches a terminal state. + Background bool + // ParentSession is the coordinator's chat session id. + ParentSession string + // ParentEmit is the SSE emitter the coordinator runtime threads + // through. Used to deliver the task_notification frame when the + // worker (background=true) finishes. nil = no notification (sync + // callers don't need it). + ParentEmit Emit + // SessionKind overrides the persisted chat_sessions.kind for the + // worker session. Empty = "work". The investigator usecase + // sets "investigation" so auto-spawned RCA transcripts stay out of + // the /chat list. + SessionKind string + // OwnerUserID overrides the persisted chat_sessions.user_id when + // the worker has no ParentSession to inherit from. The investigator + // usecase passes 0 to mark these rows as "system-owned" — they + // don't belong to any operator and the internal audience prevents + // them from appearing in the primary chat list. + OwnerUserID uint64 + // Locale is the UI language ("en", "zh-CN", ...) the worker's reply + // should use. Threaded from the parent coordinator's request so a + // sub-agent answers in the same language the user is typing in + // (otherwise GLM defaults to zh on English questions). Empty = no + // directive, back-compat with investigator/auto-spawned workers. + Locale string + // Provider + Model are the coordinator's resolved LLM choice. Threaded + // into runWorker's g.Invoke as chatModelOpts so the sub-agent runs + // against the same LLM the user picked for the coordinator — without + // this, runWorker falls through to the RoutingChatModel default and + // installs without an OpenAI key see specialists fail with + // `provider "openai" not configured`. Empty = no override; the worker + // uses whatever the routing default is. + Provider string + Model string +} + +// TaskNotification is the payload of a task_notification streaming +// frame. Mirrors the schema laid out in — coordinator's +// SSE listener fans this back through the legacy SSE envelope so the +// SPA only needs to learn one new event type. +type TaskNotification struct { + TaskID string `json:"task_id"` + Status WorkerStatus `json:"status"` + Summary string `json:"summary"` + Result string `json:"result,omitempty"` + Err string `json:"error,omitempty"` + Usage map[string]any `json:"usage,omitempty"` +} + +// EventTaskNotification is the new streaming event type the runtime +// emits when a background worker reaches a terminal state. The SPA +// listens for this in addition to the existing assistant / tool / +// done frames. +const EventTaskNotification EventType = "task_notification" + +// EventApprovalPending fires when a synchronous-blocking tool (HLD-021, +// e.g. cloud_bash) has queued a human-approval proposal and is about to +// block waiting for the decision. The frame surfaces the inline approve/ +// reject card LIVE — the tool no longer returns a pending_approval result +// blob (it blocks, then returns the real executor result), so the card +// must be driven by this frame instead. ToolCallID ties the card to the +// tool call's existing streaming card so the UI shows a single card. +const EventApprovalPending EventType = "approval_pending" + +// ApprovalPending is the EventApprovalPending payload. Emitted by the +// proposer shim (cmd/main.go) via EmitFromContext while a blocking tool +// awaits the human decision. +type ApprovalPending struct { + ApprovalID string + ToolCallID string + Kind string + ToolName string + Command string + Credentials []string +} + +// emitCtxKey is the unexported context key that threads the active +// per-request Emit through the graph layer down into AgentTool's +// InvokableRun via the WorkerSpawner shim. The shim reads it through +// EmitFromContext to populate SpawnRequest.ParentEmit so a background +// worker can fire its task_notification through the SSE channel that +// owns the user's chat session. +type emitCtxKeyT struct{} + +var emitCtxKey = emitCtxKeyT{} + +// withEmit returns ctx augmented with emit. Internal — Handle threads +// the request emitter in before invoking the graph. +func withEmit(ctx context.Context, emit Emit) context.Context { + if emit == nil { + return ctx + } + return context.WithValue(ctx, emitCtxKey, emit) +} + +// EmitFromContext retrieves the active streaming Emit from ctx, if any. +// Returns nil when no emitter was attached (e.g. blocking call without +// SSE). The wiring shim at the cmd/main.go boundary uses this to thread +// task_notification frames through the same SSE channel that owns the +// coordinator's chat session. +func EmitFromContext(ctx context.Context) Emit { + v, _ := ctx.Value(emitCtxKey).(Emit) + return v +} + +// Locale ctx propagation lives in the tools package (next to AgentTool, +// the consumer). Runtime calls tools.WithLocale before g.Invoke; the +// tool reads it via tools.LocaleFromContext inside InvokableRun. + +// SpawnWorker starts a worker. See package header for state machine. +// +// Background=false: blocks until the worker reaches completed / failed / +// killed; the returned *Worker has Result + Err filled. +// +// Background=true: returns immediately with a Worker whose Status = +// running; the graph runs in a detached goroutine and the terminal +// transition emits a task_notification through req.ParentEmit. +func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, error) { + if rt == nil { + return nil, errors.New("chatruntime: nil runtime") + } + if rt.cfg.ChatModel == nil { + return nil, errors.New("chatruntime: ChatModel is required") + } + if rt.cfg.AgentRegistry == nil { + return nil, fmt.Errorf("chatruntime: agent %q: registry not wired", req.AgentName) + } + agentDef, ok := rt.cfg.AgentRegistry.ByName(req.AgentName) + if !ok || agentDef == nil { + return nil, fmt.Errorf("chatruntime: agent %q not found", req.AgentName) + } + + id := newWorkerID() + sessID := newWorkerSessionID(id) + + // Persist the worker session row up-front so audit + worker-tree + // queries can resolve the (parent → worker) relationship without + // waiting for the worker to finish. We need the parent's UserID for + // row-level scoping; missing parent → worker session is orphan-able + // (still persisted, but parent_session_id=nil) so the LLM-driven + // flow remains usable in tests / sync calls without a parent row. + var sourceScope *aiopsmodel.APMSourceScope + var ownerUserID uint64 + var parentRefForRow *string + var rootRefForRow *string + var ownerAgentRefForRow *string + if req.ParentSession != "" && rt.cfg.Sessions != nil { + parentSess, err := rt.cfg.Sessions.GetSession(ctx, req.ParentSession) + if err == nil && parentSess != nil { + if parentSess.ParentSessionID != nil && *parentSess.ParentSessionID != "" { + return nil, fmt.Errorf("chatruntime: delegation depth exceeded for parent session %q", req.ParentSession) + } + ownerUserID = parentSess.UserID + sourceScope = parentSess.APMSource + pid := req.ParentSession + parentRefForRow = &pid + rootID := parentSess.ID + if parentSess.RootSessionID != nil && *parentSess.RootSessionID != "" { + rootID = *parentSess.RootSessionID + } + rootRefForRow = &rootID + if parentSess.OwnerAgentID != nil && *parentSess.OwnerAgentID != "" { + owner := *parentSess.OwnerAgentID + ownerAgentRefForRow = &owner + } + } + // On lookup failure we proceed without a parent ref so the + // worker can still run; the audit row simply has parent_session_id + // = nil. Logged at info — not fatal. + if err != nil && rt.log != nil { + rt.log.Info("chatruntime: worker spawn — parent session lookup failed", + "parent_session_id", req.ParentSession, + "err", err.Error()) + } + } + if rt.cfg.Sessions != nil { + agentName := agentDef.Name + // Optional caller-supplied overrides: SessionKind labels the + // transcript so /chat list can filter; OwnerUserID is used + // when there's no ParentSession to inherit from (e.g. the + // alert investigator spawns system-owned workers). + kind := aiopsmodel.SessionKindWork + if req.SessionKind != "" { + kind = req.SessionKind + } + effectiveOwner := ownerUserID + if effectiveOwner == 0 && req.OwnerUserID != 0 { + effectiveOwner = req.OwnerUserID + } + row := &aiopsmodel.Session{ + ID: sessID, + APMSource: sourceScope, + UserID: effectiveOwner, + Title: fmt.Sprintf("Worker: %s", agentDef.Name), + RootSessionID: rootRefForRow, + Initiator: aiopsmodel.SessionInitiatorAgent, + Audience: aiopsmodel.SessionAudienceInternal, + OwnerAgentID: ownerAgentRefForRow, + AgentID: &agentName, + ParentSessionID: parentRefForRow, + Background: req.Background, + Kind: kind, + CreatedAt: time.Now().UTC(), + UpdatedAt: time.Now().UTC(), + } + if err := rt.cfg.Sessions.CreateSession(ctx, row); err != nil { + return nil, fmt.Errorf("chatruntime: worker session create: %w", err) + } + } + + // Background spawn always derives from the long-lived runtime context + // so a finishing HTTP request can't tear down the worker mid-run. + // Sync spawn inherits the caller's ctx so a cancel propagates. + parent := ctx + if req.Background { + // Keep trace ancestry without inheriting cancellation or callback state. + parent = trace.ContextWithSpanContext(context.Background(), trace.SpanContextFromContext(ctx)) + } + workerCtx, cancel := context.WithCancel(parent) + + w := &Worker{ + ID: id, + AgentName: agentDef.Name, + SessionID: sessID, + ParentSessionID: req.ParentSession, + Prompt: req.Prompt, + Status: WorkerStatusPending, + Background: req.Background, + cancel: cancel, + } + + rt.workersMu.Lock() + if rt.workers == nil { + rt.workers = map[string]*Worker{} + } + rt.workers[id] = w + rt.workersMu.Unlock() + + run := func() { + // Close the session row when the goroutine terminates by any + // path (normal completion, error, kill, panic). Without this + // the chat_sessions row stays closed_at=NULL forever and + // every SpawnWorker call leaks one "looks-active" row — see + // the orphan accumulation that hit 161 rows on the test env + // before this fix landed. We use context.Background so the + // close fires even when the caller's ctx is already cancelled + // (which is the common case for the cancel-killed branch). + defer func() { + if rt.cfg.Sessions != nil && sessID != "" { + if err := rt.cfg.Sessions.CloseSession(context.Background(), sessID); err != nil && rt.log != nil { + rt.log.Warn("chatruntime: worker session close failed", + "session_id", sessID, "err", err.Error()) + } + } + }() + + w.mu.Lock() + w.Status = WorkerStatusRunning + w.StartedAt = time.Now().UTC() + w.mu.Unlock() + + // Stamp the coordinator's LLM choice onto workerCtx so runWorker + // can read it back as chatModelOpts. Without this the sub-agent + // falls through to the routing default ("openai") even though + // the user picked something else (e.g. deepseek) for the + // coordinator. SendToWorker's runWorker callsite uses ctx as-is + // because SendTo inherits the coordinator's ctx directly. + workerCtx = basetool.WithLLMChoice(workerCtx, req.Provider, req.Model) + result, err := rt.runWorker(workerCtx, agentDef, sessID, req.Prompt, req.Locale, req.ParentEmit, id) + + w.mu.Lock() + end := time.Now().UTC() + w.EndedAt = &end + switch { + case workerCtx.Err() != nil && w.Status == WorkerStatusRunning && err != nil: + // Killed via cancel(); preserve the cancellation classification + // over a stray invoke error. + w.Status = WorkerStatusKilled + w.Err = workerCtx.Err().Error() + case err != nil: + w.Status = WorkerStatusFailed + w.Err = err.Error() + default: + w.Status = WorkerStatusCompleted + w.Result = result + } + w.mu.Unlock() + + if req.Background && req.ParentEmit != nil { + req.ParentEmit(rt.notificationFor(w)) + } + } + + if req.Background { + go run() + return rt.snapshotWorker(w), nil + } + + run() + return rt.snapshotWorker(w), nil +} + +// CountWorkersByStatus returns the live worker counts split by status. +// Used by the ADR-026 self-obs sampler ticker (cmd/ongrid/main.go) to +// keep ongrid_chatruntime_worker_sessions{status="running|pending"} +// fresh. Holds the runtime mutex briefly; safe at any interval. +func (rt *Runtime) CountWorkersByStatus() (running, pending int) { + if rt == nil { + return 0, 0 + } + rt.workersMu.Lock() + defer rt.workersMu.Unlock() + for _, w := range rt.workers { + if w == nil { + continue + } + w.mu.Lock() + switch w.Status { + case WorkerStatusRunning: + running++ + case WorkerStatusPending: + pending++ + } + w.mu.Unlock() + } + return +} + +// SendToWorker continues a worker by appending a follow-up user message. +// The worker must be running or already in a terminal state — claude-code +// allows continuing a completed worker so a coordinator can refine the +// task without a fresh spawn (figure: SendMessageTool +// targets either state). For a running worker the message is queued +// until the worker yields; for a completed worker we re-run with the +// follow-up prompt. +// +// First version: we run the follow-up synchronously in the current +// goroutine (no queueing), and disallow continuing a killed worker — +// the LLM should spawn a new one in that case. +func (rt *Runtime) SendToWorker(ctx context.Context, workerID, message string) error { + if rt == nil { + return errors.New("chatruntime: nil runtime") + } + rt.workersMu.Lock() + w := rt.workers[workerID] + rt.workersMu.Unlock() + if w == nil { + return fmt.Errorf("chatruntime: worker %q not found", workerID) + } + w.mu.Lock() + status := w.Status + agentName := w.AgentName + w.mu.Unlock() + switch status { + case WorkerStatusKilled: + return fmt.Errorf("chatruntime: worker %q is killed; spawn a fresh one", workerID) + case WorkerStatusFailed: + return fmt.Errorf("chatruntime: worker %q is failed; spawn a fresh one", workerID) + } + if rt.cfg.AgentRegistry == nil { + return errors.New("chatruntime: agent registry not wired") + } + agentDef, ok := rt.cfg.AgentRegistry.ByName(agentName) + if !ok || agentDef == nil { + return fmt.Errorf("chatruntime: agent %q vanished from registry", agentName) + } + + w.mu.Lock() + w.Status = WorkerStatusRunning + sessID := w.SessionID + w.mu.Unlock() + + // SendToWorker has no SpawnRequest; inherit locale from ctx + // (chat coordinator threads it via basetool.WithLocale). + result, err := rt.runWorker(ctx, agentDef, sessID, message, basetool.LocaleFromContext(ctx), nil, w.ID) + + w.mu.Lock() + defer w.mu.Unlock() + end := time.Now().UTC() + w.EndedAt = &end + if err != nil { + w.Status = WorkerStatusFailed + w.Err = err.Error() + return err + } + w.Status = WorkerStatusCompleted + w.Result = result + return nil +} + +// StopWorker cancels a running worker. Returns nil even when the worker +// is already terminal — stopping is idempotent. +func (rt *Runtime) StopWorker(_ context.Context, workerID string) error { + if rt == nil { + return errors.New("chatruntime: nil runtime") + } + rt.workersMu.Lock() + w := rt.workers[workerID] + rt.workersMu.Unlock() + if w == nil { + return fmt.Errorf("chatruntime: worker %q not found", workerID) + } + w.mu.Lock() + defer w.mu.Unlock() + if w.Status == WorkerStatusRunning || w.Status == WorkerStatusPending { + if w.cancel != nil { + w.cancel() + } + // The goroutine flips Status to killed when it observes ctx.Done(). + // For pending workers there's no goroutine yet; stamp here. + if w.Status == WorkerStatusPending { + w.Status = WorkerStatusKilled + end := time.Now().UTC() + w.EndedAt = &end + } + } + return nil +} + +// GetWorker returns a snapshot copy of the worker by id. The returned +// *Worker is safe to read without locks; callers must NOT mutate it. +// (false, nil) when not found. +func (rt *Runtime) GetWorker(workerID string) (*Worker, bool) { + if rt == nil { + return nil, false + } + rt.workersMu.Lock() + w, ok := rt.workers[workerID] + rt.workersMu.Unlock() + if !ok { + return nil, false + } + return rt.snapshotWorker(w), true +} + +// runWorker is the inner spawn-and-invoke routine shared by SpawnWorker +// and SendToWorker. It builds a per-worker tool bag (whitelist + black- +// list filtered against the runtime's tool bag), composes the worker's +// system prompt, and drives one graph.Invoke against the agent's +// configured Model + MaxTurns. +// +// sessID is the worker's own chat_sessions row id. It is threaded into +// the persistence callback so worker assistant + tool messages are +// written into chat_messages under the worker's session (not the +// coordinator's). The user-role prompt is persisted up-front here, +// matching Handle()'s "user message lands on disk before the LLM call" +// invariant — same survival semantics on a graph crash. +func (rt *Runtime) runWorker(ctx context.Context, agentDef *Agent, sessID, userText, locale string, parentEmit Emit, workerID string) (result string, retErr error) { + session := &aiopsmodel.Session{ID: sessID} + if rt.cfg.Sessions != nil && sessID != "" { + var err error + session, err = rt.cfg.Sessions.GetSession(ctx, sessID) + if err != nil { + return "", fmt.Errorf("chatruntime: load worker source scope: %w", err) + } + ctx = basetool.WithAPMSource(ctx, session.APMSource) + } + ctx, span := startAgentSpan(ctx, session, agentDef.Name) + span.SetAttributes(attribute.String("ongrid.agent.kernel", "graph"), attribute.String("ongrid.agent.worker.id", workerID)) + tracing.RecordAgentContent(span, "input.value", userText) + defer func() { + if retErr == nil { + tracing.RecordAgentContent(span, "output.value", result) + } + tracing.EndSpan(span, retErr) + }() + + // Workers are not an authorization bypass. The coordinator stamps the + // resolved gate on context; absent wiring fails closed so a direct spawn + // cannot expose mutating tools unexpectedly. + workerTools := filterToolsForAgentRole(rt.toolBagSnapshot(), agentDef, false, !basetool.AgentWriteAllowedFromContext(ctx)) + + // Thread the persona-filtered tool view onto ctx so ToolSearch + // (which runs inside the worker graph) only returns tools the + // worker persona is allowed to see. ctx is per-request (each + // worker has its own), so concurrent workers don't race. + ctx = basetool.WithFilteredTools(ctx, workerTools) + + systemPrompt := ComposeSystemPrompt(rt.cfg.BasePrompt, nil, agentDef) + if digest := buildToolCapabilityDigest(workerTools); digest != "" { + systemPrompt = systemPrompt + "\n\n" + digest + } + + cfg := rt.cfg.GraphCfg + if agentDef.MaxTurns > 0 { + cfg.MaxIterations = agentDef.MaxTurns + } + if agentDef.Model != "" { + cfg.Model = agentDef.Model + } + + // Persist the user-role prompt under the worker's session id. Same + // invariant Handle() honours on the coordinator path — the user turn + // lives on disk before the graph runs so a crash mid-invoke leaves + // an auditable trace. + var handlers []einocallbacks.Handler + if rt.cfg.Sessions != nil && sessID != "" { + content := userText + userMsg := &aiopsmodel.Message{ + SessionID: sessID, + Role: aiopsmodel.RoleUser, + Content: &content, + CreatedAt: time.Now().UTC(), + } + if err := rt.cfg.Sessions.AppendMessage(ctx, userMsg); err != nil { + return "", fmt.Errorf("chatruntime: persist worker user msg: %w", err) + } + + // Wire the per-worker persistence callback. Reuse the runtime's + // CallbackDeps for cross-cutting context (logger, registerer) + // but override Persistence.SessionID so writes land on the + // worker row. SSE / Audit / Metrics inherit from the runtime + // config (Audit + Metrics are session-id-aware via their own + // deps; we keep the worker's session id flowing into them too). + deps := rt.cfg.CallbackDeps + deps.Persistence.SessionID = sessID + deps.Persistence.Model = agentDef.Model + if deps.Persistence.Repo == nil { + deps.Persistence.Repo = rt.cfg.Sessions + } + // Persist worker messages under the worker session, but mirror + // tool lifecycle frames to the parent stream so AgentTool's + // internal work is visible while the synchronous dispatch runs. + // Assistant frames stay private to the worker transcript; only + // tool_start/tool_end are UI breadcrumbs for the parent chat. + deps.SSE = workerToolForwarder(parentEmit, workerID) + handlers = callbacks.NewDefaultHandlers(deps) + } + cfg.ToolPersistence = callbacks.EnableSynchronousToolPersistence(handlers) + g, err := graph.BuildReActGraph(rt.cfg.ChatModel, workerTools, cfg) + if err != nil { + return "", fmt.Errorf("chatruntime: build worker graph: %w", err) + } + + // Thread the coordinator's resolved LLM choice (stamped on ctx via + // basetool.WithLLMChoice at the SpawnWorker boundary) onto the + // graph's ChatModel as eino model options. Without these, the + // RoutingChatModel falls through to its built-in default — which + // is "openai" — and installs with no OpenAI key see the worker + // fail with `provider "openai" not configured`. Empty fields add + // no option, preserving back-compat with auto-spawn paths + // (investigator) that don't carry a coordinator choice. + invokeOpts := []compose.Option{compose.WithCallbacks(handlers...)} + if mopts := workerChatModelOpts(ctx); len(mopts) > 0 { + invokeOpts = append(invokeOpts, compose.WithChatModelOption(mopts...)) + } + invokeOpts = append(invokeOpts, compose.WithToolsNodeOption( + compose.WithToolOption(graph.WithInvokeOpts( + basetool.WithUserText(userText), + basetool.WithHostWritePermission(basetool.HostWriteAllowedFromContext(ctx)), + )), + )) + // Sever the worker's callback chain from the coordinator's. The ctx we + // were handed carries the parent graph's callback manager (eino propagates + // it into nested Invokes), so WITHOUT this reset the worker's first + // ChatModel.OnStart fires the COORDINATOR's PersistenceHandler — whose + // flushIncompleteBatch then sees the still-in-flight AgentTool tool_call as + // "pending" (AgentTool is blocked right here running this worker) and + // autoheals it with an error stub. The coordinator's LLM reads that stub as + // "AgentTool unavailable" and gives up. InitCallbacks installs a fresh + // manager; the worker's own handlers are registered via WithCallbacks below. + workerCtx := einocallbacks.InitCallbacks(ctx, nil) + + // Autoheal any tool batch still open when the worker exits — same + // rationale as the parent runtime defer. context.WithoutCancel + // keeps the stub inserts running even if the caller cancelled. + defer func() { + flushCtx := context.WithoutCancel(workerCtx) + callbacks.FinalizeBatches(flushCtx, handlers) + }() + out, err := g.Invoke(workerCtx, &graph.Input{ + SystemPrompt: systemPrompt, + History: nil, + UserText: userText, + Locale: locale, + }, invokeOpts...) + if err != nil { + return "", err + } + if out == nil || out.AssistantMessage == nil { + return "", nil + } + return out.AssistantMessage.Content, nil +} + +func workerToolForwarder(parent Emit, workerID string) callbacks.SSEEmitter { + if parent == nil { + return nil + } + return func(ev callbacks.SSEEvent) { + if ev.Tool == nil { + return + } + switch ev.Type { + case callbacks.SSEEventToolStart: + parent(Event{Type: EventToolStart, Tool: &ToolEvent{ + ToolCallID: scopedWorkerToolCallID(workerID, ev.Tool.ToolCallID), + Name: ev.Tool.Name, + ArgsJSON: ev.Tool.ArgsJSON, + Status: ev.Tool.Status, + StartedAt: ev.Tool.StartedAt, + }}) + case callbacks.SSEEventToolEnd: + parent(Event{Type: EventToolEnd, Tool: &ToolEvent{ + ToolCallID: scopedWorkerToolCallID(workerID, ev.Tool.ToolCallID), + Name: ev.Tool.Name, + ArgsJSON: ev.Tool.ArgsJSON, + Status: ev.Tool.Status, + StartedAt: ev.Tool.StartedAt, + EndedAt: ev.Tool.EndedAt, + DurationMs: ev.Tool.DurationMs, + Error: ev.Tool.Error, + ResultJSON: ev.Tool.ResultJSON, + }}) + } + } +} + +func scopedWorkerToolCallID(workerID, toolCallID string) string { + workerID = strings.TrimSpace(workerID) + toolCallID = strings.TrimSpace(toolCallID) + if workerID == "" { + return toolCallID + } + if toolCallID == "" { + return workerID + } + return workerID + ":" + toolCallID +} + +// notificationFor produces a task_notification Event for the given +// terminal worker. Field shape mirrors +func (rt *Runtime) notificationFor(w *Worker) Event { + w.mu.Lock() + defer w.mu.Unlock() + summary := fmt.Sprintf("Agent %s %s", w.AgentName, w.Status) + usage := map[string]any{} + if !w.StartedAt.IsZero() && w.EndedAt != nil { + usage["duration_ms"] = w.EndedAt.Sub(w.StartedAt).Milliseconds() + } + notif := &TaskNotification{ + TaskID: w.ID, + Status: w.Status, + Summary: summary, + Result: w.Result, + Err: w.Err, + Usage: usage, + } + return Event{Type: EventTaskNotification, Notification: notif} +} + +// coordinatorOnlyTools are control-plane tools. AgentTool is selectively +// re-enabled for specialists that explicitly declare can_delegate; the +// SpawnWorker depth guard remains the authoritative recursion boundary. +var coordinatorOnlyTools = map[string]bool{ + "AgentTool": true, + "SendMessage": true, + "TaskStop": true, +} + +// alwaysAvailableTools survive the read-only strip (write gate OFF / viewer) +// despite a non-"read" Class, because they only produce a benign read-only +// artifact (host an HTML page) with no infrastructure side-effect. The write +// gate exists to guard arbitrary host exec (cloud_bash) / install (destructive), +// not page generation — so serve_page stays usable in normal chat regardless. +var alwaysAvailableTools = map[string]bool{ + "serve_page": true, +} + +// filterToolsForAgent applies the agent persona's whitelist + blacklist +// against the runtime tool bag. Black wins : +// +// - if Tools is empty, every tool is initially allowed (inherit from +// policy); otherwise only tools whose Info().Name appears in the +// whitelist are allowed. +// - DisallowedTools is then subtracted. Pattern "*_skill" is a +// suffix wildcard supported for ergonomic blacklists ( +// example). +// - SendMessage and TaskStop are root-session control tools. AgentTool is +// visible to the default assistant and to a specialist declaring +// CanDelegate; SpawnWorker enforces a maximum delegation depth of two. +// - viewerOnly==true drops every tool whose Info().Class is +// not "read". Viewer chat is degraded to single-agent read-only, +// which is fine — the coordinator can still answer with read tools +// directly (no need to dispatch since there's nothing mutating to do). +func filterToolsForAgent(bag []basetool.BaseTool, agentDef *Agent, isCoordinator bool) []basetool.BaseTool { + return filterToolsForAgentRole(bag, agentDef, isCoordinator, false) +} + +func filterToolsForAgentRole(bag []basetool.BaseTool, agentDef *Agent, isCoordinator bool, viewerOnly bool) []basetool.BaseTool { + whitelist := map[string]bool{} + blacklist := []string(nil) + if agentDef != nil { + for _, n := range agentDef.Tools { + whitelist[strings.TrimSpace(n)] = true + } + blacklist = agentDef.DisallowedTools + } + + out := make([]basetool.BaseTool, 0, len(bag)) + for _, t := range bag { + if t == nil { + continue + } + info, err := t.Info(context.Background()) + if err != nil || info == nil { + continue + } + // Viewer downgrade: drop any tool that is not Class="read". + // Applies BEFORE persona / control-tool checks so that even the + // coordinator chat for a viewer cannot dispatch (AgentTool is + // Class="write") or mutate (host_restart_service, etc.). + // Exception: alwaysAvailableTools (e.g. serve_page) produce a benign, + // read-only artifact with no infra side-effect, so the write gate / + // viewer downgrade shouldn't hide them — generating a page is not a + // dangerous write the way arbitrary host exec is. + if viewerOnly && (info.Class != "read" || info.Confirmation == basetool.ConfirmationRequired) && !alwaysAvailableTools[info.Name] { + continue + } + // Dynamic tools (runtime-discovered: MCP servers HLD-018, and any future + // installed skill / extension — identified by Origin, NOT a name prefix + // so this scales as those sources grow) can't be pre-listed in a + // persona's static Tools whitelist, so they're always in scope for every + // persona (coordinator + workers). The viewerOnly check above already + // dropped non-read (untrusted) dynamic tools; the blacklist still applies. + // + // History: we briefly excluded dynamic tools from the COORDINATOR (idea: + // force it to delegate, to stop it rabbit-holing through k8s tools for an + // ongrid-device question). The multi-turn sweep showed that BACKFIRED with + // the deployed weak model (zhipu): direct tool use is reliable but + // delegation is not, so the coordinator never reached the MCP tools at all + // — it fell back to `cloud_bash kubectl` (uninstalled) and couldn't answer + // "k8s namespaces". MCP became unreachable in chat. The real cause of the + // rabbit-hole is ROUTING (device≈node confusion), addressed in the + // coordinator system prompt, not by amputating the coordinator's reach. + if info.IsDynamic() { + if matchesAny(info.Name, blacklist) { + continue + } + out = append(out, t) + continue + } + // Coordinator-only tools survive the strip when we ARE the + // coordinator. They're always in scope for the coordinator + // regardless of any persona whitelist — control plane. + if coordinatorOnlyTools[info.Name] { + if !isCoordinator { + if info.Name != "AgentTool" || agentDef == nil || !agentDef.CanDelegate { + continue + } + } + } else if agentDef != nil && len(whitelist) > 0 && !whitelist[info.Name] { + continue + } + if matchesAny(info.Name, blacklist) { + continue + } + out = append(out, t) + } + return out +} + +// matchesAny returns true when name matches any pattern in patterns. +// Patterns: exact name, or "*" / "*" wildcard. +func matchesAny(name string, patterns []string) bool { + for _, p := range patterns { + p = strings.TrimSpace(p) + if p == "" { + continue + } + switch { + case p == name: + return true + case strings.HasPrefix(p, "*") && strings.HasSuffix(name, p[1:]): + return true + case strings.HasSuffix(p, "*") && strings.HasPrefix(name, p[:len(p)-1]): + return true + } + } + return false +} + +// snapshotWorker returns a copy of w safe to publish to a caller. The +// internal lock + cancel func are not copied — the returned *Worker is +// strictly read-only. +func (rt *Runtime) snapshotWorker(w *Worker) *Worker { + if w == nil { + return nil + } + w.mu.Lock() + defer w.mu.Unlock() + cp := &Worker{ + ID: w.ID, + AgentName: w.AgentName, + SessionID: w.SessionID, + ParentSessionID: w.ParentSessionID, + Prompt: w.Prompt, + Status: w.Status, + Background: w.Background, + StartedAt: w.StartedAt, + Result: w.Result, + Err: w.Err, + } + if w.EndedAt != nil { + t := *w.EndedAt + cp.EndedAt = &t + } + return cp +} + +// newWorkerID returns a fresh "agent-<8hex>" identifier. crypto/rand +// keeps collisions astronomically unlikely; callers don't need to retry. +func newWorkerID() string { + var b [4]byte + if _, err := rand.Read(b[:]); err != nil { + // crypto/rand failure is treated as fatal in the Go stdlib + // design; fall back to a timestamp-derived id to avoid panic. + return fmt.Sprintf("agent-%08x", time.Now().UnixNano()&0xffffffff) + } + return "agent-" + hex.EncodeToString(b[:]) +} + +// newWorkerSessionID derives a session id from the worker id. Distinct +// from the parent session so worker chat history can be persisted +// independently. The "worker-<8hex>" shape mirrors the worker id's +// hex tail and stays under chat_sessions.id (char(36)) — short, prefix- +// searchable, and visually distinct from coordinator UUIDs. +func newWorkerSessionID(workerID string) string { + // workerID is "agent-<8hex>"; rebrand the prefix to "worker-" so + // the chat_sessions row id is self-describing in audit dumps. + if strings.HasPrefix(workerID, "agent-") { + return "worker-" + workerID[len("agent-"):] + } + return "worker-" + workerID +} + +// hasTool reports whether `name` is present in the BaseTool slice. Used +// by runWorker to decide whether to run the KB prologue. +func hasTool(bag []basetool.BaseTool, name string) bool { + for _, t := range bag { + if t == nil { + continue + } + info, err := t.Info(context.Background()) + if err != nil || info == nil { + continue + } + if info.Name == name { + return true + } + } + return false +} + +// prologueKBLookup runs `query_knowledge` once with the user's task as +// the natural-language query and, when the top hit has score ≥ 0.6, +// returns a formatted KB context block ready to prepend to userText. +// +// All failure modes (tool missing, decode error, no hits, low score) +// return "" — the caller falls back to the bare user prompt. This is +// deliberately silent: the LLM should never see a "KB lookup failed" +// message, just sometimes-richer context. +func (rt *Runtime) prologueKBLookup(ctx context.Context, bag []basetool.BaseTool, userText string) string { + var kb basetool.BaseTool + for _, t := range bag { + if t == nil { + continue + } + info, err := t.Info(context.Background()) + if err != nil || info == nil || info.Name != "query_knowledge" { + continue + } + kb = t + break + } + if kb == nil { + return "" + } + // Schema across versions has used both "query" and "q"; send + // "query" and let the tool ignore extras. + args, _ := json.Marshal(map[string]any{ + "query": userText, + "top_k": 3, + "min_score": 0.6, + }) + out, err := kb.InvokableRun(ctx, string(args)) + if err != nil || out == "" { + return "" + } + // Parse loosely — the response shape is {"items":[{"title":..,"preview":..,"score":..}, ...]} + var parsed struct { + Items []struct { + Title string `json:"title"` + Preview string `json:"preview"` + Path string `json:"path"` + Score float64 `json:"score"` + URL string `json:"url"` + } `json:"items"` + } + if jerr := json.Unmarshal([]byte(out), &parsed); jerr != nil { + return "" + } + if len(parsed.Items) == 0 { + return "" + } + top := parsed.Items[0] + if top.Score < 0.6 { + return "" + } + var b strings.Builder + b.WriteString("[KB 自动注入 · 协调员认为这是相关 playbook]\n") + b.WriteString("Title: ") + b.WriteString(top.Title) + b.WriteString(fmt.Sprintf(" (score=%.2f)\n", top.Score)) + if top.Path != "" { + b.WriteString("Path: ") + b.WriteString(top.Path) + b.WriteString("\n") + } + b.WriteString("\n") + b.WriteString(top.Preview) + if top.URL != "" { + b.WriteString("\n\nSource: ") + b.WriteString(top.URL) + } + b.WriteString("\n\n请基于这份 playbook 推进诊断；末尾在答案里标注 `（参考 KB: " + top.Title + "）`。如果 playbook 不适用，可以忽略并自由发挥。") + return b.String() +} + +// workerChatModelOpts mirrors chatModelOpts but reads the (provider, +// model) pair from ctx instead of a *Request, so SpawnWorker and +// SendToWorker (which has no Request) share the same plumbing. +// RoutingChatModel.pick consumes WithProvider; the inner +// clientChatModel honours WithModel. Empty fields add no option, so +// auto-spawn paths that don't carry a coordinator choice fall through +// to the routing default unchanged. +func workerChatModelOpts(ctx context.Context) []model.Option { + var opts []model.Option + if p := strings.TrimSpace(basetool.LLMProviderFromContext(ctx)); p != "" { + opts = append(opts, llm.WithProvider(p)) + } + if m := strings.TrimSpace(basetool.LLMModelFromContext(ctx)); m != "" { + opts = append(opts, model.WithModel(m)) + } + return opts +} diff --git a/internal/manager/biz/aiops/chatruntime/worker_filter_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/worker_filter_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/worker_filter_test.go rename to internal/manager/biz/aiops/runtime/v2/chatruntime/worker_filter_test.go diff --git a/internal/manager/biz/aiops/graph/budget_stop_model.go b/internal/manager/biz/aiops/runtime/v2/graph/budget_stop_model.go similarity index 100% rename from internal/manager/biz/aiops/graph/budget_stop_model.go rename to internal/manager/biz/aiops/runtime/v2/graph/budget_stop_model.go diff --git a/internal/manager/biz/aiops/graph/callbacks/alert_draft_guard.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/alert_draft_guard.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/alert_draft_guard.go rename to internal/manager/biz/aiops/runtime/v2/graph/callbacks/alert_draft_guard.go diff --git a/internal/manager/biz/aiops/graph/callbacks/alert_draft_guard_test.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/alert_draft_guard_test.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/alert_draft_guard_test.go rename to internal/manager/biz/aiops/runtime/v2/graph/callbacks/alert_draft_guard_test.go diff --git a/internal/manager/biz/aiops/runtime/v2/graph/callbacks/audit.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/audit.go new file mode 100644 index 000000000..fd990b8c2 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/audit.go @@ -0,0 +1,288 @@ +package callbacks + +import ( + "context" + "log/slog" + "sync" + "sync/atomic" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +// AuditDeps configures the audit handler. Logger is required (a nil +// logger turns the handler into a no-op so the cutover layer can opt +// in/out per environment). SessionID is the chat session this run +// belongs to — included in every log line so operators can grep all +// stages of one conversation. UserID is logged as a numeric scope. +type AuditDeps struct { + Logger *slog.Logger + SessionID string + UserID uint64 +} + +// AuditHandler emits structured slog INFO records for every stage of +// the graph: per-ChatModel turn (prompt token estimate + reply tokens), +// per-tool call (name + duration + status). spec — and +// / 红线: the user's raw prompt content is NEVER +// included in the log line; only counts and identifiers. +// +// One handler instance per graph run. Concurrency: tool fan-out may +// call OnStart / OnEnd concurrently; per-call timestamps live in a +// mutex-guarded map. +type AuditHandler struct { + deps AuditDeps + + chatTurn atomic.Int64 + + startsMu sync.Mutex + starts map[string]auditStart +} + +type auditStart struct { + at time.Time + component components.Component + name string + estTokens int + toolCallID string +} + +// NewAuditHandler builds the handler. Returns nil when Logger is nil +// so the cutover layer can short-circuit. +func NewAuditHandler(deps AuditDeps) *AuditHandler { + if deps.Logger == nil { + return nil + } + return &AuditHandler{deps: deps, starts: make(map[string]auditStart)} +} + +// Needed gates timings. +func (h *AuditHandler) Needed(_ context.Context, info *callbacks.RunInfo, timing callbacks.CallbackTiming) bool { + if h == nil || info == nil { + return false + } + switch info.Component { + case components.ComponentOfChatModel, components.ComponentOfTool: + switch timing { + case callbacks.TimingOnStart, callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + } + return false +} + +// OnStart records the start time so OnEnd can compute duration. +func (h *AuditHandler) OnStart(ctx context.Context, info *callbacks.RunInfo, input callbacks.CallbackInput) context.Context { + if h == nil || info == nil { + return ctx + } + at := time.Now() + entry := auditStart{at: at, component: info.Component, name: info.Name} + switch info.Component { + case components.ComponentOfChatModel: + mi := einomodel.ConvCallbackInput(input) + if mi != nil { + entry.estTokens = estimatePromptTokens(mi.Messages) + } + turn := h.chatTurn.Add(1) + h.deps.Logger.Info("graph stage start", + slog.String("session_id", h.deps.SessionID), + slog.Uint64("user_id", h.deps.UserID), + slog.String("kind", "chat_model"), + slog.String("name", info.Name), + slog.Int64("iteration", turn), + slog.Int("est_prompt_tokens", entry.estTokens), + ) + case components.ComponentOfTool: + entry.toolCallID = toolCallIDFromCtx(ctx, info) + args := "" + if tin := einotool.ConvCallbackInput(input); tin != nil { + args = tin.ArgumentsInJSON + } + // Log args size, not args content — the args may carry user- + // authored text (PromQL / LogQL queries) that + // forbids us from logging in cleartext. + h.deps.Logger.Info("graph stage start", + slog.String("session_id", h.deps.SessionID), + slog.Uint64("user_id", h.deps.UserID), + slog.String("kind", "tool"), + slog.String("name", info.Name), + slog.String("tool_call_id", entry.toolCallID), + slog.Int("args_bytes", len(args)), + ) + } + h.startsMu.Lock() + h.starts[stageKey(ctx, info)] = entry + h.startsMu.Unlock() + return ctx +} + +// OnEnd logs the stage's duration and result. +func (h *AuditHandler) OnEnd(ctx context.Context, info *callbacks.RunInfo, output callbacks.CallbackOutput) context.Context { + if h == nil || info == nil { + return ctx + } + h.startsMu.Lock() + entry, ok := h.starts[stageKey(ctx, info)] + delete(h.starts, stageKey(ctx, info)) + h.startsMu.Unlock() + if !ok { + return ctx + } + dur := time.Since(entry.at) + switch info.Component { + case components.ComponentOfChatModel: + mo := einomodel.ConvCallbackOutput(output) + usage := slog.Group("token_usage") + toolCalls := 0 + if mo != nil { + if mo.TokenUsage != nil { + usage = slog.Group("token_usage", + slog.Int("prompt", mo.TokenUsage.PromptTokens), + slog.Int("completion", mo.TokenUsage.CompletionTokens), + slog.Int("total", mo.TokenUsage.TotalTokens), + ) + } else if mo.Message != nil && mo.Message.ResponseMeta != nil && mo.Message.ResponseMeta.Usage != nil { + u := mo.Message.ResponseMeta.Usage + usage = slog.Group("token_usage", + slog.Int("prompt", u.PromptTokens), + slog.Int("completion", u.CompletionTokens), + slog.Int("total", u.TotalTokens), + ) + } + if mo.Message != nil { + toolCalls = len(mo.Message.ToolCalls) + } + } + h.deps.Logger.Info("graph stage end", + slog.String("session_id", h.deps.SessionID), + slog.Uint64("user_id", h.deps.UserID), + slog.String("kind", "chat_model"), + slog.String("name", info.Name), + slog.Int64("duration_ms", dur.Milliseconds()), + slog.Int("tool_calls_emitted", toolCalls), + slog.String("status", "success"), + usage, + ) + case components.ComponentOfTool: + body := "" + if tout := einotool.ConvCallbackOutput(output); tout != nil { + body = tout.Response + } + h.deps.Logger.Info("graph stage end", + slog.String("session_id", h.deps.SessionID), + slog.Uint64("user_id", h.deps.UserID), + slog.String("kind", "tool"), + slog.String("name", info.Name), + slog.String("tool_call_id", entry.toolCallID), + slog.Int64("duration_ms", dur.Milliseconds()), + slog.Int("result_bytes", len(body)), + slog.String("status", "success"), + ) + } + return ctx +} + +// OnError logs the stage's failure. +func (h *AuditHandler) OnError(ctx context.Context, info *callbacks.RunInfo, err error) context.Context { + if h == nil || info == nil || err == nil { + return ctx + } + h.startsMu.Lock() + entry, ok := h.starts[stageKey(ctx, info)] + delete(h.starts, stageKey(ctx, info)) + h.startsMu.Unlock() + dur := time.Duration(0) + if ok { + dur = time.Since(entry.at) + } + status := "error" + if isDeadlineErr(err) { + status = "timeout" + } + h.deps.Logger.Warn("graph stage end", + slog.String("session_id", h.deps.SessionID), + slog.Uint64("user_id", h.deps.UserID), + slog.String("kind", componentKind(info.Component)), + slog.String("name", info.Name), + slog.Int64("duration_ms", dur.Milliseconds()), + slog.String("status", status), + slog.String("error", err.Error()), + ) + return ctx +} + +// OnStartWithStreamInput drains and closes the stream copy. +func (h *AuditHandler) OnStartWithStreamInput(ctx context.Context, _ *callbacks.RunInfo, in *schema.StreamReader[callbacks.CallbackInput]) context.Context { + if in != nil { + in.Close() + } + return ctx +} + +// OnEndWithStreamOutput drains and closes the stream copy. +func (h *AuditHandler) OnEndWithStreamOutput(ctx context.Context, _ *callbacks.RunInfo, out *schema.StreamReader[callbacks.CallbackOutput]) context.Context { + if out != nil { + out.Close() + } + return ctx +} + +// estimatePromptTokens is the same rule-of-thumb used by the budget +// callback (chars/4). Local copy so we don't pull the llm package in +// just for the function (avoids an import cycle with biz/aiops/graph). +func estimatePromptTokens(msgs []*schema.Message) int { + const perMsgOverhead = 4 + total := 0 + for _, m := range msgs { + if m == nil { + continue + } + total += perMsgOverhead + total += len(m.Content) / 4 + for _, tc := range m.ToolCalls { + total += len(tc.Function.Name) / 4 + total += len(tc.Function.Arguments) / 4 + } + } + return total +} + +func componentKind(c components.Component) string { + switch c { + case components.ComponentOfChatModel: + return "chat_model" + case components.ComponentOfTool: + return "tool" + default: + return string(c) + } +} + +// stageKey is the per-stage correlation key used to look up the +// matching OnStart entry from OnEnd / OnError. For tool calls we use +// the tool_call id; for chat models we use the run-info name + the +// turn index so concurrent (eino doesn't fan out chat models, but +// future SOP graphs may) calls don't collide. +func stageKey(ctx context.Context, info *callbacks.RunInfo) string { + if info == nil { + return "" + } + if info.Component == components.ComponentOfTool { + return "tool|" + toolCallIDFromCtx(ctx, info) + } + if v, ok := ctx.Value(messageIDCtxKey{}).(string); ok && v != "" { + return string(info.Component) + "|" + info.Name + "|" + v + } + return string(info.Component) + "|" + info.Name +} + +// Compile-time checks. +var ( + _ callbacks.Handler = (*AuditHandler)(nil) + _ callbacks.TimingChecker = (*AuditHandler)(nil) +) diff --git a/internal/manager/biz/aiops/runtime/v2/graph/callbacks/audit_test.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/audit_test.go new file mode 100644 index 000000000..10d0689b2 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/audit_test.go @@ -0,0 +1,166 @@ +package callbacks + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "strings" + "testing" + + "github.com/cloudwego/eino/callbacks" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +// captureLogger writes JSON log lines into an in-memory buffer the test +// can grep. We use slog.NewJSONHandler so attribute names are stable. +func captureLogger() (*slog.Logger, *bytes.Buffer) { + buf := &bytes.Buffer{} + h := slog.NewJSONHandler(buf, &slog.HandlerOptions{Level: slog.LevelInfo}) + return slog.New(h), buf +} + +func TestAuditHandler_NewNilLogger(t *testing.T) { + t.Parallel() + if NewAuditHandler(AuditDeps{}) != nil { + t.Fatalf("expected nil handler when logger is nil") + } +} + +func TestAuditHandler_ChatModelStartAndEnd(t *testing.T) { + t.Parallel() + log, buf := captureLogger() + h := NewAuditHandler(AuditDeps{Logger: log, SessionID: "sess-1", UserID: 42}) + ctx := context.Background() + in := &einomodel.CallbackInput{ + Messages: []*schema.Message{{Role: schema.User, Content: "hello world"}}, + } + h.OnStart(ctx, chatModelInfo(), in) + out := &einomodel.CallbackOutput{ + Message: &schema.Message{Role: schema.Assistant, Content: "ok"}, + TokenUsage: &einomodel.TokenUsage{PromptTokens: 5, CompletionTokens: 3, TotalTokens: 8}, + } + h.OnEnd(ctx, chatModelInfo(), out) + + lines := splitLines(buf.String()) + if len(lines) != 2 { + t.Fatalf("expected 2 log lines got %d: %v", len(lines), lines) + } + + startLine := decodeJSON(t, lines[0]) + if startLine["msg"] != "graph stage start" { + t.Errorf("msg = %v, want graph stage start", startLine["msg"]) + } + if startLine["session_id"] != "sess-1" { + t.Errorf("session_id = %v", startLine["session_id"]) + } + if startLine["kind"] != "chat_model" { + t.Errorf("kind = %v", startLine["kind"]) + } + if iter, ok := startLine["iteration"].(float64); !ok || iter != 1 { + t.Errorf("iteration = %v", startLine["iteration"]) + } + + endLine := decodeJSON(t, lines[1]) + if endLine["status"] != "success" { + t.Errorf("status = %v", endLine["status"]) + } + usage, ok := endLine["token_usage"].(map[string]any) + if !ok { + t.Fatalf("token_usage missing: %v", endLine) + } + if usage["total"] != float64(8) { + t.Errorf("total tokens = %v", usage["total"]) + } +} + +func TestAuditHandler_ToolError(t *testing.T) { + t.Parallel() + log, buf := captureLogger() + h := NewAuditHandler(AuditDeps{Logger: log, SessionID: "s1"}) + ctx := WithToolCallID(context.Background(), "tc-1") + h.OnStart(ctx, toolInfo("flaky"), &einotool.CallbackInput{ArgumentsInJSON: `{"x":1}`}) + h.OnError(ctx, toolInfo("flaky"), errors.New("boom")) + + lines := splitLines(buf.String()) + if len(lines) < 2 { + t.Fatalf("expected start+end lines, got %d", len(lines)) + } + end := decodeJSON(t, lines[len(lines)-1]) + if end["status"] != "error" { + t.Errorf("status = %v", end["status"]) + } + if end["error"] != "boom" { + t.Errorf("error = %v", end["error"]) + } +} + +func TestAuditHandler_DoesNotLogUserText(t *testing.T) { + t.Parallel() + // / 红线: user text must never appear in audit logs. + const sensitive = "私密文本-DO-NOT-LOG" + log, buf := captureLogger() + h := NewAuditHandler(AuditDeps{Logger: log, SessionID: "s"}) + in := &einomodel.CallbackInput{ + Messages: []*schema.Message{{Role: schema.User, Content: sensitive}}, + } + h.OnStart(context.Background(), chatModelInfo(), in) + h.OnEnd(context.Background(), chatModelInfo(), &einomodel.CallbackOutput{ + Message: &schema.Message{Role: schema.Assistant, Content: "reply"}, + }) + if strings.Contains(buf.String(), sensitive) { + t.Fatalf("audit log leaked user text: %s", buf.String()) + } +} + +func TestAuditHandler_TimeoutClassified(t *testing.T) { + t.Parallel() + log, buf := captureLogger() + h := NewAuditHandler(AuditDeps{Logger: log, SessionID: "s"}) + ctx := WithToolCallID(context.Background(), "tc-x") + h.OnStart(ctx, toolInfo("slow"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("slow"), context.DeadlineExceeded) + end := decodeJSON(t, splitLines(buf.String())[1]) + if end["status"] != "timeout" { + t.Errorf("status = %v, want timeout", end["status"]) + } +} + +// helpers -------------------------------------------------- + +func splitLines(s string) []string { + out := []string{} + for _, line := range strings.Split(s, "\n") { + if strings.TrimSpace(line) != "" { + out = append(out, line) + } + } + return out +} + +func decodeJSON(t *testing.T, s string) map[string]any { + t.Helper() + var m map[string]any + if err := json.Unmarshal([]byte(s), &m); err != nil { + t.Fatalf("decode line: %v\nline=%s", err, s) + } + return m +} + +func TestAuditHandler_NeededGating(t *testing.T) { + t.Parallel() + log, _ := captureLogger() + h := NewAuditHandler(AuditDeps{Logger: log}) + if !h.Needed(context.Background(), chatModelInfo(), callbacks.TimingOnStart) { + t.Errorf("ChatModel OnStart should be needed") + } + if !h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnEnd) { + t.Errorf("Tool OnEnd should be needed") + } + if !h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnError) { + t.Errorf("Tool OnError should be needed") + } +} diff --git a/internal/manager/biz/aiops/graph/callbacks/chain.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/chain.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/chain.go rename to internal/manager/biz/aiops/runtime/v2/graph/callbacks/chain.go diff --git a/internal/manager/biz/aiops/runtime/v2/graph/callbacks/chain_test.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/chain_test.go new file mode 100644 index 000000000..7aadee069 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/chain_test.go @@ -0,0 +1,83 @@ +package callbacks + +import ( + "context" + "log/slog" + "testing" + + "github.com/cloudwego/eino/callbacks" + "github.com/prometheus/client_golang/prometheus" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +func TestNewDefaultHandlers_AllWiredEmitsFive(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + reg := prometheus.NewRegistry() + deps := Deps{ + Persistence: PersistenceDeps{SessionID: "s", Repo: repo, Registerer: reg}, + SSE: func(SSEEvent) {}, + Audit: AuditDeps{Logger: slog.Default(), SessionID: "s"}, + Metrics: MetricsDeps{Registerer: reg}, + BudgetChecker: stubBudgetChecker{}, + BudgetUserID: 1, + } + got := NewDefaultHandlers(deps) + if len(got) != 5 { + t.Fatalf("expected 5 handlers, got %d", len(got)) + } + // Spot-check ordering: persistence first, budget last. + if _, ok := got[0].(*PersistenceHandler); !ok { + t.Errorf("got[0] = %T, want *PersistenceHandler", got[0]) + } + if _, ok := got[len(got)-1].(*llm.BudgetCallbackHandler); !ok { + t.Errorf("got[last] = %T, want *llm.BudgetCallbackHandler", got[len(got)-1]) + } +} + +func TestNewDefaultHandlers_PartialDepsSkipsHandlers(t *testing.T) { + t.Parallel() + got := NewDefaultHandlers(Deps{}) + if len(got) != 0 { + t.Fatalf("expected empty chain when no deps wired, got %d", len(got)) + } +} + +func TestNewDefaultHandlers_OnlyMetrics(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + got := NewDefaultHandlers(Deps{Metrics: MetricsDeps{Registerer: reg}}) + if len(got) != 1 { + t.Fatalf("expected 1 handler got %d", len(got)) + } + if _, ok := got[0].(*MetricsHandler); !ok { + t.Errorf("got[0] = %T", got[0]) + } +} + +func TestNewDefaultHandlers_AllImplementHandlerInterface(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + reg := prometheus.NewRegistry() + deps := Deps{ + Persistence: PersistenceDeps{SessionID: "s", Repo: repo, Registerer: reg}, + SSE: func(SSEEvent) {}, + Audit: AuditDeps{Logger: slog.Default(), SessionID: "s"}, + Metrics: MetricsDeps{Registerer: reg}, + BudgetChecker: stubBudgetChecker{}, + } + for i, h := range NewDefaultHandlers(deps) { + if _, ok := h.(callbacks.Handler); !ok { + t.Errorf("got[%d] = %T does not satisfy callbacks.Handler", i, h) + } + } +} + +// stubBudgetChecker is a minimal llm.BudgetChecker that allows everything. +type stubBudgetChecker struct{} + +func (stubBudgetChecker) Check(_ context.Context, _ uint64, _ int) error { return nil } +func (stubBudgetChecker) Record(_ context.Context, _ uint64, _ llm.Usage) error { + return nil +} diff --git a/internal/manager/biz/aiops/runtime/v2/graph/callbacks/metrics.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/metrics.go new file mode 100644 index 000000000..738f093d7 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/metrics.go @@ -0,0 +1,305 @@ +package callbacks + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + "github.com/cloudwego/eino/schema" + "github.com/prometheus/client_golang/prometheus" +) + +// MetricsDeps configures the metrics handler. Registerer is required; +// callers pass prometheus.DefaultRegisterer in production and a fresh +// prometheus.NewRegistry in tests so collectors do not collide. +type MetricsDeps struct { + Registerer prometheus.Registerer +} + +// metricsCollectors bundles the Prom collectors registered by the +// metrics handler. Allocated once per Registerer and shared across +// handler instances so multiple graph runs don't trip +// AlreadyRegisteredError. +// +// Cardinality red line: labels are limited to {name, result} +// (tools) / {provider, result} (chat models) / {result} (graph +// terminations). Never include user / tenant / session. +type metricsCollectors struct { + toolInvocations *prometheus.CounterVec // name, result + toolDuration *prometheus.HistogramVec // name + graphIterations *prometheus.CounterVec // result + chatTurns *prometheus.CounterVec // result +} + +var ( + metricsRegMu sync.Mutex + metricsRegMap = map[prometheus.Registerer]*metricsCollectors{} +) + +func getMetricsCollectors(reg prometheus.Registerer) *metricsCollectors { + if reg == nil { + reg = prometheus.DefaultRegisterer + } + metricsRegMu.Lock() + defer metricsRegMu.Unlock() + if mc, ok := metricsRegMap[reg]; ok { + return mc + } + // Help string MUST match tools/decorators/metric.go byte-for-byte — + // Prom panics on AlreadyRegistered if descriptors differ on Help. + // Both layers observe the same counter from different vantage points + // (dual-observation contract), so they MUST share desc. + tinv := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_tool_invocations_total", + Help: "Total ongrid tool invocations, split by name and result.", + }, + []string{"name", "result"}, + ) + tdur := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ongrid_tool_duration_seconds", + Help: "ongrid tool invocation duration in seconds.", + Buckets: prometheus.ExponentialBuckets(0.05, 2, 10), + }, + []string{"name"}, + ) + gi := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_graph_iterations_total", + Help: "Total ongrid agent graph terminations, split by result (success | max_iterations | error).", + }, + []string{"result"}, + ) + ct := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_graph_chat_turns_total", + Help: "Total ChatModel turns observed during ongrid agent graph runs.", + }, + []string{"result"}, + ) + mc := &metricsCollectors{ + toolInvocations: regOrExist(reg, tinv).(*prometheus.CounterVec), + toolDuration: regOrExist(reg, tdur).(*prometheus.HistogramVec), + graphIterations: regOrExist(reg, gi).(*prometheus.CounterVec), + chatTurns: regOrExist(reg, ct).(*prometheus.CounterVec), + } + metricsRegMap[reg] = mc + return mc +} + +// regOrExist mirrors the helper in tools/decorators/metric.go and +// llm/metrics.go: register; on AlreadyRegisteredError reuse the +// existing collector. +func regOrExist(reg prometheus.Registerer, c prometheus.Collector) prometheus.Collector { + err := reg.Register(c) + if err == nil { + return c + } + var are prometheus.AlreadyRegisteredError + if errors.As(err, &are) { + return are.ExistingCollector + } + panic(err) +} + +// MetricsHandler observes tool + chat-model invocations from the +// graph's perspective. Note: the PR-3 decorator chain already records +// ongrid_tool_invocations_total at the per-call seam — this handler +// re-uses the SAME collector via the registry-keyed cache so we never +// double-register, and the counter increments are best-effort +// duplicates (acceptable : "callback 视角 + 装饰器视 +// 角 双观察 OK"). Tests assert exactly one increment per call when +// the decorator chain is bypassed (the unit test calls the handler +// directly). +type MetricsHandler struct { + collectors *metricsCollectors + + chatTurns atomic.Int64 + toolStartsMu sync.Mutex + toolStarts map[string]time.Time + + terminated atomic.Bool +} + +// NewMetricsHandler builds a handler against the given registerer. +// Returns nil if Registerer is nil. +func NewMetricsHandler(deps MetricsDeps) *MetricsHandler { + if deps.Registerer == nil { + return nil + } + return &MetricsHandler{ + collectors: getMetricsCollectors(deps.Registerer), + toolStarts: make(map[string]time.Time), + } +} + +// Needed gates timings. +func (h *MetricsHandler) Needed(_ context.Context, info *callbacks.RunInfo, timing callbacks.CallbackTiming) bool { + if h == nil || info == nil { + return false + } + switch info.Component { + case components.ComponentOfChatModel: + switch timing { + case callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + case components.ComponentOfTool: + switch timing { + case callbacks.TimingOnStart, callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + default: + // Graph terminations. + switch timing { + case callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + } + return false +} + +// OnStart records tool start times. +func (h *MetricsHandler) OnStart(ctx context.Context, info *callbacks.RunInfo, _ callbacks.CallbackInput) context.Context { + if h == nil || info == nil { + return ctx + } + if info.Component == components.ComponentOfTool { + h.toolStartsMu.Lock() + h.toolStarts[toolCallIDFromCtx(ctx, info)] = time.Now() + h.toolStartsMu.Unlock() + } + return ctx +} + +// OnEnd increments counters for tool / chat / graph completion. +func (h *MetricsHandler) OnEnd(ctx context.Context, info *callbacks.RunInfo, output callbacks.CallbackOutput) context.Context { + if h == nil || info == nil { + return ctx + } + switch info.Component { + case components.ComponentOfChatModel: + h.chatTurns.Add(1) + h.collectors.chatTurns.WithLabelValues("success").Inc() + _ = output // ChatModel output not needed for counters + case components.ComponentOfTool: + key := toolCallIDFromCtx(ctx, info) + h.toolStartsMu.Lock() + started, ok := h.toolStarts[key] + delete(h.toolStarts, key) + h.toolStartsMu.Unlock() + if ok { + h.collectors.toolDuration.WithLabelValues(info.Name).Observe(time.Since(started).Seconds()) + } + // We can't tell from CallbackOutput alone whether the tool + // returned an error payload (it's just a JSON string). Treat + // this hook as "success" — the OnError path covers actual + // failures returned via tool.InvokableRun. + h.collectors.toolInvocations.WithLabelValues(info.Name, "success").Inc() + default: + // Graph-scope terminal: success. + if h.terminated.CompareAndSwap(false, true) { + h.collectors.graphIterations.WithLabelValues("success").Inc() + } + } + return ctx +} + +// OnError increments error counters. +func (h *MetricsHandler) OnError(ctx context.Context, info *callbacks.RunInfo, err error) context.Context { + if h == nil || info == nil || err == nil { + return ctx + } + switch info.Component { + case components.ComponentOfChatModel: + h.collectors.chatTurns.WithLabelValues("error").Inc() + case components.ComponentOfTool: + key := toolCallIDFromCtx(ctx, info) + h.toolStartsMu.Lock() + started, ok := h.toolStarts[key] + delete(h.toolStarts, key) + h.toolStartsMu.Unlock() + result := "error" + if isDeadlineErr(err) { + result = "timeout" + } + if ok { + h.collectors.toolDuration.WithLabelValues(info.Name).Observe(time.Since(started).Seconds()) + } + h.collectors.toolInvocations.WithLabelValues(info.Name, result).Inc() + default: + // Graph-scope error: classify max-iterations vs other. + result := "error" + if isMaxIterations(err) { + result = "max_iterations" + } + if h.terminated.CompareAndSwap(false, true) { + h.collectors.graphIterations.WithLabelValues(result).Inc() + } + } + return ctx +} + +// OnStartWithStreamInput / OnEndWithStreamOutput are no-ops; the +// counters fire on the non-stream OnEnd path. +func (h *MetricsHandler) OnStartWithStreamInput(ctx context.Context, _ *callbacks.RunInfo, in *schema.StreamReader[callbacks.CallbackInput]) context.Context { + if in != nil { + in.Close() + } + return ctx +} + +func (h *MetricsHandler) OnEndWithStreamOutput(ctx context.Context, _ *callbacks.RunInfo, out *schema.StreamReader[callbacks.CallbackOutput]) context.Context { + if out != nil { + out.Close() + } + return ctx +} + +// ChatTurns reports the number of ChatModel turns observed. Exposed +// for tests. +func (h *MetricsHandler) ChatTurns() int { + if h == nil { + return 0 + } + return int(h.chatTurns.Load()) +} + +// isDeadlineErr returns true when err signals a deadline exceeded. +// Exported only by virtue of being shared across handler files; lives +// in this package because it mirrors the legacy classifyToolOutcome +// used by agent.go. +func isDeadlineErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + return false +} + +// isMaxIterations reports whether err looks like the eino runtime's +// "graph reached max steps" error. eino does not export a typed error +// for this, so we string-match the standard message it produces. The +// match is loose on purpose — the metric's `max_iterations` bucket is +// a hint, not authoritative. +func isMaxIterations(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "max steps") || strings.Contains(msg, "MaxRunSteps") || strings.Contains(msg, "max iterations") +} + +// Compile-time checks. +var ( + _ callbacks.Handler = (*MetricsHandler)(nil) + _ callbacks.TimingChecker = (*MetricsHandler)(nil) +) diff --git a/internal/manager/biz/aiops/runtime/v2/graph/callbacks/metrics_test.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/metrics_test.go new file mode 100644 index 000000000..23369d42c --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/metrics_test.go @@ -0,0 +1,120 @@ +package callbacks + +import ( + "context" + "errors" + "testing" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestMetricsHandler_NewNilRegistererReturnsNil(t *testing.T) { + t.Parallel() + if NewMetricsHandler(MetricsDeps{}) != nil { + t.Fatalf("expected nil when registerer is nil") + } +} + +func TestMetricsHandler_ToolSuccessIncrements(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + ctx := WithToolCallID(context.Background(), "tc-1") + h.OnStart(ctx, toolInfo("query_promql"), &einotool.CallbackInput{}) + h.OnEnd(ctx, toolInfo("query_promql"), &einotool.CallbackOutput{Response: "{}"}) + + got := testutil.ToFloat64(h.collectors.toolInvocations.WithLabelValues("query_promql", "success")) + if got != 1 { + t.Errorf("invocations counter = %v, want 1", got) + } +} + +func TestMetricsHandler_ToolErrorIncrementsErrorBucket(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + ctx := WithToolCallID(context.Background(), "tc-2") + h.OnStart(ctx, toolInfo("flaky"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("flaky"), errors.New("boom")) + + got := testutil.ToFloat64(h.collectors.toolInvocations.WithLabelValues("flaky", "error")) + if got != 1 { + t.Errorf("invocations error counter = %v, want 1", got) + } +} + +func TestMetricsHandler_ToolTimeoutClassified(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + ctx := WithToolCallID(context.Background(), "tc-3") + h.OnStart(ctx, toolInfo("slow"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("slow"), context.DeadlineExceeded) + + if got := testutil.ToFloat64(h.collectors.toolInvocations.WithLabelValues("slow", "timeout")); got != 1 { + t.Errorf("timeout counter = %v, want 1", got) + } +} + +func TestMetricsHandler_ChatTurnCounter(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + for i := 0; i < 3; i++ { + h.OnEnd(context.Background(), chatModelInfo(), nil) + } + if got := testutil.ToFloat64(h.collectors.chatTurns.WithLabelValues("success")); got != 3 { + t.Errorf("chat_turns success = %v, want 3", got) + } + if h.ChatTurns() != 3 { + t.Errorf("ChatTurns() = %d, want 3", h.ChatTurns()) + } +} + +func TestMetricsHandler_GraphTermination(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + graphInfo := &callbacks.RunInfo{Component: components.Component("Graph")} + h.OnEnd(context.Background(), graphInfo, nil) + if got := testutil.ToFloat64(h.collectors.graphIterations.WithLabelValues("success")); got != 1 { + t.Errorf("graph success = %v, want 1", got) + } +} + +func TestMetricsHandler_GraphMaxIterationsClassified(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + graphInfo := &callbacks.RunInfo{Component: components.Component("Graph")} + h.OnError(context.Background(), graphInfo, errors.New("graph reached max steps limit")) + if got := testutil.ToFloat64(h.collectors.graphIterations.WithLabelValues("max_iterations")); got != 1 { + t.Errorf("max_iterations counter = %v, want 1", got) + } +} + +func TestMetricsHandler_RegistererCollectorReuse(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h1 := NewMetricsHandler(MetricsDeps{Registerer: reg}) + h2 := NewMetricsHandler(MetricsDeps{Registerer: reg}) + if h1.collectors != h2.collectors { + t.Fatalf("expected the same collectors to be reused for the same registerer") + } +} + +func TestMetricsHandler_NeededGating(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + if !h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnEnd) { + t.Error("Tool OnEnd should be needed") + } + if h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnStartWithStreamInput) { + t.Error("Tool OnStartWithStreamInput should NOT be needed") + } +} diff --git a/internal/manager/biz/aiops/graph/callbacks/persistence.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/persistence.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/persistence.go rename to internal/manager/biz/aiops/runtime/v2/graph/callbacks/persistence.go diff --git a/internal/manager/biz/aiops/graph/callbacks/persistence_test.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/persistence_test.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/persistence_test.go rename to internal/manager/biz/aiops/runtime/v2/graph/callbacks/persistence_test.go diff --git a/internal/manager/biz/aiops/graph/callbacks/reasoning_test.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/reasoning_test.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/reasoning_test.go rename to internal/manager/biz/aiops/runtime/v2/graph/callbacks/reasoning_test.go diff --git a/internal/manager/biz/aiops/runtime/v2/graph/callbacks/sse.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/sse.go new file mode 100644 index 000000000..07d1b0251 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/sse.go @@ -0,0 +1,455 @@ +package callbacks + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +// SSEEventType enumerates the streaming events the SSE handler emits. +// — wire shapes mirror the legacy agent.Emit contract used +// in internal/manager/server/aiops/http.go so the SPA round-trips +// without changes after cutover. +type SSEEventType string + +const ( + // SSEEventAssistantStart fires before the ChatModel produces any + // content (— assistant_start frame). + SSEEventAssistantStart SSEEventType = "assistant_start" + // SSEEventAssistantDelta is the new token-level streaming frame + // (— "新增能力"). The delta payload only carries the + // incremental Content chunk so the SPA can append to the bubble. + SSEEventAssistantDelta SSEEventType = "assistant_delta" + // SSEEventAssistantEnd fires after the ChatModel returns. Payload + // matches the legacy `assistant` frame: full content + iteration + + // pending tool count. + SSEEventAssistantEnd SSEEventType = "assistant_end" + // SSEEventToolStart fires when a tool starts. Mirrors the legacy + // `tool_start` frame. + SSEEventToolStart SSEEventType = "tool_start" + // SSEEventToolEnd fires when a tool finishes. Mirrors the legacy + // `tool_end` frame; status carries success / error / timeout. + SSEEventToolEnd SSEEventType = "tool_end" + // SSEEventDone fires once at terminal success. + SSEEventDone SSEEventType = "done" + // SSEEventError fires on any unrecoverable graph error. + SSEEventError SSEEventType = "error" + // SSEEventTaskNotification fires when a background sub-agent worker + // reaches a terminal state The coordinator + // runtime emits this through the same SSE channel as the assistant + // frames so the SPA can render a worker-result tile inline. Payload + // shape lives on TaskNotificationPayload below. + SSEEventTaskNotification SSEEventType = "task_notification" +) + +// SSEEvent is the payload-typed envelope passed to the SSEEmitter. +// The handler stays format-agnostic (it doesn't know about HTTP / +// `event:` lines / JSON encoding); the cutover layer's emitter is +// responsible for encoding into wire frames so the existing http.go +// writeSSE routine stays intact. +type SSEEvent struct { + Type SSEEventType + Iteration int + Assistant *AssistantPayload // for assistant_start / assistant_end + Delta *AssistantDelta // for assistant_delta + Tool *ToolPayload // for tool_start / tool_end + Done *DonePayload // for done + Error *ErrorPayload // for error + Notification *TaskNotificationPayload // for task_notification +} + +// AssistantPayload is the start/end frame body. ContentSoFar is empty +// at start; on end it carries the model's full reply content. +// +// MessageID is the persisted chat_messages row id assigned by the +// PersistenceHandler when it wrote the assistant row. It's filled at +// assistant_end emit time by looking it up via the chain's +// AssistantIDReader (chained ordering: persistence runs before SSE). +// Empty when persistence is disabled or hasn't completed. +type AssistantPayload struct { + Iteration int + MessageID string + Content string + PendingToolCalls int + CreatedAt time.Time +} + +// AssistantDelta is the per-chunk streaming payload. The handler +// emits one for each non-empty Content chunk delivered by eino's +// stream output. +type AssistantDelta struct { + Iteration int + Content string +} + +// ToolPayload is the tool_start / tool_end frame body. +type ToolPayload struct { + ToolCallID string + Name string + ArgsJSON string + Status string // pending | success | error | timeout + StartedAt time.Time + EndedAt *time.Time + DurationMs int64 + ResultJSON string + Error string +} + +// DonePayload is the terminal success frame. +type DonePayload struct { + Iterations int +} + +// ErrorPayload is the terminal failure frame. +type ErrorPayload struct { + Message string + Code string +} + +// TaskNotificationPayload is the body of a task_notification frame +// Emitted by the coordinator runtime when a background +// worker spawned via AgentTool reaches a terminal state. The shape is +// stable wire because the SPA reads each field directly: +// +// event: task_notification +// data: { +// "task_id": "agent-12ab34cd", // worker identifier +// "status": "completed"|"failed"|"killed", +// "summary": "Agent incident-investigator completed", +// "result": "...final assistant content...", // present on completed +// "error": "context canceled", // present on failed/killed +// "usage": { // optional, may be empty +// "duration_ms": 12345 +// } +// } +type TaskNotificationPayload struct { + TaskID string `json:"task_id"` + Status string `json:"status"` + Summary string `json:"summary"` + Result string `json:"result,omitempty"` + Err string `json:"error,omitempty"` + Usage map[string]any `json:"usage,omitempty"` +} + +// SSEEmitter is the seam between the handler and the HTTP writer. +// The cutover layer builds an emitter that wraps the http.Flusher + +// writeSSE helper from internal/manager/server/aiops/http.go so the +// wire format stays byte-compatible. emit MUST NOT block — slow +// consumers must be handled (drop or buffer) inside the implementation +// Streaming non-blocking 约束. +type SSEEmitter func(SSEEvent) + +// SSEHandler translates graph callbacks into SSEEvents. +// +// spec: +// +// OnChatModelStart → assistant_start +// OnChatModelStream(chunk)→ assistant_delta +// OnChatModelEnd → assistant_end +// OnToolStart → tool_start +// OnToolEnd → tool_end +// OnGraphComplete → done +// OnError → error +// +// The graph "complete" event is emitted from OnEnd when info.Component +// is the wrapper graph itself (components.ComponentOfGraph) — eino +// fires the same OnEnd contract on graph-scope as on node-scope. +// +// Concurrency: the emitter is called sequentially within the lifetime +// of a single graph run; tool fan-out (compose ToolsNode parallel +// execution) emits per-tool start/end events that may interleave — +// the iteration counter is atomic so the assistant_start sequence is +// consistent. Cutover-layer emitter implementations should be +// goroutine-safe (the existing http.go writeSSE serializes through +// the response writer, which is already single-goroutine). +type SSEHandler struct { + emit SSEEmitter + + // iterations counts ChatModel turns for the assistant frame's + // `iteration` field. Atomic because tool fan-out callbacks may + // interleave. + iterations atomic.Int64 + + // toolStarts records per-tool start time to compute duration in OnEnd. + toolStartsMu sync.Mutex + toolStarts map[string]toolStart + + // assistantIDRelay (optional) is the cross-handler share with + // PersistenceHandler — see chain.go. NewDefaultHandlers sets it; + // nil when SSE is used standalone (tests). + assistantIDRelay *assistantIDRelay +} + +type toolStart struct { + at time.Time + argsJSON string + name string +} + +// NewSSEHandler builds the handler. Returns nil if emit is nil so the +// cutover layer can opt into SSE on a per-request basis. +func NewSSEHandler(emit SSEEmitter) *SSEHandler { + if emit == nil { + return nil + } + return &SSEHandler{ + emit: emit, + toolStarts: make(map[string]toolStart), + } +} + +// Needed gates which timings the handler observes. +func (h *SSEHandler) Needed(_ context.Context, info *callbacks.RunInfo, timing callbacks.CallbackTiming) bool { + if h == nil || info == nil { + return false + } + switch info.Component { + case components.ComponentOfChatModel: + switch timing { + case callbacks.TimingOnStart, callbacks.TimingOnEnd, callbacks.TimingOnError, callbacks.TimingOnEndWithStreamOutput: + return true + } + case components.ComponentOfTool: + switch timing { + case callbacks.TimingOnStart, callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + default: + // Graph / Workflow / Chain — observe terminal so we can emit + // `done` / `error` frames. + switch timing { + case callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + } + return false +} + +// OnStart fires before a component runs. +func (h *SSEHandler) OnStart(ctx context.Context, info *callbacks.RunInfo, input callbacks.CallbackInput) context.Context { + if h == nil || info == nil { + return ctx + } + switch info.Component { + case components.ComponentOfChatModel: + it := h.iterations.Add(1) + h.emit(SSEEvent{ + Type: SSEEventAssistantStart, + Iteration: int(it), + Assistant: &AssistantPayload{Iteration: int(it)}, + }) + case components.ComponentOfTool: + tin := einotool.ConvCallbackInput(input) + args := "" + if tin != nil { + args = tin.ArgumentsInJSON + } + startedAt := time.Now().UTC() + key := toolCallIDFromCtx(ctx, info) + h.toolStartsMu.Lock() + h.toolStarts[key] = toolStart{at: startedAt, argsJSON: args, name: info.Name} + h.toolStartsMu.Unlock() + h.emit(SSEEvent{ + Type: SSEEventToolStart, + Tool: &ToolPayload{ + ToolCallID: key, + Name: info.Name, + ArgsJSON: args, + Status: "pending", + StartedAt: startedAt, + }, + }) + } + return ctx +} + +// OnEnd fires after a component succeeds. +func (h *SSEHandler) OnEnd(ctx context.Context, info *callbacks.RunInfo, output callbacks.CallbackOutput) context.Context { + if h == nil || info == nil { + return ctx + } + switch info.Component { + case components.ComponentOfChatModel: + mo := einomodel.ConvCallbackOutput(output) + if mo == nil { + return ctx + } + var content string + var pending int + if mo.Message != nil { + content = mo.Message.Content + pending = len(mo.Message.ToolCalls) + } + it := int(h.iterations.Load()) + // h.assistantIDRelay carries the row id PersistenceHandler + // just stored — chain.go registers Persistence before SSE so + // by the time we emit here, the AppendMessage for this iter + // is done. Empty string when persistence is disabled + // (e.g. tests that wire only SSE). + h.emit(SSEEvent{ + Type: SSEEventAssistantEnd, + Iteration: it, + Assistant: &AssistantPayload{ + Iteration: it, + MessageID: h.assistantIDRelay.load(), + Content: content, + PendingToolCalls: pending, + CreatedAt: time.Now().UTC(), + }, + }) + case components.ComponentOfTool: + tout := einotool.ConvCallbackOutput(output) + key := toolCallIDFromCtx(ctx, info) + h.toolStartsMu.Lock() + ts, ok := h.toolStarts[key] + delete(h.toolStarts, key) + h.toolStartsMu.Unlock() + endedAt := time.Now().UTC() + var dur int64 + if ok { + dur = endedAt.Sub(ts.at).Milliseconds() + } + body := "" + if tout != nil { + body = tout.Response + } + h.emit(SSEEvent{ + Type: SSEEventToolEnd, + Tool: &ToolPayload{ + ToolCallID: key, + Name: info.Name, + ArgsJSON: ts.argsJSON, + Status: "success", + StartedAt: ts.at, + EndedAt: &endedAt, + DurationMs: dur, + ResultJSON: body, + }, + }) + default: + // Graph-scope OnEnd — emit `done`. + h.emit(SSEEvent{ + Type: SSEEventDone, + Done: &DonePayload{Iterations: int(h.iterations.Load())}, + }) + } + return ctx +} + +// OnError fires when a component returns a non-nil error. +func (h *SSEHandler) OnError(ctx context.Context, info *callbacks.RunInfo, err error) context.Context { + if h == nil || info == nil || err == nil { + return ctx + } + if info.Component == components.ComponentOfTool { + key := toolCallIDFromCtx(ctx, info) + h.toolStartsMu.Lock() + ts, ok := h.toolStarts[key] + delete(h.toolStarts, key) + h.toolStartsMu.Unlock() + endedAt := time.Now().UTC() + var dur int64 + if ok { + dur = endedAt.Sub(ts.at).Milliseconds() + } + status := "error" + if isDeadlineErr(err) { + status = "timeout" + } + h.emit(SSEEvent{ + Type: SSEEventToolEnd, + Tool: &ToolPayload{ + ToolCallID: key, + Name: info.Name, + ArgsJSON: ts.argsJSON, + Status: status, + StartedAt: ts.at, + EndedAt: &endedAt, + DurationMs: dur, + Error: err.Error(), + }, + }) + return ctx + } + // Top-level / ChatModel error — emit terminal `error` frame. + h.emit(SSEEvent{ + Type: SSEEventError, + Error: &ErrorPayload{Message: err.Error()}, + }) + return ctx +} + +// OnStartWithStreamInput is a no-op (we ignore streaming input). +func (h *SSEHandler) OnStartWithStreamInput(ctx context.Context, _ *callbacks.RunInfo, in *schema.StreamReader[callbacks.CallbackInput]) context.Context { + if in != nil { + in.Close() + } + return ctx +} + +// OnEndWithStreamOutput drains the streaming output of a ChatModel and +// fans assistant_delta frames to the SSE emitter as chunks land. +// — assistant_delta 是新增的 token-level 帧。 +// +// We close the receiver-side copy when done so eino can reclaim the +// stream goroutine; the original output stream continues to flow to +// downstream graph nodes. +func (h *SSEHandler) OnEndWithStreamOutput(ctx context.Context, info *callbacks.RunInfo, out *schema.StreamReader[callbacks.CallbackOutput]) context.Context { + if h == nil || info == nil || out == nil { + if out != nil { + out.Close() + } + return ctx + } + if info.Component != components.ComponentOfChatModel { + out.Close() + return ctx + } + go h.drainStream(out) + return ctx +} + +func (h *SSEHandler) drainStream(out *schema.StreamReader[callbacks.CallbackOutput]) { + defer out.Close() + for { + chunk, err := out.Recv() + if err != nil { + return + } + mo := einomodel.ConvCallbackOutput(chunk) + if mo == nil || mo.Message == nil { + continue + } + if mo.Message.Content == "" { + continue + } + it := int(h.iterations.Load()) + h.emit(SSEEvent{ + Type: SSEEventAssistantDelta, + Iteration: it, + Delta: &AssistantDelta{Iteration: it, Content: mo.Message.Content}, + }) + } +} + +// IterationCount returns the number of ChatModel turns the handler has +// observed in this run. Exposed for tests. +func (h *SSEHandler) IterationCount() int { + if h == nil { + return 0 + } + return int(h.iterations.Load()) +} + +// Compile-time check. +var ( + _ callbacks.Handler = (*SSEHandler)(nil) + _ callbacks.TimingChecker = (*SSEHandler)(nil) +) diff --git a/internal/manager/biz/aiops/runtime/v2/graph/callbacks/sse_test.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/sse_test.go new file mode 100644 index 000000000..39779c1a0 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/sse_test.go @@ -0,0 +1,222 @@ +package callbacks + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +// captureSSE collects SSEEvents emitted by the handler. Goroutine-safe +// — the handler is meant to be invoked from a single goroutine in +// production but tool fan-out + stream draining can interleave. +type captureSSE struct { + mu sync.Mutex + events []SSEEvent +} + +func (c *captureSSE) emit(e SSEEvent) { + c.mu.Lock() + c.events = append(c.events, e) + c.mu.Unlock() +} + +func (c *captureSSE) snapshot() []SSEEvent { + c.mu.Lock() + defer c.mu.Unlock() + cp := make([]SSEEvent, len(c.events)) + copy(cp, c.events) + return cp +} + +func (c *captureSSE) hasType(tp SSEEventType) bool { + for _, e := range c.snapshot() { + if e.Type == tp { + return true + } + } + return false +} + +func TestSSEHandler_NilEmitterReturnsNil(t *testing.T) { + t.Parallel() + if NewSSEHandler(nil) != nil { + t.Fatalf("expected nil when emitter is nil") + } +} + +func TestSSEHandler_AssistantStartEnd(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + ctx := context.Background() + h.OnStart(ctx, chatModelInfo(), &einomodel.CallbackInput{}) + h.OnEnd(ctx, chatModelInfo(), &einomodel.CallbackOutput{ + Message: &schema.Message{Role: schema.Assistant, Content: "done"}, + }) + events := cap.snapshot() + if len(events) != 2 { + t.Fatalf("expected 2 events got %d", len(events)) + } + if events[0].Type != SSEEventAssistantStart { + t.Errorf("event[0] = %s, want assistant_start", events[0].Type) + } + if events[0].Iteration != 1 { + t.Errorf("iteration[0] = %d, want 1", events[0].Iteration) + } + if events[1].Type != SSEEventAssistantEnd { + t.Errorf("event[1] = %s, want assistant_end", events[1].Type) + } + if events[1].Assistant == nil || events[1].Assistant.Content != "done" { + t.Errorf("end content payload missing: %+v", events[1].Assistant) + } +} + +func TestSSEHandler_ToolLifecycle(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + ctx := WithToolCallID(context.Background(), "tc-1") + h.OnStart(ctx, toolInfo("query_promql"), &einotool.CallbackInput{ArgumentsInJSON: `{"q":"up"}`}) + h.OnEnd(ctx, toolInfo("query_promql"), &einotool.CallbackOutput{Response: `{"ok":true}`}) + events := cap.snapshot() + if len(events) != 2 { + t.Fatalf("expected 2 events got %d", len(events)) + } + if events[0].Type != SSEEventToolStart { + t.Errorf("event[0] = %s, want tool_start", events[0].Type) + } + if events[0].Tool.ArgsJSON != `{"q":"up"}` { + t.Errorf("args = %q, want %q", events[0].Tool.ArgsJSON, `{"q":"up"}`) + } + if events[1].Type != SSEEventToolEnd { + t.Errorf("event[1] = %s, want tool_end", events[1].Type) + } + if events[1].Tool.Status != "success" { + t.Errorf("status = %s, want success", events[1].Tool.Status) + } + if events[1].Tool.ResultJSON != `{"ok":true}` { + t.Errorf("result = %q", events[1].Tool.ResultJSON) + } +} + +func TestSSEHandler_ToolError(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + ctx := WithToolCallID(context.Background(), "tc-2") + h.OnStart(ctx, toolInfo("flaky"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("flaky"), errors.New("kaboom")) + events := cap.snapshot() + last := events[len(events)-1] + if last.Type != SSEEventToolEnd { + t.Fatalf("last = %s, want tool_end", last.Type) + } + if last.Tool.Status != "error" { + t.Errorf("status = %s, want error", last.Tool.Status) + } + if last.Tool.Error != "kaboom" { + t.Errorf("error = %q", last.Tool.Error) + } +} + +func TestSSEHandler_TopLevelErrorEmitsError(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + graphInfo := &callbacks.RunInfo{Name: "OngridReActAgent", Component: components.Component("Graph")} + h.OnError(context.Background(), graphInfo, errors.New("graph went bad")) + if !cap.hasType(SSEEventError) { + t.Fatalf("expected error event, got %+v", cap.snapshot()) + } +} + +func TestSSEHandler_GraphEndEmitsDone(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + graphInfo := &callbacks.RunInfo{Name: "OngridReActAgent", Component: components.Component("Graph")} + h.OnEnd(context.Background(), graphInfo, nil) + if !cap.hasType(SSEEventDone) { + t.Fatalf("expected done event, got %+v", cap.snapshot()) + } +} + +func TestSSEHandler_NeededGating(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + if !h.Needed(context.Background(), chatModelInfo(), callbacks.TimingOnStart) { + t.Error("ChatModel start should be needed") + } + if !h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnEnd) { + t.Error("Tool end should be needed") + } + graphInfo := &callbacks.RunInfo{Component: components.Component("Graph")} + if !h.Needed(context.Background(), graphInfo, callbacks.TimingOnEnd) { + t.Error("Graph end should be needed (drives done event)") + } +} + +func TestSSEHandler_IterationCount(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + ctx := context.Background() + for i := 0; i < 3; i++ { + h.OnStart(ctx, chatModelInfo(), &einomodel.CallbackInput{}) + h.OnEnd(ctx, chatModelInfo(), &einomodel.CallbackOutput{Message: &schema.Message{Role: schema.Assistant}}) + } + if h.IterationCount() != 3 { + t.Errorf("IterationCount = %d, want 3", h.IterationCount()) + } +} + +// TestSSEEvent_TaskNotification_Shape pins the wire-shape contract for +// the task_notification frame so the SPA's task_notification +// renderer doesn't drift. The emitter is invoked directly here — the +// runtime is the one that fires this event in production (it bypasses +// the graph callback chain and feeds the SSE channel directly). +func TestSSEEvent_TaskNotification_Shape(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + cap.emit(SSEEvent{ + Type: SSEEventTaskNotification, + Notification: &TaskNotificationPayload{ + TaskID: "agent-12ab34cd", + Status: "completed", + Summary: "Agent incident-investigator completed", + Result: "the answer is 42", + Usage: map[string]any{"duration_ms": int64(1234)}, + }, + }) + events := cap.snapshot() + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + if events[0].Type != SSEEventTaskNotification { + t.Errorf("type = %q, want task_notification", events[0].Type) + } + n := events[0].Notification + if n == nil { + t.Fatalf("Notification nil") + } + if n.TaskID != "agent-12ab34cd" { + t.Errorf("task_id = %q", n.TaskID) + } + if n.Status != "completed" { + t.Errorf("status = %q", n.Status) + } + if n.Result != "the answer is 42" { + t.Errorf("result = %q", n.Result) + } + if got, _ := n.Usage["duration_ms"].(int64); got != 1234 { + t.Errorf("usage.duration_ms = %v", n.Usage["duration_ms"]) + } +} diff --git a/internal/manager/biz/aiops/graph/callbacks/tracing.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/tracing.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/tracing.go rename to internal/manager/biz/aiops/runtime/v2/graph/callbacks/tracing.go diff --git a/internal/manager/biz/aiops/graph/callbacks/tracing_test.go b/internal/manager/biz/aiops/runtime/v2/graph/callbacks/tracing_test.go similarity index 100% rename from internal/manager/biz/aiops/graph/callbacks/tracing_test.go rename to internal/manager/biz/aiops/runtime/v2/graph/callbacks/tracing_test.go diff --git a/internal/manager/biz/aiops/graph/react.go b/internal/manager/biz/aiops/runtime/v2/graph/react.go similarity index 100% rename from internal/manager/biz/aiops/graph/react.go rename to internal/manager/biz/aiops/runtime/v2/graph/react.go diff --git a/internal/manager/biz/aiops/graph/react_test.go b/internal/manager/biz/aiops/runtime/v2/graph/react_test.go similarity index 100% rename from internal/manager/biz/aiops/graph/react_test.go rename to internal/manager/biz/aiops/runtime/v2/graph/react_test.go diff --git a/internal/manager/biz/aiops/graph/token_budget_test.go b/internal/manager/biz/aiops/runtime/v2/graph/token_budget_test.go similarity index 100% rename from internal/manager/biz/aiops/graph/token_budget_test.go rename to internal/manager/biz/aiops/runtime/v2/graph/token_budget_test.go diff --git a/internal/manager/biz/aiops/runtime/v2/graph/tool_adapter.go b/internal/manager/biz/aiops/runtime/v2/graph/tool_adapter.go new file mode 100644 index 000000000..22e0fe0e4 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/graph/tool_adapter.go @@ -0,0 +1,522 @@ +package graph + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" + "github.com/eino-contrib/jsonschema" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + "github.com/ongridio/ongrid/internal/pkg/tracing" +) + +// toolMemo is a per-run cache of identical (read-tool, args) calls. The +// graph — and therefore the wrapped tool set — is rebuilt per request, so +// one memo per WrapBaseTools call is naturally scoped to a single ReAct +// run. Within that run an identical call (same tool, byte-identical args, +// seconds apart) cannot yield new information and is almost always a ReAct +// loop artifact; returning the prior result skips re-executing an expensive +// tool (SSH probe, PromQL, LLM-backed query_translate) and keeps an +// identical-call loop from burning the iteration budget on real work. Only +// Class=="read" tools are memoized — write/destructive tools never touch +// this path, so the review/mutation flow is unaffected. +type toolMemo struct { + mu sync.Mutex + m map[string]string // (tool\x00args) -> result, identical-call cache + counts map[string]int // tool name -> distinct executions this run + last map[string]string // tool name -> most recent successful result this run +} + +func newToolMemo() *toolMemo { + return &toolMemo{m: make(map[string]string), counts: make(map[string]int), last: make(map[string]string)} +} + +func (t *toolMemo) get(k string) (string, bool) { + t.mu.Lock() + defer t.mu.Unlock() + v, ok := t.m[k] + return v, ok +} + +func (t *toolMemo) put(k, v string) { + t.mu.Lock() + defer t.mu.Unlock() + t.m[k] = v +} + +// count returns how many real executions of the named tool have happened +// this run; bump records one more. +func (t *toolMemo) count(name string) int { + t.mu.Lock() + defer t.mu.Unlock() + return t.counts[name] +} + +func (t *toolMemo) bump(name string) { + t.mu.Lock() + defer t.mu.Unlock() + t.counts[name]++ +} + +func (t *toolMemo) reserve(name string, perToolLimit int) (int, bool) { + t.mu.Lock() + defer t.mu.Unlock() + if perToolLimit > 0 && t.counts[name] >= perToolLimit { + return t.counts[name], false + } + t.counts[name]++ + return t.counts[name], true +} + +func (t *toolMemo) putLast(name, result string) { + t.mu.Lock() + defer t.mu.Unlock() + t.last[name] = result +} + +func (t *toolMemo) lastResult(name string) (string, bool) { + t.mu.Lock() + defer t.mu.Unlock() + result, ok := t.last[name] + return result, ok +} + +const ( + toolNameDraftConfigChange = "draft_config_change" + toolNameListMetricCatalog = "list_metric_catalog" + toolNameQueryPromQL = "query_promql" +) + +// maxToolCallsPerRun caps how many times any one tool may EXECUTE within a +// single agent run. Identical-arg repeats are served from the memo and don't +// count; this catches the other failure mode — the model calling the same +// tool many times with slightly different args (e.g. query_promql across a +// dozen metrics, query_alert_rules over and over) without converging. Past +// the cap the tool returns a "synthesize now" directive instead of running, +// which forces the agent to answer from what it already gathered. Generous +// enough that normal multi-step investigation isn't clipped. A 30-call cap +// merely turns a wrong route into a long, expensive failure, so the general +// limit is intentionally small. There is deliberately no aggregate cap: +// different tools contribute different evidence, so only repeated use of the +// same tool is circuit-broken within one user turn. +const maxToolCallsPerRun = 10 + +func maxCallsForTool(name string) int { + switch name { + case "draft_config_change": + // Only a confirmable config_draft increments this counter; + // config_validation_failed remains retryable so the model can repair + // a draft in the same user turn. + return 1 + default: + return maxToolCallsPerRun + } +} + +func countFailedToolCall(name string) bool { + switch name { + case "draft_config_change": + return false + default: + return true + } +} + +func reservesBeforeExecution(name string) bool { + return name != toolNameDraftConfigChange +} + +func countSuccessfulToolCall(name, result string) bool { + if name != toolNameDraftConfigChange { + return true + } + var raw struct { + Kind string `json:"kind"` + DraftHash string `json:"draft_hash"` + } + if err := json.Unmarshal([]byte(result), &raw); err != nil { + return true + } + return strings.EqualFold(strings.TrimSpace(raw.Kind), "config_draft") && + strings.TrimSpace(raw.DraftHash) != "" +} + +type draftConfigChangeGateArgs struct { + Domain string `json:"domain"` + Rule struct { + Kind string `json:"kind"` + Conditions []struct{} `json:"conditions"` + Spec map[string]interface{} `json:"spec"` + } `json:"rule"` +} + +func (a *einoToolAdapter) draftMetricCatalogPreflight(argumentsInJSON string) (string, bool) { + if a.memo == nil || a.cacheName != toolNameDraftConfigChange { + return "", false + } + if !draftConfigChangeNeedsMetricCatalog(argumentsInJSON) { + return "", false + } + if _, ok := a.memo.lastResult(toolNameListMetricCatalog); !ok { + return metricCatalogRequiredResult(), true + } + return "", false +} + +func draftConfigChangeNeedsMetricCatalog(argumentsInJSON string) bool { + var in draftConfigChangeGateArgs + if err := json.Unmarshal([]byte(argumentsInJSON), &in); err != nil { + return false + } + domain := strings.ToLower(strings.TrimSpace(in.Domain)) + if domain != "" && domain != "alert_rule" { + return false + } + if len(in.Rule.Conditions) > 0 { + return true + } + kind := strings.ToLower(strings.TrimSpace(in.Rule.Kind)) + switch kind { + case "metric_threshold", "metric_raw", "metric_anomaly", "metric_forecast", "metric_burn_rate", + "trace_latency", "trace_error_rate": + return true + case "log_search", "log_match", "log_volume": + return false + case "": + return len(in.Rule.Spec) > 0 + default: + return false + } +} + +func metricCatalogRequiredResult() string { + return toolResultJSON(map[string]interface{}{ + "status": "blocked", + "error": "metric_catalog_required", + "instruction": "Metric-based alert-rule drafts must call list_metric_catalog once earlier in this same user turn. Use the returned metric names and sample_labels to build the rule, then call draft_config_change again.", + }) +} + +func toolResultJSON(fields map[string]interface{}) string { + b, err := json.Marshal(fields) + if err != nil { + return `{"status":"blocked","error":"tool_result_marshal_failed"}` + } + return string(b) +} + +// toolBudgetExceeded is the synthetic tool result returned once a tool hits +// maxToolCallsPerRun. Shaped like a normal JSON tool result so the LLM reads +// it as data and (re)directs to answering. +func toolBudgetExceeded(name string, n int) string { + instruction := fmt.Sprintf("PER-TOOL CIRCUIT BREAKER. You have already called %q %d times in the current user turn — that limit applies only to this tool and expires on the next user message. Do NOT call this tool again. Synthesize from gathered evidence, or call a different tool only when it provides materially different evidence. Do not substitute another tool merely to continue the same query loop.", name, n) + if name == toolNameQueryPromQL { + instruction = fmt.Sprintf("PER-TOOL CIRCUIT BREAKER. You have already called %q %d times in the current user turn. Do not call query_promql again this turn. Synthesize from gathered metrics, or use a different telemetry tool only when it provides materially different evidence. If metric evidence is insufficient, explain what is missing and suggest one aggregated PromQL expression for the next user turn.", name, n) + } + b, err := json.Marshal(struct { + Status string `json:"status"` + Tool string `json:"tool"` + Calls int `json:"calls"` + Scope string `json:"scope"` + FinalAnswer bool `json:"final_answer_required"` + Instruction string `json:"instruction"` + }{ + Status: "call_budget_exceeded", + Tool: name, + Calls: n, + Scope: "current_user_turn", + FinalAnswer: false, + Instruction: instruction, + }) + if err != nil { + return fmt.Sprintf(`{"status":"call_budget_exceeded","scope":"current_user_turn","instruction":%q}`, instruction) + } + return string(b) +} + +// WrapBaseTool adapts an ongrid basetool.BaseTool to eino's +// components/tool.BaseTool + InvokableTool surface so the eino ToolsNode +// can dispatch to it. PR-3's basetool was deliberately mirror-shaped +// against eino (see basetool.go header comment), so this adapter is +// thin: Info is a 1-1 field copy and InvokableRun forwards the args +// JSON verbatim. +// +// graph 执行层 ToolsNode 接收的是 eino +// tool.BaseTool；本 adapter 是仓库自家 BaseTool 与 eino 之间唯一胶水点。 +// +// Per-call options (tenant / user / device id) ride on +// `basetool.InvokeOption` slots; eino's `tool.Option` system carries an +// impl-specific bag for them — see WithInvokeOpts. If the caller does +// not pass any impl-specific options the inner tool runs with its +// decorator-resolved defaults (the typical path). +func WrapBaseTool(t basetool.BaseTool) einotool.InvokableTool { + if t == nil { + return nil + } + return &einoToolAdapter{inner: t} +} + +// einoInvokeOptKey is the internal carrier for ongrid InvokeOptions +// passed through eino's `tool.Option` slot. Unexported so callers +// route through WithInvokeOpts. +type einoInvokeOptKey struct { + opts []basetool.InvokeOption + persistence ToolInvocationPersistence +} + +// ToolInvocationPersistence records the lifecycle of an actual adapter +// invocation. Eino's nested ToolsNode does not reliably propagate component +// callbacks to the outer graph, so the adapter is the only reliable place to +// persist a tool result before the next model turn is assembled. +type ToolInvocationPersistence interface { + PersistToolStart(ctx context.Context, toolName, argumentsInJSON string) + PersistToolEnd(ctx context.Context, toolName, response string) +} + +// WithToolInvocationPersistence passes the persistence sink through Eino's +// documented per-tool option channel. ToolsNode may derive a child context +// without request values, but it preserves tool options for InvokableRun. +func WithToolInvocationPersistence(sink ToolInvocationPersistence) einotool.Option { + return einotool.WrapImplSpecificOptFn(func(k *einoInvokeOptKey) { + k.persistence = sink + }) +} + +// WithInvokeOpts is the eino-side option helper that carries +// basetool.InvokeOption into a ToolsNode call. The graph wiring layer +// (PR-N chatruntime) will use this to thread per-request tenant / user +// id through the graph runtime down to each tool's InvokableRun call. +// +// Usage from a graph client: +// +// runnable.Invoke(ctx, in, compose.WithToolsNodeOption( +// compose.WithToolOption(graph.WithInvokeOpts( +// basetool.WithUserID(uid), +// basetool.WithTenant(tenantID), +// ), +// ) +func WithInvokeOpts(opts ...basetool.InvokeOption) einotool.Option { + return einotool.WrapImplSpecificOptFn(func(k *einoInvokeOptKey) { + k.opts = append(k.opts, opts...) + }) +} + +// einoToolAdapter wraps a basetool.BaseTool to satisfy eino's +// InvokableTool interface. The struct is intentionally trivial — all +// real behaviour (tenant/audit/timeout/ratelimit/metric) lives in the +// PR-3 decorator chain wrapped *around* the inner tool *before* it +// reaches this adapter. +type einoToolAdapter struct { + inner basetool.BaseTool + persistence ToolInvocationPersistence + // memo is the per-run identical-call cache (nil = memoization off, e.g. + // the single-tool WrapBaseTool path used by tests). Shared across all + // adapters built by one WrapBaseTools call. + memo *toolMemo + // Info() is resolved once (name + read-ness) for the memo key + gate. + infoOnce sync.Once + cacheName string + cacheable bool // Class == "read" +} + +// resolveInfo lazily caches the tool's name + whether it's a pure-read tool +// (the only class we memoize). Info() is otherwise called by eino at build +// time; caching here avoids a call per dispatch. +func (a *einoToolAdapter) resolveInfo(ctx context.Context) { + a.infoOnce.Do(func() { + if info, err := a.inner.Info(ctx); err == nil && info != nil { + a.cacheName = info.Name + a.cacheable = info.Class == "read" + } + }) +} + +// Info returns the eino schema.ToolInfo for this tool. WhenToUse from +// our extended ToolInfo is appended to the description (with a +// "When to use:" prefix) so the LLM sees both halves through the +// standard schema field. — Tool 层 description vs +// when_to_use 拆分。 +func (a *einoToolAdapter) Info(ctx context.Context) (*schema.ToolInfo, error) { + if a == nil || a.inner == nil { + return nil, fmt.Errorf("graph: tool adapter has nil inner tool") + } + info, err := a.inner.Info(ctx) + if err != nil { + return nil, err + } + if info == nil { + return nil, fmt.Errorf("graph: tool returned nil ToolInfo") + } + desc := info.Description + if info.WhenToUse != "" { + if desc != "" { + desc = desc + "\n\nWhen to use: " + info.WhenToUse + } else { + desc = "When to use: " + info.WhenToUse + } + } + out := &schema.ToolInfo{ + Name: info.Name, + Desc: desc, + } + if len(info.Parameters) > 0 { + // Preserve the existing JSON-Schema bytes verbatim by re-parsing + // into eino's jsonschema.Schema. PR-3's basetool.ToolInfo carries + // the schema as raw JSON; eino's ParamsOneOf wants a typed + // *jsonschema.Schema, so we deserialize. A failure here means the + // upstream tool produced invalid JSON Schema — bubble it as an + // error so the graph build refuses to compile. + js := &jsonschema.Schema{} + if err := json.Unmarshal(info.Parameters, js); err != nil { + return nil, fmt.Errorf("graph: tool %q: parse parameters JSON Schema: %w", info.Name, err) + } + out.ParamsOneOf = schema.NewParamsOneOfByJSONSchema(js) + } + return out, nil +} + +// InvokableRun forwards to the inner basetool.BaseTool. Per-call +// InvokeOptions are extracted from the eino tool.Option bag if the +// caller used WithInvokeOpts. +// +// **Tool errors are converted to a JSON envelope, never returned as a +// Go error.** Eino's ToolsNode treats Go-level errors as graph-fatal +// (terminates the whole invoke + SSE stream); ongrid's invariant is +// "tool failures are facts the LLM can recover from" — the LLM should +// see the error text as a tool result and decide to retry / switch / +// ask the user, NOT have the conversation aborted. We mirror what the +// legacy agent.go for-loop did: marshal err into a result-shaped JSON +// like {"error": "..."} so the LLM consumes it as data. +// +// True nil-receiver / unrecoverable bugs (we wrote the wrong inner) +// still surface as Go error so eino can panic-loud, since those are +// not user-fixable. +func (a *einoToolAdapter) InvokableRun(ctx context.Context, argumentsInJSON string, opts ...einotool.Option) (out string, err error) { + if a == nil || a.inner == nil { + return "", fmt.Errorf("graph: tool adapter has nil inner tool") + } + // Persist at the adapter boundary instead of depending on the nested + // ToolsNode callback. The latter may omit Tool.OnEnd, leaving a valid tool + // execution represented as an auto-healed failure in chat history. + a.resolveInfo(ctx) + resolved := einotool.GetImplSpecificOptions(&einoInvokeOptKey{}, opts...) + sink := a.persistence + if sink == nil { + sink = resolved.persistence + } + if sink != nil && a.cacheName != "" { + sink.PersistToolStart(ctx, a.cacheName, argumentsInJSON) + defer func() { sink.PersistToolEnd(ctx, a.cacheName, out) }() + } + var memoKey string + if a.memo != nil { + // 1. Identical-call memo (read tools only): a byte-identical repeat + // returns the prior result without re-executing. + if a.cacheable && a.cacheName != "" { + memoKey = a.cacheName + "\x00" + argumentsInJSON + if cached, ok := a.memo.get(memoKey); ok { + return cached, nil + } + } + // 2. Per-tool execution cap (all tools): once a tool has run + // maxToolCallsPerRun times this run, stop executing it and hand + // back a "synthesize now" directive. Catches the distinct-args + // repeat loop (query_promql/query_alert_rules called over and over) + // that the memo can't. + if blocked, ok := a.draftMetricCatalogPreflight(argumentsInJSON); ok { + return blocked, nil + } + if a.cacheName != "" { + if reservesBeforeExecution(a.cacheName) { + if n, ok := a.memo.reserve(a.cacheName, maxCallsForTool(a.cacheName)); !ok { + return toolBudgetExceeded(a.cacheName, n), nil + } + } else if a.memo.count(a.cacheName) >= maxCallsForTool(a.cacheName) { + return toolBudgetExceeded(a.cacheName, a.memo.count(a.cacheName)), nil + } + } + } + ctx, span := otel.Tracer("github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph").Start(ctx, "aiops.Tool."+a.cacheName, + trace.WithAttributes(attribute.String("aiops.tool.name", a.cacheName)), + ) + var spanErr error + defer func() { tracing.EndSpan(span, spanErr) }() + out, err = a.inner.InvokableRun(ctx, argumentsInJSON, resolved.opts...) + if err != nil { + spanErr = err + // Count most failures toward the cap so a failing tool cannot be + // hammered. draft_config_change is the exception: validation failures + // are common while the model corrects structured config args, and only + // a successful draft should consume the one-draft-per-turn budget. + if a.memo != nil && a.cacheName != "" && countFailedToolCall(a.cacheName) && !reservesBeforeExecution(a.cacheName) { + a.memo.bump(a.cacheName) + } + // Re-shape as a tool-result-style JSON so the LLM gets it as a + // message instead of having the graph terminate. Truncate long + // errors so we don't blow the context window with stack traces. + msg := err.Error() + const cap = 2048 + if len(msg) > cap { + msg = msg[:cap] + "...(truncated)" + } + envelope, mErr := json.Marshal(map[string]any{ + "error": msg, + "status": "failed", + }) + if mErr != nil { + // Marshal of a string + status into a 2-key map should be + // infallible; if it isn't, fall back to the original error. + return "", err + } + return string(envelope), nil + } + // Count this successful real execution toward the per-tool cap. A + // config_validation_failed result from draft_config_change is a normal + // repair signal, not a confirmable draft, so it remains retryable within + // the same user turn. Only a real config_draft consumes the one-draft cap. + if a.memo != nil && a.cacheName != "" { + if countSuccessfulToolCall(a.cacheName, out) { + if !reservesBeforeExecution(a.cacheName) { + a.memo.bump(a.cacheName) + } + } + a.memo.putLast(a.cacheName, out) + } + // Cache successful read-tool results only — a failed call stays + // retryable (a transient error shouldn't be pinned for the whole run). + if memoKey != "" { + a.memo.put(memoKey, out) + } + return out, nil +} + +// WrapBaseTools is the slice-flavoured WrapBaseTool. Returns a slice of +// eino tool.BaseTool ready to feed into compose.ToolsNodeConfig.Tools. +// Nil entries in the input are skipped so callers can pass a sparse +// list (e.g. from a skill activation filter). +func WrapBaseTools(tools []basetool.BaseTool, persistence ...ToolInvocationPersistence) []einotool.BaseTool { + // One memo shared by every tool in this build = scoped to one run + // (the graph is rebuilt per request). The single-tool WrapBaseTool + // path deliberately leaves memo nil (tests / non-graph callers). + memo := newToolMemo() + var sink ToolInvocationPersistence + if len(persistence) > 0 { + sink = persistence[0] + } + out := make([]einotool.BaseTool, 0, len(tools)) + for _, t := range tools { + if t == nil { + continue + } + out = append(out, &einoToolAdapter{inner: t, memo: memo, persistence: sink}) + } + return out +} diff --git a/internal/manager/biz/aiops/graph/tool_adapter_test.go b/internal/manager/biz/aiops/runtime/v2/graph/tool_adapter_test.go similarity index 100% rename from internal/manager/biz/aiops/graph/tool_adapter_test.go rename to internal/manager/biz/aiops/runtime/v2/graph/tool_adapter_test.go diff --git a/internal/manager/biz/aiops/graph/types.go b/internal/manager/biz/aiops/runtime/v2/graph/types.go similarity index 100% rename from internal/manager/biz/aiops/graph/types.go rename to internal/manager/biz/aiops/runtime/v2/graph/types.go diff --git a/internal/manager/biz/aiops/runtime/v2/source.json b/internal/manager/biz/aiops/runtime/v2/source.json new file mode 100644 index 000000000..4d9cf5520 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/source.json @@ -0,0 +1,451 @@ +{ + "release": "v0.18.2", + "commit": "ad9e52f670aa8a23976942954acbf103e1a6ef2d", + "files": [ + { + "path": "chatruntime/agent_parser.go", + "source_blob": "3d6fd34deb5ee03071d928b18757bdf40e13b1a5", + "mode": "100644" + }, + { + "path": "chatruntime/agent_parser_test.go", + "source_blob": "dc92fd7639b917990429afc2350ba9c7a0bdbfdb", + "mode": "100644" + }, + { + "path": "chatruntime/agent_registry.go", + "source_blob": "c6ca9c8d92fcc545b75b90be1c4bd571696d5cbb", + "mode": "100644" + }, + { + "path": "chatruntime/agent_registry_test.go", + "source_blob": "bbf811b7de9dc8d81c3907f1abab6d2f3b49654f", + "mode": "100644" + }, + { + "path": "chatruntime/bare_skills_test.go", + "source_blob": "390c0df30112b32a2987d1d08565fb99f7b3602d", + "mode": "100644" + }, + { + "path": "chatruntime/chatmodelopts_test.go", + "source_blob": "dcfd314dc03f42faeb82b2a29d03c45c7cebf63f", + "mode": "100644" + }, + { + "path": "chatruntime/command_parser.go", + "source_blob": "a294aa1283d3dd5c7897495e4a7052fd1baa9e85", + "mode": "100644" + }, + { + "path": "chatruntime/command_parser_test.go", + "source_blob": "19534a03fb3160d0dad4d5694f7548626d0a2f51", + "mode": "100644" + }, + { + "path": "chatruntime/config_confirm.go", + "source_blob": "173c8aaf00280911b54905417c20b4327693cc51", + "mode": "100644" + }, + { + "path": "chatruntime/decision.go", + "source_blob": "59554ebd5b1a257070db69575e7834145837fef3", + "mode": "100644" + }, + { + "path": "chatruntime/decision_test.go", + "source_blob": "e11fbb347b07b23c1e78c09bdfcf540a7ff4a802", + "mode": "100644" + }, + { + "path": "chatruntime/k8s_action_e2e_test.go", + "source_blob": "203e91bb7a0c7d64a106b4a834213989ec59433f", + "mode": "100644" + }, + { + "path": "chatruntime/load_all.go", + "source_blob": "304ef393320a1f290576f691c572208d0d0b394f", + "mode": "100644" + }, + { + "path": "chatruntime/load_all_test.go", + "source_blob": "bebe1581af2035f6073b51ed70409543ead883a7", + "mode": "100644" + }, + { + "path": "chatruntime/plugin_container.go", + "source_blob": "896cb8c758d6d6f308db9e62c313af1008d93cff", + "mode": "100644" + }, + { + "path": "chatruntime/plugin_container_test.go", + "source_blob": "93fc67b3e2d885b47ed2fbf2e3b3074dfa5aaae5", + "mode": "100644" + }, + { + "path": "chatruntime/reasoning_test.go", + "source_blob": "77ab5b2e448d9f0916d7c03ab52594cb307ed572", + "mode": "100644" + }, + { + "path": "chatruntime/runtime.go", + "source_blob": "e2c958ea31047274471f1e5a44227b22b73d42d5", + "mode": "100644" + }, + { + "path": "chatruntime/runtime_test.go", + "source_blob": "04cf4a87567c025b77953509113b908f18a458b5", + "mode": "100644" + }, + { + "path": "chatruntime/runtime_worker_test.go", + "source_blob": "c26286dea2fe2719214a96cb6bbb28aa513348c0", + "mode": "100644" + }, + { + "path": "chatruntime/skill_parser.go", + "source_blob": "0aa1f698962db68ce06356cd611a72c22ec93c32", + "mode": "100644" + }, + { + "path": "chatruntime/skill_parser_test.go", + "source_blob": "abe27da3b74fa13750101a645d5f881fd5288290", + "mode": "100644" + }, + { + "path": "chatruntime/skill_registry.go", + "source_blob": "d7ffc79c986c7575c4e8f6bfc08982864a3a6290", + "mode": "100644" + }, + { + "path": "chatruntime/skill_registry_test.go", + "source_blob": "20496349467411cd59082d37002efa4a616600e1", + "mode": "100644" + }, + { + "path": "chatruntime/system_prompt.go", + "source_blob": "032edaa61590bc33476eb6987417744b24e8227e", + "mode": "100644" + }, + { + "path": "chatruntime/system_prompt_test.go", + "source_blob": "b311121fe6492b58559dd7d3ad1656784978289d", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_parser/minimal/incident-investigator.md", + "source_blob": "0ab578a58dd808b8d932f9a412f086eef80aade2", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_parser/missing_when_to_use/foo.md", + "source_blob": "a8a7d82cf5e3186519c0b1600f3fc45fcbec350a", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md", + "source_blob": "ac8e534216a81e1281ae586bf3faf297152977d1", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md", + "source_blob": "6126be6350121eaa920551219a8512a075e25dac", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_registry/multi/README.md", + "source_blob": "fe1b6dbad2476cb6ca521af1e51416325f348eec", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_registry/multi/incident-investigator.md", + "source_blob": "3e9cd127eb07d1b576e6f702a72b29a4a42bf7f3", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/agent_registry/multi/reviewer.md", + "source_blob": "2b5b7571f4e6a37ae953da9f821dbb6df3736334", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json", + "source_blob": "35a928395bf934cb1ea3c5da3d66708cc37b1cae", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json", + "source_blob": "7c92e55c3a2fefdde62ef3a49151e99158da9b25", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json", + "source_blob": "da39e4ffafe816be90259a3f68b763a3f71b93ed", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md", + "source_blob": "2f5cd59517e9ac7c24259f30dcbb2d386af164b4", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json", + "source_blob": "90566062121c9cc4af73a1fc2b48aa8e193c443f", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json", + "source_blob": "90b0c506aefd8bc4ba2f65a9fd67ddb3dfc3e379", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/path_traversal/skills/pwned", + "source_blob": "6998f20a07b1d03c62a19b010d26e3844be24112", + "mode": "120000" + }, + { + "path": "chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md", + "source_blob": "dba27c4eb35eb7bb75233992c014a4cec75bbba5", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json", + "source_blob": "b4853c2c112529ca01548882e3ace065c7c4978e", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/.mcp.json", + "source_blob": "5392ca626579dde6e2b956e632f557d162fc9b83", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md", + "source_blob": "0265b69d1e3f28d934912f51041404b1ef569d07", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md", + "source_blob": "a38e3afedc44973b49226263f88571122bf64345", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md", + "source_blob": "2f5cd59517e9ac7c24259f30dcbb2d386af164b4", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md", + "source_blob": "14870376331ae58968a2ade178201029ae684cdb", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json", + "source_blob": "72b2c845b9b1b346754d47cbb17c0e3dce936d35", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md", + "source_blob": "b820c41153888824658ad32a49d1d65b3e81f513", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md", + "source_blob": "baf6620bd2f182e8fd49223c4c2e1aaecc8c419d", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/minimal/SKILL.md", + "source_blob": "14870376331ae58968a2ade178201029ae684cdb", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/missing_description/SKILL.md", + "source_blob": "a517a9e42c235a311ef34f9a0798f30f19bfaa65", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/name_with_dash/SKILL.md", + "source_blob": "7b9cb73116176c3cad3ade313ef3f84105b7b73b", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/typical/SKILL.md", + "source_blob": "60ce5c8422705cf8b27c2446f09cc326e204addf", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md", + "source_blob": "b5143a6a10f9b10ad2769863c918d9f8ebc5e22d", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md", + "source_blob": "c428a8e0cddf8e21e89c5c74ab874dce223f0d86", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md", + "source_blob": "e935d3aaac26cbf723acf72bde3bd1f39a3e2e5a", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md", + "source_blob": "1b5ea8da6600914beecb051517fd998da5c1711b", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md", + "source_blob": "82023343213f597789836b8de5263a4a2912d7b0", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md", + "source_blob": "a1f04f7194c7505bf096285e9f5fc6cdcfa2c81e", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md", + "source_blob": "e038d9d44835a98ca2300d95326efd1baa1b8a0f", + "mode": "100644" + }, + { + "path": "chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md", + "source_blob": "9412ba7d741aa3052fd82d77b83dc060d1a24f1b", + "mode": "100644" + }, + { + "path": "chatruntime/tracing.go", + "source_blob": "1221905213e64235588fc0227008ac5d55e2760a", + "mode": "100644" + }, + { + "path": "chatruntime/tracing_test.go", + "source_blob": "a3dcb67f60cddffbb18cf1cc4f4749925bb7f327", + "mode": "100644" + }, + { + "path": "chatruntime/types.go", + "source_blob": "81c2a78e65efcc443db2ac427ede67b803ed0458", + "mode": "100644" + }, + { + "path": "chatruntime/worker.go", + "source_blob": "f7ddcd89d786f7e1c318515287323bb55777205e", + "mode": "100644" + }, + { + "path": "chatruntime/worker_filter_test.go", + "source_blob": "038eb61b1aec33cf7074d50f236ec47a0bf6c996", + "mode": "100644" + }, + { + "path": "graph/budget_stop_model.go", + "source_blob": "828ff306d86bd9ec5a3a54ebc9e4a4323a0afa1f", + "mode": "100644" + }, + { + "path": "graph/callbacks/alert_draft_guard.go", + "source_blob": "7c769fad1c9b9476dc4083dd98cfc46f5d635777", + "mode": "100644" + }, + { + "path": "graph/callbacks/alert_draft_guard_test.go", + "source_blob": "9777a3de243413562ebfd3e344bff61798cc8f86", + "mode": "100644" + }, + { + "path": "graph/callbacks/audit.go", + "source_blob": "fd990b8c22a1d60a418c0e5c5be5d0ae549335d3", + "mode": "100644" + }, + { + "path": "graph/callbacks/audit_test.go", + "source_blob": "10d0689b2d8696541d1d0d5f07db2318a0ca0e78", + "mode": "100644" + }, + { + "path": "graph/callbacks/chain.go", + "source_blob": "3a6865fe2bf433db56e7f50e2d80afcb72fc389e", + "mode": "100644" + }, + { + "path": "graph/callbacks/chain_test.go", + "source_blob": "0b6f2ab7b1e189b2cafb75f71c3bad14a98368aa", + "mode": "100644" + }, + { + "path": "graph/callbacks/metrics.go", + "source_blob": "01736ea520bf373678ad3aba0ffc00ff7eb9d9ef", + "mode": "100644" + }, + { + "path": "graph/callbacks/metrics_test.go", + "source_blob": "23369d42cf58d06f9c76cb071c2a742885aca218", + "mode": "100644" + }, + { + "path": "graph/callbacks/persistence.go", + "source_blob": "1636939cf7e08b5fbdc49f1f23508ec09e87785d", + "mode": "100644" + }, + { + "path": "graph/callbacks/persistence_test.go", + "source_blob": "bf5881c396a95be2f9ac2144db56dd6647685cad", + "mode": "100644" + }, + { + "path": "graph/callbacks/reasoning_test.go", + "source_blob": "340ea398eaec369ded46ef5348f5a7894718908d", + "mode": "100644" + }, + { + "path": "graph/callbacks/sse.go", + "source_blob": "99432f8be298de0134c337e618ec174f88d8cbc7", + "mode": "100644" + }, + { + "path": "graph/callbacks/sse_test.go", + "source_blob": "39779c1a0dee9b4f6cfaec5b110347b080275a30", + "mode": "100644" + }, + { + "path": "graph/callbacks/tracing.go", + "source_blob": "09e4d86f228e6976a6e9fd9a3d00f3cbdcb32502", + "mode": "100644" + }, + { + "path": "graph/callbacks/tracing_test.go", + "source_blob": "00fc520fa6548a068b2dcb7556a8dc7e06668a3e", + "mode": "100644" + }, + { + "path": "graph/react.go", + "source_blob": "a1a741d22499d073959e80758b4d4dcaffbec11f", + "mode": "100644" + }, + { + "path": "graph/react_test.go", + "source_blob": "603c4f30b0fb5454fbf227dcfb2764e49f4e3eaf", + "mode": "100644" + }, + { + "path": "graph/token_budget_test.go", + "source_blob": "496b8ecd1c6811f4616cf953eb466d1d0994ce6b", + "mode": "100644" + }, + { + "path": "graph/tool_adapter.go", + "source_blob": "80e9d9744fdd6b7b53e13c0ca7eb49ec1c2b3137", + "mode": "100644" + }, + { + "path": "graph/tool_adapter_test.go", + "source_blob": "33eeccb0ac8a3f04f782376c5d2eeeab5990eb3a", + "mode": "100644" + }, + { + "path": "graph/types.go", + "source_blob": "483774abb656b3df2500632439dbf4147acbf19c", + "mode": "100644" + } + ] +} diff --git a/internal/manager/biz/aiops/runtime/v3/README.md b/internal/manager/biz/aiops/runtime/v3/README.md new file mode 100644 index 000000000..04d20ecae --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/README.md @@ -0,0 +1,5 @@ +# Agent Runtime v3 + +从草稿 PR #503 的 `cc25f29f` 迁移当前 `chatruntime/`、`graph/` 及其测试到本目录。只调整 import 与测试夹具路径，保留 tracing scope 名称和现有行为。 + +当前生产装配调用本版本。后续策略变更在本目录实施，不修改 v1/v2 的历史行为基线。业务工具、领域模型、存储和外部 agents/skills 资源仍然共享；跨版本路由与版本化装配尚待实施。 diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_parser.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_parser.go new file mode 100644 index 000000000..3d6fd34de --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_parser.go @@ -0,0 +1,111 @@ +package chatruntime + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" +) + +// ParseAgentMd reads an agent persona file (agents/.md) at path. +// Same on-disk shape as SKILL.md but the body becomes the agent's +// system prompt. — required fields are name, description, +// when_to_use; the latter is strict per the spec because a coordinator +// can't decide whether to spawn an agent without it. +// +// Snake_case field names (this is the explicit +// rename of the original camelCase shape). +// +// Forward-compat: unknown frontmatter keys are preserved into +// Agent.UnknownFields so claude-code adding new persona fields (effort, +// memory, isolation, ...) doesn't break loading. +func ParseAgentMd(path string) (*Agent, []LoadWarning, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: read %s: %w", path, err) + } + frontmatter, body, err := splitFrontmatter(raw) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: %s: %w", path, err) + } + + var warnings []LoadWarning + + var ag Agent + if len(frontmatter) > 0 { + if err := yaml.Unmarshal(frontmatter, &ag); err != nil { + return nil, nil, fmt.Errorf("chatruntime: parse %s frontmatter: %w", path, err) + } + } + + rawMap := map[string]any{} + if len(frontmatter) > 0 { + if err := yaml.Unmarshal(frontmatter, &rawMap); err != nil { + return nil, nil, fmt.Errorf("chatruntime: parse %s frontmatter (raw): %w", path, err) + } + } + ag.UnknownFields = retainUnknownAgentFields(rawMap) + + // Required fields — strict. Missing when_to_use is an + // error not a warning: a coordinator can't choose this agent without + // it. + if strings.TrimSpace(ag.Name) == "" { + return nil, warnings, fmt.Errorf("chatruntime: %s: frontmatter missing required field 'name'", path) + } + if strings.TrimSpace(ag.Description) == "" { + return nil, warnings, fmt.Errorf("chatruntime: %s: frontmatter missing required field 'description'", path) + } + if strings.TrimSpace(ag.WhenToUse) == "" { + return nil, warnings, fmt.Errorf("chatruntime: %s: frontmatter missing required field 'when_to_use'", path) + } + + // Snake-case enforcement on name. We don't auto-normalize agent names + // (they're spawn keys; rewriting silently would break references). + if !skillNameRe.MatchString(ag.Name) { + // Allow dash for agents — claude-code uses `incident-investigator` + // style. Be permissive: warn rather than rewrite. + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "name_non_snake", + Reason: fmt.Sprintf("agent name %q is not snake_case (recommends underscores)", ag.Name), + }) + } + + ag.SystemPrompt = strings.TrimRight(string(body), "\n") + return &ag, warnings, nil +} + +// retainUnknownAgentFields keeps frontmatter keys we don't model on +// Agent. Drives forward compatibility with claude-code agent schema +// additions (effort, isolation, mcp_servers, hooks, ...). +func retainUnknownAgentFields(raw map[string]any) map[string]any { + known := map[string]struct{}{ + "name": {}, + "description": {}, + "when_to_use": {}, + "capabilities": {}, + "can_delegate": {}, + "tools": {}, + "disallowed_tools": {}, + "permission_mode": {}, + "max_turns": {}, + "model": {}, + "critical_reminder": {}, + "initial_prompt": {}, + "background": {}, + "omit_claude_md": {}, + "metadata": {}, + } + out := map[string]any{} + for k, v := range raw { + if _, ok := known[k]; ok { + continue + } + out[k] = v + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_parser_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_parser_test.go new file mode 100644 index 000000000..dc92fd763 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_parser_test.go @@ -0,0 +1,106 @@ +package chatruntime + +import ( + "path/filepath" + "strings" + "testing" +) + +// fixtureAgent returns the absolute path to an agent .md fixture. +func fixtureAgent(scenario, file string) string { + return filepath.Join("testdata", "agent_parser", scenario, file) +} + +func TestParseAgentMd_Minimal(t *testing.T) { + ag, warns, err := ParseAgentMd(fixtureAgent("minimal", "incident-investigator.md")) + if err != nil { + t.Fatalf("ParseAgentMd: %v", err) + } + // "incident-investigator" uses dash; we warn but don't rewrite. + foundDashWarn := false + for _, w := range warns { + if w.Code == "name_non_snake" { + foundDashWarn = true + } + } + if !foundDashWarn { + t.Errorf("expected name_non_snake warning; got %+v", warns) + } + if ag.Name != "incident-investigator" { + t.Errorf("Name = %q (should not be rewritten)", ag.Name) + } + if ag.Description == "" { + t.Error("Description empty") + } + if ag.WhenToUse == "" { + t.Error("WhenToUse empty") + } + if len(ag.Capabilities) != 1 || ag.Capabilities[0].ID != "incident_analysis" { + t.Errorf("Capabilities = %+v", ag.Capabilities) + } + if ag.CanDelegate { + t.Error("minimal fixture should not enable delegation") + } + if !strings.Contains(ag.SystemPrompt, "ongrid alert") { + t.Errorf("SystemPrompt body lost: %q", ag.SystemPrompt) + } +} + +func TestParseAgentMd_WithDisallowedTools(t *testing.T) { + ag, _, err := ParseAgentMd(fixtureAgent("with_disallowed_tools", "reviewer.md")) + if err != nil { + t.Fatalf("ParseAgentMd: %v", err) + } + if ag.Name != "reviewer" { + t.Errorf("Name = %q", ag.Name) + } + wantDis := []string{"*_skill", "run_shell"} + if !strSliceEq(ag.DisallowedTools, wantDis) { + t.Errorf("DisallowedTools = %v, want %v", ag.DisallowedTools, wantDis) + } + if ag.PermissionMode != "read-only" { + t.Errorf("PermissionMode = %q", ag.PermissionMode) + } + if ag.MaxTurns != 5 { + t.Errorf("MaxTurns = %d, want 5", ag.MaxTurns) + } + if !strings.Contains(ag.Model, "opus") { + t.Errorf("Model = %q", ag.Model) + } + if len(ag.Tools) != 4 { + t.Errorf("Tools len = %d, want 4", len(ag.Tools)) + } +} + +func TestParseAgentMd_MissingWhenToUse(t *testing.T) { + _, _, err := ParseAgentMd(fixtureAgent("missing_when_to_use", "foo.md")) + if err == nil { + t.Fatal("expected error for missing when_to_use, got nil") + } + if !strings.Contains(err.Error(), "when_to_use") { + t.Errorf("error should mention when_to_use; got %v", err) + } +} + +func TestParseAgentMd_WithCriticalReminder(t *testing.T) { + ag, _, err := ParseAgentMd(fixtureAgent("with_critical_reminder", "auditor.md")) + if err != nil { + t.Fatalf("ParseAgentMd: %v", err) + } + if ag.CriticalReminder == "" { + t.Fatal("CriticalReminder empty") + } + if !strings.Contains(ag.CriticalReminder, "Do not approve") { + t.Errorf("CriticalReminder content unexpected: %q", ag.CriticalReminder) + } + if ag.MaxTurns != 8 { + t.Errorf("MaxTurns = %d", ag.MaxTurns) + } +} + +func TestParseAgentMd_FileMissing(t *testing.T) { + _, _, err := ParseAgentMd(fixtureAgent("does_not_exist", "x.md")) + if err == nil { + t.Fatal("expected error for missing file") + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry.go new file mode 100644 index 000000000..c6ca9c8d9 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry.go @@ -0,0 +1,201 @@ +package chatruntime + +import ( + "fmt" + "sync" +) + +// AgentRegistry holds parsed agent personas. Loaded +// once at startup from the configured agents root; coordinator looks up +// by name at spawn time. +// +// Concurrency: a sync.RWMutex guards the slices. marketplace +// Install/Uninstall calls Reload while in-flight coordinator turns may +// be running; Reload swaps a freshly-built slice atomically and +// returning slices are value copies so callers already holding a result +// don't observe mutations. +type AgentRegistry struct { + mu sync.RWMutex + agents []*Agent + warnings []LoadWarning +} + +// NewAgentRegistry returns an empty registry. +func NewAgentRegistry() *AgentRegistry { return &AgentRegistry{} } + +// Load walks agentsRoot for *.md files and plugin containers, parsing +// each agent persona it finds. Subdirectories are walked recursively; +// an `agents/team/.md` layout is fine. +// +// As of PR-skill-load (simplified loader) Load delegates +// to LoadAll, which also recurses into `.claude-plugin/` and +// `openclaw.plugin.json` containers under agentsRoot and pulls their +// agents (skills from packs in the agents root are kept on the result, +// but ignored here — AgentRegistry only owns agents). +// +// Per-file parse errors land as warnings rather than aborting the walk +// — same policy as SkillRegistry.Load. Non-existent agentsRoot is not +// an error. +func (r *AgentRegistry) Load(agentsRoot string) error { + res, err := LoadAll(LoadAllConfig{AgentsRoot: agentsRoot}) + if err != nil { + return fmt.Errorf("chatruntime: agent registry load: %w", err) + } + r.mu.Lock() + r.agents = append([]*Agent(nil), res.Agents...) + r.warnings = append([]LoadWarning(nil), res.Warnings...) + r.mu.Unlock() + return nil +} + +// Reload replaces the registry contents with a fresh scan of agentsRoot +// (primary) plus optional extras. Used by marketplace Install/Uninstall +// to hot-reload without restart. Atomic: builds the new slice outside +// the lock, then swaps under r.mu.Lock() in O(1). +// +// Variadic extras are treated as additional agent roots — each walked +// with the loose-*.md → persona rule so persona drops outside the +// primary AgentsRoot still load (used to preserve image-baked built-in +// agents through a marketplace install reload). +func (r *AgentRegistry) Reload(agentsRoot string, extras ...string) error { + cfg := LoadAllConfig{ + AgentsRoot: agentsRoot, + ExtraAgentsRoots: extras, + } + res, err := LoadAll(cfg) + if err != nil { + return fmt.Errorf("chatruntime: agent registry reload: %w", err) + } + newAgents := append([]*Agent(nil), res.Agents...) + newWarnings := append([]LoadWarning(nil), res.Warnings...) + r.mu.Lock() + r.agents = newAgents + r.warnings = newWarnings + r.mu.Unlock() + return nil +} + +// All returns a copy of every loaded agent. +func (r *AgentRegistry) All() []*Agent { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]*Agent, len(r.agents)) + copy(out, r.agents) + return out +} + +// CapabilityCards returns a stable snapshot of registered, agent-owned +// capability declarations. Resolve uses these cards to decide whether the +// default assistant can continue directly or should ask an expert for extra +// evidence; a card never transfers ownership of the root conversation. +func (r *AgentRegistry) CapabilityCards() []AgentCapability { + r.mu.RLock() + defer r.mu.RUnlock() + var out []AgentCapability + for _, ag := range r.agents { + if ag == nil { + continue + } + for _, card := range ag.Capabilities { + if card.ID == "" { + continue + } + copyCard := card + copyCard.AgentName = ag.Name + copyCard.Tools = append([]string(nil), card.Tools...) + copyCard.Skills = append([]string(nil), card.Skills...) + out = append(out, copyCard) + } + } + return out +} + +// Warnings returns a copy of every warning recorded during Load. +func (r *AgentRegistry) Warnings() []LoadWarning { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]LoadWarning, len(r.warnings)) + copy(out, r.warnings) + return out +} + +// ByName returns the agent with the given name (frontmatter `name` +// field). Returns (nil, false) when not found. Name match is exact — +// callers (the coordinator AgentTool) should already know the spawn key. +func (r *AgentRegistry) ByName(name string) (*Agent, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + for _, ag := range r.agents { + if ag.Name == name { + return ag, true + } + } + return nil, false +} + +// Add inserts a programmatically-constructed agent. Used by built-in +// agents (general-purpose) that ship in the binary rather than on disk. +func (r *AgentRegistry) Add(ag *Agent) { + if ag == nil { + return + } + r.mu.Lock() + r.agents = append(r.agents, ag) + r.mu.Unlock() +} + +// AddAll inserts a batch of agents. Nil entries are skipped. Used by +// LoadAll to merge plugin-container output into the registry. +func (r *AgentRegistry) AddAll(agents []*Agent) { + for _, ag := range agents { + r.Add(ag) + } +} + +// Replace swaps the agent with the same Name. When no row matches it +// behaves like Add (so callers can use it as upsert). Used by the +// user-agent edit path so a save mutates the live registry without a +// restart. +func (r *AgentRegistry) Replace(ag *Agent) { + if ag == nil || ag.Name == "" { + return + } + r.mu.Lock() + defer r.mu.Unlock() + for i, existing := range r.agents { + if existing != nil && existing.Name == ag.Name { + r.agents[i] = ag + return + } + } + r.agents = append(r.agents, ag) +} + +// Remove drops the agent with this name. Returns true when a row was +// removed. Used by the user-agent delete path. +func (r *AgentRegistry) Remove(name string) bool { + if name == "" { + return false + } + r.mu.Lock() + defer r.mu.Unlock() + for i, existing := range r.agents { + if existing != nil && existing.Name == name { + r.agents = append(r.agents[:i], r.agents[i+1:]...) + return true + } + } + return false +} + +// AddWarnings appends loader warnings (e.g. from LoadAll) so +// AgentRegistry.Warnings() exposes them alongside per-agent parse +// warnings. +func (r *AgentRegistry) AddWarnings(ws []LoadWarning) { + if len(ws) == 0 { + return + } + r.mu.Lock() + r.warnings = append(r.warnings, ws...) + r.mu.Unlock() +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry_test.go new file mode 100644 index 000000000..bbf811b7d --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry_test.go @@ -0,0 +1,64 @@ +package chatruntime + +import ( + "path/filepath" + "testing" +) + +func fixtureAgentRoot(scenario string) string { + return filepath.Join("testdata", "agent_registry", scenario) +} + +func TestAgentRegistry_LoadMulti(t *testing.T) { + r := NewAgentRegistry() + if err := r.Load(fixtureAgentRoot("multi")); err != nil { + t.Fatalf("Load: %v", err) + } + all := r.All() + if len(all) != 2 { + t.Fatalf("All() len = %d, want 2 (README.md skipped)", len(all)) + } + if _, ok := r.ByName("incident_investigator"); !ok { + t.Errorf("ByName(incident_investigator) miss") + } + if _, ok := r.ByName("reviewer"); !ok { + t.Errorf("ByName(reviewer) miss") + } + if _, ok := r.ByName("does_not_exist"); ok { + t.Errorf("ByName should miss for unknown name") + } +} + +func TestAgentRegistry_LoadNonExistent(t *testing.T) { + r := NewAgentRegistry() + if err := r.Load(fixtureAgentRoot("does_not_exist")); err != nil { + t.Fatalf("Load: %v", err) + } + if len(r.All()) != 0 { + t.Errorf("expected zero agents") + } +} + +func TestAgentRegistry_AddAndByName(t *testing.T) { + r := NewAgentRegistry() + r.Add(&Agent{Name: "general_purpose", Description: "default", WhenToUse: "fallback"}) + if got, ok := r.ByName("general_purpose"); !ok || got.Name != "general_purpose" { + t.Errorf("ByName failed for added agent") + } +} + +func TestAgentRegistryCapabilityCardsCarryOwner(t *testing.T) { + r := NewAgentRegistry() + r.Add(&Agent{Name: "specialist-disk", Capabilities: []AgentCapability{{ID: "disk_analysis", Tools: []string{"host_du_summary"}}}}) + cards := r.CapabilityCards() + if len(cards) != 1 { + t.Fatalf("CapabilityCards len = %d, want 1", len(cards)) + } + if cards[0].AgentName != "specialist-disk" || cards[0].ID != "disk_analysis" { + t.Errorf("card = %+v", cards[0]) + } + cards[0].Tools[0] = "changed" + if got := r.All()[0].Capabilities[0].Tools[0]; got != "host_du_summary" { + t.Errorf("registry card mutated through snapshot: %q", got) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/bare_skills_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/bare_skills_test.go new file mode 100644 index 000000000..390c0df30 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/bare_skills_test.go @@ -0,0 +1,138 @@ +package chatruntime + +import ( + "os" + "path/filepath" + "testing" +) + +// TestDetectContainer_BareSkillsDir — a skills.sh-style drop is a +// directory with skills//SKILL.md and NO plugin manifest. Should +// be recognized as ContainerBareSkills. +func TestDetectContainer_BareSkillsDir(t *testing.T) { + root := t.TempDir() + writeSkillMd(t, filepath.Join(root, "skills", "find-skills", "SKILL.md"), `--- +name: find-skills +description: Find and install agent skills when asked +--- + +Body of the skill. +`) + kind, manifest, err := DetectContainer(root) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerBareSkills { + t.Errorf("want ContainerBareSkills, got %q", kind) + } + if manifest != "" { + t.Errorf("bare-skills has no manifest path, got %q", manifest) + } +} + +// TestDetectContainer_BareSkillsRootSkillMd — a single-skill pack drops +// SKILL.md at the root of the cloned directory. +func TestDetectContainer_BareSkillsRootSkillMd(t *testing.T) { + root := t.TempDir() + writeSkillMd(t, filepath.Join(root, "SKILL.md"), `--- +name: solo-skill +description: A standalone skill at the root +--- +body +`) + kind, _, err := DetectContainer(root) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerBareSkills { + t.Errorf("want ContainerBareSkills, got %q", kind) + } +} + +// TestDetectContainer_ClaudeWins — when both .claude-plugin/plugin.json +// AND skills/ exist, the claude marker wins (existing behaviour). +func TestDetectContainer_ClaudeWins(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, ".claude-plugin"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(root, ".claude-plugin", "plugin.json"), + []byte(`{"id":"x","name":"X","version":"1.0.0"}`), 0o644); err != nil { + t.Fatalf("write manifest: %v", err) + } + writeSkillMd(t, filepath.Join(root, "skills", "s", "SKILL.md"), `--- +name: s +description: d +---`) + + kind, _, err := DetectContainer(root) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerClaude { + t.Errorf("want ContainerClaude, got %q", kind) + } +} + +// TestLoadPluginContainer_BareSkillsSynthesizesPack — bare-skill load +// produces a Pack with id derived from the directory basename. +func TestLoadPluginContainer_BareSkillsSynthesizesPack(t *testing.T) { + parent := t.TempDir() + root := filepath.Join(parent, "vercel-labs__skills") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + writeSkillMd(t, filepath.Join(root, "skills", "alpha", "SKILL.md"), `--- +name: alpha +description: First skill +--- +body alpha +`) + writeSkillMd(t, filepath.Join(root, "skills", "beta", "SKILL.md"), `--- +name: beta +description: Second skill +--- +body beta +`) + + res, err := LoadPluginContainer(root) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + if res.Pack == nil { + t.Fatalf("Pack nil") + } + if res.Pack.ID != "vercel-labs-skills" { + t.Errorf("Pack.ID = %q, want vercel-labs-skills", res.Pack.ID) + } + if len(res.Skills) != 2 { + t.Errorf("want 2 skills loaded, got %d", len(res.Skills)) + } +} + +// TestBareSkillsPackID — id normalisation rules. +func TestBareSkillsPackID(t *testing.T) { + cases := []struct{ in, want string }{ + {"vercel-labs__skills", "vercel-labs-skills"}, + {"My Awesome Pack v2", "my-awesome-pack-v2"}, + {"____", "untitled-skill-pack"}, + {"", "untitled-skill-pack"}, + {"snake_case_repo", "snake-case-repo"}, + {"already-clean", "already-clean"}, + } + for _, tc := range cases { + if got := bareSkillsPackID(tc.in); got != tc.want { + t.Errorf("bareSkillsPackID(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func writeSkillMd(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write: %v", err) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/chatmodelopts_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/chatmodelopts_test.go new file mode 100644 index 000000000..dcfd314dc --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/chatmodelopts_test.go @@ -0,0 +1,23 @@ +package chatruntime + +import "testing" + +// chatModelOpts must add an option only for a set field — the empty/default +// path adds nothing so the routing default is preserved. +func TestChatModelOpts(t *testing.T) { + if got := chatModelOpts(nil); got != nil { + t.Errorf("nil req -> nil, got %v", got) + } + if got := chatModelOpts(&Request{}); len(got) != 0 { + t.Errorf("empty req -> no options, got %d", len(got)) + } + if got := chatModelOpts(&Request{Provider: " "}); len(got) != 0 { + t.Errorf("whitespace provider -> no options, got %d", len(got)) + } + if got := chatModelOpts(&Request{Provider: "anthropic"}); len(got) != 1 { + t.Errorf("provider only -> 1 option, got %d", len(got)) + } + if got := chatModelOpts(&Request{Provider: "anthropic", Model: "claude-3-opus-latest"}); len(got) != 2 { + t.Errorf("provider+model -> 2 options, got %d", len(got)) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/command_parser.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/command_parser.go new file mode 100644 index 000000000..a294aa128 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/command_parser.go @@ -0,0 +1,151 @@ +package chatruntime + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// commandFrontmatter is the parsed YAML frontmatter of a claude/cursor +// `commands/.md` file. We model only the fields +// asks for; everything else lands in the prompt body unchanged. +// +// `allowed-tools` (claude-specific, kebab-cased — claude-code uses +// kebab-case in commands frontmatter, unlike SKILL.md / agents) is +// captured purely so we can echo it into the prompt body as a soft +// hint. ongrid does NOT enforce it as a hard restriction, since claude +// command tools (Bash / Edit / ...) don't map 1:1 to ongrid tool +// classes. +type commandFrontmatter struct { + Description string `yaml:"description"` + AllowedTools []string `yaml:"allowed-tools"` +} + +// ConvertCommandFile parses a claude/cursor `commands/.md` file +// at path and returns a Skill with mode=keyword. : +// +// - keyword = ["/", ""] (file basename, optional snake-cased +// alias when the basename contains dashes) +// - frontmatter description → Skill.Description +// - body → Skill.PromptBody (with `[能力: cmd_]` header) +// - tools: [] (commands are pure prompt injection — no new tools wired +// by ongrid; the claude-side tool restriction is informational) +// - name prefixed with `cmd_` to avoid collision with regular skills +// - frontmatter `allowed-tools: [Bash, Edit]` (claude-specific) → appended +// to the prompt body as a soft hint, NOT enforced. +// +// The basename rule mirrors how claude-code resolves slash commands: +// `commands/foo.md` → `/foo`. Nested subdirs are flattened +// (`commands/git/commit.md` → `/commit`); the loader (LoadPluginContainer) +// is responsible for walking commands/ recursively. +// +// Returns the parsed Skill, any non-fatal warnings, and a parse error +// when the file is unreadable / not markdown / missing required fields. +func ConvertCommandFile(path string) (*Skill, []LoadWarning, error) { + if !strings.EqualFold(filepath.Ext(path), ".md") { + return nil, nil, fmt.Errorf("chatruntime: %s: command file must end in .md", path) + } + + raw, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: read %s: %w", path, err) + } + + frontmatter, body, err := splitFrontmatter(raw) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: %s: %w", path, err) + } + + var fm commandFrontmatter + if len(frontmatter) > 0 { + if err := yaml.Unmarshal(frontmatter, &fm); err != nil { + return nil, nil, fmt.Errorf("chatruntime: parse %s frontmatter: %w", path, err) + } + } + + base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + base = strings.TrimSpace(base) + if base == "" { + return nil, nil, fmt.Errorf("chatruntime: %s: command file has empty basename", path) + } + + // Build keyword list + // Always include "/" and ""; if the basename contains + // dashes, also include the snake_cased alias so users typing either + // "review-pr" or "review_pr" trigger activation. + keywords := []string{"/" + base, base} + if strings.ContainsAny(base, "-. ") { + alias := normalizeSnakeName(base) + if alias != "" && alias != base { + keywords = append(keywords, alias) + } + } + + // Skill name uses cmd_ prefix to avoid colliding with regular skills. + // If the basename isn't snake_case, normalize after the prefix so the + // final name is always valid. + nameSuffix := base + if !skillNameRe.MatchString(nameSuffix) { + nameSuffix = normalizeSnakeName(nameSuffix) + } + skillName := "cmd_" + nameSuffix + + description := strings.TrimSpace(fm.Description) + if description == "" { + description = fmt.Sprintf("Slash-command equivalent of `/%s` (claude commands import).", base) + } + + // Compose the prompt body. The H1 is the canonical [能力: cmd_] + // tag so system_prompt assembly picks it up as-is. + bodyText := strings.TrimRight(string(body), "\n") + header := "[能力: " + skillName + "]" + + var b strings.Builder + b.WriteString(header) + b.WriteString("\n\n") + if bodyText != "" { + b.WriteString(bodyText) + b.WriteString("\n") + } + if len(fm.AllowedTools) > 0 { + // Soft hint, not a hard restriction. We surface the upstream + // allowed-tools list verbatim so the LLM can take it as a clue + // about the author's intent. + b.WriteString("\n") + b.WriteString(fmt.Sprintf("> 上游 (claude) 允许使用的工具: %s。ongrid 不强制此清单，仅作提示。\n", + strings.Join(fm.AllowedTools, ", "))) + } + + sk := &Skill{ + Name: skillName, + Description: description, + Activation: Activation{ + Mode: "keyword", + Keywords: keywords, + }, + PromptBody: strings.TrimRight(b.String(), "\n"), + Metadata: SkillMetadata{ + Ongrid: OngridExt{ + Activation: Activation{ + Mode: "keyword", + Keywords: keywords, + }, + }, + }, + Dir: filepath.Dir(path), + } + + var warnings []LoadWarning + if strings.TrimSpace(fm.Description) == "" { + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "command_missing_description", + Reason: fmt.Sprintf("command %q has no frontmatter description; using a generated default", base), + }) + } + + return sk, warnings, nil +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/command_parser_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/command_parser_test.go new file mode 100644 index 000000000..19534a03f --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/command_parser_test.go @@ -0,0 +1,159 @@ +package chatruntime + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// writeTemp writes content to a fresh tempdir/file and returns the path. +func writeTemp(t *testing.T, name, content string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + return path +} + +func TestConvertCommandFile_Minimal(t *testing.T) { + path := writeTemp(t, "commit.md", `--- +description: Commit the staged changes. +--- + +Run git commit with a conventional message. +`) + sk, warns, err := ConvertCommandFile(path) + if err != nil { + t.Fatalf("ConvertCommandFile: %v", err) + } + if len(warns) != 0 { + t.Errorf("unexpected warnings: %+v", warns) + } + if sk.Name != "cmd_commit" { + t.Errorf("Name = %q, want cmd_commit", sk.Name) + } + if sk.Description != "Commit the staged changes." { + t.Errorf("Description = %q", sk.Description) + } + if sk.Activation.Mode != "keyword" { + t.Errorf("Activation.Mode = %q, want keyword", sk.Activation.Mode) + } + if !hasName(sk.Activation.Keywords, "/commit") { + t.Errorf("missing /commit keyword; got %v", sk.Activation.Keywords) + } + if !hasName(sk.Activation.Keywords, "commit") { + t.Errorf("missing commit keyword; got %v", sk.Activation.Keywords) + } + // metadata.ongrid.activation should mirror the top-level activation + // so downstream consumers reading either key see keyword mode. + if sk.Metadata.Ongrid.Activation.Mode != "keyword" { + t.Errorf("metadata.ongrid.activation.mode = %q, want keyword", sk.Metadata.Ongrid.Activation.Mode) + } + if len(sk.Tools) != 0 { + t.Errorf("Tools should be empty; got %d", len(sk.Tools)) + } + if !strings.HasPrefix(strings.TrimSpace(sk.PromptBody), "[能力: cmd_commit]") { + t.Errorf("body should start with [能力: cmd_commit]; got:\n%s", sk.PromptBody) + } +} + +func TestConvertCommandFile_AllowedToolsAppendedToBody(t *testing.T) { + path := writeTemp(t, "deploy.md", `--- +description: Deploy to staging. +allowed-tools: [Bash, Edit, Read] +--- + +Run the deploy script. +`) + sk, _, err := ConvertCommandFile(path) + if err != nil { + t.Fatalf("ConvertCommandFile: %v", err) + } + for _, tool := range []string{"Bash", "Edit", "Read"} { + if !strings.Contains(sk.PromptBody, tool) { + t.Errorf("body should mention allowed tool %q; got:\n%s", tool, sk.PromptBody) + } + } + // Soft-hint, not enforcement: Tools slice is still empty. + if len(sk.Tools) != 0 { + t.Errorf("Tools should remain empty (soft hint, not hard restriction); got %d", len(sk.Tools)) + } +} + +func TestConvertCommandFile_DashedBasename_SnakeAlias(t *testing.T) { + path := writeTemp(t, "review-pr.md", `--- +description: Review the PR diff. +--- + +Walk through the diff and call out issues. +`) + sk, _, err := ConvertCommandFile(path) + if err != nil { + t.Fatalf("ConvertCommandFile: %v", err) + } + if sk.Name != "cmd_review_pr" { + t.Errorf("Name = %q, want cmd_review_pr", sk.Name) + } + for _, kw := range []string{"/review-pr", "review-pr", "review_pr"} { + if !hasName(sk.Activation.Keywords, kw) { + t.Errorf("missing keyword %q; got %v", kw, sk.Activation.Keywords) + } + } +} + +func TestConvertCommandFile_NonMarkdown_Error(t *testing.T) { + path := writeTemp(t, "weird.txt", `not markdown`) + if _, _, err := ConvertCommandFile(path); err == nil { + t.Fatalf("expected error for .txt file") + } +} + +func TestConvertCommandFile_NoFrontmatter_GeneratesDescription(t *testing.T) { + path := writeTemp(t, "raw.md", "Just a plain markdown body, no frontmatter.\n") + sk, warns, err := ConvertCommandFile(path) + if err != nil { + t.Fatalf("ConvertCommandFile: %v", err) + } + if sk.Name != "cmd_raw" { + t.Errorf("Name = %q, want cmd_raw", sk.Name) + } + if sk.Description == "" { + t.Errorf("Description should be generated when missing") + } + hasMissingDesc := false + for _, w := range warns { + if w.Code == "command_missing_description" { + hasMissingDesc = true + } + } + if !hasMissingDesc { + t.Errorf("expected command_missing_description warning; got %+v", warns) + } +} + +func TestConvertCommandFile_NestedFlattens(t *testing.T) { + // A file at commands/git/commit.md should flatten to cmd_commit by + // basename — confirms the loader's flatten policy at unit level. + dir := t.TempDir() + nested := filepath.Join(dir, "git") + if err := os.MkdirAll(nested, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + path := filepath.Join(nested, "commit.md") + if err := os.WriteFile(path, []byte(`--- +description: nested commit cmd +--- +body`), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + sk, _, err := ConvertCommandFile(path) + if err != nil { + t.Fatalf("ConvertCommandFile: %v", err) + } + if sk.Name != "cmd_commit" { + t.Errorf("Name = %q, want cmd_commit (basename flatten)", sk.Name) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/config_confirm.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/config_confirm.go new file mode 100644 index 000000000..173c8aaf0 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/config_confirm.go @@ -0,0 +1,403 @@ +package chatruntime + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "regexp" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" +) + +const applyConfigChangeToolName = "apply_config_change" + +var confirmDraftHashRE = regexp.MustCompile(`(?i)\bdraft_hash\b\s*[:=]\s*"?\s*(sha256:[a-f0-9]{64})\s*"?`) + +func (rt *Runtime) tryApplyConfirmedConfigDraft(ctx context.Context, req *Request, sess *aiopsmodel.Session, history []*aiopsmodel.Message, emit Emit) (*Reply, bool) { + if req == nil || sess == nil { + return nil, false + } + argsJSON, parseErr, ok := parseConfirmedConfigDraftApplyArgs(req.UserText) + if !ok { + if !looksLikeConfigDraftConfirmation(req.UserText) { + return nil, false + } + argsJSON, parseErr, ok = latestConfigDraftApplyArgs(history, req.UserText) + if !ok { + return nil, false + } + } + if parseErr != nil { + return rt.persistAndEmitDirectAssistant(ctx, sess.ID, emit, fmt.Sprintf("确认应用失败：%s", parseErr.Error())), true + } + tool, err := findToolByName(ctx, rt.toolBagSnapshot(), applyConfigChangeToolName) + if err != nil { + return rt.persistAndEmitDirectAssistant(ctx, sess.ID, emit, fmt.Sprintf("确认应用失败：%s", err.Error())), true + } + + callID := "direct_" + uuid.NewString() + startedAt := time.Now().UTC() + emit(Event{Type: EventToolStart, Tool: &ToolEvent{ + ToolCallID: callID, + Name: applyConfigChangeToolName, + ArgsJSON: argsJSON, + Status: "running", + StartedAt: startedAt, + }}) + result, runErr := tool.InvokableRun(ctx, argsJSON, + basetool.WithUserID(req.UserID), + basetool.WithUserText(req.UserText), + ) + endedAt := time.Now().UTC() + status := "success" + errText := "" + if runErr != nil { + status = "error" + errText = runErr.Error() + } + emit(Event{Type: EventToolEnd, Tool: &ToolEvent{ + ToolCallID: callID, + Name: applyConfigChangeToolName, + ArgsJSON: argsJSON, + ResultJSON: result, + Status: status, + StartedAt: startedAt, + EndedAt: &endedAt, + DurationMs: endedAt.Sub(startedAt).Milliseconds(), + Error: errText, + }}) + if runErr != nil { + return rt.persistAndEmitDirectAssistant(ctx, sess.ID, emit, fmt.Sprintf("确认应用失败：%s", runErr.Error())), true + } + return rt.persistAndEmitDirectAssistant(ctx, sess.ID, emit, formatConfigApplyResultMessage(result)), true +} + +func (rt *Runtime) persistAndEmitDirectAssistant(ctx context.Context, sessionID string, emit Emit, content string, toolCalls ...*aiopsmodel.ToolCall) *Reply { + if emit == nil { + emit = func(Event) {} + } + msg := &aiopsmodel.Message{ + SessionID: sessionID, + Role: aiopsmodel.RoleAssistant, + Content: &content, + CreatedAt: time.Now().UTC(), + } + if err := rt.cfg.Sessions.AppendMessage(ctx, msg); err != nil { + if rt.log != nil { + rt.log.Warn("chatruntime: persist direct assistant failed", + slog.String("session_id", sessionID), + slog.String("err", err.Error())) + } + } + emit(Event{Type: EventAssistant, Assistant: &AssistantEvent{ + MessageID: msg.ID, + Content: content, + CreatedAt: msg.CreatedAt, + }}) + reply := &Reply{Message: msg, Iterations: 1, ToolCalls: toolCalls} + emit(Event{Type: EventDone, Done: reply}) + return reply +} + +func parseConfirmedConfigDraftApplyArgs(text string) (string, error, bool) { + lower := strings.ToLower(text) + if !strings.Contains(lower, "apply_config_change") || + !strings.Contains(lower, "draft_hash") || + !strings.Contains(lower, "payload") { + return "", nil, false + } + hashMatch := confirmDraftHashRE.FindStringSubmatch(text) + if len(hashMatch) != 2 { + return "", fmt.Errorf("确认消息缺少有效 draft_hash"), true + } + payload, err := extractPayloadJSON(text) + if err != nil { + return "", err, true + } + args := struct { + Domain string `json:"domain"` + Action string `json:"action"` + Confirmed bool `json:"confirmed"` + DraftHash string `json:"draft_hash"` + Payload json.RawMessage `json:"payload"` + ConfirmationText string `json:"confirmation_text,omitempty"` + }{ + Domain: "alert_rule", + Action: "create", + Confirmed: true, + DraftHash: hashMatch[1], + Payload: payload, + ConfirmationText: "confirmed via chat confirmation button", + } + b, err := json.Marshal(args) + if err != nil { + return "", fmt.Errorf("构造 apply_config_change 参数失败: %w", err), true + } + return string(b), nil, true +} + +func latestConfigDraftApplyArgs(history []*aiopsmodel.Message, confirmationText string) (string, error, bool) { + skippedCurrentUser := false + for i := len(history) - 1; i >= 0; i-- { + msg := history[i] + if msg == nil { + continue + } + if msg.Role == aiopsmodel.RoleUser { + if !skippedCurrentUser { + skippedCurrentUser = true + continue + } + return "", nil, false + } + if !skippedCurrentUser { + continue + } + if msg.Content != nil { + args, err, ok := configDraftApplyArgsFromToolResult(*msg.Content, confirmationText) + if err != nil || ok { + return args, err, ok + } + if isConfigApplyResult(*msg.Content) { + return "", nil, false + } + } + for j := len(msg.ToolCalls) - 1; j >= 0; j-- { + tc := msg.ToolCalls[j] + if tc.ResultJSON == nil { + continue + } + args, err, ok := configDraftApplyArgsFromToolResult(*tc.ResultJSON, confirmationText) + if err != nil || ok { + return args, err, ok + } + if isConfigApplyResult(*tc.ResultJSON) { + return "", nil, false + } + } + } + return "", nil, false +} + +func configDraftApplyArgsFromToolResult(result string, confirmationText string) (string, error, bool) { + result = strings.TrimSpace(result) + if !strings.HasPrefix(result, "{") { + return "", nil, false + } + var draft struct { + Kind string `json:"kind"` + Domain string `json:"domain"` + Action string `json:"action"` + DraftHash string `json:"draft_hash"` + Payload json.RawMessage `json:"payload"` + } + if err := json.Unmarshal([]byte(result), &draft); err != nil { + return "", nil, false + } + if draft.Kind != "config_draft" { + return "", nil, false + } + if strings.TrimSpace(draft.DraftHash) == "" { + return "", fmt.Errorf("最近的 config_draft 缺少 draft_hash"), true + } + if len(draft.Payload) == 0 || !json.Valid(draft.Payload) { + return "", fmt.Errorf("最近的 config_draft payload 无效"), true + } + domain := strings.TrimSpace(draft.Domain) + if domain == "" { + domain = "alert_rule" + } + action := strings.TrimSpace(draft.Action) + if action == "" { + action = "create" + } + args := struct { + Domain string `json:"domain"` + Action string `json:"action"` + Confirmed bool `json:"confirmed"` + DraftHash string `json:"draft_hash"` + Payload json.RawMessage `json:"payload"` + ConfirmationText string `json:"confirmation_text,omitempty"` + }{ + Domain: domain, + Action: action, + Confirmed: true, + DraftHash: strings.TrimSpace(draft.DraftHash), + Payload: draft.Payload, + ConfirmationText: strings.TrimSpace(confirmationText), + } + b, err := json.Marshal(args) + if err != nil { + return "", fmt.Errorf("构造 apply_config_change 参数失败: %w", err), true + } + return string(b), nil, true +} + +func isConfigApplyResult(result string) bool { + result = strings.TrimSpace(result) + if !strings.HasPrefix(result, "{") { + return false + } + var resp struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal([]byte(result), &resp); err != nil { + return false + } + return resp.Kind == "config_apply_result" +} + +func looksLikeConfigDraftConfirmation(text string) bool { + normalized := strings.ToLower(strings.TrimSpace(text)) + normalized = strings.Trim(normalized, " \t\r\n。.!！?？") + if normalized == "" || len([]rune(normalized)) > 40 { + return false + } + if strings.Contains(normalized, "取消") || + strings.Contains(normalized, "不要") || + strings.Contains(normalized, "别") || + strings.Contains(normalized, "no") || + strings.Contains(normalized, "cancel") { + return false + } + switch normalized { + case "ok", "okay", "yes", "y", "confirm", "confirmed", "approve", "approved", "apply", + "好", "好的", "可以", "行", "嗯", "是", "确认", "确认创建", "确认应用", "应用", "创建", "同意", "通过": + return true + default: + return (strings.Contains(normalized, "确认") || strings.Contains(normalized, "同意")) && + (strings.Contains(normalized, "创建") || strings.Contains(normalized, "应用") || strings.Contains(normalized, "生效")) + } +} + +func extractPayloadJSON(text string) (json.RawMessage, error) { + lower := strings.ToLower(text) + payloadIdx := strings.Index(lower, "payload") + if payloadIdx < 0 { + return nil, fmt.Errorf("确认消息缺少 payload") + } + afterPayload := text[payloadIdx:] + if fenced := extractFencedJSON(afterPayload); len(fenced) > 0 { + return validateRawJSONObject(fenced) + } + openRel := strings.Index(afterPayload, "{") + if openRel < 0 { + return nil, fmt.Errorf("确认消息 payload 缺少 JSON 对象") + } + raw, err := extractBalancedJSONObject(afterPayload[openRel:]) + if err != nil { + return nil, err + } + return validateRawJSONObject(raw) +} + +func extractFencedJSON(text string) []byte { + fenceStart := strings.Index(text, "```") + if fenceStart < 0 { + return nil + } + bodyStart := fenceStart + len("```") + if strings.HasPrefix(strings.ToLower(text[bodyStart:]), "json") { + bodyStart += len("json") + } + body := text[bodyStart:] + fenceEnd := strings.Index(body, "```") + if fenceEnd < 0 { + return nil + } + return []byte(strings.TrimSpace(body[:fenceEnd])) +} + +func extractBalancedJSONObject(text string) ([]byte, error) { + depth := 0 + inString := false + escaped := false + for i, r := range text { + if inString { + if escaped { + escaped = false + continue + } + switch r { + case '\\': + escaped = true + case '"': + inString = false + } + continue + } + switch r { + case '"': + inString = true + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return []byte(text[:i+1]), nil + } + } + } + return nil, fmt.Errorf("确认消息 payload JSON 不完整") +} + +func validateRawJSONObject(raw []byte) (json.RawMessage, error) { + var obj map[string]any + if err := json.Unmarshal(raw, &obj); err != nil { + return nil, fmt.Errorf("确认消息 payload 不是有效 JSON: %w", err) + } + if len(obj) == 0 { + return nil, fmt.Errorf("确认消息 payload 不能为空") + } + return json.RawMessage(raw), nil +} + +func findToolByName(ctx context.Context, tools []basetool.BaseTool, name string) (basetool.BaseTool, error) { + for _, tool := range tools { + if tool == nil { + continue + } + info, err := tool.Info(ctx) + if err != nil || info == nil { + continue + } + if info.Name == name { + return tool, nil + } + } + return nil, fmt.Errorf("%s 未配置", name) +} + +func formatConfigApplyResultMessage(result string) string { + var resp struct { + Kind string `json:"kind"` + Status string `json:"status"` + ResourceID uint64 `json:"resource_id"` + Resource struct { + Name string `json:"name"` + Type string `json:"type"` + } `json:"resource"` + Message string `json:"message"` + } + if err := json.Unmarshal([]byte(result), &resp); err != nil { + return "已确认并应用配置变更。" + } + name := strings.TrimSpace(resp.Resource.Name) + if name == "" { + name = "告警规则" + } + status := strings.TrimSpace(resp.Status) + if status == "" { + status = "applied" + } + if resp.ResourceID > 0 { + return fmt.Sprintf("已确认并创建告警规则：%s（ID: %d，状态: %s）。", name, resp.ResourceID, status) + } + return fmt.Sprintf("已确认并创建告警规则：%s（状态: %s）。", name, status) +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/decision.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/decision.go new file mode 100644 index 000000000..59554ebd5 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/decision.go @@ -0,0 +1,129 @@ +package chatruntime + +import "strings" + +// TurnPhase is the system-owned state of one AgentLoop iteration. LLM output +// may supply reasoning and tool arguments, but it may not skip a phase or +// upgrade a candidate target into an executable target. +type TurnPhase string + +const ( + PhaseUnderstand TurnPhase = "understand" + PhaseResolve TurnPhase = "resolve" + PhaseDecide TurnPhase = "decide" + PhaseClarify TurnPhase = "clarify" + PhasePropose TurnPhase = "propose" + PhaseAct TurnPhase = "act" + PhaseOperate TurnPhase = "operate" + PhaseObserve TurnPhase = "observe" + PhaseComplete TurnPhase = "complete" + PhaseReject TurnPhase = "reject" +) + +// Decision is the controlled branch selected after Resolve. +type Decision string + +const ( + DecisionClarify Decision = "clarify" + DecisionPropose Decision = "propose" + DecisionAct Decision = "act" + DecisionOperate Decision = "operate" + DecisionReject Decision = "reject" +) + +// ResolvedFacts are deterministic facts collected before an execution branch +// is exposed to the model. +type ResolvedFacts struct { + Missing bool + AmbiguousTarget bool + Permitted bool + NeedsConfirmation bool + LongRunning bool + Reason string +} + +// TurnPlan records the state transition and the boundary supplied to the LLM. +// It remains request-local: durable work is represented by an Operation, not +// an in-memory loop object. +type TurnPlan struct { + Phase TurnPhase + Decision Decision + Facts ResolvedFacts + Transitions []TurnPhase +} + +func PlanTurn(facts ResolvedFacts) TurnPlan { + decision := Decide(facts) + phase := PhaseAct + switch decision { + case DecisionClarify: + phase = PhaseClarify + case DecisionPropose: + phase = PhasePropose + case DecisionOperate: + phase = PhaseOperate + case DecisionReject: + phase = PhaseReject + } + return TurnPlan{ + Phase: phase, + Decision: decision, + Facts: facts, + Transitions: []TurnPhase{PhaseUnderstand, PhaseResolve, PhaseDecide, phase}, + } +} + +// Observe advances only an executable branch. A tool result is an observation +// that returns control to the loop; the caller either plans another turn or +// completes the current response. Long-running work completes later via its +// durable Operation, never by holding this request open. +func (p TurnPlan) Observe() TurnPhase { + switch p.Phase { + case PhaseAct, PhaseOperate: + return PhaseObserve + default: + return p.Phase + } +} + +// NextAfterObserve is the only loop-back edge. A tool result is evidence, +// not a final decision, so the assistant re-enters Understand with the new +// observation before it may act again or complete the user-facing response. +func (p TurnPlan) NextAfterObserve() TurnPhase { + if p.Observe() == PhaseObserve { + return PhaseUnderstand + } + return p.Phase +} + +func (p TurnPlan) ModelBoundary() string { + switch p.Phase { + case PhaseOperate: + return "当前阶段是 Operate：只创建或更新可追踪、可取消的 Operation；不要等待长任务完成。" + case PhaseAct: + return "当前阶段是 Act：可执行短时、直接的工具调用；获得观察结果后继续推理。" + case PhasePropose: + return "当前阶段是 Propose：只展示可确认的提案，不执行变更。" + default: + return "当前阶段不允许工具执行。" + } +} + +func Decide(f ResolvedFacts) Decision { + if strings.TrimSpace(f.Reason) != "" && !f.Permitted { + return DecisionReject + } + if f.Missing || f.AmbiguousTarget { + return DecisionClarify + } + if !f.Permitted { + return DecisionReject + } + if f.NeedsConfirmation { + return DecisionPropose + } + if f.LongRunning { + return DecisionOperate + } + return DecisionAct +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/decision_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/decision_test.go new file mode 100644 index 000000000..e11fbb347 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/decision_test.go @@ -0,0 +1,159 @@ +package chatruntime + +import ( + "reflect" + "strings" + "testing" +) + +func TestDecide(t *testing.T) { + tests := []struct { + name string + facts ResolvedFacts + want Decision + }{ + {name: "ambiguous target cannot execute", facts: ResolvedFacts{AmbiguousTarget: true, Permitted: true, LongRunning: true}, want: DecisionClarify}, + {name: "missing input clarifies", facts: ResolvedFacts{Missing: true, Permitted: true}, want: DecisionClarify}, + {name: "permission rejection wins", facts: ResolvedFacts{Permitted: false, Reason: "viewer"}, want: DecisionReject}, + {name: "confirmation precedes operation", facts: ResolvedFacts{Permitted: true, NeedsConfirmation: true, LongRunning: true}, want: DecisionPropose}, + {name: "long work creates operation", facts: ResolvedFacts{Permitted: true, LongRunning: true}, want: DecisionOperate}, + {name: "short work stays in loop", facts: ResolvedFacts{Permitted: true}, want: DecisionAct}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Decide(tt.facts); got != tt.want { + t.Fatalf("Decide(%+v) = %q, want %q", tt.facts, got, tt.want) + } + }) + } +} + +func TestConfirmedDeviceIDs(t *testing.T) { + got := confirmedDeviceIDs([]Mention{{Type: "device", ID: "24"}, {Type: "alert", ID: "9"}, {Type: "device", ID: "bad"}, {Type: "DEVICE", ID: "25"}}) + if len(got) != 2 || got[0] != 24 || got[1] != 25 { + t.Fatalf("confirmedDeviceIDs = %v", got) + } +} + +func TestResolveTurnProposesPacketCapture(t *testing.T) { + plan, clarification := resolveTurn(&Request{UserText: "抓 60 秒 HTTPS 流量", Role: "admin"}) + if plan.Decision != DecisionPropose || plan.Phase != PhasePropose || clarification != "" { + t.Fatalf("resolveTurn missing current-turn target = %+v, %q; want semantic resolution then proposal", plan, clarification) + } + plan, _ = resolveTurn(&Request{UserText: "抓包", Role: "admin", Mentions: []Mention{{Type: "device", ID: "24"}}}) + if plan.Decision != DecisionPropose || plan.Phase != PhasePropose || plan.Observe() != PhasePropose { + t.Fatalf("resolveTurn selected target = %+v", plan) + } + plan, clarification = resolveTurn(&Request{UserText: "抓 60 秒 tcp port 443 的包", Role: "admin"}) + if plan.Decision != DecisionPropose || clarification != "" { + t.Fatalf("resolveTurn packet wording = %+v, %q", plan, clarification) + } + plan, clarification = resolveTurn(&Request{UserText: "Capture packets on this device", Role: "admin"}) + if plan.Decision != DecisionPropose || clarification != "" { + t.Fatalf("resolveTurn English capture without current target = %+v, %q", plan, clarification) + } + plan, clarification = resolveTurn(&Request{UserText: "Capture packets on device_id=24 interface eth0", Role: "admin"}) + if plan.Decision != DecisionPropose || clarification != "" { + t.Fatalf("resolveTurn English capture with current target = %+v, %q", plan, clarification) + } + plan, clarification = resolveTurn(&Request{UserText: "停止刚才的抓包任务", Role: "admin"}) + if plan.Decision == DecisionClarify || clarification != "" { + t.Fatalf("resolveTurn stop capture = %+v, %q; want agent loop", plan, clarification) + } +} + +func TestResolveTurnAllowsReadOnlyPacketAndGlobalRequests(t *testing.T) { + tests := []string{ + "在源码里找 packet capture 会话列表 API 的入口文件，只给路径和函数名。", + "分析最近 packet capture 页面为什么可能 page not found，从产物关联角度回答。", + "跨 Edge 抓包能不能直接用时间差判断网络延迟？请基于当前产品能力回答。", + "用 PromQL 算出各设备根分区磁盘使用率，按高到低排序。", + "列出当前可用的磁盘、CPU、内存相关指标名。", + "有没有 Linux 磁盘占用排查的内置知识？给我相关条目标题。", + "在源码里搜索 ToolSearch 的实现位置，给出文件路径。", + } + for _, text := range tests { + t.Run(text, func(t *testing.T) { + plan, clarification := resolveTurn(&Request{UserText: text, Role: "admin"}) + if plan.Decision == DecisionClarify || clarification != "" { + t.Fatalf("resolveTurn(%q) = %+v, %q; want agent loop", text, plan, clarification) + } + }) + } +} + +func TestResolveTurnClarifiesMissingPacketCaptureSessionID(t *testing.T) { + for _, text := range []string{ + "告诉我最近的数据包会话有哪些，给出会话名、pcap 数量和分析入口。", + "分析最近一个 HTTPS 排障抓包会话，说明主要通信端点。", + } { + t.Run(text, func(t *testing.T) { + plan, clarification := resolveTurn(&Request{UserText: text, Role: "admin"}) + if plan.Decision != DecisionClarify || !strings.Contains(clarification, "pcap-session") { + t.Fatalf("resolveTurn(%q) = %+v, %q; want pcap-session clarification", text, plan, clarification) + } + }) + } +} + +func TestResolveTurnClarifiesAmbiguousPacketPath(t *testing.T) { + plan, clarification := resolveTurn(&Request{UserText: "这个包是不是经过 NAT 或网关了，帮我看一下链路。", Role: "admin"}) + if plan.Decision != DecisionClarify || !strings.Contains(clarification, "源/目的地址") { + t.Fatalf("resolveTurn ambiguous packet path = %+v, %q; want packet details clarification", plan, clarification) + } +} + +func TestResolveTurnLetsLLMUnderstandReadOnlyHostRequests(t *testing.T) { + tests := []struct { + name string + text string + }{ + {name: "disk", text: "磁盘快满了，帮我看看"}, + {name: "directory", text: "看看 /var 哪些目录最大"}, + {name: "process", text: "找一下占内存最高的进程"}, + {name: "interface", text: "检查网络接口错误包"}, + {name: "dns", text: "DNS 解析失败了，帮我排查"}, + {name: "english disk", text: "Which disk is using the most space on this device?"}, + {name: "english directory", text: "Show me the largest directories on this host"}, + {name: "english process", text: "Find the process using the most memory"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + plan, clarification := resolveTurn(&Request{UserText: tt.text, Role: "admin"}) + if plan.Decision == DecisionClarify || clarification != "" { + t.Fatalf("resolveTurn(%q) = %+v, %q; want LLM understand path", tt.text, plan, clarification) + } + }) + } +} + +func TestResolveTurnAllowsExplicitHostTarget(t *testing.T) { + tests := []Request{ + {UserText: "看看 edge-001 的磁盘", Role: "admin"}, + {UserText: "检查 10.0.0.5 的网络接口", Role: "admin"}, + {UserText: "看看目录 /var", Role: "admin", Mentions: []Mention{{Type: "device", ID: "24"}}}, + {UserText: "device_id=24 查大文件", Role: "admin"}, + } + for _, req := range tests { + plan, clarification := resolveTurn(&req) + if plan.Decision == DecisionClarify || clarification != "" { + t.Fatalf("resolveTurn(%q) = %+v, %q; want non-clarify", req.UserText, plan, clarification) + } + } +} + +func TestTurnPlanRecordsAndLoopsThroughSystemStates(t *testing.T) { + plan := PlanTurn(ResolvedFacts{Permitted: true, LongRunning: true}) + want := []TurnPhase{PhaseUnderstand, PhaseResolve, PhaseDecide, PhaseOperate} + if !reflect.DeepEqual(plan.Transitions, want) { + t.Fatalf("Transitions = %v, want %v", plan.Transitions, want) + } + if plan.Observe() != PhaseObserve || plan.NextAfterObserve() != PhaseUnderstand { + t.Fatalf("observe loop = %q -> %q, want observe -> understand", plan.Observe(), plan.NextAfterObserve()) + } + + clarify := PlanTurn(ResolvedFacts{Permitted: true, Missing: true}) + if clarify.NextAfterObserve() != PhaseClarify { + t.Fatalf("non-executable plan looped to %q, want clarify", clarify.NextAfterObserve()) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/k8s_action_e2e_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/k8s_action_e2e_test.go new file mode 100644 index 000000000..e1f7b829e --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/k8s_action_e2e_test.go @@ -0,0 +1,324 @@ +package chatruntime + +import ( + "context" + "encoding/json" + "log/slog" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph" + aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" + k8sbiz "github.com/ongridio/ongrid/internal/manager/biz/k8s" + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" + k8smodel "github.com/ongridio/ongrid/internal/manager/model/k8s" + "github.com/ongridio/ongrid/internal/pkg/errs" + "github.com/ongridio/ongrid/internal/pkg/tenantctx" + "github.com/ongridio/ongrid/internal/pkg/tunnel" +) + +func TestRuntimeK8sActionToolCallPassesReviewGateAndController(t *testing.T) { + replicas := 2 + args := mustJSON(t, map[string]any{ + "cluster_id": 1, + "action": "scale", + "kind": "Deployment", + "namespace": "default", + "name": "api", + "replicas": replicas, + "dry_run": true, + "reason": "e2e chat smoke", + }) + model := newScriptedChatModel( + &schema.Message{ + Role: schema.Assistant, + ToolCalls: []schema.ToolCall{{ + ID: "call_k8s_scale", + Type: "function", + Function: schema.FunctionCall{ + Name: aiopstools.ToolNameExecuteK8sAction, + Arguments: args, + }, + }}, + }, + &schema.Message{ + Role: schema.Assistant, + Content: "dry run preflight completed", + }, + ) + controllerEdgeID := uint64(77) + caller := &e2eK8sActionCaller{ + resp: mustJSON(t, tunnel.KubernetesActionResponse{ + ClusterID: 1, + Action: "scale", + Kind: "Deployment", + APIVersion: "apps/v1", + Namespace: "default", + Name: "api", + DryRun: true, + Applied: false, + Preflight: tunnel.KubernetesActionPreflight{ + Kind: "Deployment", + APIVersion: "apps/v1", + Namespace: "default", + Name: "api", + UID: "deploy-uid", + ResourceVersion: "10", + Exists: true, + }, + ResultResourceVersion: "10", + Message: "dry run only", + }), + } + reader := &e2eK8sSnapshotReader{ + clusters: []*k8smodel.Cluster{{ + ID: 1, + Name: "kind-local", + Mode: k8smodel.ModeFullNode, + Status: k8smodel.ClusterStatusOnline, + ControllerEdgeID: &controllerEdgeID, + }}, + } + reviewer := &e2eReviewSpawner{ + result: "Decision: approve\nReason: scoped dry-run scale preflight is safe", + } + sink := &e2eProposalSink{} + inner := aiopstools.NewExecuteK8sActionTool(caller, reader, slog.Default()) + k8sActionTool := decorators.WithTenantBind(decorators.WithReviewGate(inner, reviewer, decorators.ReviewGateConfig{ + Sink: sink, + Timeout: time.Second, + })) + + sess := &aiopsmodel.Session{ID: "s-k8s", UserID: 42} + store := newMemSessions(sess) + rt, err := NewRuntime(Config{ + Sessions: store, + ChatModel: model, + ToolBag: []basetool.BaseTool{k8sActionTool}, + GraphCfg: graph.Config{MaxIterations: 5}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + var events []Event + ctx := tenantctx.With(context.Background(), tenantctx.Tenant{UserID: 42, Role: "admin"}) + reply, err := rt.Handle(ctx, &Request{ + SessionID: sess.ID, + UserID: sess.UserID, + Role: "admin", + UserText: "把 default/api Deployment 扩到 2 个副本，先 dry-run", + Emit: func(ev Event) { + events = append(events, ev) + }, + }) + if err != nil { + t.Fatalf("Handle: %v", err) + } + if reply == nil || reply.Message == nil || reply.Message.Content == nil || *reply.Message.Content != "dry run preflight completed" { + t.Fatalf("reply = %+v, want final assistant content", reply) + } + if model.calls.Load() != 2 { + t.Fatalf("LLM Generate calls = %d, want 2", model.calls.Load()) + } + + if caller.lastEdgeID != controllerEdgeID { + t.Fatalf("controller edge id = %d, want %d", caller.lastEdgeID, controllerEdgeID) + } + if caller.lastMethod != tunnel.MethodExecuteK8sAction { + t.Fatalf("controller method = %q, want %q", caller.lastMethod, tunnel.MethodExecuteK8sAction) + } + var sent tunnel.KubernetesActionRequest + if err := json.Unmarshal(caller.lastBody, &sent); err != nil { + t.Fatalf("decode controller request: %v", err) + } + if sent.ClusterID != 1 || sent.Action != "scale" || sent.Kind != "Deployment" || sent.Namespace != "default" || sent.Name != "api" { + t.Fatalf("unexpected controller request: %+v", sent) + } + if sent.Replicas == nil || *sent.Replicas != replicas || !sent.DryRun || sent.Reason != "e2e chat smoke" { + t.Fatalf("missing normalized write args: %+v", sent) + } + + if reviewer.calls.Load() != 1 { + t.Fatalf("reviewer calls = %d, want 1", reviewer.calls.Load()) + } + if reviewer.lastAgent != decorators.DefaultReviewerAgent { + t.Fatalf("reviewer agent = %q, want %q", reviewer.lastAgent, decorators.DefaultReviewerAgent) + } + if reviewer.lastPrompt == "" || !containsAll(reviewer.lastPrompt, aiopstools.ToolNameExecuteK8sAction, `"dry_run": true`, `"action": "scale"`) { + t.Fatalf("reviewer prompt missing proposal details: %s", reviewer.lastPrompt) + } + + if len(sink.inserts) != 1 { + t.Fatalf("proposal inserts = %d, want 1", len(sink.inserts)) + } + insert := sink.inserts[0] + if insert.ToolName != aiopstools.ToolNameExecuteK8sAction || insert.ToolClass != "write" || insert.OperatorUserID != 42 || insert.SessionID != sess.ID { + t.Fatalf("unexpected proposal insert: %+v", insert) + } + if sink.decisions["proposal-1"] != "approve" || !sink.executed["proposal-1"] { + t.Fatalf("proposal decision/execution = decisions=%v executed=%v", sink.decisions, sink.executed) + } + + store.mu.Lock() + defer store.mu.Unlock() + if len(store.toolCalls) != 1 { + t.Fatalf("tool calls = %d, want 1", len(store.toolCalls)) + } + tc := store.toolCalls[0] + if tc.ToolName != aiopstools.ToolNameExecuteK8sAction || tc.Status != aiopsmodel.StatusSuccess { + t.Fatalf("unexpected persisted tool call: %+v", tc) + } + if tc.LLMCallID == nil || *tc.LLMCallID != "call_k8s_scale" { + t.Fatalf("llm_call_id = %v, want call_k8s_scale", tc.LLMCallID) + } + if tc.ResultJSON == nil || !containsAll(*tc.ResultJSON, `"source":"kubernetes_api"`, `"controller_edge_id":77`, `"dry_run":true`) { + t.Fatalf("tool result json = %v", tc.ResultJSON) + } + if !sawToolEvent(events, EventToolStart, aiopstools.ToolNameExecuteK8sAction) || + !sawToolEvent(events, EventToolEnd, aiopstools.ToolNameExecuteK8sAction) { + t.Fatalf("missing tool lifecycle events: %+v", events) + } +} + +type e2eK8sActionCaller struct { + resp string + lastEdgeID uint64 + lastMethod string + lastBody []byte +} + +func (c *e2eK8sActionCaller) Call(_ context.Context, edgeID uint64, method string, body []byte) ([]byte, error) { + c.lastEdgeID = edgeID + c.lastMethod = method + c.lastBody = append([]byte(nil), body...) + return []byte(c.resp), nil +} + +type e2eK8sSnapshotReader struct { + clusters []*k8smodel.Cluster +} + +func (r *e2eK8sSnapshotReader) ListClusters(_ context.Context, _ k8sbiz.ListClustersFilter) ([]*k8smodel.Cluster, error) { + return r.clusters, nil +} + +func (r *e2eK8sSnapshotReader) GetCluster(_ context.Context, id uint64) (*k8smodel.Cluster, error) { + for _, cluster := range r.clusters { + if cluster.ID == id { + cp := *cluster + return &cp, nil + } + } + return nil, errs.ErrNotFound +} + +func (r *e2eK8sSnapshotReader) ListNodes(context.Context, uint64) ([]*k8smodel.Node, error) { + return nil, nil +} + +func (r *e2eK8sSnapshotReader) CountNodes(context.Context, uint64) (int64, error) { return 0, nil } + +func (r *e2eK8sSnapshotReader) ListWorkloads(context.Context, k8sbiz.ListWorkloadsFilter) ([]*k8smodel.Workload, error) { + return nil, nil +} + +func (r *e2eK8sSnapshotReader) CountWorkloads(context.Context, k8sbiz.ListWorkloadsFilter) (int64, error) { + return 0, nil +} + +func (r *e2eK8sSnapshotReader) ListPods(context.Context, k8sbiz.ListPodsFilter) ([]*k8smodel.Pod, error) { + return nil, nil +} + +func (r *e2eK8sSnapshotReader) CountPods(context.Context, k8sbiz.ListPodsFilter) (int64, error) { + return 0, nil +} + +func (r *e2eK8sSnapshotReader) ListEvents(context.Context, k8sbiz.ListEventsFilter) ([]*k8smodel.Event, error) { + return nil, nil +} + +func (r *e2eK8sSnapshotReader) CountEvents(context.Context, k8sbiz.ListEventsFilter) (int64, error) { + return 0, nil +} + +type e2eReviewSpawner struct { + result string + lastAgent string + lastPrompt string + calls atomic.Int32 +} + +func (s *e2eReviewSpawner) SpawnReviewer(_ context.Context, req decorators.ReviewSpawnRequest) (*decorators.ReviewSpawnResult, error) { + s.calls.Add(1) + s.lastAgent = req.AgentName + s.lastPrompt = req.Prompt + return &decorators.ReviewSpawnResult{TaskID: "review-e2e-1", Result: s.result}, nil +} + +type e2eProposalSink struct { + inserts []decorators.MutatingProposalEvent + decisions map[string]string + executed map[string]bool +} + +func (s *e2eProposalSink) Insert(_ context.Context, ev decorators.MutatingProposalEvent) (string, error) { + s.inserts = append(s.inserts, ev) + if s.decisions == nil { + s.decisions = map[string]string{} + } + if s.executed == nil { + s.executed = map[string]bool{} + } + return "proposal-" + strconv.Itoa(len(s.inserts)), nil +} + +func (s *e2eProposalSink) UpdateDecision(_ context.Context, id, decision string, _ string) error { + if s.decisions == nil { + s.decisions = map[string]string{} + } + s.decisions[id] = decision + return nil +} + +func (s *e2eProposalSink) MarkExecuted(_ context.Context, id string, _ time.Time) error { + if s.executed == nil { + s.executed = map[string]bool{} + } + s.executed[id] = true + return nil +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal json: %v", err) + } + return string(b) +} + +func containsAll(s string, wants ...string) bool { + for _, want := range wants { + if !contains(s, want) { + return false + } + } + return true +} + +func sawToolEvent(events []Event, typ EventType, name string) bool { + for _, ev := range events { + if ev.Type == typ && ev.Tool != nil && ev.Tool.Name == name { + return true + } + } + return false +} diff --git a/internal/manager/biz/aiops/chatruntime/load_all.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/load_all.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/load_all.go rename to internal/manager/biz/aiops/runtime/v3/chatruntime/load_all.go diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/load_all_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/load_all_test.go new file mode 100644 index 000000000..bebe1581a --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/load_all_test.go @@ -0,0 +1,146 @@ +package chatruntime + +import ( + "os" + "path/filepath" + "testing" +) + +// TestLoadAll_WiresPluginContainersFromSkillsRoot drops a recursive_claude +// pack under skillsRoot and asserts both its skills AND its agents reach +// the LoadResult. +func TestLoadAll_WiresPluginContainersFromSkillsRoot(t *testing.T) { + root := t.TempDir() + // Mirror the recursive_claude fixture into root/recursive so that + // SkillsRoot is the conventional "parent of packs" layout. + src, err := filepath.Abs(filepath.Join("testdata", "plugin_container", "recursive_claude")) + if err != nil { + t.Fatalf("abs: %v", err) + } + dst := filepath.Join(root, "recursive") + if err := copyTree(src, dst); err != nil { + t.Fatalf("copyTree: %v", err) + } + + res, err := LoadAll(LoadAllConfig{SkillsRoot: root}) + if err != nil { + t.Fatalf("LoadAll: %v", err) + } + if len(res.Skills) == 0 { + t.Fatalf("LoadAll returned no skills; warns=%+v", res.Warnings) + } + hasHello := false + hasCmd := false + for _, sk := range res.Skills { + if sk.Name == "hello_world" { + hasHello = true + } + if sk.Name == "cmd_commit" { + hasCmd = true + } + } + if !hasHello { + t.Errorf("hello_world not loaded from pack under skills root; got %v", skillNames(res.Skills)) + } + if !hasCmd { + t.Errorf("cmd_commit not loaded from pack under skills root; got %v", skillNames(res.Skills)) + } + // Agents from the pack should also surface. + hasAgent := false + for _, ag := range res.Agents { + if ag.Name == "pr_summary" { + hasAgent = true + } + } + if !hasAgent { + t.Errorf("pr_summary agent should reach LoadAll result; got %d agents", len(res.Agents)) + } +} + +func TestLoadAll_NonExistentRoots_NoError(t *testing.T) { + res, err := LoadAll(LoadAllConfig{ + SkillsRoot: filepath.Join(t.TempDir(), "missing-skills"), + AgentsRoot: filepath.Join(t.TempDir(), "missing-agents"), + }) + if err != nil { + t.Fatalf("LoadAll on missing roots should be nil-error, got %v", err) + } + if len(res.Skills) != 0 || len(res.Agents) != 0 { + t.Errorf("expected empty result; got skills=%d agents=%d", len(res.Skills), len(res.Agents)) + } +} + +func TestLoadAll_AgentsRootLoadsLoosePersonas(t *testing.T) { + src, err := filepath.Abs(filepath.Join("testdata", "agent_registry", "multi")) + if err != nil { + t.Fatalf("abs: %v", err) + } + res, err := LoadAll(LoadAllConfig{AgentsRoot: src}) + if err != nil { + t.Fatalf("LoadAll: %v", err) + } + if len(res.Agents) != 2 { + t.Errorf("expected 2 agents, got %d", len(res.Agents)) + } +} + +// TestLoadAll_ExtraAgentsRoots — regression for the marketplace reload +// path. When the AgentRegistry reloads after a pack install, the +// marketplace passes the builtin agents directory as an extra. The +// previous implementation routed extras through ExtraSkillsRoots which +// silently dropped loose *.md agent personas (only SKILL.md matches +// the wantSkills=true filter). Result: every marketplace install +// emptied the assistant page until manager restart. +func TestLoadAll_ExtraAgentsRoots(t *testing.T) { + personas, err := filepath.Abs(filepath.Join("testdata", "agent_registry", "multi")) + if err != nil { + t.Fatalf("abs: %v", err) + } + // Treat the persona dir as an EXTRA — primary is empty. Should + // still load both .md personas. + res, err := LoadAll(LoadAllConfig{ExtraAgentsRoots: []string{personas}}) + if err != nil { + t.Fatalf("LoadAll: %v", err) + } + if len(res.Agents) != 2 { + t.Errorf("ExtraAgentsRoots: want 2 agents, got %d", len(res.Agents)) + } + // Sanity: ExtraSkillsRoots on the same dir would NOT yield the + // agents (only SKILL.md matches the skills filter). This guards + // against future refactors collapsing the two extra-lists back + // into one without restoring the semantic split. + res2, err := LoadAll(LoadAllConfig{ExtraSkillsRoots: []string{personas}}) + if err != nil { + t.Fatalf("LoadAll (skills extras): %v", err) + } + if len(res2.Agents) != 0 { + t.Errorf("ExtraSkillsRoots: agent personas leaked into skills walk, got %d", len(res2.Agents)) + } +} + +// copyTree recursively copies the contents of src to dst. Used by +// LoadAll tests so the test's tempdir is a self-contained skills root +// (no symlink escape). +func copyTree(src, dst string) error { + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + if info.IsDir() { + return os.MkdirAll(target, 0o755) + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + return os.WriteFile(target, raw, 0o600) + }) +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/plugin_container.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/plugin_container.go new file mode 100644 index 000000000..896cb8c75 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/plugin_container.go @@ -0,0 +1,686 @@ +package chatruntime + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// PluginManifest is the on-disk shape of `.claude-plugin/plugin.json` +// (claude-code) and `openclaw.plugin.json` (openclaw). +// ongrid recognizes both: the former is treated as the canonical, the +// latter is treated as a superset whose ongrid-irrelevant fields are +// preserved into UIMetadata["openclaw_legacy"]. +// +// The struct is deliberately permissive — we model only the fields +// ongrid currently consumes; any other field is captured into Extras +// and (for openclaw) routed into Pack.UIMetadata["openclaw_legacy"]. +type PluginManifest struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + ConfigSchema json.RawMessage `json:"configSchema"` + + // Extras captures every JSON field not modeled above. We keep it + // raw so we can stash openclaw-specific keys (providers / channels + // / cliBackends / providerAuthChoices, ...) into Pack.UIMetadata. + Extras map[string]json.RawMessage `json:"-"` +} + +// ContainerKind labels which marker file we found in a candidate dir. +type ContainerKind string + +const ( + // ContainerClaude is the `.claude-plugin/plugin.json` form. + ContainerClaude ContainerKind = "claude" + // ContainerOpenclaw is the `openclaw.plugin.json` form. + ContainerOpenclaw ContainerKind = "openclaw" + // ContainerBareSkills is the skills.sh / vercel-labs/skills form: + // a directory containing one or more `skills//SKILL.md` + // (or a root-level SKILL.md) and NO manifest file. The pack ID + // and display name are synthesized from the directory name. This + // is what `npx skills add owner/repo` produces. + ContainerBareSkills ContainerKind = "bare_skills" + // ContainerNone means no recognized layout was found. + ContainerNone ContainerKind = "" +) + +// DetectContainer probes a directory for plugin container markers. +// Returns the marker kind and absolute path to the manifest file +// (empty for ContainerBareSkills, which has no manifest). Resolution +// order: openclaw > claude-plugin > bare-skills > none. +// +// ongrid recognizes the openclaw and claude-plugin manifest forms +// plus the bare-skills (skills.sh / vercel-labs/skills) layout +// where there's no manifest and the pack is just `skills//SKILL.md` +// drops under a parent dir. +func DetectContainer(dir string) (ContainerKind, string, error) { + openclawPath := filepath.Join(dir, "openclaw.plugin.json") + if info, err := os.Stat(openclawPath); err == nil && !info.IsDir() { + return ContainerOpenclaw, openclawPath, nil + } + claudePath := filepath.Join(dir, ".claude-plugin", "plugin.json") + if info, err := os.Stat(claudePath); err == nil && !info.IsDir() { + return ContainerClaude, claudePath, nil + } + if hasBareSkills(dir) { + return ContainerBareSkills, "", nil + } + return ContainerNone, "", nil +} + +// hasBareSkills reports whether dir looks like a skills.sh-style pack — +// either a root-level SKILL.md or at least one `skills//SKILL.md`. +// Scans at most one level deep under `skills/` to keep the probe cheap. +func hasBareSkills(dir string) bool { + if info, err := os.Stat(filepath.Join(dir, "SKILL.md")); err == nil && !info.IsDir() { + return true + } + skillsDir := filepath.Join(dir, "skills") + entries, err := os.ReadDir(skillsDir) + if err != nil { + return false + } + for _, e := range entries { + if !e.IsDir() { + continue + } + if info, err := os.Stat(filepath.Join(skillsDir, e.Name(), "SKILL.md")); err == nil && !info.IsDir() { + return true + } + } + return false +} + +// LoadPluginContainer parses the marker manifest in dir, then recursively +// loads every SKILL.md / agent .md / command .md found under the pack +// root. / — claude-code-style bundle dropped into the +// skills root has all sub-content auto-registered. +// +// Sub-roots: +// - claude pack: /skills/, /agents/, /commands/ +// - openclaw pack: paths from manifest.skills[] (relative); also +// /skills/, /agents/, /commands/ when present +// +// Path safety: every sub-path is resolved with filepath.EvalSymlinks +// and verified to land back inside the pack root before parse — symlink +// hops out of the tree and `..` components are rejected with an +// "escapes_root" warning. The pack itself is loaded; only the offending +// file is dropped. +// +// Per-file parse failures are recorded as warnings (code "parse_failed") +// rather than aborting the load — a single bad SKILL.md should not take +// the whole pack offline. +// +// hooks/ subdirs are walked once for diagnostics (per-hook warning code +// "hooks_dropped"); .mcp.json triggers a single "mcp_unsupported" +// warning. Neither is acted upon +func LoadPluginContainer(dir string) (*LoadResult, error) { + kind, manifestPath, err := DetectContainer(dir) + if err != nil { + return nil, fmt.Errorf("chatruntime: detect container in %s: %w", dir, err) + } + if kind == ContainerNone { + return nil, fmt.Errorf("chatruntime: no recognized pack layout in %s (need .claude-plugin/plugin.json, openclaw.plugin.json, or at least one skills//SKILL.md)", dir) + } + + var ( + pack *Pack + warnings []LoadWarning + raw []byte + ) + if kind == ContainerBareSkills { + // skills.sh-style drop: no manifest file, synthesize one from the + // directory name. There's nothing to hash, and hashing the nil + // `raw` would give EVERY bare pack the SAME empty-input sha — so + // the installer's content-dedupe (GetByManifestSHA) rejects the + // 2nd bare skills.sh pack as "identical content already installed". + // Derive the manifest sha from the synthesized pack id (directory + // name) so distinct bare packs get distinct shas, while the same + // dir re-installs to a stable sha (idempotent). + pack = synthesizeBareSkillsPack(dir) + raw = []byte("bare-skills:" + pack.ID) + } else { + raw, err = os.ReadFile(manifestPath) + if err != nil { + return nil, fmt.Errorf("chatruntime: read %s: %w", manifestPath, err) + } + pack, warnings, err = parsePluginManifest(raw, kind) + if err != nil { + return nil, fmt.Errorf("chatruntime: parse %s: %w", manifestPath, err) + } + } + pack.Dir = mustAbs(dir) + pack.ManifestSHA256 = sha256Hex(raw) + + // Resolve the pack root — used as the allowlist boundary for every + // sub-file we touch. EvalSymlinks lets the user place the pack + // itself behind a symlink (cp -r dance) without breaking the root, + // while still rejecting symlinks under it that escape outward. + resolvedRoot, rootErr := filepath.EvalSymlinks(dir) + if rootErr != nil { + // If we can't even resolve the root we can't safely walk; + // return what we have (metadata only) plus a warning. + warnings = append(warnings, LoadWarning{ + Path: dir, + Code: "root_resolve_failed", + Reason: fmt.Sprintf("eval symlinks on pack root: %v; sub-tree skipped", rootErr), + }) + return &LoadResult{Pack: pack, Warnings: warnings}, nil + } + + // Skills sub-tree. + skillRoots := resolveSkillSubRoots(dir, kind, raw, &warnings) + var skills []*Skill + for _, sr := range skillRoots { + more, w := walkSkillsRoot(sr, resolvedRoot) + skills = append(skills, more...) + warnings = append(warnings, w...) + } + + // Agents sub-tree (claude only — openclaw doesn't ship agents/). + agentsDir := filepath.Join(dir, "agents") + agents, agentWarns := walkAgentsRoot(agentsDir, resolvedRoot) + warnings = append(warnings, agentWarns...) + + // Commands sub-tree → keyword skills. + commandsDir := filepath.Join(dir, "commands") + commandSkills, commandWarns := walkCommandsRoot(commandsDir, resolvedRoot) + skills = append(skills, commandSkills...) + warnings = append(warnings, commandWarns...) + + // hooks/ + .mcp.json — diagnostics only. + warnings = append(warnings, scanHooksDir(filepath.Join(dir, "hooks"), resolvedRoot)...) + if info, err := os.Stat(filepath.Join(dir, ".mcp.json")); err == nil && !info.IsDir() { + warnings = append(warnings, LoadWarning{ + Path: filepath.Join(dir, ".mcp.json"), + Code: "mcp_unsupported", + Reason: ".mcp.json found; ongrid does not run MCP servers yet", + }) + } + + return &LoadResult{ + Pack: pack, + Skills: skills, + Agents: agents, + Warnings: warnings, + }, nil +} + +// resolveSkillSubRoots returns the directories we should walk for +// SKILL.md files. Claude packs always use /skills; openclaw packs +// MAY declare a manifest.skills[] array of relative paths — we follow +// those, plus the conventional /skills if it exists. +// +// Every returned path is verified to be a child of `dir` (string-level +// guard against `..` in manifest.skills[]); deeper symlink-vs-root +// validation happens during the walk itself. +func resolveSkillSubRoots(dir string, kind ContainerKind, manifestRaw []byte, warnings *[]LoadWarning) []string { + var roots []string + seen := map[string]bool{} + + add := func(p string) { + clean := filepath.Clean(p) + if seen[clean] { + return + } + seen[clean] = true + if info, err := os.Stat(clean); err == nil && info.IsDir() { + roots = append(roots, clean) + } + } + + if kind == ContainerOpenclaw { + // Pull the optional skills[] array. When present it's authoritative — + // we follow only those paths (treats manifest.skills[] + // and the conventional skills/ directory as equivalent forms; a pack + // that lists both would otherwise double-load). + var generic struct { + Skills []string `json:"skills"` + } + hasManifestSkills := false + if err := json.Unmarshal(manifestRaw, &generic); err == nil { + for _, raw := range generic.Skills { + if strings.TrimSpace(raw) == "" { + continue + } + hasManifestSkills = true + // Disallow absolute paths and `..` components in the + // manifest declaration itself — purely string check; + // EvalSymlinks during the walk catches symlink escape. + if filepath.IsAbs(raw) || strings.Contains(raw, "..") { + *warnings = append(*warnings, LoadWarning{ + Path: raw, + Code: "skills_path_rejected", + Reason: fmt.Sprintf("manifest skills[] entry %q is absolute or contains '..'; rejected", raw), + }) + continue + } + add(filepath.Join(dir, raw)) + } + } + // Fall through to the conventional dir only if the manifest + // didn't declare its own skills[] list. + if !hasManifestSkills { + add(filepath.Join(dir, "skills")) + } + } else if kind == ContainerBareSkills { + // skills.sh layout: either a multi-skill repo with + // skills//SKILL.md or a single-skill repo with a + // root-level SKILL.md. Pick the right root — adding BOTH + // causes the walker to double-load every skills/* file + // (once via skills/, once via the recursive dir walk). + if info, err := os.Stat(filepath.Join(dir, "skills")); err == nil && info.IsDir() { + add(filepath.Join(dir, "skills")) + } else { + add(dir) + } + } else { + // Claude pack: conventional /skills only. + add(filepath.Join(dir, "skills")) + } + return roots +} + +// walkSkillsRoot walks skillsDir for SKILL.md files, returning the +// parsed skills + non-fatal warnings. resolvedRoot is the eval-symlinked +// pack root used to detect escape via symlinks. +func walkSkillsRoot(skillsDir, resolvedRoot string) ([]*Skill, []LoadWarning) { + info, err := os.Stat(skillsDir) + if err != nil || !info.IsDir() { + return nil, nil + } + var skills []*Skill + var warnings []LoadWarning + + walkErr := filepath.Walk(skillsDir, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: path, Code: "walk_error", Reason: walkErr.Error()}) + return nil + } + if info == nil { + return nil + } + // Symlinks (Lstat: ModeSymlink) need explicit policy — filepath.Walk + // does NOT follow them. We must validate that the link target is + // inside resolvedRoot; if it escapes, emit an escapes_root warning + // so the test (and humans) see the rejection. If it lands inside, + // we still don't auto-recurse (avoid loops); operators can land + // SKILL.md files at real paths. + if info.Mode()&os.ModeSymlink != 0 { + if !pathSafeUnderRoot(path, resolvedRoot, &warnings) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + return nil + } + if info.IsDir() { + return nil + } + if filepath.Base(path) != "SKILL.md" { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &warnings) { + return nil + } + sk, ws, err := ParseSkillMd(path) + warnings = append(warnings, ws...) + if err != nil { + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "parse_failed", + Reason: err.Error(), + }) + return nil + } + sk.Dir = filepath.Dir(path) + // Default activation = always when neither + // top-level nor metadata.ongrid.activation set a mode. + if sk.Activation.Mode == "" { + sk.Activation.Mode = "always" + } + if sk.Metadata.Ongrid.Activation.Mode == "" { + sk.Metadata.Ongrid.Activation.Mode = sk.Activation.Mode + } + skills = append(skills, sk) + return nil + }) + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: skillsDir, Code: "walk_error", Reason: walkErr.Error()}) + } + return skills, warnings +} + +// walkAgentsRoot walks agentsDir for *.md agent personas. +// README.md is skipped (documentation, not a persona) — same policy as +// AgentRegistry.Load. +func walkAgentsRoot(agentsDir, resolvedRoot string) ([]*Agent, []LoadWarning) { + info, err := os.Stat(agentsDir) + if err != nil || !info.IsDir() { + return nil, nil + } + var agents []*Agent + var warnings []LoadWarning + + walkErr := filepath.Walk(agentsDir, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: path, Code: "walk_error", Reason: walkErr.Error()}) + return nil + } + if info == nil || info.IsDir() { + return nil + } + if !strings.EqualFold(filepath.Ext(info.Name()), ".md") { + return nil + } + if strings.EqualFold(info.Name(), "README.md") { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &warnings) { + return nil + } + ag, ws, err := ParseAgentMd(path) + warnings = append(warnings, ws...) + if err != nil { + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "parse_failed", + Reason: err.Error(), + }) + return nil + } + ag.Dir = filepath.Dir(path) + agents = append(agents, ag) + return nil + }) + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: agentsDir, Code: "walk_error", Reason: walkErr.Error()}) + } + return agents, warnings +} + +// walkCommandsRoot walks commandsDir for *.md command files and +// converts each into a keyword skill. Nested subdirs +// are flattened — `commands/git/commit.md` becomes `cmd_commit`. +// README.md is skipped. +func walkCommandsRoot(commandsDir, resolvedRoot string) ([]*Skill, []LoadWarning) { + info, err := os.Stat(commandsDir) + if err != nil || !info.IsDir() { + return nil, nil + } + var skills []*Skill + var warnings []LoadWarning + + walkErr := filepath.Walk(commandsDir, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: path, Code: "walk_error", Reason: walkErr.Error()}) + return nil + } + if info == nil || info.IsDir() { + return nil + } + if !strings.EqualFold(filepath.Ext(info.Name()), ".md") { + return nil + } + if strings.EqualFold(info.Name(), "README.md") { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &warnings) { + return nil + } + sk, ws, err := ConvertCommandFile(path) + warnings = append(warnings, ws...) + if err != nil { + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "command_parse_failed", + Reason: err.Error(), + }) + return nil + } + skills = append(skills, sk) + return nil + }) + if walkErr != nil { + warnings = append(warnings, LoadWarning{Path: commandsDir, Code: "walk_error", Reason: walkErr.Error()}) + } + return skills, warnings +} + +// scanHooksDir walks the hooks/ subdir (if present) and emits one +// warning per hook file plus one summary warning. ongrid never executes +// hooks. +func scanHooksDir(hooksDir, resolvedRoot string) []LoadWarning { + info, err := os.Stat(hooksDir) + if err != nil || !info.IsDir() { + return nil + } + var warnings []LoadWarning + // Top-level summary warning so callers grepping for "hooks_unsupported" + // keep working (back-compat with PR-2 test). + warnings = append(warnings, LoadWarning{ + Path: hooksDir, + Code: "hooks_unsupported", + Reason: "hooks/ subdir found; ongrid does not run plugin hooks", + }) + + var dropped []string + _ = filepath.Walk(hooksDir, func(path string, info os.FileInfo, _ error) error { + if info == nil || info.IsDir() { + return nil + } + if !pathSafeUnderRoot(path, resolvedRoot, &warnings) { + return nil + } + dropped = append(dropped, path) + return nil + }) + sort.Strings(dropped) + for _, p := range dropped { + warnings = append(warnings, LoadWarning{ + Path: p, + Code: "hooks_dropped", + Reason: "hook file ignored; ongrid never executes plugin hooks", + }) + } + return warnings +} + +// pathSafeUnderRoot validates that path, after resolving symlinks, sits +// inside resolvedRoot. Failures append an "escapes_root" or +// "symlink_error" warning to *warnings and return false so the caller +// can skip the offending file. +// +// Implementation note: we use filepath.EvalSymlinks rather than a pure +// string-prefix check. A string check would miss symlink hops (the +// classic /tmp/evil → /etc dodge), and absent EvalSymlinks any +// `/skills/` symlink that resolves outside the pack would be +// silently followed by ParseSkillMd. The trade-off is one extra +// stat-per-file plus a hard fail when the path itself can't be +// resolved (broken symlink, race-deleted file) — surfaced as +// symlink_error rather than crashing the load. +func pathSafeUnderRoot(path, resolvedRoot string, warnings *[]LoadWarning) bool { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + *warnings = append(*warnings, LoadWarning{ + Path: path, + Code: "symlink_error", + Reason: err.Error(), + }) + return false + } + if !pathHasPrefix(resolved, resolvedRoot) { + *warnings = append(*warnings, LoadWarning{ + Path: path, + Code: "escapes_root", + Reason: fmt.Sprintf("path traversal: resolved path %s escapes pack root %s", resolved, resolvedRoot), + }) + return false + } + return true +} + +// parsePluginManifest decodes the manifest JSON and produces a Pack. +// openclawLegacyKeys are the openclaw-only fields ongrid stuffs into +// UIMetadata["openclaw_legacy"] for SPA display. +func parsePluginManifest(raw []byte, kind ContainerKind) (*Pack, []LoadWarning, error) { + var manifest PluginManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, nil, fmt.Errorf("decode: %w", err) + } + + // Re-decode into a generic map to capture extras. + var generic map[string]json.RawMessage + if err := json.Unmarshal(raw, &generic); err != nil { + return nil, nil, fmt.Errorf("decode generic: %w", err) + } + + known := map[string]struct{}{ + "id": {}, + "name": {}, + "version": {}, + "description": {}, + "configSchema": {}, + } + extras := map[string]json.RawMessage{} + for k, v := range generic { + if _, ok := known[k]; ok { + continue + } + extras[k] = v + } + manifest.Extras = extras + + var warnings []LoadWarning + + pack := &Pack{ + ID: manifest.ID, + DisplayName: manifest.Name, + Version: manifest.Version, + Description: manifest.Description, + ConfigSchema: manifest.ConfigSchema, + SignatureState: "unsigned", + } + + // id/name/version/description go straight into Pack. + // Required: id is the only must-have. + if pack.ID == "" { + // Some claude-code plugin.json files use `name` as the id when + // id is absent — fall back to that for compat. + if manifest.Name != "" { + pack.ID = manifest.Name + warnings = append(warnings, LoadWarning{ + Code: "id_fallback_from_name", + Reason: "plugin manifest missing 'id'; using 'name' as id", + }) + } else { + return nil, warnings, fmt.Errorf("manifest missing required field 'id'") + } + } + + // openclaw-only fields are routed into ui_metadata.openclaw_legacy + // so the SPA can render them, but ongrid's runtime ignores them. + if kind == ContainerOpenclaw && len(extras) > 0 { + legacy := map[string]any{} + for k, v := range extras { + // Keep raw JSON; SPA may want the structure verbatim. + var decoded any + if err := json.Unmarshal(v, &decoded); err == nil { + legacy[k] = decoded + } else { + legacy[k] = string(v) + } + } + pack.UIMetadata = map[string]any{ + "openclaw_legacy": legacy, + } + warnings = append(warnings, LoadWarning{ + Code: "openclaw_legacy_preserved", + Reason: fmt.Sprintf("preserved %d openclaw-specific manifest fields under ui_metadata.openclaw_legacy", len(legacy)), + }) + } else if len(extras) > 0 { + // Claude-plugin manifest with extra unrecognized keys — keep + // them too, but under ui_metadata.extra. + extraMap := map[string]any{} + for k, v := range extras { + var decoded any + if err := json.Unmarshal(v, &decoded); err == nil { + extraMap[k] = decoded + } else { + extraMap[k] = string(v) + } + } + pack.UIMetadata = map[string]any{ + "extra": extraMap, + } + } + + return pack, warnings, nil +} + +// synthesizeBareSkillsPack builds a minimal Pack for a skills.sh-style +// drop where there's no plugin.json / openclaw.plugin.json manifest. +// ID = sanitized directory basename (lowercase, hyphens), DisplayName +// = same, Version = "0.0.0". The pack-level ConfigSchema / UIMetadata +// stay empty — the individual SKILL.md files are the canonical metadata +// source for this layout. +func synthesizeBareSkillsPack(dir string) *Pack { + base := strings.TrimSpace(filepath.Base(strings.TrimRight(dir, string(filepath.Separator)))) + id := bareSkillsPackID(base) + return &Pack{ + ID: id, + DisplayName: base, + Version: "0.0.0", + Description: "Bare skills.sh-style pack (no manifest).", + } +} + +// bareSkillsPackID normalises a directory basename into a pack id. +// Rules mirror the SKILL.md skillNameRe: lowercase, digits, hyphens; +// non-conformant chars collapse to a single hyphen; leading/trailing +// hyphens trimmed; empty fallback = "untitled-skill-pack". +func bareSkillsPackID(base string) string { + if base == "" { + return "untitled-skill-pack" + } + var b strings.Builder + prevDash := false + for _, r := range strings.ToLower(base) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + prevDash = false + default: + if !prevDash && b.Len() > 0 { + b.WriteByte('-') + prevDash = true + } + } + } + out := strings.Trim(b.String(), "-") + if out == "" { + return "untitled-skill-pack" + } + return out +} + +func sha256Hex(raw []byte) string { + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) +} + +func mustAbs(dir string) string { + abs, err := filepath.Abs(dir) + if err != nil { + return dir + } + return abs +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/plugin_container_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/plugin_container_test.go new file mode 100644 index 000000000..93fc67b3e --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/plugin_container_test.go @@ -0,0 +1,302 @@ +package chatruntime + +import ( + "path/filepath" + "strings" + "testing" +) + +func fixturePack(name string) string { + return filepath.Join("testdata", "plugin_container", name) +} + +func TestDetectContainer_Claude(t *testing.T) { + kind, path, err := DetectContainer(fixturePack("claude_pack")) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerClaude { + t.Errorf("kind = %q, want claude", kind) + } + if filepath.Base(path) != "plugin.json" { + t.Errorf("path = %q", path) + } +} + +func TestDetectContainer_Openclaw(t *testing.T) { + kind, path, err := DetectContainer(fixturePack("openclaw_pack")) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerOpenclaw { + t.Errorf("kind = %q, want openclaw", kind) + } + if filepath.Base(path) != "openclaw.plugin.json" { + t.Errorf("path = %q", path) + } +} + +func TestDetectContainer_None(t *testing.T) { + kind, _, err := DetectContainer(t.TempDir()) + if err != nil { + t.Fatalf("DetectContainer: %v", err) + } + if kind != ContainerNone { + t.Errorf("kind = %q, want none", kind) + } +} + +func TestLoadPluginContainer_Claude(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("claude_pack")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + if res.Pack == nil { + t.Fatal("Pack is nil") + } + if res.Pack.ID != "acme-tools" { + t.Errorf("ID = %q", res.Pack.ID) + } + if res.Pack.Version != "1.0.0" { + t.Errorf("Version = %q", res.Pack.Version) + } + if res.Pack.DisplayName != "Acme Tools" { + t.Errorf("DisplayName = %q", res.Pack.DisplayName) + } + if len(res.Pack.ConfigSchema) == 0 { + t.Error("ConfigSchema empty") + } + if res.Pack.ManifestSHA256 == "" { + t.Error("ManifestSHA256 empty") + } + if res.Pack.SignatureState != "unsigned" { + t.Errorf("SignatureState = %q", res.Pack.SignatureState) + } +} + +func TestLoadPluginContainer_OpenclawLegacyPreserved(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("openclaw_pack")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + if res.Pack.ID != "openclaw-bundle" { + t.Errorf("ID = %q", res.Pack.ID) + } + legacy, ok := res.Pack.UIMetadata["openclaw_legacy"].(map[string]any) + if !ok { + t.Fatalf("openclaw_legacy missing or wrong type: %#v", res.Pack.UIMetadata) + } + for _, k := range []string{"providers", "channels", "cliBackends", "providerAuthChoices"} { + if _, ok := legacy[k]; !ok { + t.Errorf("openclaw_legacy missing key %q; got %v", k, mapKeys(legacy)) + } + } + hasLegacyWarn := false + for _, w := range res.Warnings { + if w.Code == "openclaw_legacy_preserved" { + hasLegacyWarn = true + } + } + if !hasLegacyWarn { + t.Errorf("expected openclaw_legacy_preserved warning; got %+v", res.Warnings) + } +} + +func TestLoadPluginContainer_HooksAndMcpWarnings(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("claude_pack_with_extras")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + codes := map[string]bool{} + for _, w := range res.Warnings { + codes[w.Code] = true + } + if !codes["hooks_unsupported"] { + t.Errorf("expected hooks_unsupported warning; got %+v", res.Warnings) + } + if !codes["mcp_unsupported"] { + t.Errorf("expected mcp_unsupported warning; got %+v", res.Warnings) + } +} + +func TestLoadPluginContainer_NoMarker(t *testing.T) { + _, err := LoadPluginContainer(t.TempDir()) + if err == nil { + t.Fatal("expected error when no marker present") + } +} + +// --- recursive load (PR-skill-load) --- + +func TestLoadPluginContainer_ClaudeRecursive_Skills(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_claude")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + // Expect: 1 SKILL.md (hello_world) + 1 command-converted skill (cmd_commit). + var helloFound, cmdCommitFound bool + for _, sk := range res.Skills { + if sk.Name == "hello_world" { + helloFound = true + if sk.Dir == "" { + t.Errorf("hello_world.Dir empty") + } + if sk.Activation.Mode != "always" { + t.Errorf("hello_world activation = %q, want always (default)", sk.Activation.Mode) + } + } + if sk.Name == "cmd_commit" { + cmdCommitFound = true + } + } + if !helloFound { + t.Errorf("hello_world skill not loaded; got %v", skillNames(res.Skills)) + } + if !cmdCommitFound { + t.Errorf("cmd_commit (from commands/) not loaded; got %v", skillNames(res.Skills)) + } +} + +func TestLoadPluginContainer_ClaudeRecursive_Agents(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_claude")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + if len(res.Agents) != 1 { + t.Fatalf("Agents len = %d, want 1", len(res.Agents)) + } + if res.Agents[0].Name != "pr_summary" { + t.Errorf("agent name = %q, want pr_summary", res.Agents[0].Name) + } + if res.Agents[0].Dir == "" { + t.Errorf("agent.Dir empty") + } +} + +func TestLoadPluginContainer_ClaudeRecursive_Commands(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_claude")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + var cmd *Skill + for _, sk := range res.Skills { + if sk.Name == "cmd_commit" { + cmd = sk + break + } + } + if cmd == nil { + t.Fatalf("cmd_commit not in skills; got %v", skillNames(res.Skills)) + } + if cmd.Activation.Mode != "keyword" { + t.Errorf("cmd_commit activation = %q, want keyword", cmd.Activation.Mode) + } + wantKw := map[string]bool{"/commit": false, "commit": false} + for _, kw := range cmd.Activation.Keywords { + if _, ok := wantKw[kw]; ok { + wantKw[kw] = true + } + } + for kw, found := range wantKw { + if !found { + t.Errorf("cmd_commit missing keyword %q; got %v", kw, cmd.Activation.Keywords) + } + } + // Body should carry the soft-hint about claude allowed-tools. + if !strings.Contains(cmd.PromptBody, "Bash") || !strings.Contains(cmd.PromptBody, "Edit") { + t.Errorf("cmd_commit prompt body should include allowed-tools hint; got:\n%s", cmd.PromptBody) + } + if !strings.HasPrefix(strings.TrimSpace(cmd.PromptBody), "[能力: cmd_commit]") { + t.Errorf("cmd_commit body should start with [能力: cmd_commit] header; got:\n%s", cmd.PromptBody) + } +} + +func TestLoadPluginContainer_HookSubdir_Warning(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_claude")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + codes := map[string]int{} + for _, w := range res.Warnings { + codes[w.Code]++ + } + if codes["hooks_unsupported"] == 0 { + t.Errorf("expected hooks_unsupported warning; got %v", codes) + } + if codes["hooks_dropped"] == 0 { + t.Errorf("expected hooks_dropped warning per hook file; got %v", codes) + } +} + +func TestLoadPluginContainer_McpFile_Warning(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_claude")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + hasMcp := false + for _, w := range res.Warnings { + if w.Code == "mcp_unsupported" { + hasMcp = true + } + } + if !hasMcp { + t.Errorf("expected mcp_unsupported warning; got %+v", res.Warnings) + } +} + +func TestLoadPluginContainer_OpenclawRecursive_RelativeSkills(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("recursive_openclaw")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + if len(res.Skills) != 2 { + t.Fatalf("Skills len = %d, want 2 (foo, bar); got %v", len(res.Skills), skillNames(res.Skills)) + } + names := map[string]bool{} + for _, sk := range res.Skills { + names[sk.Name] = true + } + for _, expected := range []string{"foo", "bar"} { + if !names[expected] { + t.Errorf("skill %q not loaded; got %v", expected, skillNames(res.Skills)) + } + } +} + +func TestLoadPluginContainer_PathTraversal_Rejected(t *testing.T) { + res, err := LoadPluginContainer(fixturePack("path_traversal")) + if err != nil { + t.Fatalf("LoadPluginContainer: %v", err) + } + // The pwned skill (under a symlink that escapes the pack root) MUST + // NOT make it into res.Skills. + for _, sk := range res.Skills { + if sk.Name == "pwned" { + t.Fatalf("pwned skill loaded despite path-traversal symlink: %+v", sk) + } + } + // Expect an escapes_root warning. + hasEscape := false + for _, w := range res.Warnings { + if w.Code == "escapes_root" { + hasEscape = true + if !strings.Contains(strings.ToLower(w.Reason), "path traversal") && + !strings.Contains(strings.ToLower(w.Reason), "escapes") { + t.Errorf("escapes_root reason should mention traversal/escapes; got %q", w.Reason) + } + } + } + if !hasEscape { + t.Errorf("expected escapes_root warning; got %+v", res.Warnings) + } +} + +// --- helpers --- + +func mapKeys(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/reasoning_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/reasoning_test.go new file mode 100644 index 000000000..77ab5b2e4 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/reasoning_test.go @@ -0,0 +1,23 @@ +package chatruntime + +import ( + "testing" + + "github.com/stretchr/testify/require" + + model "github.com/ongridio/ongrid/internal/manager/model/aiops" +) + +func TestHistory_ReasoningSurvivesNextUserTurn(t *testing.T) { + for _, reason := range []*string{nil, strPtr("original protocol field")} { + rows := []*model.Message{{ID: "a", Role: model.RoleAssistant, Content: strPtr("answer"), ReasoningContent: reason}, {ID: "u", Role: model.RoleUser, Content: strPtr("next")}} + got := buildEinoHistory(rows) + require.Len(t, got, 1) + require.Equal(t, "answer", got[0].Content) + if reason != nil { + require.Equal(t, *reason, got[0].ReasoningContent) + } else { + require.Empty(t, got[0].ReasoningContent) + } + } +} diff --git a/internal/manager/biz/aiops/chatruntime/runtime.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go similarity index 99% rename from internal/manager/biz/aiops/chatruntime/runtime.go rename to internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go index 1e0f69453..cf7b853b1 100644 --- a/internal/manager/biz/aiops/chatruntime/runtime.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go @@ -22,8 +22,8 @@ import ( "go.opentelemetry.io/otel/attribute" biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph/callbacks" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph/callbacks" "github.com/ongridio/ongrid/internal/manager/biz/aiops/toolreplay" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" diff --git a/internal/manager/biz/aiops/chatruntime/runtime_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_test.go similarity index 99% rename from internal/manager/biz/aiops/chatruntime/runtime_test.go rename to internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_test.go index 4bfedc12e..fc206e061 100644 --- a/internal/manager/biz/aiops/chatruntime/runtime_test.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_test.go @@ -12,8 +12,8 @@ import ( "github.com/cloudwego/eino/schema" biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph/callbacks" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph/callbacks" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" model "github.com/ongridio/ongrid/internal/manager/model/aiops" "github.com/ongridio/ongrid/internal/pkg/errs" diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_worker_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_worker_test.go new file mode 100644 index 000000000..e7c32d24a --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_worker_test.go @@ -0,0 +1,633 @@ +package chatruntime + +import ( + "context" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" +) + +// TestSpawnWorker_Sync drives SpawnWorker with Background=false and +// verifies the returned worker is in WorkerStatusCompleted with the +// scripted assistant content as Result. Also asserts the worker's +// chat_sessions row was persisted with the audit columns +// (agent_id / parent_session_id / background). +func TestSpawnWorker_Sync(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "worker-result", + }) + store, parentID := newSeededRuntimeStore(t, 7) + rt := newRuntimeWithStore(t, "incident-investigator", scripted, nil, store) + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "incident-investigator", + Prompt: "diagnose host-01", + ParentSession: parentID, + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + if w == nil { + t.Fatalf("expected non-nil worker") + } + if w.Status != WorkerStatusCompleted { + t.Errorf("status = %q, want completed", w.Status) + } + if w.Result != "worker-result" { + t.Errorf("result = %q, want worker-result", w.Result) + } + if w.AgentName != "incident-investigator" { + t.Errorf("agent = %q", w.AgentName) + } + if !strings.HasPrefix(w.ID, "agent-") { + t.Errorf("id = %q, want agent-* prefix", w.ID) + } + if !strings.HasPrefix(w.SessionID, "worker-") { + t.Errorf("session_id = %q, want worker-* prefix", w.SessionID) + } + + // chat_sessions audit row — assert all three columns are set. + row, err := store.GetSession(context.Background(), w.SessionID) + if err != nil || row == nil { + t.Fatalf("worker chat_sessions row missing: %v", err) + } + if row.AgentID == nil || *row.AgentID != "incident-investigator" { + t.Errorf("row.AgentID = %v, want incident-investigator", row.AgentID) + } + if row.ParentSessionID == nil || *row.ParentSessionID != parentID { + t.Errorf("row.ParentSessionID = %v, want %q", row.ParentSessionID, parentID) + } + if row.Background { + t.Errorf("row.Background = true, want false (sync spawn)") + } + if row.UserID != 7 { + t.Errorf("row.UserID = %d, want 7 (inherited from parent)", row.UserID) + } + if row.RootSessionID == nil || *row.RootSessionID != parentID { + t.Errorf("row.RootSessionID = %v, want %q", row.RootSessionID, parentID) + } + if row.OwnerAgentID == nil || *row.OwnerAgentID != model.DefaultSessionOwnerAgent { + t.Errorf("row.OwnerAgentID = %v, want default", row.OwnerAgentID) + } + if row.Initiator != model.SessionInitiatorAgent || row.Audience != model.SessionAudienceInternal { + t.Errorf("row session boundary = initiator=%q audience=%q", row.Initiator, row.Audience) + } + + // ListByParent surfaces the worker session under the parent. + kids, err := store.ListByParent(context.Background(), parentID) + if err != nil { + t.Fatalf("ListByParent: %v", err) + } + if len(kids) != 1 || kids[0].ID != w.SessionID { + t.Errorf("ListByParent = %+v, want one row %q", kids, w.SessionID) + } +} + +func TestSpawnWorker_Sync_ForwardsWorkerToolEventsToParent(t *testing.T) { + scripted := newScriptedChatModel( + &schema.Message{ + Role: schema.Assistant, + Content: "checking metric", + ToolCalls: []schema.ToolCall{{ + ID: "call_worker_metric", + Type: "function", + Function: schema.FunctionCall{Name: "query_promql", Arguments: `{"query":"up"}`}, + }}, + }, + &schema.Message{Role: schema.Assistant, Content: "worker-result"}, + ) + store, parentID := newSeededRuntimeStore(t, 7) + rt := newRuntimeWithStore(t, "specialist-sre", scripted, []basetool.BaseTool{ + &fakeTool{name: "query_promql", schema: `{"type":"object","properties":{"query":{"type":"string"}}}`}, + }, store) + + var events []Event + emit := func(ev Event) { + events = append(events, ev) + } + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "specialist-sre", + Prompt: "check prometheus", + ParentSession: parentID, + ParentEmit: emit, + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + if w.Status != WorkerStatusCompleted { + t.Fatalf("status = %q, want completed", w.Status) + } + + var sawStart, sawEnd bool + for _, ev := range events { + if ev.Type == EventAssistant { + t.Fatalf("worker assistant event leaked to parent stream: %+v", ev) + } + if ev.Tool == nil || ev.Tool.Name != "query_promql" { + continue + } + if !strings.HasPrefix(ev.Tool.ToolCallID, w.ID+":") { + t.Fatalf("worker tool_call_id = %q, want %q prefix", ev.Tool.ToolCallID, w.ID+":") + } + switch ev.Type { + case EventToolStart: + sawStart = true + if ev.Tool.ArgsJSON != `{"query":"up"}` { + t.Fatalf("start args = %q", ev.Tool.ArgsJSON) + } + case EventToolEnd: + sawEnd = true + if ev.Tool.Status != "success" { + t.Fatalf("end status = %q, want success", ev.Tool.Status) + } + if !strings.Contains(ev.Tool.ResultJSON, `"ok":true`) { + t.Fatalf("end result = %q", ev.Tool.ResultJSON) + } + } + } + if !sawStart || !sawEnd { + t.Fatalf("missing forwarded worker tool events: sawStart=%v sawEnd=%v events=%+v", sawStart, sawEnd, events) + } +} + +// TestSpawnWorker_Async drives SpawnWorker with Background=true and +// verifies (a) immediate return with status running OR completed (race) +// + (b) eventual TaskNotification fires through ParentEmit. +func TestSpawnWorker_Async(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "async-result", + }) + rt := newRuntimeWithAgent(t, "general-purpose", scripted, nil) + + var ( + mu sync.Mutex + events []Event + done = make(chan struct{}, 1) + ) + emit := func(ev Event) { + mu.Lock() + events = append(events, ev) + mu.Unlock() + if ev.Type == EventTaskNotification { + select { + case done <- struct{}{}: + default: + } + } + } + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "general-purpose", + Prompt: "go", + Background: true, + ParentEmit: emit, + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + if w == nil { + t.Fatalf("expected worker") + } + // On the goroutine race the snapshot can already be running OR + // completed (scripted model is fast). Both are valid post-spawn + // states. + if w.Status != WorkerStatusRunning && + w.Status != WorkerStatusPending && + w.Status != WorkerStatusCompleted { + t.Errorf("immediate status = %q", w.Status) + } + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for task_notification") + } + + mu.Lock() + defer mu.Unlock() + var notif *TaskNotification + for _, e := range events { + if e.Type == EventTaskNotification { + notif = e.Notification + break + } + } + if notif == nil { + t.Fatalf("no task_notification in events: %+v", events) + } + if notif.TaskID != w.ID { + t.Errorf("notif.TaskID = %q, worker.ID = %q", notif.TaskID, w.ID) + } + if notif.Status != WorkerStatusCompleted { + t.Errorf("notif.status = %q, want completed", notif.Status) + } + if notif.Result != "async-result" { + t.Errorf("notif.Result = %q", notif.Result) + } + if !strings.Contains(notif.Summary, "general-purpose") { + t.Errorf("notif.Summary = %q (expected agent name)", notif.Summary) + } +} + +// TestSendToWorker_Continuation continues a completed worker with a +// follow-up message and checks the Result is updated. +func TestSendToWorker_Continuation(t *testing.T) { + scripted := newScriptedChatModel( + &schema.Message{Role: schema.Assistant, Content: "first"}, + &schema.Message{Role: schema.Assistant, Content: "second"}, + ) + rt := newRuntimeWithAgent(t, "general-purpose", scripted, nil) + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "general-purpose", + Prompt: "step 1", + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + if w.Result != "first" { + t.Fatalf("first result = %q, want first", w.Result) + } + if err := rt.SendToWorker(context.Background(), w.ID, "step 2 please"); err != nil { + t.Fatalf("SendToWorker: %v", err) + } + got, ok := rt.GetWorker(w.ID) + if !ok || got == nil { + t.Fatalf("GetWorker after send: not found") + } + if got.Result != "second" { + t.Errorf("after send, result = %q, want second", got.Result) + } + if got.Status != WorkerStatusCompleted { + t.Errorf("status = %q", got.Status) + } +} + +// TestStopWorker_Idempotent confirms StopWorker is safe to call on a +// terminal worker. +func TestStopWorker_Idempotent(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "x", + }) + rt := newRuntimeWithAgent(t, "general-purpose", scripted, nil) + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "general-purpose", + Prompt: "go", + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + if err := rt.StopWorker(context.Background(), w.ID); err != nil { + t.Errorf("Stop on completed: %v", err) + } + // Stop a non-existent id returns an error per contract. + if err := rt.StopWorker(context.Background(), "agent-nonsuch"); err == nil { + t.Errorf("expected error for unknown worker id") + } +} + +// TestSpawnWorker_ClosesSession is the regression test for the orphan +// worker session bug that accumulated 161 closed_at=NULL rows on the +// test env before the defer-close was added in worker.go. Both the +// sync and async termination paths must call CloseSession; this drives +// both and asserts ClosedAt is non-nil after termination. +func TestSpawnWorker_ClosesSession(t *testing.T) { + t.Run("sync", func(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, Content: "done", + }) + store, parentID := newSeededRuntimeStore(t, 1) + rt := newRuntimeWithStore(t, "incident-investigator", scripted, nil, store) + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "incident-investigator", + Prompt: "go", + ParentSession: parentID, + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + row, err := store.GetSession(context.Background(), w.SessionID) + if err != nil || row == nil { + t.Fatalf("session row missing: %v", err) + } + if row.ClosedAt == nil { + t.Fatalf("closed_at = nil after sync spawn; want non-nil so the row stops looking active") + } + }) + + t.Run("async", func(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, Content: "done", + }) + store, parentID := newSeededRuntimeStore(t, 1) + rt := newRuntimeWithStore(t, "general-purpose", scripted, nil, store) + + done := make(chan struct{}, 1) + emit := func(ev Event) { + if ev.Type == EventTaskNotification { + select { + case done <- struct{}{}: + default: + } + } + } + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "general-purpose", + Prompt: "go", + Background: true, + ParentSession: parentID, + ParentEmit: emit, + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for task_notification") + } + // CloseSession is deferred after the notify fires, so give the + // goroutine a tick to return. + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + row, _ := store.GetSession(context.Background(), w.SessionID) + if row != nil && row.ClosedAt != nil { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("closed_at still nil after async spawn — defer didn't fire") + }) +} + +// TestSpawnWorker_UnknownAgent confirms SpawnWorker rejects a name not +// in the AgentRegistry. +func TestSpawnWorker_UnknownAgent(t *testing.T) { + scripted := newScriptedChatModel() + rt := newRuntimeWithAgent(t, "general-purpose", scripted, nil) + _, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "no-such-agent", + Prompt: "x", + }) + if err == nil { + t.Fatalf("expected error for unknown agent") + } +} + +// TestFilterToolsForAgent verifies whitelist + blacklist + the +// implicit coordinator-only strip. +func TestFilterToolsForAgent(t *testing.T) { + bag := []basetool.BaseTool{ + &fakeTool{name: "AgentTool", schema: `{"type":"object"}`}, // coordinator-only — always stripped + &fakeTool{name: "SendMessage", schema: `{"type":"object"}`}, // coordinator-only — always stripped + &fakeTool{name: "TaskStop", schema: `{"type":"object"}`}, // coordinator-only — always stripped + &fakeTool{name: "query_promql", schema: `{"type":"object"}`}, // whitelisted + &fakeTool{name: "query_logql", schema: `{"type":"object"}`}, // not whitelisted + &fakeTool{name: "host_restart_service", schema: `{"type":"object"}`}, // blacklisted + &fakeTool{name: "kill_process", schema: `{"type":"object"}`}, // blacklisted via *_process + } + ag := &Agent{ + Name: "test-worker", + Tools: []string{"query_promql"}, + DisallowedTools: []string{"host_restart_service", "*_process"}, + } + // Worker path: isCoordinator=false → AgentTool/SendMessage/TaskStop stripped. + out := filterToolsForAgent(bag, ag, false) + names := make([]string, 0, len(out)) + for _, t := range out { + info, _ := t.Info(context.Background()) + names = append(names, info.Name) + } + if len(names) != 1 || names[0] != "query_promql" { + t.Errorf("worker filtered names = %v, want [query_promql]", names) + } + + // Coordinator path: isCoordinator=true → coordinator-only tools + // survive even when the persona's Tools whitelist doesn't include + // them. (The whitelist still gates non-control-plane tools.) + out = filterToolsForAgent(bag, ag, true) + names = names[:0] + for _, t := range out { + info, _ := t.Info(context.Background()) + names = append(names, info.Name) + } + gotCoord := map[string]bool{} + for _, n := range names { + gotCoord[n] = true + } + if !gotCoord["AgentTool"] || !gotCoord["SendMessage"] || !gotCoord["TaskStop"] || !gotCoord["query_promql"] { + t.Errorf("coordinator filtered names = %v; want all of AgentTool / SendMessage / TaskStop / query_promql", names) + } + if gotCoord["query_logql"] || gotCoord["host_restart_service"] || gotCoord["kill_process"] { + t.Errorf("coordinator filtered names = %v; whitelist or blacklist leaked", names) + } +} + +func TestFilterToolsForAgent_AllowsDeclaredWorkerDelegationOnly(t *testing.T) { + bag := []basetool.BaseTool{ + &fakeTool{name: "AgentTool", schema: `{"type":"object"}`}, + &fakeTool{name: "SendMessage", schema: `{"type":"object"}`}, + &fakeTool{name: "TaskStop", schema: `{"type":"object"}`}, + } + delegating := filteredToolNames(t, filterToolsForAgent(bag, &Agent{CanDelegate: true}, false)) + if len(delegating) != 1 || delegating[0] != "AgentTool" { + t.Fatalf("delegating worker tools = %v, want [AgentTool]", delegating) + } + nonDelegating := filteredToolNames(t, filterToolsForAgent(bag, &Agent{}, false)) + if len(nonDelegating) != 0 { + t.Fatalf("non-delegating worker tools = %v, want none", nonDelegating) + } +} + +func TestSpawnWorkerRejectsThirdDelegationLevel(t *testing.T) { + rootID := "root" + workerID := "worker-1" + owner := model.DefaultSessionOwnerAgent + store := newMemSessions(&model.Session{ID: rootID, UserID: 7, RootSessionID: &rootID, OwnerAgentID: &owner}) + for _, session := range []*model.Session{ + {ID: workerID, UserID: 7, RootSessionID: &rootID, ParentSessionID: &rootID, OwnerAgentID: &owner}, + {ID: "worker-2", UserID: 7, RootSessionID: &rootID, ParentSessionID: &workerID, OwnerAgentID: &owner}, + } { + if err := store.CreateSession(context.Background(), session); err != nil { + t.Fatalf("seed session: %v", err) + } + } + rt := newRuntimeWithStore(t, "specialist-disk", newScriptedChatModel(), nil, store) + _, err := rt.SpawnWorker(context.Background(), SpawnRequest{AgentName: "specialist-disk", Prompt: "delegate again", ParentSession: "worker-2"}) + if err == nil || !strings.Contains(err.Error(), "delegation depth exceeded") { + t.Fatalf("SpawnWorker error = %v, want depth rejection", err) + } +} + +func filteredToolNames(t *testing.T, tools []basetool.BaseTool) []string { + t.Helper() + names := make([]string, 0, len(tools)) + for _, tool := range tools { + info, err := tool.Info(context.Background()) + if err != nil { + t.Fatalf("tool info: %v", err) + } + names = append(names, info.Name) + } + return names +} + +func TestSpawnWorker_DoesNotForceKnowledgePrologue(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "worker-result", + }) + var calls atomic.Int32 + reg := NewAgentRegistry() + reg.Add(&Agent{ + Name: "kb-capable-worker", + Description: "test agent", + WhenToUse: "test", + MaxTurns: 3, + Tools: []string{"query_knowledge"}, + }) + rt, err := NewRuntime(Config{ + Sessions: newMemSessions(&model.Session{ID: "s1", UserID: 7}), + ChatModel: scripted, + ToolBag: []basetool.BaseTool{&countingTool{name: "query_knowledge", calls: &calls}}, + AgentRegistry: reg, + GraphCfg: graph.Config{MaxIterations: 3}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "kb-capable-worker", + Prompt: "check cpu", + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + if w.Status != WorkerStatusCompleted { + t.Fatalf("status = %q, want completed", w.Status) + } + if calls.Load() != 0 { + t.Fatalf("query_knowledge was called by runtime prologue %d time(s), want 0", calls.Load()) + } +} + +// TestNotificationFor_FieldShape locks down the wire shape consumers +// (the SPA's task_notification renderer) depend on. +func TestNotificationFor_FieldShape(t *testing.T) { + scripted := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "shape-test", + }) + rt := newRuntimeWithAgent(t, "general-purpose", scripted, nil) + w, err := rt.SpawnWorker(context.Background(), SpawnRequest{ + AgentName: "general-purpose", + Prompt: "shape", + }) + if err != nil { + t.Fatalf("SpawnWorker: %v", err) + } + // Reach into the internal map to invoke notificationFor on the + // canonical worker ptr (snapshots have nil mu). + rt.workersMu.Lock() + internal := rt.workers[w.ID] + rt.workersMu.Unlock() + ev := rt.notificationFor(internal) + if ev.Type != EventTaskNotification { + t.Errorf("type = %q", ev.Type) + } + if ev.Notification == nil { + t.Fatalf("Notification nil") + } + if ev.Notification.TaskID != w.ID { + t.Errorf("TaskID = %q", ev.Notification.TaskID) + } + if ev.Notification.Status != WorkerStatusCompleted { + t.Errorf("Status = %q", ev.Notification.Status) + } + if ev.Notification.Result != "shape-test" { + t.Errorf("Result = %q", ev.Notification.Result) + } + if _, ok := ev.Notification.Usage["duration_ms"]; !ok { + t.Errorf("usage.duration_ms missing") + } +} + +type countingTool struct { + name string + calls *atomic.Int32 +} + +func (t *countingTool) Info(_ context.Context) (*basetool.ToolInfo, error) { + return &basetool.ToolInfo{ + Name: t.name, + Description: "fake", + Parameters: []byte(`{"type":"object","properties":{"query":{"type":"string"}}}`), + Class: "read", + }, nil +} + +func (t *countingTool) InvokableRun(_ context.Context, _ string, _ ...basetool.InvokeOption) (string, error) { + t.calls.Add(1) + return `{"items":[{"title":"kb","score":0.9}]}`, nil +} + +// newRuntimeWithAgent builds a Runtime with one persona registered and +// no tool bag. cfg.AgentRegistry's Add is used to inject the persona +// inline so the test doesn't need a fixture file. Default +// graph.MaxIterations is bumped down to keep tests snappy. +func newRuntimeWithAgent(t *testing.T, name string, cm *scriptedChatModel, toolBag []basetool.BaseTool) *Runtime { + t.Helper() + sess := &model.Session{ID: "s1", UserID: 7} + return newRuntimeWithStore(t, name, cm, toolBag, newMemSessions(sess)) +} + +// newRuntimeWithStore is the shape used by tests that need to inspect +// the SessionRepo after the spawn (worker session row, ListByParent). +// Caller passes the pre-seeded *memSessions so the test can read it back. +func newRuntimeWithStore(t *testing.T, name string, cm *scriptedChatModel, toolBag []basetool.BaseTool, store *memSessions) *Runtime { + t.Helper() + reg := NewAgentRegistry() + reg.Add(&Agent{ + Name: name, + Description: "test agent", + WhenToUse: "test", + MaxTurns: 3, + Tools: []string{}, // empty whitelist = inherit all tools (minus coordinator-only) + }) + rt, err := NewRuntime(Config{ + Sessions: store, + ChatModel: cm, + ToolBag: toolBag, + AgentRegistry: reg, + GraphCfg: graph.Config{MaxIterations: 3}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + return rt +} + +// newSeededRuntimeStore returns a memSessions with one parent (coordinator) +// session pre-inserted, plus that parent's id. UserID is the owner. +func newSeededRuntimeStore(t *testing.T, userID uint64) (*memSessions, string) { + t.Helper() + parentID := "parent-sess-1" + rootID := parentID + owner := model.DefaultSessionOwnerAgent + store := newMemSessions(&model.Session{ID: parentID, UserID: userID, Title: "parent", RootSessionID: &rootID, OwnerAgentID: &owner}) + return store, parentID +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_parser.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_parser.go new file mode 100644 index 000000000..0aa1f6989 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_parser.go @@ -0,0 +1,248 @@ +package chatruntime + +import ( + "bufio" + "bytes" + "fmt" + "os" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +// frontmatterDelimiter is the YAML frontmatter fence used by SKILL.md / +// agents/*.md and every other Markdown frontmatter dialect. +const frontmatterDelimiter = "---" + +// skillNameRe is the snake_case validator. says the name is +// lowercase + digits + underscore; this matches what openclaw enforces too. +var skillNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9_]*$`) + +// nonSnakeRe matches characters that should be normalized to underscore +// in a non-conforming name (dashes, dots, spaces). +var nonSnakeRe = regexp.MustCompile(`[^a-z0-9_]+`) + +// h1Re matches the leading `#`-style markdown H1 header. +var h1Re = regexp.MustCompile(`^#\s+.+\s*$`) + +// ParseSkillMd reads a SKILL.md file at path and returns the parsed +// Skill plus any non-fatal warnings. Required fields +// are name + description; missing them produces an error rather than a +// warning (a SKILL.md without a description is unusable). +// +// Behavioral notes: +// - Frontmatter is preserved into Skill.UnknownFields for any keys we +// don't model — explicitly requires this so upstream +// schema additions don't break ongrid loading. +// - The body H1 is auto-normalized to `[能力: ]` +// - name is snake_case validated; non-conforming names are normalized +// and a warning with code "name_normalized" is emitted. +// +// The returned Skill.Dir is the directory containing path. PromptBody is +// the markdown body verbatim (after H1 normalization). Tag-less fields +// (Provenance, Dir) stay zero-valued — the install path fills them. +func ParseSkillMd(path string) (*Skill, []LoadWarning, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: read %s: %w", path, err) + } + frontmatter, body, err := splitFrontmatter(raw) + if err != nil { + return nil, nil, fmt.Errorf("chatruntime: %s: %w", path, err) + } + + var warnings []LoadWarning + + // Decode into the typed struct first, then again into a map so we + // can compute "unknown" keys for forward compatibility. + var sk Skill + if len(frontmatter) > 0 { + if err := yaml.Unmarshal(frontmatter, &sk); err != nil { + return nil, nil, fmt.Errorf("chatruntime: parse %s frontmatter: %w", path, err) + } + } + + rawMap := map[string]any{} + if len(frontmatter) > 0 { + if err := yaml.Unmarshal(frontmatter, &rawMap); err != nil { + return nil, nil, fmt.Errorf("chatruntime: parse %s frontmatter (raw): %w", path, err) + } + } + sk.UnknownFields = retainUnknownSkillFields(rawMap) + + // Required fields — + if strings.TrimSpace(sk.Name) == "" { + return nil, warnings, fmt.Errorf("chatruntime: %s: frontmatter missing required field 'name'", path) + } + if strings.TrimSpace(sk.Description) == "" { + return nil, warnings, fmt.Errorf("chatruntime: %s: frontmatter missing required field 'description'", path) + } + + // Snake-case enforcement with auto-normalization. + if !skillNameRe.MatchString(sk.Name) { + original := sk.Name + sk.Name = normalizeSnakeName(sk.Name) + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "name_normalized", + Reason: fmt.Sprintf("name %q is not snake_case; normalized to %q", original, sk.Name), + }) + } + + // Body: normalize the leading H1 to [能力: ] + sk.PromptBody = normalizeSkillBodyH1(body, sk.Name) + + // Reconcile metadata.ongrid.activation vs top-level activation. + // — both are accepted; ongrid one wins on conflict, with + // a warning so the author knows. + if sk.Metadata.Ongrid.Activation.Mode != "" { + if sk.Activation.Mode != "" && sk.Metadata.Ongrid.Activation.Mode != sk.Activation.Mode { + warnings = append(warnings, LoadWarning{ + Path: path, + Code: "activation_conflict", + Reason: "both top-level activation and metadata.ongrid.activation are set; using metadata.ongrid.activation", + }) + } + sk.Activation = sk.Metadata.Ongrid.Activation + } + + return &sk, warnings, nil +} + +// splitFrontmatter splits a SKILL.md / agent .md byte slice into +// (frontmatterYAML, bodyMarkdown). When no frontmatter is present (no +// leading `---` line) it returns frontmatter=nil and body=full input. +// +// A malformed frontmatter (opening `---` but no closing `---`) is an error. +func splitFrontmatter(raw []byte) ([]byte, []byte, error) { + scanner := bufio.NewScanner(bytes.NewReader(raw)) + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) + + // Peek the first non-empty line. If it isn't `---`, treat the whole + // thing as body (no frontmatter). + var firstLine string + var leading []string + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "" { + leading = append(leading, line) + continue + } + firstLine = line + break + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("scan: %w", err) + } + if strings.TrimRight(firstLine, " \t") != frontmatterDelimiter { + // No frontmatter — return the entire raw as body. + return nil, raw, nil + } + + var fmLines []string + closed := false + for scanner.Scan() { + line := scanner.Text() + if strings.TrimRight(line, " \t") == frontmatterDelimiter { + closed = true + break + } + fmLines = append(fmLines, line) + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("scan: %w", err) + } + if !closed { + return nil, nil, fmt.Errorf("frontmatter started with --- but never closed") + } + + var bodyLines []string + for scanner.Scan() { + bodyLines = append(bodyLines, scanner.Text()) + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("scan: %w", err) + } + + frontmatter := []byte(strings.Join(fmLines, "\n")) + body := []byte(strings.Join(bodyLines, "\n")) + _ = leading + return frontmatter, body, nil +} + +// normalizeSkillBodyH1 rewrites the leading H1 (`# Title`) to +// `[能力: ]` If there's no H1, prepends one. +func normalizeSkillBodyH1(body []byte, name string) string { + const tagPrefix = "[能力: " + header := tagPrefix + name + "]" + + text := string(body) + if strings.TrimSpace(text) == "" { + return header + } + + lines := strings.Split(text, "\n") + // Skip leading blanks to find the first content line; we drop those + // leading blanks so the canonical header is always the first byte. + idx := 0 + for idx < len(lines) && strings.TrimSpace(lines[idx]) == "" { + idx++ + } + if idx >= len(lines) { + return header + } + lines = lines[idx:] + first := lines[0] + if h1Re.MatchString(strings.TrimSpace(first)) { + // Replace the H1 with the canonical tag. + lines[0] = header + } else if strings.HasPrefix(strings.TrimSpace(first), tagPrefix) { + // Already normalized — leave as is. + } else { + // Prepend the canonical tag as a new first line. + newLines := []string{header, ""} + newLines = append(newLines, lines...) + lines = newLines + } + return strings.TrimRight(strings.Join(lines, "\n"), "\n") +} + +// normalizeSnakeName converts dashes / dots / spaces / etc. to underscores +// and lowercases the result. Used when the author wrote `hello-world` etc. +func normalizeSnakeName(in string) string { + out := strings.ToLower(strings.TrimSpace(in)) + out = nonSnakeRe.ReplaceAllString(out, "_") + out = strings.Trim(out, "_") + if out == "" { + out = "unnamed" + } + return out +} + +// retainUnknownSkillFields returns a map of frontmatter keys that we +// don't model on the Skill struct. wants these preserved so +// upstream openclaw / claude-code schema additions don't break loading. +func retainUnknownSkillFields(raw map[string]any) map[string]any { + known := map[string]struct{}{ + "name": {}, + "version": {}, + "description": {}, + "when_to_use": {}, + "activation": {}, + "config_section": {}, + "tools": {}, + "metadata": {}, + } + out := map[string]any{} + for k, v := range raw { + if _, ok := known[k]; ok { + continue + } + out[k] = v + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_parser_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_parser_test.go new file mode 100644 index 000000000..abe27da3b --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_parser_test.go @@ -0,0 +1,230 @@ +package chatruntime + +import ( + "path/filepath" + "strings" + "testing" +) + +// fixtureSkill returns the absolute path to a SKILL.md fixture under +// testdata/skill_parser//SKILL.md. +func fixtureSkill(scenario string) string { + return filepath.Join("testdata", "skill_parser", scenario, "SKILL.md") +} + +func TestParseSkillMd_Minimal(t *testing.T) { + sk, warns, err := ParseSkillMd(fixtureSkill("minimal")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if len(warns) != 0 { + t.Fatalf("expected no warnings, got %+v", warns) + } + if sk.Name != "hello_world" { + t.Errorf("Name = %q, want hello_world", sk.Name) + } + if sk.Description == "" { + t.Error("Description is empty") + } + // Body's H1 should be normalized to [能力: hello_world]. + if !strings.HasPrefix(sk.PromptBody, "[能力: hello_world]") { + t.Errorf("PromptBody = %q, want H1 normalized", sk.PromptBody) + } + if !strings.Contains(sk.PromptBody, "Hello") { + t.Errorf("PromptBody lost the body content: %q", sk.PromptBody) + } +} + +func TestParseSkillMd_Typical(t *testing.T) { + sk, warns, err := ParseSkillMd(fixtureSkill("typical")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if len(warns) != 0 { + t.Fatalf("expected no warnings, got %+v", warns) + } + if sk.Version != "1.0.0" { + t.Errorf("Version = %q, want 1.0.0", sk.Version) + } + wantOS := []string{"darwin", "linux"} + if got := sk.Metadata.OS; !strSliceEq(got, wantOS) { + t.Errorf("Metadata.OS = %v, want %v", got, wantOS) + } + if got := sk.Metadata.Requires.Bins; !strSliceEq(got, []string{"find", "du"}) { + t.Errorf("Metadata.Requires.Bins = %v", got) + } + if got := sk.Metadata.Requires.Config; !strSliceEq(got, []string{"accountsPath"}) { + t.Errorf("Metadata.Requires.Config = %v", got) + } + if len(sk.Tools) != 2 { + t.Fatalf("Tools len = %d, want 2", len(sk.Tools)) + } + if sk.Tools[0].Name != "find_files" || sk.Tools[0].Class != ClassRead { + t.Errorf("Tools[0] = %+v", sk.Tools[0]) + } + if sk.Tools[0].WhenToUse == "" { + t.Error("Tools[0].WhenToUse is empty — snake_case decode failed") + } +} + +func TestParseSkillMd_WithWhenToUse(t *testing.T) { + sk, _, err := ParseSkillMd(fixtureSkill("with_when_to_use")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if sk.WhenToUse == "" { + t.Fatal("WhenToUse is empty") + } + if !strings.Contains(sk.WhenToUse, "log content") { + t.Errorf("WhenToUse content unexpected: %q", sk.WhenToUse) + } +} + +func TestParseSkillMd_WithOngridExt(t *testing.T) { + sk, warns, err := ParseSkillMd(fixtureSkill("with_ongrid_ext")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if len(warns) != 0 { + t.Fatalf("expected no warnings, got %+v", warns) + } + if sk.Metadata.Ongrid.Scope != "edge" { + t.Errorf("Ongrid.Scope = %q, want edge", sk.Metadata.Ongrid.Scope) + } + if sk.Metadata.Ongrid.EdgeRuntime != "subprocess" { + t.Errorf("Ongrid.EdgeRuntime = %q", sk.Metadata.Ongrid.EdgeRuntime) + } + if len(sk.Metadata.Ongrid.EdgeCapabilities) != 2 { + t.Errorf("EdgeCapabilities len = %d, want 2", len(sk.Metadata.Ongrid.EdgeCapabilities)) + } + if sk.Metadata.Ongrid.MinOngridVersion != ">=0.7.30" { + t.Errorf("MinOngridVersion = %q", sk.Metadata.Ongrid.MinOngridVersion) + } + // metadata.ongrid.activation should win and be reflected in + // top-level Activation per the parser's reconciliation rule. + if sk.Activation.Mode != "keyword" { + t.Errorf("Activation.Mode = %q, want keyword (from ongrid ext)", sk.Activation.Mode) + } + if !strSliceEq(sk.Activation.Keywords, []string{"disk", "磁盘", "du"}) { + t.Errorf("Activation.Keywords = %v", sk.Activation.Keywords) + } +} + +func TestParseSkillMd_WithUnknownFields(t *testing.T) { + sk, _, err := ParseSkillMd(fixtureSkill("with_unknown_fields")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if sk.UnknownFields == nil { + t.Fatal("UnknownFields is nil; expected upstream-only keys preserved") + } + if _, ok := sk.UnknownFields["custom_field_one"]; !ok { + t.Errorf("custom_field_one not preserved; got keys %v", keys(sk.UnknownFields)) + } + if _, ok := sk.UnknownFields["custom_field_two"]; !ok { + t.Errorf("custom_field_two not preserved") + } + if _, ok := sk.UnknownFields["upstream_only_array"]; !ok { + t.Errorf("upstream_only_array not preserved") + } + // Known fields should NOT leak into UnknownFields. + if _, ok := sk.UnknownFields["name"]; ok { + t.Errorf("'name' should not appear in UnknownFields") + } +} + +func TestParseSkillMd_NameWithDash(t *testing.T) { + sk, warns, err := ParseSkillMd(fixtureSkill("name_with_dash")) + if err != nil { + t.Fatalf("ParseSkillMd: %v", err) + } + if sk.Name != "hello_world" { + t.Errorf("Name = %q, want normalized hello_world", sk.Name) + } + found := false + for _, w := range warns { + if w.Code == "name_normalized" { + found = true + break + } + } + if !found { + t.Errorf("expected name_normalized warning; got %+v", warns) + } +} + +func TestParseSkillMd_MissingDescription(t *testing.T) { + _, _, err := ParseSkillMd(fixtureSkill("missing_description")) + if err == nil { + t.Fatal("expected error for missing description, got nil") + } + if !strings.Contains(err.Error(), "description") { + t.Errorf("error should mention 'description'; got %v", err) + } +} + +func TestParseSkillMd_FileMissing(t *testing.T) { + _, _, err := ParseSkillMd(filepath.Join("testdata", "skill_parser", "does_not_exist", "SKILL.md")) + if err == nil { + t.Fatal("expected error for missing file") + } +} + +func TestSplitFrontmatter_NoFrontmatter(t *testing.T) { + body := []byte("# Just a body\n\nNo frontmatter here.\n") + fm, b, err := splitFrontmatter(body) + if err != nil { + t.Fatalf("splitFrontmatter: %v", err) + } + if fm != nil { + t.Errorf("expected nil frontmatter, got %q", fm) + } + if string(b) != string(body) { + t.Errorf("body should be untouched") + } +} + +func TestSplitFrontmatter_MalformedUnclosed(t *testing.T) { + body := []byte("---\nname: x\n\nno closing fence\n") + _, _, err := splitFrontmatter(body) + if err == nil { + t.Fatal("expected error for unclosed frontmatter") + } +} + +func TestNormalizeSkillBodyH1_NoH1Adds(t *testing.T) { + out := normalizeSkillBodyH1([]byte("Just a paragraph.\n"), "abc") + if !strings.HasPrefix(out, "[能力: abc]") { + t.Errorf("expected canonical header prepended; got %q", out) + } +} + +func TestNormalizeSkillBodyH1_AlreadyTagged(t *testing.T) { + body := []byte("[能力: abc]\n\nbody\n") + out := normalizeSkillBodyH1(body, "abc") + if strings.Count(out, "[能力: abc]") != 1 { + t.Errorf("expected single tag header; got %q", out) + } +} + +// --- helpers --- + +func strSliceEq(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func keys(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/manager/biz/aiops/chatruntime/skill_registry.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_registry.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/skill_registry.go rename to internal/manager/biz/aiops/runtime/v3/chatruntime/skill_registry.go diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_registry_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_registry_test.go new file mode 100644 index 000000000..204963494 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_registry_test.go @@ -0,0 +1,166 @@ +package chatruntime + +import ( + "path/filepath" + "testing" +) + +func fixtureSkillRoot(scenario string) string { + return filepath.Join("testdata", "skill_registry", scenario) +} + +func TestSkillRegistry_LoadMulti(t *testing.T) { + r := NewSkillRegistry() + if err := r.Load(fixtureSkillRoot("multi")); err != nil { + t.Fatalf("Load: %v", err) + } + all := r.All() + if len(all) != 3 { + t.Fatalf("All() len = %d, want 3 (always_skill, keyword_skill, buried_skill)", len(all)) + } + names := map[string]bool{} + for _, sk := range all { + names[sk.Name] = true + if sk.Dir == "" { + t.Errorf("skill %q has empty Dir", sk.Name) + } + } + for _, expected := range []string{"always_skill", "keyword_skill", "buried_skill"} { + if !names[expected] { + t.Errorf("skill %q not loaded", expected) + } + } +} + +func TestSkillRegistry_LoadNonExistent(t *testing.T) { + r := NewSkillRegistry() + if err := r.Load(fixtureSkillRoot("does_not_exist")); err != nil { + t.Fatalf("Load on missing dir should be nil-error, got %v", err) + } + if len(r.All()) != 0 { + t.Errorf("expected zero skills") + } +} + +func TestSkillRegistry_BadSkillIsWarning(t *testing.T) { + r := NewSkillRegistry() + if err := r.Load(fixtureSkillRoot("with_bad_skill")); err != nil { + t.Fatalf("Load: %v", err) + } + if len(r.All()) != 1 { + t.Errorf("All() len = %d, want 1 (only 'good' parses)", len(r.All())) + } + hasParseFailed := false + for _, w := range r.Warnings() { + if w.Code == "parse_failed" { + hasParseFailed = true + } + } + if !hasParseFailed { + t.Errorf("expected parse_failed warning for the bad skill; got %+v", r.Warnings()) + } +} + +func TestSkillRegistry_ResolveAlwaysVsKeyword(t *testing.T) { + r := NewSkillRegistry() + if err := r.Load(fixtureSkillRoot("multi")); err != nil { + t.Fatalf("Load: %v", err) + } + policy := Policy{AllowedClasses: []string{"*"}} + + // Without any keyword in the query, only always-mode + buried (default + // always) should resolve. keyword_skill should be excluded. + got := r.Resolve("hello world", policy) + names := skillNames(got) + if hasName(names, "keyword_skill") { + t.Errorf("keyword_skill should NOT resolve without keyword; got %v", names) + } + if !hasName(names, "always_skill") { + t.Errorf("always_skill should resolve; got %v", names) + } + if !hasName(names, "buried_skill") { + t.Errorf("buried_skill (default-always) should resolve; got %v", names) + } + + // With "logs" in the query, keyword_skill should activate. + got = r.Resolve("show me the logs from edge X", policy) + if !hasName(skillNames(got), "keyword_skill") { + t.Errorf("keyword_skill should activate on 'logs'; got %v", skillNames(got)) + } +} + +func TestSkillRegistry_ResolvePolicyDropsTools(t *testing.T) { + r := NewSkillRegistry() + if err := r.Load(fixtureSkillRoot("multi")); err != nil { + t.Fatalf("Load: %v", err) + } + + // Read-only policy on a 'logs' query: keyword_skill activates, but its + // destructive tool (delete_logs) should be filtered out. + policy := Policy{AllowedClasses: []string{"read"}} + got := r.Resolve("logs please", policy) + for _, sk := range got { + if sk.Name != "keyword_skill" { + continue + } + if len(sk.Tools) != 1 { + t.Fatalf("keyword_skill Tools len = %d, want 1 (delete_logs filtered)", len(sk.Tools)) + } + if sk.Tools[0].Name != "query_logs" { + t.Errorf("expected query_logs to survive; got %q", sk.Tools[0].Name) + } + } +} + +func TestSkillRegistry_ResolveDropsSkillIfAllToolsFiltered(t *testing.T) { + r := NewSkillRegistry() + // Load a synthetic skill with only destructive tools; under a + // read-only policy the skill should be dropped entirely. + r.Add(&Skill{ + Name: "destructive_only", + Description: "Has only destructive tools.", + Activation: Activation{Mode: "always"}, + Tools: []ToolDecl{ + {Name: "wipe", Impl: "x", Class: ClassDestructive, Description: "Wipes the disk."}, + }, + }) + policy := Policy{AllowedClasses: []string{"read"}} + got := r.Resolve("hi", policy) + if hasName(skillNames(got), "destructive_only") { + t.Errorf("destructive_only should be dropped; got %v", skillNames(got)) + } +} + +func TestSkillRegistry_ResolvePromptOnlySkillSurvives(t *testing.T) { + r := NewSkillRegistry() + r.Add(&Skill{ + Name: "pure_prompt", + Description: "Prompt-only skill, no tools.", + Activation: Activation{Mode: "always"}, + PromptBody: "guidance text", + }) + policy := Policy{AllowedClasses: []string{"read"}} + got := r.Resolve("anything", policy) + if !hasName(skillNames(got), "pure_prompt") { + t.Errorf("pure_prompt skill should survive policy with no tools; got %v", skillNames(got)) + } +} + +// --- helpers --- + +func skillNames(skills []*Skill) []string { + out := make([]string, 0, len(skills)) + for _, sk := range skills { + out = append(out, sk.Name) + } + return out +} + +func hasName(list []string, name string) bool { + for _, n := range list { + if n == name { + return true + } + } + return false +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/system_prompt.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/system_prompt.go new file mode 100644 index 000000000..032edaa61 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/system_prompt.go @@ -0,0 +1,275 @@ +package chatruntime + +import ( + "context" + "fmt" + "sort" + "strings" + + aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// ComposeSystemPrompt assembles the SystemPrompt block ChatRuntime feeds +// to the LLM. Layered (the ComposeSystemPrompt arrow +// in the main reference figure): +// +// 1. basePrompt (the runtime's universal preamble; can be empty) +// 2. agentProfile.SystemPrompt + agentProfile.CriticalReminder +// (only when a worker is being constructed; nil for coordinator +// unless the coordinator itself is also persona-driven) +// 3. for each active skill, a `[能力: ]` header + skill PromptBody +// +// Pure string assembly — no per-turn system-reminder injection. That +// happens in the graph layer (graph.buildSystemReminder is called by +// graph.assembleMessages on every turn so the block survives long- +// session attention drift), not here. +// coordinatorToolRouting steers the coordinator to the RIGHT tool family so it +// doesn't (a) rabbit-hole through k8s tools for an ongrid-device question, or +// (b) fall back to an uninstalled `kubectl` for a genuine k8s question instead +// of the MCP tools. Injected only into the coordinator prompt (agentProfile==nil). +const coordinatorToolRouting = `## 工具选型补充 +- ongrid device/edge ≠ k8s node。问 ongrid 设备用 ongrid 工具；问 k8s 集群用 ` + + "`mcp__k8s__*`" + `，不要猜 ` + "`kubectl`" + `。 +- 产品行为、交互设计、页面/产物关联、ToolSearch/Operation/AgentLoop 策略解释，默认直接解释；不要为了这类问题发散调用 ` + "`host_bash`" + ` / 拓扑 / 源码工具。若缺少运行时产物索引，直接说明能力缺口。 +- 复杂跨域任务可同轮并行多个 ` + "`AgentTool`" + `；简单 topN / 快照 / 列表仍直接查。错误来源连续 2 次不匹配就换路或说明缺口。` + +func ComposeSystemPrompt(basePrompt string, activeSkills []*Skill, agentProfile *Agent) string { + var parts []string + + if base := strings.TrimSpace(basePrompt); base != "" { + parts = append(parts, base) + } + + // Coordinator-only tool-routing rules (workers are already scoped to a focused + // toolbag by their persona, so they don't need this). + if agentProfile == nil { + parts = append(parts, coordinatorToolRouting) + } + + if agentProfile != nil { + if body := strings.TrimSpace(agentProfile.SystemPrompt); body != "" { + parts = append(parts, body) + } + if reminder := strings.TrimSpace(agentProfile.CriticalReminder); reminder != "" { + // — the per-turn injection wraps this in + // ...; here we just plant + // it once into the system prompt as the persona-level + // constant. The graph layer will additionally re-inject + // per-turn. + parts = append(parts, "\n"+reminder+"\n") + } + } + + for _, sk := range activeSkills { + if sk == nil { + continue + } + header := "[能力: " + sk.Name + "]" + body := strings.TrimSpace(sk.PromptBody) + // PromptBody is already H1-normalized at parse time ( + // If the body already starts with the canonical header + // we skip prepending again. + if strings.HasPrefix(body, header) { + parts = append(parts, body) + continue + } + if body == "" { + parts = append(parts, header) + continue + } + parts = append(parts, header+"\n\n"+body) + } + + return strings.Join(parts, "\n\n") +} + +const maxCapabilityDigestTools = 12 + +type capabilityDigestRow struct { + name, origin, class, desc string +} + +// buildToolCapabilityDigest renders a compact, per-request inventory of the +// tools that survived persona / role filtering. This is the dynamic +// counterpart to the static base prompt: built-in routing rules stay small, +// while MCP / installed-skill tools surface automatically as soon as they are +// appended to the runtime toolbag. +func buildToolCapabilityDigest(tools []basetool.BaseTool) string { + if len(tools) == 0 { + return "" + } + rows := make([]capabilityDigestRow, 0, len(tools)) + counts := map[string]int{} + for _, t := range tools { + if t == nil { + continue + } + info, err := t.Info(context.Background()) + if err != nil || info == nil || strings.TrimSpace(info.Name) == "" { + continue + } + origin := strings.TrimSpace(info.Origin) + if origin == "" { + origin = "builtin" + } + class := strings.TrimSpace(info.Class) + if class == "" { + class = "read" + } + counts[origin+"/"+class]++ + if info.Origin == "" && !isDigestBuiltin(info.Name) { + continue + } + rows = append(rows, capabilityDigestRow{ + name: info.Name, + origin: origin, + class: class, + desc: compactOneLine(firstNonEmpty(info.WhenToUse, info.Description), 72), + }) + } + if len(rows) == 0 && len(counts) == 0 { + return "" + } + sort.Slice(rows, func(i, j int) bool { + if rows[i].origin != rows[j].origin { + return rows[i].origin < rows[j].origin + } + return rows[i].name < rows[j].name + }) + + var b strings.Builder + b.WriteString("## 本轮可见能力（动态）\n") + if len(counts) > 0 { + keys := make([]string, 0, len(counts)) + for k := range counts { + keys = append(keys, k) + } + sort.Strings(keys) + b.WriteString("- counts: ") + for i, k := range keys { + if i > 0 { + b.WriteString(", ") + } + b.WriteString(k) + b.WriteString("=") + b.WriteString(fmt.Sprint(counts[k])) + } + b.WriteString("\n") + } + if direct := directReadToolNames(rows); len(direct) > 0 { + b.WriteString("- direct_read_tools: ") + for i, name := range direct { + if i > 0 { + b.WriteString(", ") + } + b.WriteString("`") + b.WriteString(name) + b.WriteString("`") + } + b.WriteString("\n") + } + limit := len(rows) + truncated := 0 + if limit > maxCapabilityDigestTools { + truncated = limit - maxCapabilityDigestTools + limit = maxCapabilityDigestTools + } + for i := 0; i < limit; i++ { + r := rows[i] + b.WriteString("- ") + b.WriteString(r.name) + b.WriteString(" [") + b.WriteString(r.origin) + b.WriteString("/") + b.WriteString(r.class) + b.WriteString("]") + if r.desc != "" { + b.WriteString(": ") + b.WriteString(r.desc) + } + b.WriteString("\n") + } + if truncated > 0 { + b.WriteString("- ... ") + b.WriteString(fmt.Sprint(truncated)) + b.WriteString(" more; use ToolSearch/select: if a schema is redacted or the exact tool is unclear.\n") + } + return strings.TrimSpace(b.String()) +} + +func directReadToolNames(rows []capabilityDigestRow) []string { + names := make([]string, 0, len(rows)) + for _, r := range rows { + if r.origin != "builtin" || r.class != "read" { + continue + } + if !aiopstools.IsCoreToolName(r.name) { + continue + } + names = append(names, r.name) + } + sort.Strings(names) + return names +} + +func isDigestBuiltin(name string) bool { + return aiopstools.IsCoreToolName(name) +} + +func compactOneLine(s string, max int) string { + s = strings.Join(strings.Fields(s), " ") + if max <= 0 || len(s) <= max { + return s + } + return strings.TrimSpace(s[:max-1]) + "…" +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +// buildAgentCatalog renders optional expert capability cards for the default +// assistant. The cards explain what additional evidence an expert can obtain; +// they do not turn the expert into the owner of the user conversation. +func buildAgentCatalog(reg *AgentRegistry) string { + if reg == nil { + return "" + } + cards := reg.CapabilityCards() + if len(cards) == 0 { + return "" + } + type row struct{ agent, id, desc string } + rows := make([]row, 0, len(cards)) + for _, card := range cards { + if card.AgentName == "" || card.AgentName == "reviewer" || card.AgentName == "default" { + continue + } + rows = append(rows, row{agent: card.AgentName, id: card.ID, desc: strings.TrimSpace(card.Description)}) + } + if len(rows) == 0 { + return "" + } + var sb strings.Builder + sb.WriteString("## 可选专家能力（AgentTool 的 subagent_type）\n\n") + sb.WriteString("默认助理负责最终答复并可直接使用本轮工具。需要独立复核、额外领域证据或并行分析时，才调用 AgentTool 请求以下专家工作会话：\n\n") + for _, r := range rows { + sb.WriteString("- `") + sb.WriteString(r.agent) + sb.WriteString("` (`") + sb.WriteString(r.id) + sb.WriteString("`) — ") + if r.desc != "" { + sb.WriteString(r.desc) + } + sb.WriteString("\n") + } + sb.WriteString("\n派活时：在 prompt 里写清 incident_id / device_id / 用户原话——worker 看不到你的上下文。") + return sb.String() +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/system_prompt_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/system_prompt_test.go new file mode 100644 index 000000000..b311121fe --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/system_prompt_test.go @@ -0,0 +1,151 @@ +package chatruntime + +import ( + "fmt" + "strings" + "testing" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +func TestComposeSystemPrompt_Empty(t *testing.T) { + // agentProfile==nil ⇒ coordinator ⇒ the coordinatorToolRouting block is + // injected, so even an otherwise-empty compose carries it. + got := ComposeSystemPrompt("", nil, nil) + if got != coordinatorToolRouting { + t.Errorf("expected just the routing block, got %q", got) + } +} + +func TestComposeSystemPrompt_BaseOnly(t *testing.T) { + got := ComposeSystemPrompt("you are ongrid.", nil, nil) + if got != "you are ongrid.\n\n"+coordinatorToolRouting { + t.Errorf("got %q", got) + } +} + +func TestComposeSystemPrompt_MultipleSkills(t *testing.T) { + skills := []*Skill{ + {Name: "alpha", PromptBody: "[能力: alpha]\n\nalpha guide"}, + {Name: "beta", PromptBody: "beta guide"}, + } + got := ComposeSystemPrompt("base prompt", skills, nil) + if !strings.Contains(got, "base prompt") { + t.Error("base missing") + } + if !strings.Contains(got, "[能力: alpha]") { + t.Error("alpha header missing") + } + // beta wasn't pre-tagged in PromptBody, ComposeSystemPrompt should + // prepend the canonical tag. + if !strings.Contains(got, "[能力: beta]") { + t.Error("beta header missing") + } + if !strings.Contains(got, "alpha guide") || !strings.Contains(got, "beta guide") { + t.Error("skill bodies missing") + } + // Order: base before alpha before beta. + if strings.Index(got, "base prompt") > strings.Index(got, "alpha guide") { + t.Error("base should come before alpha") + } + if strings.Index(got, "alpha guide") > strings.Index(got, "beta guide") { + t.Error("alpha should come before beta") + } +} + +func TestComposeSystemPrompt_AgentProfile(t *testing.T) { + ag := &Agent{ + Name: "reviewer", + SystemPrompt: "you review proposals.", + CriticalReminder: "do not approve mutations on your own.", + } + got := ComposeSystemPrompt("base", nil, ag) + if !strings.Contains(got, "base") { + t.Error("base missing") + } + if !strings.Contains(got, "you review proposals.") { + t.Error("system prompt body missing") + } + if !strings.Contains(got, "") { + t.Error("critical-reminder block missing") + } + if !strings.Contains(got, "do not approve mutations") { + t.Error("reminder content missing") + } +} + +func TestComposeSystemPrompt_AgentAndSkills(t *testing.T) { + ag := &Agent{Name: "worker", SystemPrompt: "worker prompt", CriticalReminder: ""} + skills := []*Skill{{Name: "tool_x", PromptBody: "x guide"}} + got := ComposeSystemPrompt("base", skills, ag) + if !strings.Contains(got, "base") || !strings.Contains(got, "worker prompt") || !strings.Contains(got, "[能力: tool_x]") { + t.Errorf("missing parts; got %q", got) + } + // Order: base, then agent, then skills. + bIdx := strings.Index(got, "base") + aIdx := strings.Index(got, "worker prompt") + sIdx := strings.Index(got, "[能力: tool_x]") + if !(bIdx < aIdx && aIdx < sIdx) { + t.Errorf("order should be base→agent→skills; bIdx=%d aIdx=%d sIdx=%d", bIdx, aIdx, sIdx) + } +} + +func TestComposeSystemPrompt_NilSkillEntrySkipped(t *testing.T) { + skills := []*Skill{nil, {Name: "z", PromptBody: "z body"}} + got := ComposeSystemPrompt("", skills, nil) + if !strings.Contains(got, "[能力: z]") { + t.Errorf("z skill should be present despite preceding nil; got %q", got) + } +} + +func TestComposeSystemPrompt_EmptyPromptBody(t *testing.T) { + skills := []*Skill{{Name: "empty"}} + got := ComposeSystemPrompt("", skills, nil) + // nil agentProfile ⇒ coordinator routing block precedes the skill header. + if got != coordinatorToolRouting+"\n\n[能力: empty]" { + t.Errorf("got %q", got) + } +} + +func TestBuildToolCapabilityDigest_IncludesDynamicTools(t *testing.T) { + bag := []basetool.BaseTool{ + &originTool{name: "query_devices", class: "read", origin: basetool.OriginBuiltin}, + &originTool{name: "query_traceql", class: "read", origin: basetool.OriginBuiltin}, + &originTool{name: "mcp__k8s__namespaces_list", class: "read", origin: basetool.OriginMCP}, + &originTool{name: "skill__custom__foo", class: "read", origin: basetool.OriginSkill}, + &originTool{name: "internal_big_tool", class: "read", origin: basetool.OriginBuiltin}, + } + got := buildToolCapabilityDigest(bag) + for _, want := range []string{ + "本轮可见能力", + "query_devices [builtin/read]", + "query_traceql [builtin/read]", + "mcp__k8s__namespaces_list [mcp/read]", + "skill__custom__foo [skill/read]", + "mcp/read=1", + "skill/read=1", + } { + if !strings.Contains(got, want) { + t.Fatalf("capability digest missing %q:\n%s", want, got) + } + } + if !strings.Contains(got, "direct_read_tools:") || + !strings.Contains(got, "`query_devices`") || + !strings.Contains(got, "`query_traceql`") { + t.Fatalf("capability digest should derive direct read tools from visible toolbag:\n%s", got) + } + if strings.Contains(got, "internal_big_tool [builtin/read]") { + t.Fatalf("digest should not list arbitrary builtin tools; got:\n%s", got) + } +} + +func TestBuildToolCapabilityDigest_TruncatesLargeDynamicSets(t *testing.T) { + bag := make([]basetool.BaseTool, 0, maxCapabilityDigestTools+3) + for i := 0; i < maxCapabilityDigestTools+3; i++ { + bag = append(bag, &originTool{name: fmt.Sprintf("mcp__srv__tool_%02d", i), class: "read", origin: basetool.OriginMCP}) + } + got := buildToolCapabilityDigest(bag) + if !strings.Contains(got, "more; use ToolSearch") { + t.Fatalf("digest should truncate and point to ToolSearch, got:\n%s", got) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/minimal/incident-investigator.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/minimal/incident-investigator.md new file mode 100644 index 000000000..0ab578a58 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/minimal/incident-investigator.md @@ -0,0 +1,21 @@ +--- +name: incident-investigator +description: Alert root-cause investigator. +when_to_use: | + Use when the user asks "what is the root cause / how to investigate / blast + radius" of an alert. Provide incident_id or device_id; returns three + sections: phenomenon / signals / hypotheses. +capabilities: + - id: incident_analysis + description: Correlate alert evidence into a root-cause analysis. + tools: [query_promql, query_loki] + skills: [incident-response] + max_tool_calls: 8 +--- + +You are the ongrid alert root-cause investigation agent. + +Workflow: +1. Fetch the incident detail. +2. Correlate metric / log / trace. +3. Output three sections: phenomenon, related signals, hypotheses. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md new file mode 100644 index 000000000..a8a7d82cf --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/missing_when_to_use/foo.md @@ -0,0 +1,6 @@ +--- +name: foo +description: Missing when_to_use to test required-field error. +--- + +This file is intentionally missing the when_to_use field. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md new file mode 100644 index 000000000..ac8e53421 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/with_critical_reminder/auditor.md @@ -0,0 +1,23 @@ +--- +name: auditor +description: Read-only auditor that watches mutating decisions for compliance breaches. +when_to_use: Spawn whenever a mutating proposal lands in audit mode. +tools: + - get_audit_log + - query_promql +permission_mode: read-only +max_turns: 8 +critical_reminder: | + - Do not approve or reject; only flag. + - Quote SOP IDs verbatim; never paraphrase. + - If unsure, say so explicitly. +background: false +omit_claude_md: false +--- + +You are the ongrid compliance auditor agent. + +Workflow: +1. Pull recent audit_log entries for the incident in scope. +2. Cross-check each mutating action against the SOP catalogue. +3. Output a flag list (no approve/reject). diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md new file mode 100644 index 000000000..6126be635 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_parser/with_disallowed_tools/reviewer.md @@ -0,0 +1,23 @@ +--- +name: reviewer +description: SOP dual-sign reviewer for mutating proposals. +when_to_use: Spawn only when the user proposes a mutating / destructive operation. +tools: + - get_incident_detail + - query_promql + - query_logql + - get_sop_text +disallowed_tools: + - "*_skill" + - run_shell +permission_mode: read-only +max_turns: 5 +model: anthropic/claude-opus-4-7 +--- + +You are the ongrid high-risk-operation second-review agent. + +After receiving a proposal {action, target, reason, blast_radius}: +1. Verify a relevant SOP exists. +2. Verify target device's current state. +3. Return approve / reject with one-line rationale. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/README.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/README.md new file mode 100644 index 000000000..fe1b6dbad --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/README.md @@ -0,0 +1,4 @@ +# agents/ + +Test fixture root for AgentRegistry.Load. README.md is intentionally +present to verify the loader skips it. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/incident-investigator.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/incident-investigator.md new file mode 100644 index 000000000..3e9cd127e --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/incident-investigator.md @@ -0,0 +1,7 @@ +--- +name: incident_investigator +description: Alert root-cause investigator. +when_to_use: When the user asks about an alert's root cause or blast radius. +--- + +You are the ongrid alert root-cause investigation agent. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/reviewer.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/reviewer.md new file mode 100644 index 000000000..2b5b7571f --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/reviewer.md @@ -0,0 +1,9 @@ +--- +name: reviewer +description: SOP dual-sign reviewer. +when_to_use: Spawn for any mutating proposal that needs HLD-003 dual sign. +permission_mode: read-only +max_turns: 5 +--- + +You are the ongrid SOP reviewer. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json new file mode 100644 index 000000000..35a928395 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack/.claude-plugin/plugin.json @@ -0,0 +1,13 @@ +{ + "id": "acme-tools", + "name": "Acme Tools", + "version": "1.0.0", + "description": "A bundle of acme tools.", + "configSchema": { + "type": "object", + "properties": { + "endpoint": { "type": "string" } + }, + "required": ["endpoint"] + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json new file mode 100644 index 000000000..7c92e55c3 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/.claude-plugin/plugin.json @@ -0,0 +1,7 @@ +{ + "id": "acme-extras", + "name": "Acme Extras", + "version": "1.0.0", + "description": "Bundle that ships hooks and an MCP config.", + "configSchema": {} +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json new file mode 100644 index 000000000..da39e4ffa --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/.mcp.json @@ -0,0 +1,3 @@ +{ + "mcpServers": {} +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md new file mode 100644 index 000000000..2f5cd5951 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/claude_pack_with_extras/hooks/PRE_PROMPT.md @@ -0,0 +1,3 @@ +# pre-prompt hook + +(unused, present only to test that the hooks/ subdir is detected and warned about) diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json new file mode 100644 index 000000000..905660621 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/openclaw_pack/openclaw.plugin.json @@ -0,0 +1,18 @@ +{ + "id": "openclaw-bundle", + "name": "OpenClaw Bundle", + "version": "2.3.4", + "description": "OpenClaw plugin with provider-specific extras.", + "providers": { + "openai": { "baseUrl": "https://api.openai.com/v1" }, + "anthropic": { "baseUrl": "https://api.anthropic.com" } + }, + "channels": ["feishu", "cli"], + "cliBackends": ["claude", "codex"], + "providerAuthChoices": { + "preferred": "cliFlag" + }, + "configSchema": { + "type": "object" + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json new file mode 100644 index 000000000..90b0c506a --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal/.claude-plugin/plugin.json @@ -0,0 +1,6 @@ +{ + "id": "evil", + "name": "Evil Plugin", + "version": "0.0.1", + "description": "A plugin whose skills/ directory contains a symlink that escapes the pack root." +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal/skills/pwned b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal/skills/pwned new file mode 120000 index 000000000..6998f20a0 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal/skills/pwned @@ -0,0 +1 @@ +../../path_traversal_target \ No newline at end of file diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md new file mode 100644 index 000000000..dba27c4eb --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/path_traversal_target/SKILL.md @@ -0,0 +1,8 @@ +--- +name: pwned +description: I am outside the pack root and should not be loaded. +--- + +# Pwned + +This SKILL.md is outside the path_traversal pack — loader must reject access via symlink. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json new file mode 100644 index 000000000..b4853c2c1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/.claude-plugin/plugin.json @@ -0,0 +1,6 @@ +{ + "id": "recursive-test", + "name": "Recursive Test", + "version": "1.0.0", + "description": "Bundle that ships skills, agents, commands, hooks and an MCP config; used to test recursive sub-tree load." +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json new file mode 100644 index 000000000..5392ca626 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "example": { + "command": "node", + "args": ["./mcp-server.js"] + } + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md new file mode 100644 index 000000000..0265b69d1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/agents/pr-summary.md @@ -0,0 +1,7 @@ +--- +name: pr_summary +description: Summarizes the diff of a pull request. +when_to_use: When the user asks for a summary of the current PR or branch diff. +--- + +You are the PR summary agent. Read the diff, then output a 3-bullet summary. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md new file mode 100644 index 000000000..a38e3afed --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/commands/commit.md @@ -0,0 +1,7 @@ +--- +description: Commit the staged changes with a conventional commit message. +allowed-tools: [Bash, Edit] +--- + +Commit the currently staged changes. Generate a conventional commit message +based on the diff. Then run `git status` to show the result. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md new file mode 100644 index 000000000..2f5cd5951 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/hooks/pre-prompt/HOOK.md @@ -0,0 +1,3 @@ +# pre-prompt hook + +(unused, present only to test that the hooks/ subdir is detected and warned about) diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md new file mode 100644 index 000000000..148703763 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_claude/skills/hello-world/SKILL.md @@ -0,0 +1,8 @@ +--- +name: hello_world +description: A simple skill that says hello. +--- + +# Hello World + +When the user asks for a greeting, respond with "Hello". diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json new file mode 100644 index 000000000..72b2c845b --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/openclaw.plugin.json @@ -0,0 +1,7 @@ +{ + "id": "openclaw-recursive", + "name": "OpenClaw Recursive", + "version": "0.1.0", + "description": "OpenClaw bundle declaring its skills via the manifest skills[] array.", + "skills": ["./skills/foo", "./skills/bar"] +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md new file mode 100644 index 000000000..b820c4115 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/skills/bar/SKILL.md @@ -0,0 +1,8 @@ +--- +name: bar +description: The bar skill. +--- + +# Bar + +Do bar things. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md new file mode 100644 index 000000000..baf6620bd --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/plugin_container/recursive_openclaw/skills/foo/SKILL.md @@ -0,0 +1,8 @@ +--- +name: foo +description: The foo skill. +--- + +# Foo + +Do foo things. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/minimal/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/minimal/SKILL.md new file mode 100644 index 000000000..148703763 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/minimal/SKILL.md @@ -0,0 +1,8 @@ +--- +name: hello_world +description: A simple skill that says hello. +--- + +# Hello World + +When the user asks for a greeting, respond with "Hello". diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/missing_description/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/missing_description/SKILL.md new file mode 100644 index 000000000..a517a9e42 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/missing_description/SKILL.md @@ -0,0 +1,7 @@ +--- +name: nope +--- + +# No Description + +Frontmatter is missing description; this should be a parse error. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md new file mode 100644 index 000000000..7b9cb7311 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/name_with_dash/SKILL.md @@ -0,0 +1,9 @@ +--- +name: hello-world +description: Skill whose name uses a dash, expected to be normalized. +--- + +# Hello World Dashed + +This skill name is `hello-world` which is not snake_case. The parser +should normalize it to `hello_world` and emit a warning. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/typical/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/typical/SKILL.md new file mode 100644 index 000000000..60ce5c842 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/typical/SKILL.md @@ -0,0 +1,29 @@ +--- +name: file_inspector +version: 1.0.0 +description: Inspect files on the host filesystem. +metadata: + os: ["darwin", "linux"] + requires: + bins: [find, du] + config: [accountsPath] +tools: + - name: find_files + impl: builtin:filetools.Find + class: read + description: Search files by name pattern. + when_to_use: When the user asks to find files. + - name: du_summary + impl: builtin:filetools.DuSummary + class: read + description: Disk usage summary. +--- + +# File Inspector + +Use this skill to find files on disk and report disk usage. + +## Workflow + +1. Use `find_files` to locate candidate files. +2. Use `du_summary` to size up the matches. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md new file mode 100644 index 000000000..b5143a6a1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_ongrid_ext/SKILL.md @@ -0,0 +1,27 @@ +--- +name: edge_disk_audit +description: Audit disk usage on edge devices. +metadata: + os: [linux] + requires: + bins: [find, du] + ongrid: + scope: edge + edge_runtime: subprocess + edge_capabilities: + - filesystem.read.path: ["/data", "/var/log"] + - process.exec: [find, du] + activation: + mode: keyword + keywords: [disk, 磁盘, du] + min_ongrid_version: ">=0.7.30" +tools: + - name: edge_du + impl: edge:disk.DuSummary + class: read + description: Disk usage on edge. +--- + +# Edge Disk Audit + +Inspect disk consumption on connected edges via the plugin runtime. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md new file mode 100644 index 000000000..c428a8e0c --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_unknown_fields/SKILL.md @@ -0,0 +1,16 @@ +--- +name: future_skill +description: A skill that uses fields ongrid does not yet recognize. +custom_field_one: hello +custom_field_two: + nested: + value: 42 +upstream_only_array: + - a + - b +--- + +# Future Skill + +Newer SKILL.md spec adds fields ongrid hasn't modeled yet — they should +survive into UnknownFields. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md new file mode 100644 index 000000000..e935d3aaa --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_parser/with_when_to_use/SKILL.md @@ -0,0 +1,16 @@ +--- +name: log_search +description: Search Loki logs. +when_to_use: | + When the user asks about log content, error patterns, or log volume per device. + NOT for: file-system search (use file_inspector), metric queries (use metric_query). +tools: + - name: query_logql + impl: builtin:loki.Query + class: read + description: Run LogQL. +--- + +# Log Search + +Use LogQL to search distributed logs. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md new file mode 100644 index 000000000..1b5ea8da6 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/always_skill/SKILL.md @@ -0,0 +1,15 @@ +--- +name: always_skill +description: Always active. +activation: + mode: always +tools: + - name: ping + impl: builtin:ping.Ping + class: read + description: Ping a host. +--- + +# Always Skill + +Always loaded into the toolBag. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md new file mode 100644 index 000000000..820233432 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/keyword_skill/SKILL.md @@ -0,0 +1,20 @@ +--- +name: keyword_skill +description: Activated by the word "logs". +activation: + mode: keyword + keywords: [logs, 日志] +tools: + - name: query_logs + impl: builtin:loki.Query + class: read + description: Search logs. + - name: delete_logs + impl: builtin:loki.Delete + class: destructive + description: Wipe a log range. +--- + +# Keyword Skill + +Only loaded when the user mentions logs / 日志. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md new file mode 100644 index 000000000..a1f04f719 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/multi/nested/deeper/buried_skill/SKILL.md @@ -0,0 +1,8 @@ +--- +name: buried_skill +description: A skill nested several directories deep, to verify recursive walk. +--- + +# Buried Skill + +Recursive walk should still find this. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md new file mode 100644 index 000000000..e038d9d44 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/with_bad_skill/bad/SKILL.md @@ -0,0 +1,7 @@ +--- +name: bad +--- + +# No Description + +Missing description — should produce a parse_failed warning, not crash the walk. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md new file mode 100644 index 000000000..9412ba7d7 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/skill_registry/with_bad_skill/good/SKILL.md @@ -0,0 +1,8 @@ +--- +name: good +description: A well-formed skill. +--- + +# Good + +This one parses fine. diff --git a/internal/manager/biz/aiops/chatruntime/tool_lifecycle.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/tool_lifecycle.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/tool_lifecycle.go rename to internal/manager/biz/aiops/runtime/v3/chatruntime/tool_lifecycle.go diff --git a/internal/manager/biz/aiops/chatruntime/tool_lifecycle_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/tool_lifecycle_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/tool_lifecycle_test.go rename to internal/manager/biz/aiops/runtime/v3/chatruntime/tool_lifecycle_test.go diff --git a/internal/manager/biz/aiops/chatruntime/tracing.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/tracing.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/tracing.go rename to internal/manager/biz/aiops/runtime/v3/chatruntime/tracing.go diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/tracing_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/tracing_test.go new file mode 100644 index 000000000..93135db56 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/tracing_test.go @@ -0,0 +1,336 @@ +package chatruntime + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + einomodel "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + + "github.com/ongridio/ongrid/internal/manager/biz/agenttrace" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph/callbacks" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +func recordAgentSpans(t *testing.T) *tracetest.SpanRecorder { + t.Helper() + recorder := tracetest.NewSpanRecorder() + previous := otel.GetTracerProvider() + options := []sdktrace.TracerProviderOption{sdktrace.WithSpanProcessor(recorder)} + if endpoint := os.Getenv("ONGRID_AGENT_TRACE_SMOKE_OTLP"); endpoint != "" { + exporter, err := otlptracehttp.New(context.Background(), otlptracehttp.WithEndpointURL(endpoint)) + if err != nil { + t.Fatal(err) + } + options = append(options, sdktrace.WithBatcher(exporter), sdktrace.WithResource(resource.NewSchemaless( + attribute.String("service.name", "ongrid-eino-smoke"), attribute.String("service.namespace", "ongrid-test"), + attribute.String("deployment.environment.name", "test"), attribute.Bool("ongrid.test.synthetic", true)))) + } + provider := sdktrace.NewTracerProvider(options...) + otel.SetTracerProvider(provider) + t.Cleanup(func() { + otel.SetTracerProvider(previous) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := provider.Shutdown(ctx); err != nil { + t.Error(err) + } + }) + return recorder +} + +func spanAttrs(span sdktrace.ReadOnlySpan) map[string]attribute.Value { + attrs := map[string]attribute.Value{} + for _, attr := range span.Attributes() { + attrs[string(attr.Key)] = attr.Value + } + return attrs +} + +func newTestTraceHandler(t *testing.T) *callbacks.TraceHandler { + t.Helper() + handler, err := callbacks.NewTraceHandler(context.Background()) + if err != nil || handler == nil { + t.Fatalf("trace callback: handler=%v err=%v", handler, err) + } + t.Cleanup(func() { + if err := handler.Shutdown(context.Background()); err != nil { + t.Error(err) + } + }) + return handler +} + +func TestAgentTracingRealGraphModelAndTool(t *testing.T) { + for _, capture := range []bool{false, true} { + name := "metadata" + if capture { + name = "content" + } + t.Run(name, func(t *testing.T) { + t.Setenv("ONGRID_AGENT_TRACE_CAPTURE_CONTENT", strconv.FormatBool(capture)) + testAgentTracingRealGraphModelAndTool(t, capture, false) + }) + } + t.Run("large_content", func(t *testing.T) { + t.Setenv("ONGRID_AGENT_TRACE_CAPTURE_CONTENT", "true") + testAgentTracingRealGraphModelAndTool(t, true, true) + }) +} + +func testAgentTracingRealGraphModelAndTool(t *testing.T, capture, large bool) { + recorder := recordAgentSpans(t) + handler := newTestTraceHandler(t) + userPrompt, modelOutput, basePrompt := "合成测试：检查设备状态并总结结果", "合成测试完成：设备状态正常。", "" + if large { + userPrompt = "合成测试：验证大正文截断与显示" + modelOutput = strings.Repeat("这是一段大型合成输出，用于验证正文预算与中文字符边界。😀\n", 4000) + basePrompt = strings.Repeat("private-system-instructions ", 10000) + } + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := calls.Add(1) + message := map[string]any{"role": "assistant", "content": modelOutput} + if n == 1 { + message["tool_calls"] = []any{map[string]any{"id": "call-1", "type": "function", "function": map[string]string{"name": "query_traceql", "arguments": `{}`}}} + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": fmt.Sprintf("reply-%d", n), "object": "chat.completion", "model": "resolved-model", + "choices": []any{map[string]any{"index": 0, "message": message, "finish_reason": "stop"}}, + "usage": map[string]int{"prompt_tokens": 42, "completion_tokens": 8, "total_tokens": 50}, + }) + })) + defer server.Close() + client := llm.New(llm.Config{APIKey: "private-api-key", BaseURL: server.URL, Model: "requested-model"}, nil, prometheus.NewRegistry()) + chatModel, err := llm.NewClientChatModel(llm.ClientChatModelConfig{Client: client}) + if err != nil { + t.Fatal(err) + } + session := &aiopsmodel.Session{ID: "conversation-1", UserID: 7} + runtime, err := NewRuntime(Config{ + Sessions: newMemSessions(session), ChatModel: chatModel.(einomodel.ToolCallingChatModel), + BasePrompt: basePrompt, + ToolBag: []basetool.BaseTool{&fakeTool{name: "query_traceql", schema: `{"type":"object","properties":{}}`}}, + GraphCfg: graph.Config{MaxIterations: 5}, + CallbackDeps: callbacks.Deps{Tracing: handler}, + }) + if err != nil { + t.Fatal(err) + } + ctx, parent := otel.Tracer("test").Start(context.Background(), "HTTP request") + reply, err := runtime.Handle(ctx, &Request{SessionID: session.ID, UserID: 7, UserText: userPrompt}) + parent.End() + if flushErr := handler.Flush(context.Background()); flushErr != nil { + t.Fatal(flushErr) + } + if err != nil || reply == nil || reply.Message == nil { + t.Fatalf("Handle: reply=%v err=%v", reply, err) + } + if calls.Load() != 2 { + t.Fatalf("model calls = %d, want exactly 2 (tracing must not execute the agent)", calls.Load()) + } + spans := recorder.Ended() + byID := map[string]sdktrace.ReadOnlySpan{} + var agent sdktrace.ReadOnlySpan + for _, span := range spans { + byID[span.SpanContext().SpanID().String()] = span + if spanAttrs(span)["gen_ai.operation.name"].AsString() == "invoke_agent" { + agent = span + } + } + if agent == nil || agent.Parent().SpanID() != parent.SpanContext().SpanID() { + t.Fatal("missing Agent span or HTTP ancestry") + } + t.Logf("synthetic trace_id=%s entry_span_id=%s", agent.SpanContext().TraceID(), agent.SpanContext().SpanID()) + if spanAttrs(agent)["gen_ai.conversation.id"].AsString() != session.ID { + t.Fatal("conversation ID lost") + } + if spanAttrs(agent)["ongrid.agent.content.capture"].AsBool() != capture { + t.Fatal("capture policy missing from run") + } + counts := map[string]int{} + for _, span := range spans { + attrs := spanAttrs(span) + operation := attrs["gen_ai.operation.name"].AsString() + if attrs["langfuse.observation.type"].AsString() == "tool" { + operation = "execute_tool" + } + counts[operation]++ + if operation == "chat" || operation == "execute_tool" { + ancestor := byID[span.Parent().SpanID().String()] + for ancestor != nil && ancestor != agent { + ancestor = byID[ancestor.Parent().SpanID().String()] + } + if ancestor != agent { + t.Errorf("%s is not a descendant of Agent", span.Name()) + } + if span.InstrumentationScope().Name != "github.com/cloudwego/eino-ext/callbacks/langfuse/v2" { + t.Errorf("%s is not owned by the official callback", span.Name()) + } + } + if operation == "chat" && (attrs["gen_ai.usage.total_tokens"].AsInt64() != 50 || attrs["gen_ai.response.model"].AsString() != "resolved-model") { + t.Errorf("model usage or resolved model missing: %v", attrs) + } + for key, value := range attrs { + if strings.Contains(value.Emit(), "private-api-key") || strings.Contains(value.Emit(), "private-system-instructions") || (!capture && (strings.Contains(key, "messages") || strings.Contains(value.Emit(), userPrompt) || strings.Contains(value.Emit(), modelOutput))) { + t.Errorf("private content captured in attribute %s", key) + } + if (key == "input.value" || key == "output.value" || key == "langfuse.observation.input" || key == "langfuse.observation.output") && len(value.AsString()) > 8*1024 { + t.Errorf("content exceeded per-value byte budget: %s", key) + } + } + } + if counts["invoke_agent"] != 1 || counts["chat"] != 2 || counts["execute_tool"] != 1 { + t.Fatalf("unexpected lifecycle counts: %v", counts) + } + groups := []any{} + for _, span := range spans { + attrs := []any{} + for _, attr := range span.Attributes() { + value := map[string]any{} + switch attr.Value.Type() { + case attribute.STRING: + value["stringValue"] = attr.Value.AsString() + case attribute.INT64: + value["intValue"] = strconv.FormatInt(attr.Value.AsInt64(), 10) + case attribute.BOOL: + value["boolValue"] = attr.Value.AsBool() + default: + t.Fatalf("unexpected test attribute type: %v", attr.Value.Type()) + } + attrs = append(attrs, map[string]any{"key": string(attr.Key), "value": value}) + } + groups = append(groups, map[string]any{"scope": map[string]string{"name": span.InstrumentationScope().Name}, "spans": []any{map[string]any{ + "spanId": span.SpanContext().SpanID().String(), "parentSpanId": span.Parent().SpanID().String(), "name": span.Name(), + "startTimeUnixNano": strconv.FormatInt(span.StartTime().UnixNano(), 10), "endTimeUnixNano": strconv.FormatInt(span.EndTime().UnixNano(), 10), "attributes": attrs, + }}}) + } + body, err := json.Marshal(map[string]any{"resourceSpans": []any{map[string]any{"scopeSpans": groups}}}) + if err != nil { + t.Fatal(err) + } + normalized, err := agenttrace.Normalize(agent.SpanContext().TraceID().String(), body) + if err != nil || len(normalized.Runs) != 1 { + t.Fatalf("normalize: trace=%v err=%v", normalized, err) + } + run := normalized.Runs[0] + if run.Usage.TotalTokens == nil || *run.Usage.TotalTokens != 100 || run.Usage.Conflict || run.ContentAvailable != capture { + t.Fatalf("official callback must preserve the capture policy and exactly 100 tokens: %+v", run) + } + if capture { + if run.InputPreview != userPrompt || (!large && run.OutputPreview != modelOutput) || (large && run.OutputPreview == "") { + t.Fatalf("root content must come directly from its recorded span: %+v", run) + } + for _, step := range normalized.Steps { + if large && (step.Kind == "agent" || step.Kind == "llm") && !step.Output.Truncated { + t.Errorf("%s large output must declare truncation", step.Name) + } + if step.Kind == "llm" || step.Kind == "tool" { + if step.Input.State != "available" || step.Output.State != "available" { + t.Errorf("%s content missing: input=%s output=%s", step.Name, step.Input.State, step.Output.State) + } + } + } + } +} + +func TestAgentTracingMarksSoftModelFailure(t *testing.T) { + recorder := recordAgentSpans(t) + handler := newTestTraceHandler(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "model unavailable", http.StatusServiceUnavailable) + })) + defer server.Close() + client := llm.New(llm.Config{APIKey: "test-key", BaseURL: server.URL, Model: "test-model"}, nil, prometheus.NewRegistry()) + chatModel, err := llm.NewClientChatModel(llm.ClientChatModelConfig{Client: client}) + if err != nil { + t.Fatal(err) + } + runtime, err := NewRuntime(Config{Sessions: newMemSessions(&aiopsmodel.Session{ID: "error-session", UserID: 7}), ChatModel: chatModel.(einomodel.ToolCallingChatModel), CallbackDeps: callbacks.Deps{Tracing: handler}}) + if err != nil { + t.Fatal(err) + } + // The chat API returns an apology successfully; the trace must retain failure. + if _, err := runtime.Handle(context.Background(), &Request{SessionID: "error-session", UserID: 7, UserText: "test"}); err != nil { + t.Fatal(err) + } + if err := handler.Flush(context.Background()); err != nil { + t.Fatal(err) + } + spans := recorder.Ended() + count := 0 + for _, span := range spans { + op := spanAttrs(span)["gen_ai.operation.name"].AsString() + if op != "invoke_agent" && op != "chat" { + continue + } + count++ + if span.Status().Code != codes.Error { + t.Errorf("%s status = %v", span.Name(), span.Status()) + } + } + if count != 2 { + t.Fatalf("model/Agent spans = %d, want 2", count) + } +} + +func TestAgentTracingBackgroundWorkerKeepsAncestry(t *testing.T) { + t.Setenv("ONGRID_AGENT_TRACE_CAPTURE_CONTENT", "true") + recorder := recordAgentSpans(t) + store, parentID := newSeededRuntimeStore(t, 7) + runtime := newRuntimeWithStore(t, "specialist-sre", newScriptedChatModel(&schema.Message{Role: schema.Assistant, Content: "done"}), nil, store) + runtime.cfg.CallbackDeps.Tracing = newTestTraceHandler(t) + ctx, parent := otel.Tracer("test").Start(context.Background(), "coordinator") + ctx, cancel := context.WithCancel(ctx) + worker, err := runtime.SpawnWorker(ctx, SpawnRequest{AgentName: "specialist-sre", Prompt: "test", ParentSession: parentID, Background: true}) + if err != nil { + t.Fatal(err) + } + cancel() + parent.End() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + for _, span := range recorder.Ended() { + attrs := spanAttrs(span) + if attrs["ongrid.agent.worker.id"].AsString() != worker.ID { + continue + } + if span.Parent().SpanID() != parent.SpanContext().SpanID() || span.SpanContext().TraceID() != parent.SpanContext().TraceID() { + t.Fatal("background worker lost ancestry") + } + if attrs["gen_ai.conversation.id"].AsString() != parentID || attrs["ongrid.agent.session.id"].AsString() != worker.SessionID { + t.Fatal("worker must share the root conversation and retain its own session") + } + if !attrs["ongrid.agent.content.capture"].AsBool() || attrs["input.value"].AsString() != `"test"` || attrs["output.value"].AsString() != `"done"` { + t.Fatal("worker must record its input and output directly") + } + if span.Status().Code == codes.Error { + t.Fatal("caller cancellation leaked into background worker") + } + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("worker did not finish") +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/types.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/types.go new file mode 100644 index 000000000..81c2a78e6 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/types.go @@ -0,0 +1,458 @@ +// Package chatruntime is the in-process orchestration layer for the ongrid +// AIOps agent. It owns the data shape of a "skill bundle" and an +// "agent persona": both are SKILL.md / *.md files with YAML +// frontmatter + markdown body. This file defines just the in-memory structs; +// parsing lives in skill_parser.go / agent_parser.go. +// +// PR-2 of / / scaffolding only — no graph, no LLM +// wiring, no tool execution. Old internal/skill stays running for compat. +// +// Field naming convention: every YAML / JSON tag uses snake_case per +// — the spec explicitly aligns ongrid with claude-code agent +// frontmatter, openclaw SKILL.md, and the SKILL.md body we ship to users. +package chatruntime + +import ( + "encoding/json" + "time" +) + +// ToolClass categorizes a tool's blast radius. Policies filter tools by +// class so the LLM never sees tools that aren't allowed by the channel / +// agent profile. Mirrors sealsuite-agent/internal/chatruntime.ToolClass. +type ToolClass string + +const ( + // ClassRead is read-only — no side effects on the system or device. + ClassRead ToolClass = "read" + // ClassWrite mutates state (creates / updates resources) — requires a + // permissionMode >= mutating-with-confirm. + ClassWrite ToolClass = "write" + // ClassDestructive is irreversible (deletes, restarts, exec-arbitrary) + // — requires dual-sign-required (SOP) + ClassDestructive ToolClass = "destructive" +) + +// Policy is the per-request capability gate. The HTTP layer / agent +// persona constructs a Policy; ChatRuntime feeds it into ToolRegistry.Build +// and SkillRegistry.Resolve. PR-2 only defines the data shape — the +// actual filtering logic lives in skill_registry.go (and a future +// tool_registry.go in PR-3). +type Policy struct { + // AllowedClasses lists which ToolClass values are permitted. + // ["*"] means unrestricted. Empty defaults to read-only (matches + // sealsuite-agent semantics — see Allows()). + AllowedClasses []string + + // ConfirmationRequired lists classes that need a two-phase commit + // before execution. PR-2 defines the field but does not implement + // the gate (reviewer flow lands in a later PR). + ConfirmationRequired []string + + // MaxOutputChars caps the final reply length. 0 = no cap. + MaxOutputChars int + + // MaxIterations bounds the ReAct loop. 0 = use agent default. + MaxIterations int +} + +// Allows reports whether class is permitted under this policy. +// Empty AllowedClasses defaults to read-only (safe default). +func (p Policy) Allows(class ToolClass) bool { + if len(p.AllowedClasses) == 0 { + return class == ClassRead + } + for _, c := range p.AllowedClasses { + if c == "*" || c == string(class) { + return true + } + } + return false +} + +// RequiresConfirmation reports whether class must pass a confirmation +// gate (permissionMode = mutating-with-confirm). +func (p Policy) RequiresConfirmation(class ToolClass) bool { + for _, c := range p.ConfirmationRequired { + if c == "*" || c == string(class) { + return true + } + } + return false +} + +// Activation controls when a skill is mounted into the toolBag. See +// (metadata.ongrid.activation) and +type Activation struct { + // Mode = "always" (default) | "keyword". + // + sealsuite chatruntime semantics. + Mode string `yaml:"mode" json:"mode"` + + // Keywords trigger activation when Mode == "keyword". Case-insensitive + // substring match against the user query. + Keywords []string `yaml:"keywords" json:"keywords"` +} + +// Requires expresses host-level dependencies of a skill. Matches openclaw +// SKILL.md `metadata.requires` shape. ongrid reads this verbatim per +type Requires struct { + // Bins lists native binaries the skill expects on PATH (e.g. ["find", "du"]). + // They go into the sandbox process whitelist. + Bins []string `yaml:"bins" json:"bins"` + + // Config lists settings keys that must be filled by an admin before + // the skill can run (e.g. ["accountsPath"]). Lands in system_settings. + Config []string `yaml:"config" json:"config"` + + // Credentials declares the credential SLOTS this skill needs and HOW + // each one's fields inject into the skill's exec environment (HLD-017, + // the env/file analog of n8n's credential `authenticate`). A per-skill + // BINDING (installed_skills.bindings) picks WHICH stored credential + // fills each slot; at exec ongrid resolves the {{field}} templates from + // that credential's fields. ongrid stays semantics-agnostic — the skill + // author owns the mapping, the operator owns the choice. + Credentials []CredentialRequirement `yaml:"credentials" json:"credentials"` +} + +// CredentialRequirement is one credential slot declared by a skill/MCP. +type CredentialRequirement struct { + // Slot is the logical key the binding references (e.g. "tencentcloud"). + // Unique within the skill. + Slot string `yaml:"slot" json:"slot"` + + // Label is the human display name for the binding UI. + Label string `yaml:"label" json:"label"` + + // Fields lists the field names this slot expects in the bound + // credential (e.g. ["secret_id","secret_key","region"]) — drives the + // "missing field" check and the create-credential hint. + Fields []string `yaml:"fields" json:"fields"` + + // Inject is the declarative mapping of the credential's fields into + // the skill's exec environment. + Inject CredentialInject `yaml:"inject" json:"inject"` +} + +// CredentialInject declares where a slot's fields go at exec time. Each +// value is a template over the bound credential's fields ("{{secret_id}}"). +type CredentialInject struct { + // Env maps ENV_VAR_NAME -> "{{field}}" template. + Env map[string]string `yaml:"env" json:"env"` + + // Files materializes a templated blob into a file before exec (removed + // after) — for tools that read a credentials file (~/.aws/credentials, + // kubeconfig) rather than env vars. + Files []CredentialFile `yaml:"files" json:"files"` +} + +// CredentialFile is one credential file to materialize for a skill run. +type CredentialFile struct { + Path string `yaml:"path" json:"path"` // target path in the skill's exec dir + Content string `yaml:"content" json:"content"` // template over the credential fields + Mode string `yaml:"mode" json:"mode"` // octal perms, default "0600" +} + +// OngridExt is the ongrid-private extension subtree under metadata.ongrid. +// — these fields are ignored by openclaw / claude-code, so +// the same SKILL.md file runs on multiple platforms. +type OngridExt struct { + // Scope = "manager" (default) | "edge". Edge-scope skills run on the + // edge agent via frontier tunnel (plugin runtime). + Scope string `yaml:"scope" json:"scope"` + + // EdgeRuntime is a free-form hint for the edge plugin runtime + // (e.g. "subprocess", "wasm"). Empty = subprocess default. + EdgeRuntime string `yaml:"edge_runtime" json:"edge_runtime"` + + // EdgeCapabilities is the edge-side sandbox declaration — a list of + // capability statements Kept as raw map slice so + // future capability shapes don't break the parser. + EdgeCapabilities []map[string]any `yaml:"edge_capabilities" json:"edge_capabilities"` + + // Activation overrides the skill-level Activation (one is canonical, + // the other is for ergonomics). Loader prefers OngridExt.Activation + // when both are set, with a warning. + Activation Activation `yaml:"activation" json:"activation"` + + // MinOngridVersion gates loading on the running ongrid build (e.g. + // ">=0.7.30"). Failed gate produces a warning, not an error, in PR-2. + MinOngridVersion string `yaml:"min_ongrid_version" json:"min_ongrid_version"` +} + +// SkillMetadata is the `metadata` frontmatter subtree common to both +// openclaw and claude-code SKILL.md files plus the ongrid extension. +type SkillMetadata struct { + // OS gates loading on GOOS (— openclaw field). Empty = any. + OS []string `yaml:"os" json:"os"` + + // Requires is the openclaw `metadata.requires` shape. + Requires Requires `yaml:"requires" json:"requires"` + + // Ongrid is the ongrid-private extension subtree. + Ongrid OngridExt `yaml:"ongrid" json:"ongrid"` +} + +// ToolDecl is a single tool declared by a skill. PR-2 stores the +// declaration only — actual factory binding lives in PR-3. +type ToolDecl struct { + // Name is the wire-level tool name the LLM sees. + Name string `yaml:"name" json:"name"` + + // Impl is the registered factory key (e.g. "builtin:filetools.Find"). + Impl string `yaml:"impl" json:"impl"` + + // Class is the blast-radius classification — see ToolClass constants. + Class ToolClass `yaml:"class" json:"class"` + + // Description is the LLM-facing summary. + Description string `yaml:"description" json:"description"` + + // WhenToUse is the LLM-facing decision hint — + // claude-code splits "what is it" (description) from "when to pick it" + // (when_to_use). ongrid follows the same split at tool granularity. + WhenToUse string `yaml:"when_to_use" json:"when_to_use"` + + // Confirmation = "required" | "" — flips the per-tool confirmation + // gate even if the policy class would otherwise allow it. + Confirmation string `yaml:"confirmation" json:"confirmation"` + + // TimeoutSeconds caps the per-call wall time. 0 = use runtime default. + TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"` +} + +// Provenance records where a Skill / Agent / Pack came from. Filled by +// the marketplace install path; empty for filesystem drops. +type Provenance struct { + // Source = "fs" | "tgz" | "git" | "marketplace" | "" (unknown). + Source string `yaml:"source" json:"source"` + + // SourceURL is the original location (git URL, tgz fetch URL, + // marketplace slug). Diagnostic only. + SourceURL string `yaml:"source_url" json:"source_url"` + + // InstalledAt is the wall time of install. Zero for built-ins. + InstalledAt time.Time `yaml:"installed_at" json:"installed_at"` + + // ManifestSHA256 is hex of the SKILL.md / agent .md / plugin.json + // content — used by signature verification. + ManifestSHA256 string `yaml:"manifest_sha256" json:"manifest_sha256"` + + // SignatureState = "unsigned" | "valid" | "invalid" | "unknown". + SignatureState string `yaml:"signature_state" json:"signature_state"` +} + +// Skill is a parsed SKILL.md descriptor in memory. The on-disk format +// is YAML frontmatter + markdown body. The body is captured +// verbatim into PromptBody and gets prepended with `[能力: ]` +// during system-prompt assembly. +type Skill struct { + // Name is the snake_case unique key. Required. + Name string `yaml:"name" json:"name"` + + // Version is optional; "" if author didn't declare one. + Version string `yaml:"version" json:"version"` + + // Description is the one-liner shown to the LLM at listing time. + // Required. + Description string `yaml:"description" json:"description"` + + // WhenToUse is the longer "when should the LLM pick this" text. + // Recommended. + WhenToUse string `yaml:"when_to_use" json:"when_to_use"` + + // PromptBody is the markdown body of SKILL.md verbatim (after H1 + // normalization ). Loaded outside frontmatter so + // the YAML tag is `-`. + PromptBody string `yaml:"-" json:"prompt_body"` + + // Activation is the top-level activation hint (matches sealsuite + // shape). also allows it under metadata.ongrid; the + // ongrid one wins when both are set. + Activation Activation `yaml:"activation" json:"activation"` + + // ConfigSection is the legacy openclaw-style settings binding key + // (e.g. for "config_section: feilian.openapi"). Kept for compat. + ConfigSection string `yaml:"config_section" json:"config_section"` + + // Tools is the per-skill tool declaration list. + Tools []ToolDecl `yaml:"tools" json:"tools"` + + // Metadata is the `metadata` subtree (os, requires, ongrid.*). + Metadata SkillMetadata `yaml:"metadata" json:"metadata"` + + // Provenance is filled by the install path; empty for built-ins + // and direct fs drops. + Provenance Provenance `yaml:"-" json:"provenance"` + + // Dir is the absolute directory the skill was loaded from. Used + // by factories that need to resolve relative paths (openapi.yaml, + // domains/*.md, etc.). + Dir string `yaml:"-" json:"dir"` + + // UnknownFields preserves any frontmatter keys we don't recognize. + // explicitly calls for this so that openclaw / claude-code + // adding new fields doesn't break ongrid loading. Map values are + // kept as decoded YAML scalars / sequences / mappings. + UnknownFields map[string]any `yaml:"-" json:"unknown_fields"` +} + +// AgentCapability is a schedulable promise made by an Agent. It names the +// user task the agent can close and the tool/skill surface it needs; the +// default assistant may declare the same capability with a smaller budget. +type AgentCapability struct { + // AgentName is populated by AgentRegistry when exposing a card. It is not + // authored in frontmatter because the containing Agent is authoritative. + AgentName string `yaml:"-" json:"agent_name"` + ID string `yaml:"id" json:"id"` + Description string `yaml:"description" json:"description"` + Tools []string `yaml:"tools" json:"tools"` + Skills []string `yaml:"skills" json:"skills"` + MaxToolCalls int `yaml:"max_tool_calls" json:"max_tool_calls"` +} + +// Agent is a parsed agent persona (frontmatter markdown). +// — same on-disk shape as SKILL.md but the body is the agent's system +// prompt rather than skill prose. Snake_case YAML tags +type Agent struct { + // Name is the agent identifier used at spawn time. Required. + Name string `yaml:"name" json:"name"` + + // Description is the human listing string. Required. + Description string `yaml:"description" json:"description"` + + // WhenToUse is the coordinator's spawn-decision hint. Required — + // calls this the strict field a coordinator reads. + WhenToUse string `yaml:"when_to_use" json:"when_to_use"` + + // Capabilities are structured, registration-time declarations of the + // tasks this agent can help close. They are used by Resolve; Tool names + // remain the executable implementation details. + Capabilities []AgentCapability `yaml:"capabilities" json:"capabilities"` + // CanDelegate allows this specialist to request one further internal work + // session when its own evidence points to another domain. Runtime depth + // limits still apply; this flag never transfers root-session ownership. + CanDelegate bool `yaml:"can_delegate" json:"can_delegate"` + + // Tools is the explicit tool whitelist. Empty = inherit from policy. + Tools []string `yaml:"tools" json:"tools"` + + // DisallowedTools is the blacklist (applied after whitelist). Black + // wins over white + DisallowedTools []string `yaml:"disallowed_tools" json:"disallowed_tools"` + + // PermissionMode = "read-only" | "mutating-with-confirm" | "dual-sign-required". + PermissionMode string `yaml:"permission_mode" json:"permission_mode"` + + // MaxTurns caps the worker's internal ReAct loop. + MaxTurns int `yaml:"max_turns" json:"max_turns"` + + // Model is the LLM identifier (e.g. "anthropic/claude-sonnet-4-6"). + // Empty = inherit coordinator default. + Model string `yaml:"model" json:"model"` + + // CriticalReminder is the system-reminder block injected on every + // turn. — anti-drift mechanism. + CriticalReminder string `yaml:"critical_reminder" json:"critical_reminder"` + + // InitialPrompt is prepended to the first user message at spawn. + InitialPrompt string `yaml:"initial_prompt" json:"initial_prompt"` + + // Background = true forces async execution (long-running workers). + Background bool `yaml:"background" json:"background"` + + // OmitClaudeMd skips inheriting the global system context. Used + // for tightly-scoped reviewer agents. + OmitClaudeMd bool `yaml:"omit_claude_md" json:"omit_claude_md"` + + // SystemPrompt is the markdown body — the actual system prompt + // the worker LLM sees. Loaded outside frontmatter. + SystemPrompt string `yaml:"-" json:"system_prompt"` + + // Source tells the API/UI where this persona came from: + // "builtin" — shipped in the binary (programmatic Add) + // "disk" — loaded from agents/*.md + // "user" — created by the user via /v1/agents/custom (Phase 3) + // "user" agents are editable + deletable from the UI; the others + // stay read-only. + Source string `yaml:"-" json:"source,omitempty"` + + // Metadata mirrors the SKILL.md metadata subtree — agents may + // declare os / requires / ongrid extensions too. + Metadata SkillMetadata `yaml:"metadata" json:"metadata"` + + // Provenance fills on install. + Provenance Provenance `yaml:"-" json:"provenance"` + + // Dir is the directory of the .md file. + Dir string `yaml:"-" json:"dir"` + + // UnknownFields preserves unrecognized frontmatter keys. + UnknownFields map[string]any `yaml:"-" json:"unknown_fields"` +} + +// Pack is a parsed plugin container — a directory shipping zero or more +// skills + agents + commands together. ongrid recognizes both +// claude-code's `.claude-plugin/plugin.json` and openclaw's +// `openclaw.plugin.json`. PR-2 only parses the manifest +// metadata; recursive load of skills/ agents/ commands/ subdirs is a +// later PR (TODO in plugin_container.go). +type Pack struct { + // ID is the pack key (e.g. "acme-tools"). Required. + ID string `yaml:"id" json:"id"` + + // DisplayName is the human-friendly title. + DisplayName string `yaml:"name" json:"display_name"` + + // Version follows semver (e.g. "1.2.3"). + Version string `yaml:"version" json:"version"` + + // Description is a one-liner. + Description string `yaml:"description" json:"description"` + + // ConfigSchema is a JSON Schema describing tenant-level config that + // admins fill in via the SPA install dialog. Optional. + ConfigSchema json.RawMessage `yaml:"config_schema" json:"config_schema"` + + // UIMetadata holds platform-specific extras the SPA may show. We + // stuff openclaw legacy fields under UIMetadata["openclaw_legacy"] + // so future SPA can render them as-is. + UIMetadata map[string]any `yaml:"ui_metadata" json:"ui_metadata"` + + // Source is the install source ("fs" | "tgz" | "git" | "marketplace"). + Source string `yaml:"source" json:"source"` + + // ManifestSHA256 is hex of the manifest content. + ManifestSHA256 string `yaml:"manifest_sha256" json:"manifest_sha256"` + + // SignatureState = "unsigned" | "valid" | "invalid" | "unknown". + SignatureState string `yaml:"signature_state" json:"signature_state"` + + // Dir is the absolute pack root. + Dir string `yaml:"-" json:"dir"` +} + +// LoadResult is what plugin_container / registry loaders return after a +// directory walk: the discovered Pack (if any), the skills + agents +// found inside, and accumulated warnings (one per skipped / normalized +// entry). +type LoadResult struct { + Pack *Pack `json:"pack"` + Skills []*Skill `json:"skills"` + Agents []*Agent `json:"agents"` + Warnings []LoadWarning `json:"warnings"` +} + +// LoadWarning is a non-fatal load issue. Code is a stable identifier +// (e.g. "name_normalized", "missing_when_to_use") so SPA / tests can +// filter without scraping Reason text. +type LoadWarning struct { + // Path is the file the warning applies to (absolute or relative + // to the loader's root — loader convention). + Path string `json:"path"` + + // Reason is human-readable English text. + Reason string `json:"reason"` + + // Code is the stable identifier. + Code string `json:"code"` +} diff --git a/internal/manager/biz/aiops/chatruntime/worker.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go similarity index 99% rename from internal/manager/biz/aiops/chatruntime/worker.go rename to internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go index f7c87a40f..a7fb64b21 100644 --- a/internal/manager/biz/aiops/chatruntime/worker.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go @@ -53,8 +53,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph/callbacks" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph/callbacks" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" "github.com/ongridio/ongrid/internal/pkg/llm" diff --git a/internal/manager/biz/aiops/chatruntime/worker_context_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker_context_test.go similarity index 100% rename from internal/manager/biz/aiops/chatruntime/worker_context_test.go rename to internal/manager/biz/aiops/runtime/v3/chatruntime/worker_context_test.go diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/worker_filter_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker_filter_test.go new file mode 100644 index 000000000..038eb61b1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker_filter_test.go @@ -0,0 +1,103 @@ +package chatruntime + +import ( + "context" + "testing" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// originTool is a minimal BaseTool whose Info carries a configurable Origin / +// Class, for exercising the role filter. +type originTool struct{ name, class, origin, confirmation string } + +func (o *originTool) Info(context.Context) (*basetool.ToolInfo, error) { + return &basetool.ToolInfo{Name: o.name, Description: "x", Class: o.class, Origin: o.origin, Confirmation: o.confirmation}, nil +} + +func TestFilterToolsForAgentRole_ViewerDropsSensitiveReadTool(t *testing.T) { + bag := []basetool.BaseTool{ + &originTool{name: "query_devices", class: "read"}, + &originTool{name: "capture_pcap", class: "read", confirmation: basetool.ConfirmationRequired}, + } + view := filterToolsForAgentRole(bag, nil, true, true) + if !toolbagHas(view, "query_devices") || toolbagHas(view, "capture_pcap") { + t.Fatalf("viewer tools = %+v; sensitive read tool must be removed", view) + } +} +func (o *originTool) InvokableRun(context.Context, string, ...basetool.InvokeOption) (string, error) { + return "{}", nil +} + +func toolbagHas(tools []basetool.BaseTool, name string) bool { + for _, t := range tools { + if i, _ := t.Info(context.Background()); i != nil && i.Name == name { + return true + } + } + return false +} + +// Dynamic (runtime-discovered) read tools are in scope for BOTH the coordinator +// and specialists (they can't be pre-listed in a persona whitelist). A viewer +// still drops non-read dynamic tools. (We briefly excluded dynamic tools from +// the coordinator to force delegation, but that made MCP unreachable with the +// weak deployed model — see worker.go history note — so they're back.) +func TestFilterToolsForAgentRole_DynamicReadTools(t *testing.T) { + bag := []basetool.BaseTool{ + &originTool{name: "rank_edges", class: "read", origin: basetool.OriginBuiltin}, + &originTool{name: "mcp__k8s__namespaces_list", class: "read", origin: basetool.OriginMCP}, + &originTool{name: "mcp__k8s__nodes_drain", class: "destructive", origin: basetool.OriginMCP}, + &originTool{name: "skill__custom__foo", class: "read", origin: basetool.OriginSkill}, + } + + // coordinator + specialist both keep dynamic READ tools (so MCP is reachable) + for _, isCoord := range []bool{true, false} { + got := filterToolsForAgentRole(bag, nil, isCoord, false) + for _, n := range []string{"rank_edges", "mcp__k8s__namespaces_list", "skill__custom__foo"} { + if !toolbagHas(got, n) { + t.Errorf("isCoordinator=%v should keep %s", isCoord, n) + } + } + } + + // viewer: non-read dynamic tools are dropped (read ones survive) + view := filterToolsForAgentRole(bag, nil, false, true /*viewerOnly*/) + if !toolbagHas(view, "mcp__k8s__namespaces_list") { + t.Errorf("viewer should keep a READ dynamic tool") + } + if toolbagHas(view, "mcp__k8s__nodes_drain") { + t.Errorf("viewer must drop a DESTRUCTIVE dynamic tool") + } +} + +func TestFilterToolsForAgentRole_NetworkInventoryStaysWithNetworkSpecialist(t *testing.T) { + bag := []basetool.BaseTool{ + &originTool{name: "query_devices", class: "read", origin: basetool.OriginBuiltin}, + &originTool{name: "query_network_devices", class: "read", origin: basetool.OriginBuiltin}, + &originTool{name: "query_network_interfaces", class: "read", origin: basetool.OriginBuiltin}, + &originTool{name: "get_network_neighbors", class: "read", origin: basetool.OriginBuiltin}, + &originTool{name: "ToolSearch", class: "read", origin: basetool.OriginBuiltin}, + } + + coordinator := &Agent{Tools: []string{"query_devices"}} + coordinatorTools := filterToolsForAgentRole(bag, coordinator, true, false) + for _, name := range []string{"query_network_devices", "query_network_interfaces", "get_network_neighbors"} { + if toolbagHas(coordinatorTools, name) { + t.Errorf("coordinator must not receive network inventory tool %q", name) + } + } + + networkSpecialist := &Agent{Tools: []string{ + "query_network_devices", + "query_network_interfaces", + "get_network_neighbors", + "ToolSearch", + }} + networkTools := filterToolsForAgentRole(bag, networkSpecialist, false, false) + for _, name := range networkSpecialist.Tools { + if !toolbagHas(networkTools, name) { + t.Errorf("network specialist missing tool %q", name) + } + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/budget_stop_model.go b/internal/manager/biz/aiops/runtime/v3/graph/budget_stop_model.go new file mode 100644 index 000000000..828ff306d --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/budget_stop_model.go @@ -0,0 +1,577 @@ +package graph + +import ( + "context" + "encoding/json" + "regexp" + "strconv" + "strings" + + einomodel "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +type budgetStopModel struct { + inner einomodel.ToolCallingChatModel +} + +func wrapBudgetStopModel(inner einomodel.ToolCallingChatModel) einomodel.ToolCallingChatModel { + if inner == nil { + return nil + } + return &budgetStopModel{inner: inner} +} + +func (m *budgetStopModel) Generate(ctx context.Context, input []*schema.Message, opts ...einomodel.Option) (*schema.Message, error) { + // 回调只能标记预算拒绝；在普通推理和工具熔断总结之前终止模型调用。 + if err := llm.BudgetRejectionFromContext(ctx); err != nil { + return nil, err + } + if env, ok := latestTerminalToolBudget(input); ok { + return m.synthesizeAfterToolBudget(ctx, input, env.Tool, opts...) + } + msg, err := m.inner.Generate(ctx, input, opts...) + if err != nil { + return nil, err + } + pruned, exhaustedTool := pruneToolCallsForBudget(input, msg) + if exhaustedTool != "" { + return m.synthesizeAfterToolBudget(ctx, input, exhaustedTool, opts...) + } + return pruned, nil +} + +func (m *budgetStopModel) Stream(ctx context.Context, input []*schema.Message, opts ...einomodel.Option) (*schema.StreamReader[*schema.Message], error) { + if err := llm.BudgetRejectionFromContext(ctx); err != nil { + return nil, err + } + if env, ok := latestTerminalToolBudget(input); ok { + msg, err := m.synthesizeAfterToolBudget(ctx, input, env.Tool, opts...) + if err != nil { + return nil, err + } + return schema.StreamReaderFromArray([]*schema.Message{msg}), nil + } + return m.inner.Stream(ctx, input, opts...) +} + +func (m *budgetStopModel) synthesizeAfterToolBudget(ctx context.Context, input []*schema.Message, tool string, opts ...einomodel.Option) (*schema.Message, error) { + tool = strings.TrimSpace(tool) + if tool == "" { + tool = "tool" + } + reminder := schema.SystemMessage("The per-tool circuit breaker for `" + tool + "` has been reached in this user turn. Do not call that tool again. Either synthesize the answer from existing evidence or call a different tool only when it provides materially different evidence. Follow the configured response language, explain evidence limits honestly, and never paste raw tool JSON or internal reasoning.") + prompt := make([]*schema.Message, 0, len(input)+1) + prompt = append(prompt, input...) + prompt = append(prompt, reminder) + msg, err := m.inner.Generate(ctx, prompt, opts...) + if err != nil { + return nil, err + } + if msg == nil { + return &schema.Message{Role: schema.Assistant, Content: budgetPrunedFinalContent(input, tool)}, nil + } + if len(msg.ToolCalls) == 0 && strings.TrimSpace(msg.Content) != "" { + return msg, nil + } + pruned, stillExhausted := pruneToolCallsForBudget(input, msg) + if stillExhausted == "" && pruned != nil && (len(pruned.ToolCalls) > 0 || strings.TrimSpace(pruned.Content) != "") { + return pruned, nil + } + cp := *msg + cp.Role = schema.Assistant + cp.ToolCalls = nil + cp.Content = budgetPrunedFinalContent(input, tool) + return &cp, nil +} + +func pruneToolCallsForBudget(history []*schema.Message, msg *schema.Message) (*schema.Message, string) { + if msg == nil || len(msg.ToolCalls) == 0 { + return msg, "" + } + if guarded := guardUnresolvedDeviceTarget(history, msg); guarded != nil { + return guarded, "" + } + perToolCounts := priorToolCounts(history) + kept := make([]schema.ToolCall, 0, len(msg.ToolCalls)) + exhaustedTool := "" + for _, call := range msg.ToolCalls { + name := strings.TrimSpace(call.Function.Name) + if name == "" { + continue + } + if perToolCounts[name] >= maxCallsForTool(name) { + if exhaustedTool == "" { + exhaustedTool = name + } + continue + } + perToolCounts[name]++ + kept = append(kept, call) + } + if len(kept) == len(msg.ToolCalls) { + return msg, "" + } + cp := *msg + cp.ToolCalls = kept + if len(kept) == 0 { + cp.Content = budgetPrunedFinalContent(history, exhaustedTool) + return &cp, exhaustedTool + } + return &cp, "" +} + +func guardUnresolvedDeviceTarget(history []*schema.Message, msg *schema.Message) *schema.Message { + target := currentNamedDeviceTarget(history) + if target == "" || !currentTurnHasEmptyDeviceLookup(history) || !containsHostScopedToolCall(msg.ToolCalls) { + return nil + } + cp := *msg + cp.ToolCalls = nil + if wantsEnglishResponse(history) { + cp.Content = "I could not find device `" + target + "` in the fleet. I will not substitute another online device automatically. Please confirm the correct device or select it with @ before I run host-level checks." + } else { + cp.Content = "没有在设备清单中找到 `" + target + "`。我不会自动改用其他在线设备执行主机检查；请确认正确设备，或使用 @ 选择目标设备后我再继续。" + } + return &cp +} + +var namedDeviceTargetRe = regexp.MustCompile(`(?i)\b(?:edge|vm|node|host|server|device)-[a-z0-9_.-]+\b`) + +func currentNamedDeviceTarget(messages []*schema.Message) string { + for i := len(messages) - 1; i >= 0; i-- { + msg := messages[i] + if msg == nil || msg.Role != schema.User || isSystemReminderMessage(msg.Content) { + continue + } + if strings.Contains(strings.ToLower(msg.Content), "device_id") || strings.Contains(msg.Content, "@") { + return "" + } + return namedDeviceTargetRe.FindString(msg.Content) + } + return "" +} + +func currentTurnHasEmptyDeviceLookup(messages []*schema.Message) bool { + for i := len(messages) - 1; i >= 0; i-- { + msg := messages[i] + if msg == nil { + continue + } + if msg.Role == schema.User && !isSystemReminderMessage(msg.Content) { + break + } + if msg.Role != schema.Tool || msg.ToolName != "query_devices" { + continue + } + var payload map[string]any + if err := json.Unmarshal([]byte(msg.Content), &payload); err != nil { + continue + } + count, ok := jsonNumber(payload["count"]) + if ok && count == 0 { + return true + } + } + return false +} + +func containsHostScopedToolCall(calls []schema.ToolCall) bool { + for _, call := range calls { + switch strings.TrimSpace(call.Function.Name) { + case "get_host_load", "get_host_processes", "host_bash", "host_find_large_files", "host_du_summary", "host_stat_file", "host_restart_service", "capture_pcap": + return true + } + } + return false +} + +func priorToolCounts(messages []*schema.Message) map[string]int { + counts := make(map[string]int) + for i := len(messages) - 1; i >= 0; i-- { + msg := messages[i] + if msg == nil { + continue + } + if msg.Role == schema.User && !isSystemReminderMessage(msg.Content) { + break + } + if msg.Role != schema.Tool { + continue + } + name := strings.TrimSpace(msg.ToolName) + if name == "" { + continue + } + counts[name]++ + } + return counts +} + +func budgetPrunedFinalContent(messages []*schema.Message, tool string) string { + if strings.TrimSpace(tool) == "" { + tool = "tool" + } + if wantsEnglishResponse(messages) { + return "This turn reached the per-tool safety limit for `" + tool + "`, and the model did not produce a usable synthesis. No raw tool output is shown. Send a narrower target or time window to continue in a new turn." + } + return "本轮 `" + tool + "` 已达到单工具安全上限，模型未能生成可用的归纳结论。这里不会展示工具原始结果；请在下一条消息给出更窄的目标或时间窗后继续。" +} + +func summarizeRecentToolEvidence(messages []*schema.Message) string { + lines := make([]string, 0, 6) + for i := len(messages) - 1; i >= 0 && len(lines) < 6; i-- { + msg := messages[i] + if msg == nil { + continue + } + if msg.Role == schema.User && !isSystemReminderMessage(msg.Content) { + break + } + if msg.Role != schema.Tool { + continue + } + line := summarizeToolMessage(msg.ToolName, msg.Content) + if line == "" { + continue + } + lines = append(lines, line) + } + if len(lines) == 0 { + return "" + } + for i, j := 0, len(lines)-1; i < j; i, j = i+1, j-1 { + lines[i], lines[j] = lines[j], lines[i] + } + return "- " + strings.Join(lines, "\n- ") +} + +func summarizeToolMessage(toolName, content string) string { + toolName = strings.TrimSpace(toolName) + if toolName == "" { + toolName = "tool" + } + switch toolName { + case "ToolSearch", "get_edge_summary", "query_knowledge", "grep_source", "list_repo_sources": + return "" + } + var payload map[string]any + if err := json.Unmarshal([]byte(content), &payload); err == nil { + switch toolName { + case "query_incidents": + return summarizeIncidentPayload(toolName, payload) + case "correlate_incident": + return summarizeCorrelationPayload(toolName, payload) + case "host_du_summary": + return summarizeDiskUsagePayload(toolName, payload) + case "host_find_large_files": + return summarizeLargeFilesPayload(toolName, payload) + case "host_bash": + return summarizeHostBashPayload(toolName, payload) + } + if count, ok := jsonNumber(payload["count"]); ok { + return toolName + ": 返回 " + formatNumber(count) + " 条结果。" + } + } + compact := strings.Join(strings.Fields(content), " ") + if compact == "" { + return "" + } + if len(compact) > 180 { + compact = compact[:180] + "..." + } + return toolName + ": " + compact +} + +func summarizeIncidentPayload(toolName string, payload map[string]any) string { + count, _ := jsonNumber(payload["count"]) + incidents, _ := payload["incidents"].([]any) + titles := make([]string, 0, 3) + for _, item := range incidents { + obj, ok := item.(map[string]any) + if !ok { + continue + } + title, _ := obj["title"].(string) + if title == "" { + continue + } + titles = append(titles, trimRunes(title, 72)) + if len(titles) >= 3 { + break + } + } + if len(titles) == 0 { + return toolName + ": 返回 " + formatNumber(count) + " 条告警。" + } + return toolName + ": 返回 " + formatNumber(count) + " 条告警，代表项：" + strings.Join(titles, "；") +} + +func summarizeCorrelationPayload(toolName string, payload map[string]any) string { + results, _ := payload["results"].([]any) + parts := make([]string, 0, 3) + for _, item := range results { + obj, ok := item.(map[string]any) + if !ok { + continue + } + id, _ := jsonNumber(obj["incident_id"]) + bundle, _ := obj["bundle"].(map[string]any) + incident, _ := bundle["incident"].(map[string]any) + title, _ := incident["title"].(string) + value, hasValue := jsonNumber(incident["value"]) + part := "incident " + formatNumber(id) + if title != "" { + part += " " + trimRunes(title, 56) + } + if hasValue { + part += "，value=" + formatNumber(value) + } + parts = append(parts, part) + if len(parts) >= 3 { + break + } + } + if len(parts) == 0 { + return toolName + ": 返回关联分析结果。" + } + return toolName + ": " + strings.Join(parts, "；") +} + +func summarizeDiskUsagePayload(toolName string, payload map[string]any) string { + results, _ := payload["results"].([]any) + parts := make([]string, 0, 4) + for _, item := range results { + obj, ok := item.(map[string]any) + if !ok { + continue + } + path, _ := obj["path"].(string) + subpaths, _ := obj["subpaths"].([]any) + if path == "" || len(subpaths) == 0 { + continue + } + top, _ := subpaths[0].(map[string]any) + subpath, _ := top["subpath"].(string) + size, _ := top["size_human"].(string) + if subpath == "" { + continue + } + if size != "" { + parts = append(parts, path+" 最大项 "+subpath+"="+size) + } else { + parts = append(parts, path+" 最大项 "+subpath) + } + if len(parts) >= 4 { + break + } + } + if len(parts) == 0 { + return toolName + ": 返回磁盘占用分析结果。" + } + return toolName + ": " + strings.Join(parts, "；") +} + +func summarizeLargeFilesPayload(toolName string, payload map[string]any) string { + results, _ := payload["results"].([]any) + parts := make([]string, 0, 5) + for _, item := range results { + obj, ok := item.(map[string]any) + if !ok { + continue + } + files, _ := obj["files"].([]any) + for _, file := range files { + fileObj, ok := file.(map[string]any) + if !ok { + continue + } + path, _ := fileObj["path"].(string) + size, _ := fileObj["size_human"].(string) + if path == "" { + continue + } + if size != "" { + parts = append(parts, path+"="+size) + } else { + parts = append(parts, path) + } + if len(parts) >= 5 { + return toolName + ": 大文件 " + strings.Join(parts, "；") + } + } + } + if len(parts) == 0 { + return toolName + ": 返回大文件扫描结果。" + } + return toolName + ": 大文件 " + strings.Join(parts, "；") +} + +func summarizeHostBashPayload(toolName string, payload map[string]any) string { + cmd, _ := payload["cmd"].(string) + results, _ := payload["results"].([]any) + stdouts := make([]string, 0, len(results)) + for _, item := range results { + obj, ok := item.(map[string]any) + if !ok { + continue + } + stdout, _ := obj["stdout"].(string) + stdout = strings.TrimSpace(stdout) + if stdout != "" { + stdouts = append(stdouts, stdout) + } + } + if len(stdouts) == 0 { + return toolName + ": 命令 `" + trimRunes(cmd, 60) + "` 无有效输出。" + } + joined := strings.Join(stdouts, "\n") + if strings.Contains(cmd, "df ") { + line := firstDataLine(joined) + if line != "" { + return toolName + ": `df` 显示 " + trimRunes(line, 120) + } + } + if strings.Contains(cmd, "du ") { + tops := topOutputLines(joined, 4) + if len(tops) > 0 { + return toolName + ": `du` Top 项 " + strings.Join(tops, "；") + } + } + lines := topOutputLines(joined, 3) + if len(lines) == 0 { + return toolName + ": 命令 `" + trimRunes(cmd, 60) + "` 已执行。" + } + return toolName + ": `" + trimRunes(cmd, 60) + "` 输出 " + strings.Join(lines, "；") +} + +func firstDataLine(output string) string { + for _, line := range strings.Split(output, "\n") { + line = strings.Join(strings.Fields(line), " ") + if line == "" || strings.HasPrefix(strings.ToLower(line), "filesystem ") { + continue + } + return line + } + return "" +} + +func topOutputLines(output string, max int) []string { + lines := make([]string, 0, max) + for _, line := range strings.Split(output, "\n") { + line = strings.Join(strings.Fields(line), " ") + if line == "" { + continue + } + lines = append(lines, trimRunes(line, 80)) + if len(lines) >= max { + break + } + } + return lines +} + +func jsonNumber(value any) (float64, bool) { + switch v := value.(type) { + case float64: + return v, true + case int: + return float64(v), true + case int64: + return float64(v), true + case json.Number: + n, err := v.Float64() + return n, err == nil + default: + return 0, false + } +} + +func formatNumber(value float64) string { + return strconv.FormatFloat(value, 'f', -1, 64) +} + +func trimRunes(value string, max int) string { + runes := []rune(strings.TrimSpace(value)) + if len(runes) <= max { + return string(runes) + } + return string(runes[:max]) + "..." +} + +func (m *budgetStopModel) WithTools(tools []*schema.ToolInfo) (einomodel.ToolCallingChatModel, error) { + next, err := m.inner.WithTools(tools) + if err != nil { + return nil, err + } + return &budgetStopModel{inner: next}, nil +} + +func finalAnswerAfterToolBudget(messages []*schema.Message) (*schema.Message, bool) { + env, ok := latestTerminalToolBudget(messages) + if !ok { + return nil, false + } + tool := strings.TrimSpace(env.Tool) + if tool == "" { + tool = "the tool" + } + content := "我已收敛本轮排查，没有继续反复调用 `" + tool + "`。基于本轮已经拿到的结果：如果上面的数据已经出现异常信号，就按这些信号给出结论和下一步；如果结果为空或报错，本轮缺少可判定证据，请在下一条消息补充更具体的时间窗、service 或 device_id 后再查。" + if wantsEnglishResponse(messages) { + content = "I have converged this investigation instead of repeatedly calling `" + tool + "`. Based on the evidence already collected: if the earlier results show an abnormal signal, use that signal for the conclusion and next step; if they were empty or errored, this turn lacks decisive evidence, so send a narrower time window, service, or device_id in the next message and I can query again." + } + return &schema.Message{Role: schema.Assistant, Content: content}, true +} + +type toolBudgetEnvelope struct { + Status string `json:"status"` + Tool string `json:"tool"` + FinalAnswer bool `json:"final_answer_required"` +} + +func latestTerminalToolBudget(messages []*schema.Message) (toolBudgetEnvelope, bool) { + var zero toolBudgetEnvelope + for i := len(messages) - 1; i >= 0; i-- { + msg := messages[i] + if msg == nil { + continue + } + if msg.Role == schema.User && !isSystemReminderMessage(msg.Content) { + return zero, false + } + if msg.Role != schema.Tool { + continue + } + var env toolBudgetEnvelope + if err := json.Unmarshal([]byte(msg.Content), &env); err != nil { + continue + } + if env.Status == "call_budget_exceeded" && env.FinalAnswer { + return env, true + } + } + return zero, false +} + +func isSystemReminderMessage(content string) bool { + trimmed := strings.TrimSpace(strings.ToLower(content)) + return strings.HasPrefix(trimmed, "") +} + +func wantsEnglishResponse(messages []*schema.Message) bool { + for i := len(messages) - 1; i >= 0; i-- { + msg := messages[i] + if msg == nil { + continue + } + if msg.Role != schema.System && msg.Role != schema.User { + continue + } + if strings.Contains(strings.ToLower(msg.Content), "respond in english") { + return true + } + } + return false +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/alert_draft_guard.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/alert_draft_guard.go new file mode 100644 index 000000000..7c769fad1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/alert_draft_guard.go @@ -0,0 +1,310 @@ +package callbacks + +import ( + "context" + "encoding/json" + "regexp" + "strings" + "sync/atomic" + + einocallbacks "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +const draftConfigChangeToolName = "draft_config_change" + +const AlertDraftGuardBlockedMessage = "这次没有通过 draft_config_change 生成可应用的 config_draft/draft_hash，我已拦截文字草案，避免把它误当成可确认的告警规则。继续这个创建需求时，我会生成正式草案并完成验证。" +const alertDraftGuardBlockedMessage = AlertDraftGuardBlockedMessage + +var collectedSourceMentionRE = regexp.MustCompile(`\b(?:db|custom):[A-Za-z0-9_.:-]+`) + +type AlertDraftGuardDeps struct { + UserText string +} + +// AlertDraftGuardHandler prevents a model-only "draft" from being +// persisted when the user is creating an alert rule. A real alert-rule +// draft must come from draft_config_change so the payload, preview, and +// draft_hash are generated by backend code rather than free-form text. +type AlertDraftGuardHandler struct { + userText string + draftSucceeded atomic.Bool + hideSampleSources atomic.Bool +} + +func NewAlertDraftGuardHandler(deps AlertDraftGuardDeps) *AlertDraftGuardHandler { + if !looksLikeAlertRuleCreationRequest(deps.UserText) { + return nil + } + return &AlertDraftGuardHandler{userText: deps.UserText} +} + +func (h *AlertDraftGuardHandler) Needed(_ context.Context, info *einocallbacks.RunInfo, timing einocallbacks.CallbackTiming) bool { + if h == nil || info == nil { + return false + } + switch info.Component { + case components.ComponentOfChatModel, components.ComponentOfTool: + return timing == einocallbacks.TimingOnEnd + default: + return false + } +} + +func (h *AlertDraftGuardHandler) OnEnd(ctx context.Context, info *einocallbacks.RunInfo, output einocallbacks.CallbackOutput) context.Context { + if h == nil || info == nil { + return ctx + } + switch info.Component { + case components.ComponentOfTool: + tout := einotool.ConvCallbackOutput(output) + if tout == nil { + return ctx + } + switch info.Name { + case draftConfigChangeToolName: + if toolResultLooksLikeConfigDraft(tout.Response) { + h.draftSucceeded.Store(true) + if draftResultShouldHideSampleSources(tout.Response) { + h.hideSampleSources.Store(true) + } + } + } + case components.ComponentOfChatModel: + mo := einomodel.ConvCallbackOutput(output) + if mo == nil || mo.Message == nil { + return ctx + } + if messageCallsTool(mo.Message, draftConfigChangeToolName) { + return ctx + } + mo.Message.Content = h.SanitizeAssistantContent(mo.Message.Content) + } + return ctx +} + +func (h *AlertDraftGuardHandler) OnStart(ctx context.Context, _ *einocallbacks.RunInfo, _ einocallbacks.CallbackInput) context.Context { + return ctx +} + +func (h *AlertDraftGuardHandler) OnError(ctx context.Context, _ *einocallbacks.RunInfo, _ error) context.Context { + return ctx +} + +func (h *AlertDraftGuardHandler) OnStartWithStreamInput(ctx context.Context, _ *einocallbacks.RunInfo, _ *schema.StreamReader[einocallbacks.CallbackInput]) context.Context { + return ctx +} + +func (h *AlertDraftGuardHandler) OnEndWithStreamOutput(ctx context.Context, _ *einocallbacks.RunInfo, _ *schema.StreamReader[einocallbacks.CallbackOutput]) context.Context { + return ctx +} + +func (h *AlertDraftGuardHandler) SanitizeAssistantContent(content string) string { + if h == nil { + return content + } + if h.draftSucceeded.Load() { + if h.hideSampleSources.Load() { + return sanitizeCollectedSampleSourceMentions(content) + } + return content + } + if !looksLikeConfirmableAlertDraft(content) { + return content + } + return alertDraftGuardBlockedMessage +} + +func AlertDraftGuardFromHandlers(handlers []einocallbacks.Handler) *AlertDraftGuardHandler { + for _, h := range handlers { + if guard, ok := h.(*AlertDraftGuardHandler); ok { + return guard + } + } + return nil +} + +func LooksLikeAlertRuleCreationText(text string) bool { + return looksLikeAlertRuleCreationRequest(text) +} + +func messageCallsTool(msg *schema.Message, toolName string) bool { + if msg == nil { + return false + } + for _, tc := range msg.ToolCalls { + if strings.EqualFold(strings.TrimSpace(tc.Function.Name), toolName) { + return true + } + } + return false +} + +func LooksLikeAlertDraftGuardBlockedMessage(content string) bool { + lower := strings.ToLower(content) + return strings.Contains(lower, "draft_config_change") && + strings.Contains(lower, "config_draft") && + (strings.Contains(content, "拦截文字草案") || + strings.Contains(content, "没有通过 draft_config_change") || + strings.Contains(content, "我还没有通过 draft_config_change")) +} + +func looksLikeAlertRuleCreationRequest(text string) bool { + lower := strings.ToLower(text) + if !containsAnyNeedle(text, lower, []string{ + "创建", "新增", "添加", "配置", "生成", + "create", "add", "configure", "new", "generate", + }) { + return false + } + return containsAnyNeedle(text, lower, []string{ + "告警", "告警规则", "监控规则", "规则", "链路", "日志", + "alert", "rule", "monitoring rule", "slo", "burn rate", "metric", "log", "trace", + }) +} + +func looksLikeConfirmableAlertDraft(content string) bool { + lower := strings.ToLower(content) + if containsAnyNeedle(content, lower, []string{ + "config_draft", + "apply_config_change", + }) { + return true + } + hasAlertRule := strings.Contains(content, "告警") || + strings.Contains(content, "规则键") || + strings.Contains(content, "规则设计") || + strings.Contains(strings.ToLower(content), "规则 key") || + strings.Contains(lower, "alert") || + strings.Contains(lower, "rule_key") || + strings.Contains(lower, "draft_hash") + if !hasAlertRule { + return false + } + confirmableNeedles := []string{ + "草案哈希", + "确认应用", + "告警规则草案", + "draft_hash", + "sha256:", + "config_draft", + "apply_config_change", + } + for _, needle := range confirmableNeedles { + if strings.Contains(lower, needle) || strings.Contains(content, needle) { + return true + } + } + hasRuleDesign := containsAnyNeedle(content, lower, []string{ + "规则设计", + "规则 key", + "规则key", + "promql", + "触发条件", + "生效范围", + }) + asksForConfirm := containsAnyNeedle(content, lower, []string{ + "需要确认", + "确认创建", + "要确认", + "确认吗", + "confirm", + "approve", + }) + if hasRuleDesign && asksForConfirm { + return true + } + return false +} + +func containsAnyNeedle(text, lowerText string, needles []string) bool { + for _, needle := range needles { + if strings.Contains(lowerText, needle) || strings.Contains(text, needle) { + return true + } + } + return false +} + +func toolResultLooksLikeConfigDraft(result string) bool { + var resp struct { + Kind string `json:"kind"` + DraftHash string `json:"draft_hash"` + } + if err := json.Unmarshal([]byte(result), &resp); err == nil && + strings.TrimSpace(resp.Kind) == "config_draft" && + strings.TrimSpace(resp.DraftHash) != "" { + return true + } + lower := strings.ToLower(result) + return strings.Contains(lower, "config_draft") && + strings.Contains(lower, "draft_hash") +} + +func draftResultShouldHideSampleSources(result string) bool { + var resp struct { + Payload struct { + Rule struct { + Kind string `json:"kind"` + Spec map[string]interface{} `json:"spec"` + } `json:"rule"` + } `json:"payload"` + } + if err := json.Unmarshal([]byte(result), &resp); err != nil { + return false + } + if strings.TrimSpace(resp.Payload.Rule.Kind) != "metric_raw" || resp.Payload.Rule.Spec == nil { + return false + } + if boolish(resp.Payload.Rule.Spec["source_explicit"]) { + return false + } + for _, key := range []string{"expr", "promql", "query", "selector", "label_selector"} { + if containsExplicitSourceMatcher(resp.Payload.Rule.Spec[key]) { + return false + } + } + return true +} + +func boolish(raw interface{}) bool { + switch v := raw.(type) { + case bool: + return v + case string: + switch strings.ToLower(strings.TrimSpace(v)) { + case "true", "yes", "y", "1", "explicit", "user", "specified", "requested": + return true + default: + return false + } + default: + return false + } +} + +func containsExplicitSourceMatcher(raw interface{}) bool { + text, ok := raw.(string) + if !ok { + return false + } + return strings.Contains(text, `ongrid_source="`) || + strings.Contains(text, `device_id="`) || + strings.Contains(text, `instance="`) || + strings.Contains(text, `service="`) +} + +func sanitizeCollectedSampleSourceMentions(content string) string { + return collectedSourceMentionRE.ReplaceAllStringFunc(content, func(match string) string { + if strings.HasPrefix(match, "custom:") { + return "某个自定义采集源" + } + return "某个数据库采集源" + }) +} + +var _ einocallbacks.Handler = (*AlertDraftGuardHandler)(nil) +var _ einocallbacks.TimingChecker = (*AlertDraftGuardHandler)(nil) diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/alert_draft_guard_test.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/alert_draft_guard_test.go new file mode 100644 index 000000000..9777a3de2 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/alert_draft_guard_test.go @@ -0,0 +1,193 @@ +package callbacks + +import ( + "context" + "strings" + "testing" + + einocallbacks "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +func TestAlertDraftGuard_ReplacesPlainTextAlertDraftWithoutToolDraft(t *testing.T) { + guard := NewAlertDraftGuardHandler(AlertDraftGuardDeps{ + UserText: "创建一个 MongoDB 数据库告警规则", + }) + if guard == nil { + t.Fatal("guard = nil") + } + out := &einomodel.CallbackOutput{Message: &schema.Message{ + Role: schema.Assistant, + Content: "## 告警规则草案\n规则键 nl_db_mongo\n草案哈希: sha256:fake\n请确认应用。", + }} + + guard.OnEnd(context.Background(), chatModelRunInfo(), out) + + if out.Message.Content != alertDraftGuardBlockedMessage { + t.Fatalf("content = %q, want guard message", out.Message.Content) + } + if strings.Contains(out.Message.Content, "重新发送创建需求") { + t.Fatalf("guard message should not ask the user to resend the request: %q", out.Message.Content) + } +} + +func TestAlertDraftGuard_ReplacesConfirmableRuleDesignWithoutToolDraft(t *testing.T) { + guard := NewAlertDraftGuardHandler(AlertDraftGuardDeps{ + UserText: "创建一条 Redis 告警：缓存命中率明显偏低并持续一段时间时提醒我。", + }) + if guard == nil { + t.Fatal("guard = nil") + } + out := &einomodel.CallbackOutput{Message: &schema.Message{ + Role: schema.Assistant, + Content: "**规则设计：Redis 缓存命中率偏低**（Warning 级别）\n\n" + + "| 项目 | 内容 |\n|------|------|\n" + + "| **规则 Key** | `redis_cache_hit_rate_low` |\n" + + "| **触发条件** | 最近 5 分钟缓存命中率 < 90%，且持续 5 分钟 |\n" + + "| **PromQL** | `100 * rate(redis_keyspace_hits_total[5m]) / (rate(redis_keyspace_hits_total[5m]) + rate(redis_keyspace_misses_total[5m])) < 90` |\n\n" + + "需要确认创建吗？", + }} + + guard.OnEnd(context.Background(), chatModelRunInfo(), out) + + if out.Message.Content != alertDraftGuardBlockedMessage { + t.Fatalf("content = %q, want guard message", out.Message.Content) + } +} + +func TestAlertDraftGuard_InstallsForRuleRequestWithoutAlertWord(t *testing.T) { + guard := NewAlertDraftGuardHandler(AlertDraftGuardDeps{ + UserText: "配置 SLO burn rate 规则，1 小时窗口超过 14.4 倍就触发", + }) + if guard == nil { + t.Fatal("guard = nil") + } + out := &einomodel.CallbackOutput{Message: &schema.Message{ + Role: schema.Assistant, + Content: "config_draft 已准备好，draft_hash: sha256:fake，请确认后调用 apply_config_change。", + }} + + guard.OnEnd(context.Background(), chatModelRunInfo(), out) + + if out.Message.Content != alertDraftGuardBlockedMessage { + t.Fatalf("content = %q, want guard message", out.Message.Content) + } +} + +func TestAlertDraftGuard_AllowsAlertDraftAfterSuccessfulDraftTool(t *testing.T) { + guard := NewAlertDraftGuardHandler(AlertDraftGuardDeps{ + UserText: "创建一个 MongoDB 数据库告警规则", + }) + if guard == nil { + t.Fatal("guard = nil") + } + guard.OnEnd(context.Background(), toolRunInfo(draftConfigChangeToolName), &einotool.CallbackOutput{ + Response: `{"kind":"config_draft","draft_hash":"sha256:real"}`, + }) + out := &einomodel.CallbackOutput{Message: &schema.Message{ + Role: schema.Assistant, + Content: "告警规则草案已创建，草案哈希: sha256:real，请确认应用。", + }} + + guard.OnEnd(context.Background(), chatModelRunInfo(), out) + + if out.Message.Content != "告警规则草案已创建，草案哈希: sha256:real，请确认应用。" { + t.Fatalf("content = %q, want unchanged real draft text", out.Message.Content) + } +} + +func TestAlertDraftGuard_DoesNotBlockPendingDraftToolCall(t *testing.T) { + guard := NewAlertDraftGuardHandler(AlertDraftGuardDeps{ + UserText: "创建一条主机 CPU 和内存告警", + }) + if guard == nil { + t.Fatal("guard = nil") + } + out := &einomodel.CallbackOutput{Message: &schema.Message{ + Role: schema.Assistant, + Content: "告警规则草案已生成，等待 draft_config_change 返回 config_draft。", + ToolCalls: []schema.ToolCall{ + { + ID: "call_draft", + Type: "function", + Function: schema.FunctionCall{Name: draftConfigChangeToolName}, + }, + }, + }} + + guard.OnEnd(context.Background(), chatModelRunInfo(), out) + + if out.Message.Content == alertDraftGuardBlockedMessage { + t.Fatalf("content was blocked before draft tool executed") + } + if !strings.Contains(out.Message.Content, "等待 draft_config_change") { + t.Fatalf("content = %q, want unchanged pending draft text", out.Message.Content) + } +} + +func TestAlertDraftGuard_HidesSampleSourceMentionsForUnscopedMetricDraft(t *testing.T) { + guard := NewAlertDraftGuardHandler(AlertDraftGuardDeps{ + UserText: "创建一个 MySQL 连接使用率告警，不限定数据库源", + }) + if guard == nil { + t.Fatal("guard = nil") + } + guard.OnEnd(context.Background(), toolRunInfo(draftConfigChangeToolName), &einotool.CallbackOutput{ + Response: `{"kind":"config_draft","draft_hash":"sha256:real","payload":{"rule":{"kind":"metric_raw","spec":{"expr":"max by (device_id, ongrid_source) (mysql_global_status_threads_connected) > 75"}}}}`, + }) + out := &einomodel.CallbackOutput{Message: &schema.Message{ + Role: schema.Assistant, + Content: "预览数据：当前 db:mysql-test 实例连接使用率约 66%，但规则覆盖所有 source。", + }} + + guard.OnEnd(context.Background(), chatModelRunInfo(), out) + + if strings.Contains(out.Message.Content, "db:mysql-test") { + t.Fatalf("content = %q, want sample source hidden", out.Message.Content) + } + if !strings.Contains(out.Message.Content, "某个数据库采集源") { + t.Fatalf("content = %q, want generic source wording", out.Message.Content) + } +} + +func TestAlertDraftGuard_KeepsExplicitSourceMentionsForScopedMetricDraft(t *testing.T) { + guard := NewAlertDraftGuardHandler(AlertDraftGuardDeps{ + UserText: "为 db:mysql-test 创建 MySQL 连接使用率告警", + }) + if guard == nil { + t.Fatal("guard = nil") + } + guard.OnEnd(context.Background(), toolRunInfo(draftConfigChangeToolName), &einotool.CallbackOutput{ + Response: `{"kind":"config_draft","draft_hash":"sha256:real","payload":{"rule":{"kind":"metric_raw","spec":{"source_explicit":true,"expr":"mysql_global_status_threads_connected{ongrid_source=\"db:mysql-test\"} > 75"}}}}`, + }) + out := &einomodel.CallbackOutput{Message: &schema.Message{ + Role: schema.Assistant, + Content: "预览数据：当前 db:mysql-test 实例连接使用率约 66%。", + }} + + guard.OnEnd(context.Background(), chatModelRunInfo(), out) + + if !strings.Contains(out.Message.Content, "db:mysql-test") { + t.Fatalf("content = %q, want explicit source preserved", out.Message.Content) + } +} + +func TestAlertDraftGuard_DoesNotInstallForNonCreateAlertQuestion(t *testing.T) { + guard := NewAlertDraftGuardHandler(AlertDraftGuardDeps{ + UserText: "现在有哪些告警规则", + }) + if guard != nil { + t.Fatalf("guard = %#v, want nil", guard) + } +} + +func chatModelRunInfo() *einocallbacks.RunInfo { + return &einocallbacks.RunInfo{Name: "ChatModel", Component: components.ComponentOfChatModel} +} + +func toolRunInfo(name string) *einocallbacks.RunInfo { + return &einocallbacks.RunInfo{Name: name, Component: components.ComponentOfTool} +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/audit.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/audit.go new file mode 100644 index 000000000..fd990b8c2 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/audit.go @@ -0,0 +1,288 @@ +package callbacks + +import ( + "context" + "log/slog" + "sync" + "sync/atomic" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +// AuditDeps configures the audit handler. Logger is required (a nil +// logger turns the handler into a no-op so the cutover layer can opt +// in/out per environment). SessionID is the chat session this run +// belongs to — included in every log line so operators can grep all +// stages of one conversation. UserID is logged as a numeric scope. +type AuditDeps struct { + Logger *slog.Logger + SessionID string + UserID uint64 +} + +// AuditHandler emits structured slog INFO records for every stage of +// the graph: per-ChatModel turn (prompt token estimate + reply tokens), +// per-tool call (name + duration + status). spec — and +// / 红线: the user's raw prompt content is NEVER +// included in the log line; only counts and identifiers. +// +// One handler instance per graph run. Concurrency: tool fan-out may +// call OnStart / OnEnd concurrently; per-call timestamps live in a +// mutex-guarded map. +type AuditHandler struct { + deps AuditDeps + + chatTurn atomic.Int64 + + startsMu sync.Mutex + starts map[string]auditStart +} + +type auditStart struct { + at time.Time + component components.Component + name string + estTokens int + toolCallID string +} + +// NewAuditHandler builds the handler. Returns nil when Logger is nil +// so the cutover layer can short-circuit. +func NewAuditHandler(deps AuditDeps) *AuditHandler { + if deps.Logger == nil { + return nil + } + return &AuditHandler{deps: deps, starts: make(map[string]auditStart)} +} + +// Needed gates timings. +func (h *AuditHandler) Needed(_ context.Context, info *callbacks.RunInfo, timing callbacks.CallbackTiming) bool { + if h == nil || info == nil { + return false + } + switch info.Component { + case components.ComponentOfChatModel, components.ComponentOfTool: + switch timing { + case callbacks.TimingOnStart, callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + } + return false +} + +// OnStart records the start time so OnEnd can compute duration. +func (h *AuditHandler) OnStart(ctx context.Context, info *callbacks.RunInfo, input callbacks.CallbackInput) context.Context { + if h == nil || info == nil { + return ctx + } + at := time.Now() + entry := auditStart{at: at, component: info.Component, name: info.Name} + switch info.Component { + case components.ComponentOfChatModel: + mi := einomodel.ConvCallbackInput(input) + if mi != nil { + entry.estTokens = estimatePromptTokens(mi.Messages) + } + turn := h.chatTurn.Add(1) + h.deps.Logger.Info("graph stage start", + slog.String("session_id", h.deps.SessionID), + slog.Uint64("user_id", h.deps.UserID), + slog.String("kind", "chat_model"), + slog.String("name", info.Name), + slog.Int64("iteration", turn), + slog.Int("est_prompt_tokens", entry.estTokens), + ) + case components.ComponentOfTool: + entry.toolCallID = toolCallIDFromCtx(ctx, info) + args := "" + if tin := einotool.ConvCallbackInput(input); tin != nil { + args = tin.ArgumentsInJSON + } + // Log args size, not args content — the args may carry user- + // authored text (PromQL / LogQL queries) that + // forbids us from logging in cleartext. + h.deps.Logger.Info("graph stage start", + slog.String("session_id", h.deps.SessionID), + slog.Uint64("user_id", h.deps.UserID), + slog.String("kind", "tool"), + slog.String("name", info.Name), + slog.String("tool_call_id", entry.toolCallID), + slog.Int("args_bytes", len(args)), + ) + } + h.startsMu.Lock() + h.starts[stageKey(ctx, info)] = entry + h.startsMu.Unlock() + return ctx +} + +// OnEnd logs the stage's duration and result. +func (h *AuditHandler) OnEnd(ctx context.Context, info *callbacks.RunInfo, output callbacks.CallbackOutput) context.Context { + if h == nil || info == nil { + return ctx + } + h.startsMu.Lock() + entry, ok := h.starts[stageKey(ctx, info)] + delete(h.starts, stageKey(ctx, info)) + h.startsMu.Unlock() + if !ok { + return ctx + } + dur := time.Since(entry.at) + switch info.Component { + case components.ComponentOfChatModel: + mo := einomodel.ConvCallbackOutput(output) + usage := slog.Group("token_usage") + toolCalls := 0 + if mo != nil { + if mo.TokenUsage != nil { + usage = slog.Group("token_usage", + slog.Int("prompt", mo.TokenUsage.PromptTokens), + slog.Int("completion", mo.TokenUsage.CompletionTokens), + slog.Int("total", mo.TokenUsage.TotalTokens), + ) + } else if mo.Message != nil && mo.Message.ResponseMeta != nil && mo.Message.ResponseMeta.Usage != nil { + u := mo.Message.ResponseMeta.Usage + usage = slog.Group("token_usage", + slog.Int("prompt", u.PromptTokens), + slog.Int("completion", u.CompletionTokens), + slog.Int("total", u.TotalTokens), + ) + } + if mo.Message != nil { + toolCalls = len(mo.Message.ToolCalls) + } + } + h.deps.Logger.Info("graph stage end", + slog.String("session_id", h.deps.SessionID), + slog.Uint64("user_id", h.deps.UserID), + slog.String("kind", "chat_model"), + slog.String("name", info.Name), + slog.Int64("duration_ms", dur.Milliseconds()), + slog.Int("tool_calls_emitted", toolCalls), + slog.String("status", "success"), + usage, + ) + case components.ComponentOfTool: + body := "" + if tout := einotool.ConvCallbackOutput(output); tout != nil { + body = tout.Response + } + h.deps.Logger.Info("graph stage end", + slog.String("session_id", h.deps.SessionID), + slog.Uint64("user_id", h.deps.UserID), + slog.String("kind", "tool"), + slog.String("name", info.Name), + slog.String("tool_call_id", entry.toolCallID), + slog.Int64("duration_ms", dur.Milliseconds()), + slog.Int("result_bytes", len(body)), + slog.String("status", "success"), + ) + } + return ctx +} + +// OnError logs the stage's failure. +func (h *AuditHandler) OnError(ctx context.Context, info *callbacks.RunInfo, err error) context.Context { + if h == nil || info == nil || err == nil { + return ctx + } + h.startsMu.Lock() + entry, ok := h.starts[stageKey(ctx, info)] + delete(h.starts, stageKey(ctx, info)) + h.startsMu.Unlock() + dur := time.Duration(0) + if ok { + dur = time.Since(entry.at) + } + status := "error" + if isDeadlineErr(err) { + status = "timeout" + } + h.deps.Logger.Warn("graph stage end", + slog.String("session_id", h.deps.SessionID), + slog.Uint64("user_id", h.deps.UserID), + slog.String("kind", componentKind(info.Component)), + slog.String("name", info.Name), + slog.Int64("duration_ms", dur.Milliseconds()), + slog.String("status", status), + slog.String("error", err.Error()), + ) + return ctx +} + +// OnStartWithStreamInput drains and closes the stream copy. +func (h *AuditHandler) OnStartWithStreamInput(ctx context.Context, _ *callbacks.RunInfo, in *schema.StreamReader[callbacks.CallbackInput]) context.Context { + if in != nil { + in.Close() + } + return ctx +} + +// OnEndWithStreamOutput drains and closes the stream copy. +func (h *AuditHandler) OnEndWithStreamOutput(ctx context.Context, _ *callbacks.RunInfo, out *schema.StreamReader[callbacks.CallbackOutput]) context.Context { + if out != nil { + out.Close() + } + return ctx +} + +// estimatePromptTokens is the same rule-of-thumb used by the budget +// callback (chars/4). Local copy so we don't pull the llm package in +// just for the function (avoids an import cycle with biz/aiops/graph). +func estimatePromptTokens(msgs []*schema.Message) int { + const perMsgOverhead = 4 + total := 0 + for _, m := range msgs { + if m == nil { + continue + } + total += perMsgOverhead + total += len(m.Content) / 4 + for _, tc := range m.ToolCalls { + total += len(tc.Function.Name) / 4 + total += len(tc.Function.Arguments) / 4 + } + } + return total +} + +func componentKind(c components.Component) string { + switch c { + case components.ComponentOfChatModel: + return "chat_model" + case components.ComponentOfTool: + return "tool" + default: + return string(c) + } +} + +// stageKey is the per-stage correlation key used to look up the +// matching OnStart entry from OnEnd / OnError. For tool calls we use +// the tool_call id; for chat models we use the run-info name + the +// turn index so concurrent (eino doesn't fan out chat models, but +// future SOP graphs may) calls don't collide. +func stageKey(ctx context.Context, info *callbacks.RunInfo) string { + if info == nil { + return "" + } + if info.Component == components.ComponentOfTool { + return "tool|" + toolCallIDFromCtx(ctx, info) + } + if v, ok := ctx.Value(messageIDCtxKey{}).(string); ok && v != "" { + return string(info.Component) + "|" + info.Name + "|" + v + } + return string(info.Component) + "|" + info.Name +} + +// Compile-time checks. +var ( + _ callbacks.Handler = (*AuditHandler)(nil) + _ callbacks.TimingChecker = (*AuditHandler)(nil) +) diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/audit_test.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/audit_test.go new file mode 100644 index 000000000..10d0689b2 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/audit_test.go @@ -0,0 +1,166 @@ +package callbacks + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "strings" + "testing" + + "github.com/cloudwego/eino/callbacks" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +// captureLogger writes JSON log lines into an in-memory buffer the test +// can grep. We use slog.NewJSONHandler so attribute names are stable. +func captureLogger() (*slog.Logger, *bytes.Buffer) { + buf := &bytes.Buffer{} + h := slog.NewJSONHandler(buf, &slog.HandlerOptions{Level: slog.LevelInfo}) + return slog.New(h), buf +} + +func TestAuditHandler_NewNilLogger(t *testing.T) { + t.Parallel() + if NewAuditHandler(AuditDeps{}) != nil { + t.Fatalf("expected nil handler when logger is nil") + } +} + +func TestAuditHandler_ChatModelStartAndEnd(t *testing.T) { + t.Parallel() + log, buf := captureLogger() + h := NewAuditHandler(AuditDeps{Logger: log, SessionID: "sess-1", UserID: 42}) + ctx := context.Background() + in := &einomodel.CallbackInput{ + Messages: []*schema.Message{{Role: schema.User, Content: "hello world"}}, + } + h.OnStart(ctx, chatModelInfo(), in) + out := &einomodel.CallbackOutput{ + Message: &schema.Message{Role: schema.Assistant, Content: "ok"}, + TokenUsage: &einomodel.TokenUsage{PromptTokens: 5, CompletionTokens: 3, TotalTokens: 8}, + } + h.OnEnd(ctx, chatModelInfo(), out) + + lines := splitLines(buf.String()) + if len(lines) != 2 { + t.Fatalf("expected 2 log lines got %d: %v", len(lines), lines) + } + + startLine := decodeJSON(t, lines[0]) + if startLine["msg"] != "graph stage start" { + t.Errorf("msg = %v, want graph stage start", startLine["msg"]) + } + if startLine["session_id"] != "sess-1" { + t.Errorf("session_id = %v", startLine["session_id"]) + } + if startLine["kind"] != "chat_model" { + t.Errorf("kind = %v", startLine["kind"]) + } + if iter, ok := startLine["iteration"].(float64); !ok || iter != 1 { + t.Errorf("iteration = %v", startLine["iteration"]) + } + + endLine := decodeJSON(t, lines[1]) + if endLine["status"] != "success" { + t.Errorf("status = %v", endLine["status"]) + } + usage, ok := endLine["token_usage"].(map[string]any) + if !ok { + t.Fatalf("token_usage missing: %v", endLine) + } + if usage["total"] != float64(8) { + t.Errorf("total tokens = %v", usage["total"]) + } +} + +func TestAuditHandler_ToolError(t *testing.T) { + t.Parallel() + log, buf := captureLogger() + h := NewAuditHandler(AuditDeps{Logger: log, SessionID: "s1"}) + ctx := WithToolCallID(context.Background(), "tc-1") + h.OnStart(ctx, toolInfo("flaky"), &einotool.CallbackInput{ArgumentsInJSON: `{"x":1}`}) + h.OnError(ctx, toolInfo("flaky"), errors.New("boom")) + + lines := splitLines(buf.String()) + if len(lines) < 2 { + t.Fatalf("expected start+end lines, got %d", len(lines)) + } + end := decodeJSON(t, lines[len(lines)-1]) + if end["status"] != "error" { + t.Errorf("status = %v", end["status"]) + } + if end["error"] != "boom" { + t.Errorf("error = %v", end["error"]) + } +} + +func TestAuditHandler_DoesNotLogUserText(t *testing.T) { + t.Parallel() + // / 红线: user text must never appear in audit logs. + const sensitive = "私密文本-DO-NOT-LOG" + log, buf := captureLogger() + h := NewAuditHandler(AuditDeps{Logger: log, SessionID: "s"}) + in := &einomodel.CallbackInput{ + Messages: []*schema.Message{{Role: schema.User, Content: sensitive}}, + } + h.OnStart(context.Background(), chatModelInfo(), in) + h.OnEnd(context.Background(), chatModelInfo(), &einomodel.CallbackOutput{ + Message: &schema.Message{Role: schema.Assistant, Content: "reply"}, + }) + if strings.Contains(buf.String(), sensitive) { + t.Fatalf("audit log leaked user text: %s", buf.String()) + } +} + +func TestAuditHandler_TimeoutClassified(t *testing.T) { + t.Parallel() + log, buf := captureLogger() + h := NewAuditHandler(AuditDeps{Logger: log, SessionID: "s"}) + ctx := WithToolCallID(context.Background(), "tc-x") + h.OnStart(ctx, toolInfo("slow"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("slow"), context.DeadlineExceeded) + end := decodeJSON(t, splitLines(buf.String())[1]) + if end["status"] != "timeout" { + t.Errorf("status = %v, want timeout", end["status"]) + } +} + +// helpers -------------------------------------------------- + +func splitLines(s string) []string { + out := []string{} + for _, line := range strings.Split(s, "\n") { + if strings.TrimSpace(line) != "" { + out = append(out, line) + } + } + return out +} + +func decodeJSON(t *testing.T, s string) map[string]any { + t.Helper() + var m map[string]any + if err := json.Unmarshal([]byte(s), &m); err != nil { + t.Fatalf("decode line: %v\nline=%s", err, s) + } + return m +} + +func TestAuditHandler_NeededGating(t *testing.T) { + t.Parallel() + log, _ := captureLogger() + h := NewAuditHandler(AuditDeps{Logger: log}) + if !h.Needed(context.Background(), chatModelInfo(), callbacks.TimingOnStart) { + t.Errorf("ChatModel OnStart should be needed") + } + if !h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnEnd) { + t.Errorf("Tool OnEnd should be needed") + } + if !h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnError) { + t.Errorf("Tool OnError should be needed") + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/chain.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/chain.go new file mode 100644 index 000000000..3a6865fe2 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/chain.go @@ -0,0 +1,178 @@ +package callbacks + +import ( + "context" + "errors" + "sync/atomic" + + "github.com/cloudwego/eino/callbacks" + "github.com/prometheus/client_golang/prometheus" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// assistantIDRelay is the shared slot the PersistenceHandler writes +// the freshly-persisted chat_messages.id into; SSEHandler reads it +// inside OnEnd to attach to the assistant_end SSE frame. Both handlers +// run on the same goroutine for a given graph iteration (persistence +// is registered before SSE in NewDefaultHandlers and eino preserves +// registration order on OnEnd), so a plain string with atomic Load/ +// Store is sufficient — no race between writer and reader within an +// iteration. +// +// Frontend ChatThread used to dedupe by a synthetic assistant-iter-N +// id (v0.7.63 hotfix) because this field was empty. Now it can use +// the real DB id. +type assistantIDRelay struct { + id atomic.Pointer[string] +} + +func (r *assistantIDRelay) store(id string) { + if r == nil { + return + } + r.id.Store(&id) +} + +func (r *assistantIDRelay) load() string { + if r == nil { + return "" + } + if p := r.id.Load(); p != nil { + return *p + } + return "" +} + +// Deps bundles every dependency the default callback chain needs. +// Each handler treats nil dependencies as "skip me" — pass partial +// Deps to wire only some handlers (e.g. tests that only want metrics). +// +// Reference: Callback 链 + 主参考图 callback 链. +type Deps struct { + // Tracing shares one process-scoped Eino callback and exporter. + Tracing callbacks.Handler + + // AlertDraftGuard blocks model-only alert-rule drafts before they are + // persisted or streamed. It must run before Persistence/SSE. + AlertDraftGuard AlertDraftGuardDeps + + // Persistence wiring (chat_messages / chat_tool_calls writes). + Persistence PersistenceDeps + + // SSE emitter (when nil, no SSE handler is wired). Cutover layer + // builds an emitter that pipes into the existing http.go writeSSE. + SSE SSEEmitter + + // Audit (slog INFO records). Logger required. + Audit AuditDeps + + // Metrics (Prom counters). Registerer required. + Metrics MetricsDeps + + // Budget gate (PR-1). Both fields required to wire it. + BudgetChecker llm.BudgetChecker + BudgetUserID uint64 +} + +// NewDefaultHandlers builds the default ordered callback chain for an +// agent graph run. Order is incidental — eino does not document a +// guaranteed handler ordering — but the slice is stable across calls +// so tests can assert positional indices. +// +// Handlers with nil dependencies are skipped, so passing a partial +// Deps yields a sparser chain. The PR-1 BudgetCallbackHandler is +// included whenever Deps.BudgetChecker is non-nil. +// +// Cutover layer (NEXT PR) calls this once per request, threads the +// returned slice into compose.WithCallbacks at Invoke time, and +// discards the handlers when the request finishes (so per-call state +// is bounded by the request lifetime). +func NewDefaultHandlers(deps Deps) []callbacks.Handler { + out := make([]callbacks.Handler, 0, 6) + if deps.Tracing != nil { + out = append(out, deps.Tracing) + } + + if h := NewAlertDraftGuardHandler(deps.AlertDraftGuard); h != nil { + out = append(out, h) + } + + // Shared relay so Persistence (runs first) can hand the freshly- + // written assistant row id to SSE (runs second) for the + // assistant_end frame. + relay := &assistantIDRelay{} + + if h := NewPersistenceHandler(deps.Persistence); h != nil { + h.assistantIDRelay = relay + out = append(out, h) + } + if h := NewSSEHandler(deps.SSE); h != nil { + h.assistantIDRelay = relay + out = append(out, h) + } + if h := NewAuditHandler(deps.Audit); h != nil { + out = append(out, h) + } + if h := NewMetricsHandler(deps.Metrics); h != nil { + out = append(out, h) + } + if deps.BudgetChecker != nil { + out = append(out, llm.NewBudgetCallbackHandler(deps.BudgetChecker, deps.BudgetUserID)) + } + return out +} + +// EnableSynchronousToolPersistence returns the request-scoped persistence +// handler after switching its tool writes to graph's adapter boundary. This +// is required because nested ToolsNode callbacks are not guaranteed to reach +// the outer callback chain in the supported Eino runtime. +func EnableSynchronousToolPersistence(handlers []callbacks.Handler) *PersistenceHandler { + for _, handler := range handlers { + if p, ok := handler.(*PersistenceHandler); ok { + p.directToolPersistence = true + return p + } + } + return nil +} + +// FinalizeBatches runs end-of-request bookkeeping on every handler in +// the chain that wants one. Currently this is just PersistenceHandler: +// see flushIncompleteBatch — the ChatModel.OnStart hook autoheals the +// previous batch DURING a session, but a session that ends mid-batch +// (user closes browser, request cancels) never gets the next OnStart. +// chatruntime defers FinalizeBatches after compose.Invoke returns so +// the batch always gets a final flush. +func FinalizeBatches(ctx context.Context, handlers []callbacks.Handler) { + for _, h := range handlers { + if p, ok := h.(*PersistenceHandler); ok { + p.FinalizeBatch(ctx) + } + } +} + +// ErrNoHandlers is returned by helpers that require at least one +// handler in the chain. Reserved for future use; PR-6 callers tolerate +// an empty slice. +var ErrNoHandlers = errors.New("graph/callbacks: no handlers configured") + +// registerOrExisting is the package-private register helper. Mirrors +// llm/metrics.go's registerOrExisting + tools/decorators/metric.go's +// regOrExist; lives here so persistence.go can use it without +// triggering an import cycle (both packages are siblings of llm but +// llm/metrics.go's helper is unexported). +func registerOrExisting(reg prometheus.Registerer, c prometheus.Collector) prometheus.Collector { + if reg == nil { + reg = prometheus.DefaultRegisterer + } + err := reg.Register(c) + if err == nil { + return c + } + var are prometheus.AlreadyRegisteredError + if errors.As(err, &are) { + return are.ExistingCollector + } + panic(err) +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/chain_test.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/chain_test.go new file mode 100644 index 000000000..7aadee069 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/chain_test.go @@ -0,0 +1,83 @@ +package callbacks + +import ( + "context" + "log/slog" + "testing" + + "github.com/cloudwego/eino/callbacks" + "github.com/prometheus/client_golang/prometheus" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +func TestNewDefaultHandlers_AllWiredEmitsFive(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + reg := prometheus.NewRegistry() + deps := Deps{ + Persistence: PersistenceDeps{SessionID: "s", Repo: repo, Registerer: reg}, + SSE: func(SSEEvent) {}, + Audit: AuditDeps{Logger: slog.Default(), SessionID: "s"}, + Metrics: MetricsDeps{Registerer: reg}, + BudgetChecker: stubBudgetChecker{}, + BudgetUserID: 1, + } + got := NewDefaultHandlers(deps) + if len(got) != 5 { + t.Fatalf("expected 5 handlers, got %d", len(got)) + } + // Spot-check ordering: persistence first, budget last. + if _, ok := got[0].(*PersistenceHandler); !ok { + t.Errorf("got[0] = %T, want *PersistenceHandler", got[0]) + } + if _, ok := got[len(got)-1].(*llm.BudgetCallbackHandler); !ok { + t.Errorf("got[last] = %T, want *llm.BudgetCallbackHandler", got[len(got)-1]) + } +} + +func TestNewDefaultHandlers_PartialDepsSkipsHandlers(t *testing.T) { + t.Parallel() + got := NewDefaultHandlers(Deps{}) + if len(got) != 0 { + t.Fatalf("expected empty chain when no deps wired, got %d", len(got)) + } +} + +func TestNewDefaultHandlers_OnlyMetrics(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + got := NewDefaultHandlers(Deps{Metrics: MetricsDeps{Registerer: reg}}) + if len(got) != 1 { + t.Fatalf("expected 1 handler got %d", len(got)) + } + if _, ok := got[0].(*MetricsHandler); !ok { + t.Errorf("got[0] = %T", got[0]) + } +} + +func TestNewDefaultHandlers_AllImplementHandlerInterface(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + reg := prometheus.NewRegistry() + deps := Deps{ + Persistence: PersistenceDeps{SessionID: "s", Repo: repo, Registerer: reg}, + SSE: func(SSEEvent) {}, + Audit: AuditDeps{Logger: slog.Default(), SessionID: "s"}, + Metrics: MetricsDeps{Registerer: reg}, + BudgetChecker: stubBudgetChecker{}, + } + for i, h := range NewDefaultHandlers(deps) { + if _, ok := h.(callbacks.Handler); !ok { + t.Errorf("got[%d] = %T does not satisfy callbacks.Handler", i, h) + } + } +} + +// stubBudgetChecker is a minimal llm.BudgetChecker that allows everything. +type stubBudgetChecker struct{} + +func (stubBudgetChecker) Check(_ context.Context, _ uint64, _ int) error { return nil } +func (stubBudgetChecker) Record(_ context.Context, _ uint64, _ llm.Usage) error { + return nil +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/metrics.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/metrics.go new file mode 100644 index 000000000..738f093d7 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/metrics.go @@ -0,0 +1,305 @@ +package callbacks + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + "github.com/cloudwego/eino/schema" + "github.com/prometheus/client_golang/prometheus" +) + +// MetricsDeps configures the metrics handler. Registerer is required; +// callers pass prometheus.DefaultRegisterer in production and a fresh +// prometheus.NewRegistry in tests so collectors do not collide. +type MetricsDeps struct { + Registerer prometheus.Registerer +} + +// metricsCollectors bundles the Prom collectors registered by the +// metrics handler. Allocated once per Registerer and shared across +// handler instances so multiple graph runs don't trip +// AlreadyRegisteredError. +// +// Cardinality red line: labels are limited to {name, result} +// (tools) / {provider, result} (chat models) / {result} (graph +// terminations). Never include user / tenant / session. +type metricsCollectors struct { + toolInvocations *prometheus.CounterVec // name, result + toolDuration *prometheus.HistogramVec // name + graphIterations *prometheus.CounterVec // result + chatTurns *prometheus.CounterVec // result +} + +var ( + metricsRegMu sync.Mutex + metricsRegMap = map[prometheus.Registerer]*metricsCollectors{} +) + +func getMetricsCollectors(reg prometheus.Registerer) *metricsCollectors { + if reg == nil { + reg = prometheus.DefaultRegisterer + } + metricsRegMu.Lock() + defer metricsRegMu.Unlock() + if mc, ok := metricsRegMap[reg]; ok { + return mc + } + // Help string MUST match tools/decorators/metric.go byte-for-byte — + // Prom panics on AlreadyRegistered if descriptors differ on Help. + // Both layers observe the same counter from different vantage points + // (dual-observation contract), so they MUST share desc. + tinv := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_tool_invocations_total", + Help: "Total ongrid tool invocations, split by name and result.", + }, + []string{"name", "result"}, + ) + tdur := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ongrid_tool_duration_seconds", + Help: "ongrid tool invocation duration in seconds.", + Buckets: prometheus.ExponentialBuckets(0.05, 2, 10), + }, + []string{"name"}, + ) + gi := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_graph_iterations_total", + Help: "Total ongrid agent graph terminations, split by result (success | max_iterations | error).", + }, + []string{"result"}, + ) + ct := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_graph_chat_turns_total", + Help: "Total ChatModel turns observed during ongrid agent graph runs.", + }, + []string{"result"}, + ) + mc := &metricsCollectors{ + toolInvocations: regOrExist(reg, tinv).(*prometheus.CounterVec), + toolDuration: regOrExist(reg, tdur).(*prometheus.HistogramVec), + graphIterations: regOrExist(reg, gi).(*prometheus.CounterVec), + chatTurns: regOrExist(reg, ct).(*prometheus.CounterVec), + } + metricsRegMap[reg] = mc + return mc +} + +// regOrExist mirrors the helper in tools/decorators/metric.go and +// llm/metrics.go: register; on AlreadyRegisteredError reuse the +// existing collector. +func regOrExist(reg prometheus.Registerer, c prometheus.Collector) prometheus.Collector { + err := reg.Register(c) + if err == nil { + return c + } + var are prometheus.AlreadyRegisteredError + if errors.As(err, &are) { + return are.ExistingCollector + } + panic(err) +} + +// MetricsHandler observes tool + chat-model invocations from the +// graph's perspective. Note: the PR-3 decorator chain already records +// ongrid_tool_invocations_total at the per-call seam — this handler +// re-uses the SAME collector via the registry-keyed cache so we never +// double-register, and the counter increments are best-effort +// duplicates (acceptable : "callback 视角 + 装饰器视 +// 角 双观察 OK"). Tests assert exactly one increment per call when +// the decorator chain is bypassed (the unit test calls the handler +// directly). +type MetricsHandler struct { + collectors *metricsCollectors + + chatTurns atomic.Int64 + toolStartsMu sync.Mutex + toolStarts map[string]time.Time + + terminated atomic.Bool +} + +// NewMetricsHandler builds a handler against the given registerer. +// Returns nil if Registerer is nil. +func NewMetricsHandler(deps MetricsDeps) *MetricsHandler { + if deps.Registerer == nil { + return nil + } + return &MetricsHandler{ + collectors: getMetricsCollectors(deps.Registerer), + toolStarts: make(map[string]time.Time), + } +} + +// Needed gates timings. +func (h *MetricsHandler) Needed(_ context.Context, info *callbacks.RunInfo, timing callbacks.CallbackTiming) bool { + if h == nil || info == nil { + return false + } + switch info.Component { + case components.ComponentOfChatModel: + switch timing { + case callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + case components.ComponentOfTool: + switch timing { + case callbacks.TimingOnStart, callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + default: + // Graph terminations. + switch timing { + case callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + } + return false +} + +// OnStart records tool start times. +func (h *MetricsHandler) OnStart(ctx context.Context, info *callbacks.RunInfo, _ callbacks.CallbackInput) context.Context { + if h == nil || info == nil { + return ctx + } + if info.Component == components.ComponentOfTool { + h.toolStartsMu.Lock() + h.toolStarts[toolCallIDFromCtx(ctx, info)] = time.Now() + h.toolStartsMu.Unlock() + } + return ctx +} + +// OnEnd increments counters for tool / chat / graph completion. +func (h *MetricsHandler) OnEnd(ctx context.Context, info *callbacks.RunInfo, output callbacks.CallbackOutput) context.Context { + if h == nil || info == nil { + return ctx + } + switch info.Component { + case components.ComponentOfChatModel: + h.chatTurns.Add(1) + h.collectors.chatTurns.WithLabelValues("success").Inc() + _ = output // ChatModel output not needed for counters + case components.ComponentOfTool: + key := toolCallIDFromCtx(ctx, info) + h.toolStartsMu.Lock() + started, ok := h.toolStarts[key] + delete(h.toolStarts, key) + h.toolStartsMu.Unlock() + if ok { + h.collectors.toolDuration.WithLabelValues(info.Name).Observe(time.Since(started).Seconds()) + } + // We can't tell from CallbackOutput alone whether the tool + // returned an error payload (it's just a JSON string). Treat + // this hook as "success" — the OnError path covers actual + // failures returned via tool.InvokableRun. + h.collectors.toolInvocations.WithLabelValues(info.Name, "success").Inc() + default: + // Graph-scope terminal: success. + if h.terminated.CompareAndSwap(false, true) { + h.collectors.graphIterations.WithLabelValues("success").Inc() + } + } + return ctx +} + +// OnError increments error counters. +func (h *MetricsHandler) OnError(ctx context.Context, info *callbacks.RunInfo, err error) context.Context { + if h == nil || info == nil || err == nil { + return ctx + } + switch info.Component { + case components.ComponentOfChatModel: + h.collectors.chatTurns.WithLabelValues("error").Inc() + case components.ComponentOfTool: + key := toolCallIDFromCtx(ctx, info) + h.toolStartsMu.Lock() + started, ok := h.toolStarts[key] + delete(h.toolStarts, key) + h.toolStartsMu.Unlock() + result := "error" + if isDeadlineErr(err) { + result = "timeout" + } + if ok { + h.collectors.toolDuration.WithLabelValues(info.Name).Observe(time.Since(started).Seconds()) + } + h.collectors.toolInvocations.WithLabelValues(info.Name, result).Inc() + default: + // Graph-scope error: classify max-iterations vs other. + result := "error" + if isMaxIterations(err) { + result = "max_iterations" + } + if h.terminated.CompareAndSwap(false, true) { + h.collectors.graphIterations.WithLabelValues(result).Inc() + } + } + return ctx +} + +// OnStartWithStreamInput / OnEndWithStreamOutput are no-ops; the +// counters fire on the non-stream OnEnd path. +func (h *MetricsHandler) OnStartWithStreamInput(ctx context.Context, _ *callbacks.RunInfo, in *schema.StreamReader[callbacks.CallbackInput]) context.Context { + if in != nil { + in.Close() + } + return ctx +} + +func (h *MetricsHandler) OnEndWithStreamOutput(ctx context.Context, _ *callbacks.RunInfo, out *schema.StreamReader[callbacks.CallbackOutput]) context.Context { + if out != nil { + out.Close() + } + return ctx +} + +// ChatTurns reports the number of ChatModel turns observed. Exposed +// for tests. +func (h *MetricsHandler) ChatTurns() int { + if h == nil { + return 0 + } + return int(h.chatTurns.Load()) +} + +// isDeadlineErr returns true when err signals a deadline exceeded. +// Exported only by virtue of being shared across handler files; lives +// in this package because it mirrors the legacy classifyToolOutcome +// used by agent.go. +func isDeadlineErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + return false +} + +// isMaxIterations reports whether err looks like the eino runtime's +// "graph reached max steps" error. eino does not export a typed error +// for this, so we string-match the standard message it produces. The +// match is loose on purpose — the metric's `max_iterations` bucket is +// a hint, not authoritative. +func isMaxIterations(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "max steps") || strings.Contains(msg, "MaxRunSteps") || strings.Contains(msg, "max iterations") +} + +// Compile-time checks. +var ( + _ callbacks.Handler = (*MetricsHandler)(nil) + _ callbacks.TimingChecker = (*MetricsHandler)(nil) +) diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/metrics_test.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/metrics_test.go new file mode 100644 index 000000000..23369d42c --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/metrics_test.go @@ -0,0 +1,120 @@ +package callbacks + +import ( + "context" + "errors" + "testing" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestMetricsHandler_NewNilRegistererReturnsNil(t *testing.T) { + t.Parallel() + if NewMetricsHandler(MetricsDeps{}) != nil { + t.Fatalf("expected nil when registerer is nil") + } +} + +func TestMetricsHandler_ToolSuccessIncrements(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + ctx := WithToolCallID(context.Background(), "tc-1") + h.OnStart(ctx, toolInfo("query_promql"), &einotool.CallbackInput{}) + h.OnEnd(ctx, toolInfo("query_promql"), &einotool.CallbackOutput{Response: "{}"}) + + got := testutil.ToFloat64(h.collectors.toolInvocations.WithLabelValues("query_promql", "success")) + if got != 1 { + t.Errorf("invocations counter = %v, want 1", got) + } +} + +func TestMetricsHandler_ToolErrorIncrementsErrorBucket(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + ctx := WithToolCallID(context.Background(), "tc-2") + h.OnStart(ctx, toolInfo("flaky"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("flaky"), errors.New("boom")) + + got := testutil.ToFloat64(h.collectors.toolInvocations.WithLabelValues("flaky", "error")) + if got != 1 { + t.Errorf("invocations error counter = %v, want 1", got) + } +} + +func TestMetricsHandler_ToolTimeoutClassified(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + ctx := WithToolCallID(context.Background(), "tc-3") + h.OnStart(ctx, toolInfo("slow"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("slow"), context.DeadlineExceeded) + + if got := testutil.ToFloat64(h.collectors.toolInvocations.WithLabelValues("slow", "timeout")); got != 1 { + t.Errorf("timeout counter = %v, want 1", got) + } +} + +func TestMetricsHandler_ChatTurnCounter(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + for i := 0; i < 3; i++ { + h.OnEnd(context.Background(), chatModelInfo(), nil) + } + if got := testutil.ToFloat64(h.collectors.chatTurns.WithLabelValues("success")); got != 3 { + t.Errorf("chat_turns success = %v, want 3", got) + } + if h.ChatTurns() != 3 { + t.Errorf("ChatTurns() = %d, want 3", h.ChatTurns()) + } +} + +func TestMetricsHandler_GraphTermination(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + graphInfo := &callbacks.RunInfo{Component: components.Component("Graph")} + h.OnEnd(context.Background(), graphInfo, nil) + if got := testutil.ToFloat64(h.collectors.graphIterations.WithLabelValues("success")); got != 1 { + t.Errorf("graph success = %v, want 1", got) + } +} + +func TestMetricsHandler_GraphMaxIterationsClassified(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + graphInfo := &callbacks.RunInfo{Component: components.Component("Graph")} + h.OnError(context.Background(), graphInfo, errors.New("graph reached max steps limit")) + if got := testutil.ToFloat64(h.collectors.graphIterations.WithLabelValues("max_iterations")); got != 1 { + t.Errorf("max_iterations counter = %v, want 1", got) + } +} + +func TestMetricsHandler_RegistererCollectorReuse(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h1 := NewMetricsHandler(MetricsDeps{Registerer: reg}) + h2 := NewMetricsHandler(MetricsDeps{Registerer: reg}) + if h1.collectors != h2.collectors { + t.Fatalf("expected the same collectors to be reused for the same registerer") + } +} + +func TestMetricsHandler_NeededGating(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() + h := NewMetricsHandler(MetricsDeps{Registerer: reg}) + if !h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnEnd) { + t.Error("Tool OnEnd should be needed") + } + if h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnStartWithStreamInput) { + t.Error("Tool OnStartWithStreamInput should NOT be needed") + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/persistence.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/persistence.go new file mode 100644 index 000000000..1636939cf --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/persistence.go @@ -0,0 +1,1037 @@ +// Package callbacks bundles the eino callbacks.Handler implementations +// that wire ongrid cross-cutting concerns into the new compose.Graph +// agent kernel: persistence, SSE streaming, audit, metrics, and +// (carried over from PR-1) budget gating. The handler set is documented +// in (Callback 链) + (主参考图 — callback chain +// 横切区块). +// +// Each file in this package owns exactly one handler; chain.go combines +// them into a default list. Handlers are wired at Invoke time via +// compose.WithCallbacks (see graph.BuildReActGraph header comment) — no +// global registration is performed by these constructors so a graph +// run cannot accidentally pick up handlers from another tenant or +// session. PR-6 of scaffolding only — the cutover layer (NEXT +// PR) will assemble Deps and pass the chain through compose.Invoke. +package callbacks + +import ( + "context" + "errors" + "log/slog" + "sync" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/schema" + "github.com/prometheus/client_golang/prometheus" + + biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" +) + +// PersistenceDeps bundles the persistence handler's collaborators. +// SessionID identifies which chat row to write into. Repo is the +// SessionRepo binding (— write chat_messages and +// chat_tool_calls). Logger is optional — used to record best-effort +// persistence failures so they don't fail user-visible requests but +// stay observable. Registerer is optional — when set, the handler +// increments ongrid_persist_errors_total{kind=...} for any persist +// failure (错误处理 — persist 失败不阻断 graph). +type PersistenceDeps struct { + SessionID string + Repo biz.SessionRepo + Logger *slog.Logger + Registerer prometheus.Registerer + // Model is the LLM model id the cutover layer routed this run to. + // Persisted on each role=assistant chat_messages row so the SPA can + // surface per-message provenance. Empty → column stays NULL. + Model string +} + +const persistenceWriteTimeout = 5 * time.Second + +type pendingToolCallFinalizer interface { + FinalizePendingToolCalls(ctx context.Context, sessionID string, resultJSON, errStr string, endedAt time.Time) (int64, error) +} + +func persistenceWriteContext(parent context.Context) (context.Context, context.CancelFunc) { + if parent == nil { + parent = context.Background() + } + return context.WithTimeout(context.WithoutCancel(parent), persistenceWriteTimeout) +} + +// PersistenceHandler writes chat_messages + chat_tool_calls rows as +// the graph executes. Mirrors the persistence side-effects the legacy +// agent.go for-loop performs synchronously; spec: +// +// OnChatModelStart → flushIncompleteBatch (autoheal stubs for +// any tool_call from the PREVIOUS assistant turn +// whose OnEnd never fired — see currentBatch) +// OnChatModelEnd → INSERT chat_messages (role=assistant) +// OnToolStart → INSERT chat_tool_calls (status=pending) +// OnToolEnd → UPDATE chat_tool_calls + INSERT chat_messages (role=tool) +// +// User-message persistence stays at the chatruntime entry point (as +// agent.go does today) — by the time the graph starts, the user's +// turn is already on disk. This handler is only responsible for the +// agent's own writes. +// +// Concurrency: handler instances are designed for ONE graph run each. +// The cutover layer constructs a fresh PersistenceHandler per request +// so the SessionID + per-call state stay scoped. Tool start times are +// stashed on context (per-call) which is goroutine-safe. +// +// Autoheal background (currentBatch). eino's ToolsNode runs sibling +// tool_calls from one assistant turn in parallel. In live production +// (.91, session b528bfb0 on 2026-06-06) we observed an assistant +// turn emitting 4 host_bash tool_calls where only 2 of the 4 OnEnd +// callbacks fired — chat_tool_calls had all 4 rows but chat_messages +// was missing 2 role=tool rows, with NO recorded persistence error. +// Replay then produced an envelope strict providers (DeepSeek v4+) +// reject with HTTP 400 "insufficient tool messages following +// tool_calls". The replay-side defense (toolreplay precheck + +// MarkDependentToolsForSkip) keeps the user out of the loop, but the +// dropped tool results are still lost. To stop this from happening +// silently we track per-assistant batches here: persistAssistant +// records the expected llm_call_id set; persistToolEnd marks each +// seen; the next ChatModel.OnStart (signalling the previous batch is +// terminally done) flushes any leftover llm_call_ids as autoheal +// stub role=tool rows so the next replay envelope is structurally +// valid AND the LLM gets honest "tool result was not captured" +// signal instead of a fabricated success. +// +// Errors: persist failures NEVER abort the graph — the handler logs + +// counts and returns ctx unchanged. 可观测性: audit/ +// persist best-effort. +type PersistenceHandler struct { + deps PersistenceDeps + + // directToolPersistence moves tool lifecycle writes to graph's adapter + // boundary. Eino does not reliably emit nested ToolsNode callbacks through + // the outer graph, so leaving callback persistence enabled can create a + // duplicate row on versions where it does propagate them. + directToolPersistence bool + + // errCounter is the lazy collector for persist failures. Resolved + // at construction; nil when Registerer is nil. + errCounter *prometheus.CounterVec + // lossCounter is the autoheal observability collector. Labels: + // outcome="autoheal_stub" — a stub row was inserted + // tool_name= + // Cardinality: tool_name set is small (bounded by registered tools). + lossCounter *prometheus.CounterVec + + // toolCalls maps eino tool_call_id → the chat_tool_calls row id we + // inserted on OnStart, so OnEnd can update the same row. eino + // guarantees the tool_call_id is unique within a graph run; we + // scope the map to this handler instance, not globally. + // Entries are deleted on OnEnd/OnError; what remains here at flush + // time is exactly the set of tool calls whose end-callback never + // fired — those are what flushIncompleteBatch heals. + toolCalls map[string]toolCallEntry + toolCallsMu sync.Mutex + // replyToolCalls is the request-scoped projection returned to the + // synchronous HTTP caller. Persistence remains the canonical audit trail; + // this avoids re-querying a concurrent session just to reconstruct the + // legacy post-message response. + replyToolCalls []model.ToolCall + + // asstMu protects iteration-counting state used by OnEnd to + // distinguish the terminal assistant turn from intermediate ones. + asstMu sync.Mutex + asstWrites int + + // assistantIDRelay (optional) is the cross-handler share with + // SSEHandler — see chain.go. Set by NewDefaultHandlers; nil when + // tests use NewPersistenceHandler directly. + assistantIDRelay *assistantIDRelay + + // lastIDsMu protects the two pieces of cross-callback state below. + // Both flow ChatModel.OnEnd → Tool.OnStart/OnEnd within a single + // ReAct iteration, which is serial — no two tool calls overlap — + // so a plain FIFO + scalar is correct. + lastIDsMu sync.Mutex + // lastAssistantID is the chat_messages.id of the most recent + // assistant row this handler wrote. Used as MessageID for the + // chat_tool_calls rows the FOLLOWING tools persist; MessageID is + // NOT NULL on chat_tool_calls so without this every insert + // silently fails the NOT NULL check and the table stays empty — + // the exact state observed on test sessions pre-fix. + lastAssistantID string + // pendingLLMCalls is the FIFO of {llm_call_id, tool_name} pairs + // extracted from ChatModel.OnEnd's msg.ToolCalls. Tool.OnStart + // pops the head to associate the upcoming chat_tool_calls row + // with the real LLM-assigned id (e.g. "call_abc123"). Without + // this, persistToolEnd writes the synthetic + // "|" fallback into chat_messages.tool_call_id + // and history replay loses the linkage strict providers need. + pendingLLMCalls []pendingLLMCall + + // batchMu guards currentBatch. Held only while opening / sealing / + // flushing — individual mark-seen calls inside persistToolEnd take + // it just long enough to update one map. Tool callbacks fire in + // parallel from eino's ToolsNode so the mutex is non-trivial. + batchMu sync.Mutex + currentBatch *assistantBatch +} + +// assistantBatch tracks the tool_call → tool_response pairing for one +// assistant turn so flushIncompleteBatch can emit stub responses for +// any tool_call whose OnEnd never fired. expected is the set of +// LLM-assigned call ids the assistant emitted; seen accumulates as +// persistToolEnd writes role=tool rows; toolName lets the stub carry +// the right tool_name into chat_messages for replay parity. Per the +// per-request handler scoping (see handler doc), at most one batch +// is live at a time. +type assistantBatch struct { + assistantID string + expected map[string]string // llmCallID → toolName (recorded on register) + seen map[string]struct{} + openedAt time.Time +} + +// pendingLLMCall carries the LLM-assigned call id (e.g. +// "call_abc123") + the tool name from the assistant turn that emitted +// the tool_calls slot. ReAct serializes tool execution so we can +// dequeue by order; the toolName is checked for sanity (mismatch = +// graph wiring change, surface as a recorded error). +type pendingLLMCall struct { + llmCallID string + toolName string +} + +// toolCallEntry tracks per-call state across OnStart → OnEnd. +type toolCallEntry struct { + rowID string + startedAt time.Time + toolName string + argsJSON string + toolCallID string // eino's per-graph-run tool_call_id (or synthetic fallback) + // llmCallID is the LLM-assigned id (e.g. "call_abc123") flowed + // from ChatModel.OnEnd via the handler's pendingLLMCalls FIFO. + // Empty when the wiring didn't propagate (synthetic fallback); + // persistToolEnd uses it for chat_messages.tool_call_id so history + // replay can pair the tool turn with the assistant tool_calls slot. + llmCallID string +} + +// NewPersistenceHandler builds the handler. Returns nil if SessionID +// or Repo is missing — the cutover layer treats nil as "persistence +// disabled" (matching the existing optional-handler pattern). +func NewPersistenceHandler(deps PersistenceDeps) *PersistenceHandler { + if deps.SessionID == "" || deps.Repo == nil { + return nil + } + h := &PersistenceHandler{ + deps: deps, + toolCalls: make(map[string]toolCallEntry), + } + if deps.Registerer != nil { + h.errCounter = registerOrExisting(deps.Registerer, prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_persist_errors_total", + Help: "Total ongrid AIOps persistence failures (chat_messages / chat_tool_calls writes).", + }, + []string{"kind"}, + )).(*prometheus.CounterVec) + h.lossCounter = registerOrExisting(deps.Registerer, prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_chat_tool_response_loss_total", + Help: "Total ongrid AIOps tool responses lost (OnEnd never fired) and autohealed via stub.", + }, + []string{"outcome", "tool_name"}, + )).(*prometheus.CounterVec) + } + return h +} + +// Needed signals which timings to wire so eino can short-circuit the +// expensive ones we don't use. +// +// ChatModel.OnStart is wired (in addition to OnEnd) so flushIncompleteBatch +// can autoheal the previous turn before the next LLM call goes out — its +// position in the callback stream is the cleanest "previous batch is +// terminally done" signal we have. +func (h *PersistenceHandler) Needed(_ context.Context, info *callbacks.RunInfo, timing callbacks.CallbackTiming) bool { + if h == nil { + return false + } + if info == nil { + return false + } + switch info.Component { + case components.ComponentOfChatModel: + return timing == callbacks.TimingOnStart || timing == callbacks.TimingOnEnd + case components.ComponentOfTool: + if h.directToolPersistence { + return false + } + return timing == callbacks.TimingOnStart || timing == callbacks.TimingOnEnd || timing == callbacks.TimingOnError + default: + return false + } +} + +// OnStart is the hook fired before a component runs. For PersistenceHandler: +// - ChatModel start: flush any incomplete tool batch from the previous +// assistant turn (autoheal stub) BEFORE this LLM call goes out. Also +// traces the entry for instrumentation. +// - Tool start: write a chat_tool_calls row in pending state and +// stash the row id on the handler so OnEnd can find it. +func (h *PersistenceHandler) OnStart(ctx context.Context, info *callbacks.RunInfo, input callbacks.CallbackInput) context.Context { + if h == nil || info == nil { + return ctx + } + if info.Component == components.ComponentOfChatModel { + h.trace(ctx, "chat_model_start", info, "") + h.flushIncompleteBatch(ctx) + return ctx + } + if info.Component != components.ComponentOfTool { + return ctx + } + tin := einotool.ConvCallbackInput(input) + if tin == nil { + h.trace(ctx, "tool_start_skip_nil_input", info, "") + return ctx + } + h.persistToolStart(ctx, info.Name, tin.ArgumentsInJSON) + return ctx +} + +func (h *PersistenceHandler) persistToolStart(ctx context.Context, toolName, argumentsInJSON string) { + if h == nil || toolName == "" { + return + } + info := &callbacks.RunInfo{Name: toolName, Component: components.ComponentOfTool} + h.trace(ctx, "tool_start", info, "") + startedAt := time.Now().UTC() + // Resolve MessageID + LLM call id from the assistant turn that + // just emitted this tool_call. Explicit ctx seams (WithMessageID / + // WithToolCallID) take precedence for test isolation; production + // falls back to the handler's internal tracking populated in + // persistAssistant. + messageID := messageIDFromCtx(ctx) + if messageID == "" { + messageID = h.currentAssistantID() + } + // ROOT FIX: eino's ToolsNode stamps the authoritative LLM call id + // (call_XX) onto ctx before each tool runs — compose.GetToolCallID. + // Read it directly. It's exact and order-independent, so we never + // reconstruct the id from completion order (which mis-pairs under + // parallel out-of-order completion and was the source of the + // synthetic-id orphans → provider 400). The ctx seam + FIFO are kept + // only as defensive fallbacks for tools invoked outside a ToolsNode. + llmCallID := compose.GetToolCallID(ctx) + if llmCallID == "" { + if v := ctxToolCallID(ctx); v != "" && v != fallbackToolCallID(info) { + llmCallID = v + } else { + llmCallID = h.popPendingLLMCall(info.Name) + } + } + row := &model.ToolCall{ + MessageID: messageID, + ToolName: toolName, + ArgumentsJSON: argumentsInJSON, + Status: model.StatusPending, + StartedAt: startedAt, + CreatedAt: startedAt, + } + if llmCallID != "" { + id := llmCallID + row.LLMCallID = &id + } + writeCtx, cancel := persistenceWriteContext(ctx) + defer cancel() + if err := h.deps.Repo.CreateToolCall(writeCtx, row); err != nil { + h.recordErr("tool_call_insert", err) + return + } + // stash by the eino tool_call_id so OnEnd can update the same row; + // llmCallID (if any) flows downstream so the role=tool chat_messages + // row carries the real id strict providers expect. + einoTCID := toolCallIDFromCtx(ctx, info) + h.toolCallsMu.Lock() + h.toolCalls[einoTCID] = toolCallEntry{ + rowID: row.ID, + startedAt: startedAt, + toolName: toolName, + argsJSON: argumentsInJSON, + toolCallID: einoTCID, + llmCallID: llmCallID, + } + h.replyToolCalls = append(h.replyToolCalls, *row) + h.toolCallsMu.Unlock() +} + +// PersistToolStart is called by graph's synchronous tool adapter. It avoids +// relying on nested ToolsNode callbacks, which can be lost before OnEnd. +func (h *PersistenceHandler) PersistToolStart(ctx context.Context, toolName, argumentsInJSON string) { + h.persistToolStart(ctx, toolName, argumentsInJSON) +} + +// PersistToolEnd is called by graph's synchronous tool adapter after every +// return path, including memoised and budget-limited results. +func (h *PersistenceHandler) PersistToolEnd(ctx context.Context, toolName, response string) { + if h == nil || toolName == "" { + return + } + info := &callbacks.RunInfo{Name: toolName, Component: components.ComponentOfTool} + h.trace(ctx, "tool_end_success", info, toolCallIDFromCtx(ctx, info)) + h.persistToolEnd(ctx, info, &einotool.CallbackOutput{Response: response}, nil) +} + +// ctxToolCallID returns the WithToolCallID seam value, or "" when +// unset. Pulled out so OnStart's logic can ignore the seam when its +// only value is the synthetic "name|type" fallback (which carries no +// real id information). +func ctxToolCallID(ctx context.Context) string { + v, _ := ctx.Value(toolCallIDCtxKey{}).(string) + return v +} + +// fallbackToolCallID is the synthetic id toolCallIDFromCtx returns +// when no ctx value is set. Exposed for the comparison inside OnStart +// so we don't mistake the fallback for a real LLM-assigned id. +func fallbackToolCallID(info *callbacks.RunInfo) string { + if info == nil { + return "" + } + return info.Name + "|" + info.Type +} + +// OnEnd is the hook fired after a component succeeds. +// - ChatModel end: append a chat_messages row (role=assistant) with +// the model's content + token usage. +// - Tool end: update the chat_tool_calls row with status=success + +// the result JSON, AND insert a chat_messages row (role=tool) so +// history replay sees the tool result on the next turn. +func (h *PersistenceHandler) OnEnd(ctx context.Context, info *callbacks.RunInfo, output callbacks.CallbackOutput) context.Context { + if h == nil || info == nil { + return ctx + } + switch info.Component { + case components.ComponentOfChatModel: + mo := einomodel.ConvCallbackOutput(output) + if mo == nil || mo.Message == nil { + h.trace(ctx, "chat_model_end_skip_nil", info, "") + return ctx + } + h.trace(ctx, "chat_model_end", info, "") + h.persistAssistant(ctx, mo) + case components.ComponentOfTool: + tout := einotool.ConvCallbackOutput(output) + if tout == nil { + h.trace(ctx, "tool_end_skip_nil_output", info, "") + return ctx + } + h.trace(ctx, "tool_end_success", info, toolCallIDFromCtx(ctx, info)) + h.persistToolEnd(ctx, info, tout, nil) + } + return ctx +} + +// OnError fires when a component returns a non-nil error. For tool +// errors we still update the chat_tool_calls row to status=error so +// the audit trail captures the failure; ChatModel errors are recorded +// by the audit handler (this layer doesn't produce a chat row for +// them — there's no message to persist). +func (h *PersistenceHandler) OnError(ctx context.Context, info *callbacks.RunInfo, err error) context.Context { + if h == nil || info == nil || err == nil { + return ctx + } + if info.Component != components.ComponentOfTool { + return ctx + } + h.trace(ctx, "tool_end_error", info, toolCallIDFromCtx(ctx, info)) + h.persistToolEnd(ctx, info, nil, err) + return ctx +} + +// OnStartWithStreamInput is a no-op (we only consume the eventual end value). +func (h *PersistenceHandler) OnStartWithStreamInput(ctx context.Context, _ *callbacks.RunInfo, in *schema.StreamReader[callbacks.CallbackInput]) context.Context { + if in != nil { + in.Close() + } + return ctx +} + +// OnEndWithStreamOutput is a no-op for PR-6. token-level streaming +// persistence (writing the final assembled message at stream-end) is +// owned by the cutover layer in PR-7; for now we drain + close so we +// don't leak goroutines. +func (h *PersistenceHandler) OnEndWithStreamOutput(ctx context.Context, _ *callbacks.RunInfo, out *schema.StreamReader[callbacks.CallbackOutput]) context.Context { + if out != nil { + out.Close() + } + return ctx +} + +func (h *PersistenceHandler) persistAssistant(ctx context.Context, mo *einomodel.CallbackOutput) { + msg := mo.Message + row := &model.Message{ + SessionID: h.deps.SessionID, + Role: string(msg.Role), + CreatedAt: time.Now().UTC(), + } + if h.deps.Model != "" { + m := h.deps.Model + row.Model = &m + } + if msg.Content != "" { + c := msg.Content + row.Content = &c + } + if msg.ReasoningContent != "" { + reasoning := msg.ReasoningContent + row.ReasoningContent = &reasoning + } + if mo.TokenUsage != nil { + pt := mo.TokenUsage.PromptTokens + ct := mo.TokenUsage.CompletionTokens + row.PromptTokens = &pt + row.CompletionTokens = &ct + } else if msg.ResponseMeta != nil && msg.ResponseMeta.Usage != nil { + pt := msg.ResponseMeta.Usage.PromptTokens + ct := msg.ResponseMeta.Usage.CompletionTokens + row.PromptTokens = &pt + row.CompletionTokens = &ct + } + writeCtx, cancel := persistenceWriteContext(ctx) + defer cancel() + if err := h.deps.Repo.AppendMessage(writeCtx, row); err != nil { + h.recordErr("assistant_insert", err) + return + } + // Hand the persisted row id to SSEHandler so the assistant_end + // frame can ship it. AppendMessage stamps row.ID via gorm's auto- + // generated string id (see model.Message ID field). + h.assistantIDRelay.store(row.ID) + h.asstMu.Lock() + h.asstWrites++ + h.asstMu.Unlock() + + // Tool-call linkage for downstream Tool.OnStart: + // - lastAssistantID is the MessageID the next chat_tool_calls + // rows should attach to. + // - pendingLLMCalls carries the real LLM call ids emitted in + // msg.ToolCalls so the FOLLOWING tool callbacks can persist + // them as llm_call_id (and replay can pair correctly). + // Both reset on every assistant turn — the next assistant + // supersedes the previous turn's tool_calls slot. + pending := make([]pendingLLMCall, 0, len(msg.ToolCalls)) + for _, tc := range msg.ToolCalls { + if tc.ID == "" || tc.Function.Name == "" { + continue + } + pending = append(pending, pendingLLMCall{ + llmCallID: tc.ID, + toolName: tc.Function.Name, + }) + } + h.lastIDsMu.Lock() + h.lastAssistantID = row.ID + h.pendingLLMCalls = pending + h.lastIDsMu.Unlock() + // Open a new batch tracker — flushIncompleteBatch will heal it on + // the next ChatModel.OnStart. If the assistant emitted no tool_calls + // (terminal text answer), expected is empty and the batch is a + // no-op on flush. + h.openBatch(row.ID, pending) +} + +// openBatch starts tracking the tool-call → tool-response pairing for +// a fresh assistant turn. Replaces any prior batch (which means the +// previous batch was abandoned without its OnEnds firing AND without +// a ChatModel.OnStart flushing it — drop counter so we can see it). +func (h *PersistenceHandler) openBatch(assistantID string, calls []pendingLLMCall) { + expected := make(map[string]string, len(calls)) + for _, c := range calls { + if c.llmCallID == "" { + continue + } + expected[c.llmCallID] = c.toolName + } + h.batchMu.Lock() + if h.currentBatch != nil && len(h.currentBatch.expected) > 0 { + // Surface as recordErr so we know if openBatch ever races with + // an unflushed prior batch (would mean ChatModel.OnEnd fired + // twice without an intervening OnStart — very unusual). + h.recordErr("batch_overwrite_unflushed", errBatchOverwrite{ + prevAssistant: h.currentBatch.assistantID, + newAssistant: assistantID, + lost: len(h.currentBatch.expected) - len(h.currentBatch.seen), + }) + } + h.currentBatch = &assistantBatch{ + assistantID: assistantID, + expected: expected, + seen: make(map[string]struct{}, len(expected)), + openedAt: time.Now().UTC(), + } + h.batchMu.Unlock() +} + +// markSeen records that a tool_call_id's response has been persisted. +// Called from persistToolEnd AFTER the AppendMessage succeeds. +func (h *PersistenceHandler) markSeen(llmCallID string) { + if llmCallID == "" { + return + } + h.batchMu.Lock() + if h.currentBatch != nil { + h.currentBatch.seen[llmCallID] = struct{}{} + } + h.batchMu.Unlock() +} + +// flushIncompleteBatch is the autoheal step. Called at the start of +// the NEXT ChatModel call (and from FinalizeBatch on handler teardown), +// which is the cleanest signal that the previous assistant turn's +// tool batch is terminally done. For every expected llm_call_id that +// never landed a seen mark, write a stub role=tool row so the next +// replay envelope is structurally valid AND the LLM gets honest +// signal that the tool result was lost. +// +// Idempotent: clears currentBatch on exit, so a redundant call is a +// no-op. +func (h *PersistenceHandler) flushIncompleteBatch(ctx context.Context) { + h.batchMu.Lock() + batch := h.currentBatch + h.currentBatch = nil + h.batchMu.Unlock() + if batch == nil || len(batch.expected) == 0 { + return + } + missing := make(map[string]string) // llmCallID → toolName + for id, tname := range batch.expected { + if _, ok := batch.seen[id]; ok { + continue + } + missing[id] = tname + } + if len(missing) == 0 { + return + } + if h.deps.Logger != nil { + h.deps.Logger.Warn("tool response loss detected; autohealing with stub rows", + slog.String("session_id", h.deps.SessionID), + slog.String("assistant_id", batch.assistantID), + slog.Int("expected", len(batch.expected)), + slog.Int("seen", len(batch.seen)), + slog.Int("missing", len(missing)), + slog.Int64("batch_age_ms", time.Since(batch.openedAt).Milliseconds()), + ) + } + for callID, tname := range missing { + body := `{"error":"tool response was not persisted (eino ToolsNode OnEnd loss); placeholder synthesised by manager to keep history envelope valid","autoheal":true}` + if entry, ok := h.popUnendedToolCall(callID); ok { + errText := "tool response was not persisted (eino ToolsNode OnEnd loss); autohealed by manager" + result := body + endedAt := time.Now().UTC() + writeCtx, cancel := persistenceWriteContext(ctx) + err := h.deps.Repo.UpdateToolCallResult(writeCtx, entry.rowID, model.StatusError, &result, &errText, endedAt) + cancel() + if err != nil { + h.recordErr("autoheal_tool_call_update", err) + } + } + cid := callID + tn := tname + row := &model.Message{ + SessionID: h.deps.SessionID, + Role: model.RoleTool, + Content: &body, + ToolCallID: &cid, + ToolName: &tn, + CreatedAt: time.Now().UTC(), + } + writeCtx, cancel := persistenceWriteContext(ctx) + err := h.deps.Repo.AppendMessage(writeCtx, row) + cancel() + if err != nil { + h.recordErr("autoheal_stub_insert", err) + continue + } + if h.lossCounter != nil { + h.lossCounter.WithLabelValues("autoheal_stub", tname).Inc() + } + } +} + +func (h *PersistenceHandler) popUnendedToolCall(llmCallID string) (toolCallEntry, bool) { + if h == nil || llmCallID == "" { + return toolCallEntry{}, false + } + h.toolCallsMu.Lock() + defer h.toolCallsMu.Unlock() + for key, entry := range h.toolCalls { + if entry.llmCallID != llmCallID { + continue + } + delete(h.toolCalls, key) + return entry, true + } + return toolCallEntry{}, false +} + +// FinalizeBatch flushes any in-flight batch before the handler is +// retired. Called from chatruntime after the graph invoke returns so +// "user closed browser mid-batch" doesn't leave orphan tool_calls +// behind (the ChatModel.OnStart hook only catches in-session cases). +func (h *PersistenceHandler) FinalizeBatch(ctx context.Context) { + if h == nil { + return + } + h.flushIncompleteBatch(ctx) + h.finalizePendingToolCalls(ctx) +} + +func (h *PersistenceHandler) finalizePendingToolCalls(ctx context.Context) { + if h == nil || h.deps.Repo == nil || h.deps.SessionID == "" { + return + } + finalizer, ok := h.deps.Repo.(pendingToolCallFinalizer) + if !ok { + return + } + body := `{"error":"tool response was not persisted before graph shutdown; marked failed by manager finalizer","autoheal":true}` + errText := "tool response was not persisted before graph shutdown; autohealed by manager" + writeCtx, cancel := persistenceWriteContext(ctx) + defer cancel() + if _, err := finalizer.FinalizePendingToolCalls(writeCtx, h.deps.SessionID, body, errText, time.Now().UTC()); err != nil { + h.recordErr("pending_tool_call_finalize", err) + } +} + +type errBatchOverwrite struct { + prevAssistant, newAssistant string + lost int +} + +func (e errBatchOverwrite) Error() string { + return "batch overwrite prev=" + e.prevAssistant + " new=" + e.newAssistant +} + +// popPendingLLMCall returns the head of the FIFO matching toolName, +// or "" if the queue is empty / next entry doesn't match (which would +// indicate a graph wiring change we want to surface, not silently +// mispair). +func (h *PersistenceHandler) popPendingLLMCall(toolName string) string { + h.lastIDsMu.Lock() + defer h.lastIDsMu.Unlock() + if len(h.pendingLLMCalls) == 0 { + return "" + } + // Match by tool name ANYWHERE in the queue, not just the head. + // Parallel tool calls complete out of issue-order (e.g. query_promql + // finishes before get_host_processes even though it was listed + // second), so a strict head match mis-pairs and the row falls back to + // a synthetic "|einoToolAdapter" id — which has no matching + // tool_calls on the assistant turn, so replay hits the provider's + // 400 "tool must follow tool_calls". Pop the first pending entry with + // this name; duplicate names still pair in issue order (FIFO among + // same-named peers). + for i, c := range h.pendingLLMCalls { + if c.toolName == toolName { + id := c.llmCallID + h.pendingLLMCalls = append(h.pendingLLMCalls[:i:i], h.pendingLLMCalls[i+1:]...) + return id + } + } + // Genuinely no pending call for this tool — a real wiring drift. + // Surface it; the row falls back to the synthetic id and the replay + // builder drops the orphan tool message so the turn stays valid. + h.recordErr("tool_call_fifo_mismatch", errToolFIFOMismatch{ + expected: h.pendingLLMCalls[0].toolName, + got: toolName, + }) + return "" +} + +func (h *PersistenceHandler) currentAssistantID() string { + h.lastIDsMu.Lock() + defer h.lastIDsMu.Unlock() + return h.lastAssistantID +} + +type errToolFIFOMismatch struct { + expected, got string +} + +func (e errToolFIFOMismatch) Error() string { + return "tool_call FIFO head=" + e.expected + " but tool=" + e.got +} + +func (h *PersistenceHandler) persistToolEnd(ctx context.Context, info *callbacks.RunInfo, tout *einotool.CallbackOutput, execErr error) { + tcID := toolCallIDFromCtx(ctx, info) + h.toolCallsMu.Lock() + entry, ok := h.toolCalls[tcID] + delete(h.toolCalls, tcID) + h.toolCallsMu.Unlock() + if !ok { + // No matching OnStart — chatruntime didn't fire it OR the same + // tcID was double-deleted by a callback race. Used to be a + // silent return; record it now so we can correlate against + // flushIncompleteBatch stub events. + h.recordErr("tool_call_lookup_miss", errToolCallLookupMiss{ + toolCallID: tcID, + toolName: runInfoName(info), + }) + return + } + + endedAt := time.Now().UTC() + status := model.StatusSuccess + var errStr *string + var resultPtr *string + if execErr != nil { + s := execErr.Error() + errStr = &s + status = model.StatusError + if errors.Is(execErr, context.DeadlineExceeded) { + status = model.StatusTimeout + } + } else if tout != nil { + s := tout.Response + resultPtr = &s + } + + writeCtx, cancel := persistenceWriteContext(ctx) + defer cancel() + if err := h.deps.Repo.UpdateToolCallResult(writeCtx, entry.rowID, status, resultPtr, errStr, endedAt); err != nil { + h.recordErr("tool_call_update", err) + } + h.updateReplyToolCall(entry.rowID, status, resultPtr, errStr, endedAt) + + // Append a role=tool message so history replay sees the tool + // result. Use the LLM-assigned call id (entry.llmCallID) when + // available — that's what strict providers (DeepSeek v4+) match + // against the assistant turn's tool_calls slot. Falls back to the + // eino-internal tool_call_id only when the wiring didn't capture + // the real id (e.g. test or pre-fix data); toolreplay.Resolve + // drops the assistant turn for those, preserving correctness over + // orphan tool replay. + tname := entry.toolName + tcallID := entry.llmCallID + if tcallID == "" { + tcallID = entry.toolCallID + } + body := "" + if resultPtr != nil { + body = *resultPtr + } else if errStr != nil { + body = `{"error":"` + *errStr + `"}` + } else { + body = `{}` + } + row := &model.Message{ + SessionID: h.deps.SessionID, + Role: model.RoleTool, + Content: &body, + ToolCallID: &tcallID, + ToolName: &tname, + CreatedAt: endedAt, + } + if err := h.deps.Repo.AppendMessage(writeCtx, row); err != nil { + h.recordErr("tool_msg_insert", err) + return + } + // Mark seen AFTER AppendMessage succeeds so a failed insert doesn't + // fool flushIncompleteBatch into thinking this call landed. Tracked + // by llmCallID (what the batch's expected set is keyed on) — falls + // back to the synthetic id if the wiring lost the real one, which + // preserves the at-least-one-mark invariant for sanity. + if entry.llmCallID != "" { + h.markSeen(entry.llmCallID) + } else { + h.markSeen(entry.toolCallID) + } +} + +type errToolCallLookupMiss struct { + toolCallID, toolName string +} + +func (e errToolCallLookupMiss) Error() string { + return "tool_call lookup miss id=" + e.toolCallID + " name=" + e.toolName +} + +func runInfoName(info *callbacks.RunInfo) string { + if info == nil { + return "" + } + return info.Name +} + +// trace emits a single info-level breadcrumb for one callback firing. +// Used to detect "OnEnd never fired" cases by absence in logs — every +// expected callback should emit exactly one trace line. Cheap (one +// slog.Info call); operators can drop the log level to filter out if +// noise becomes a concern. +func (h *PersistenceHandler) trace(_ context.Context, event string, info *callbacks.RunInfo, toolCallID string) { + if h == nil || h.deps.Logger == nil { + return + } + name := "" + typ := "" + if info != nil { + name = info.Name + typ = string(info.Component) + } + h.deps.Logger.Info("persistence callback", + slog.String("event", event), + slog.String("session_id", h.deps.SessionID), + slog.String("component", typ), + slog.String("name", name), + slog.String("tool_call_id", toolCallID), + ) +} + +func (h *PersistenceHandler) recordErr(kind string, err error) { + if h.deps.Logger != nil { + h.deps.Logger.Warn("persistence handler write failed", + slog.String("kind", kind), + slog.String("session_id", h.deps.SessionID), + slog.String("error", err.Error())) + } + if h.errCounter != nil { + h.errCounter.WithLabelValues(kind).Inc() + } +} + +// AssistantWriteCount reports how many assistant rows this handler +// has persisted. Exposed for tests; production callers should not +// depend on it. +func (h *PersistenceHandler) AssistantWriteCount() int { + if h == nil { + return 0 + } + h.asstMu.Lock() + defer h.asstMu.Unlock() + return h.asstWrites +} + +// ToolCalls returns a stable snapshot of this graph run's tool lifecycle for +// the blocking chat response. It intentionally mirrors the legacy Reply +// contract while the database remains the source of truth for history. +func (h *PersistenceHandler) ToolCalls() []*model.ToolCall { + if h == nil { + return nil + } + h.toolCallsMu.Lock() + defer h.toolCallsMu.Unlock() + out := make([]*model.ToolCall, 0, len(h.replyToolCalls)) + for _, call := range h.replyToolCalls { + copy := call + if call.ResultJSON != nil { + result := *call.ResultJSON + copy.ResultJSON = &result + } + if call.Error != nil { + errText := *call.Error + copy.Error = &errText + } + if call.EndedAt != nil { + endedAt := *call.EndedAt + copy.EndedAt = &endedAt + } + out = append(out, ©) + } + return out +} + +func (h *PersistenceHandler) updateReplyToolCall(id, status string, resultJSON, errText *string, endedAt time.Time) { + h.toolCallsMu.Lock() + defer h.toolCallsMu.Unlock() + for i := range h.replyToolCalls { + call := &h.replyToolCalls[i] + if call.ID != id { + continue + } + call.Status = status + call.ResultJSON = resultJSON + call.Error = errText + call.EndedAt = &endedAt + return + } +} + +// Compile-time assertion. +var ( + _ callbacks.Handler = (*PersistenceHandler)(nil) + _ callbacks.TimingChecker = (*PersistenceHandler)(nil) +) + +// --- ctx helpers --------------------------------------------------------- + +// messageIDCtxKey is the context key carrying the assistant chat_messages +// row id under which tool calls should be parented. The cutover layer +// will set this; PR-6 leaves the field zero-valued when the key is +// absent (the SQL schema permits NULL message_id during the migration +// window — chatruntime will backfill). +type messageIDCtxKey struct{} + +// toolCallIDCtxKey lets the chatruntime + assembler pass eino's +// tool_call_id (the ChatGPT-style call id) through to the persistence +// handler so OnStart and OnEnd correlate via the same key. +type toolCallIDCtxKey struct{} + +// WithMessageID returns ctx with mid stashed as the parent assistant +// row id. Used by the cutover layer to link chat_tool_calls rows back +// to their owning assistant turn. +func WithMessageID(ctx context.Context, mid string) context.Context { + if mid == "" { + return ctx + } + return context.WithValue(ctx, messageIDCtxKey{}, mid) +} + +func messageIDFromCtx(ctx context.Context) string { + if v, ok := ctx.Value(messageIDCtxKey{}).(string); ok { + return v + } + return "" +} + +// WithToolCallID returns ctx with the eino tool_call_id stashed so +// PersistenceHandler.OnStart / OnEnd can correlate using a stable key. +// In production wiring eino sets the per-call id internally; for tests +// we expose this seam so handlers can be exercised in isolation. +func WithToolCallID(ctx context.Context, id string) context.Context { + if id == "" { + return ctx + } + return context.WithValue(ctx, toolCallIDCtxKey{}, id) +} + +// toolCallIDFromCtx prefers an explicit context value; otherwise it +// falls back to a tuple of (run-info name, run-info type) which is +// stable for the duration of a single tool invocation. eino guarantees +// the same RunInfo pointer is reused across OnStart/OnEnd of one call, +// but inspecting it via Type+Name keeps us decoupled from internal +// pointer identity. +func toolCallIDFromCtx(ctx context.Context, info *callbacks.RunInfo) string { + // ROOT FIX: prefer eino's authoritative tool_call_id. The ToolsNode + // sets it per call (compose.GetToolCallID), so OnStart/OnEnd of one + // tool share the real call_XX and the h.toolCalls map pairs exactly — + // even when parallel tools finish out of order. This is also what the + // role=tool chat_messages row carries, so replay never sees a + // synthetic-id orphan. + if rid := compose.GetToolCallID(ctx); rid != "" { + return rid + } + if v, ok := ctx.Value(toolCallIDCtxKey{}).(string); ok && v != "" { + return v + } + if info == nil { + return "" + } + return info.Name + "|" + info.Type +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/persistence_test.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/persistence_test.go new file mode 100644 index 000000000..bf5881c39 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/persistence_test.go @@ -0,0 +1,704 @@ +package callbacks + +import ( + "context" + "errors" + "strconv" + "sync" + "testing" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" + "github.com/prometheus/client_golang/prometheus" + + biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +// fakeSessionRepo is a goroutine-safe in-memory SessionRepo used by the +// persistence handler tests. Only the methods PersistenceHandler calls +// are exercised; the rest are no-ops to satisfy the interface. +type fakeSessionRepo struct { + mu sync.Mutex + messages []*model.Message + toolCalls map[string]*model.ToolCall + nextID int + + failAppend bool + failCreate bool + failUpdate bool + respectCanceledContext bool +} + +func newFakeSessionRepo() *fakeSessionRepo { + return &fakeSessionRepo{toolCalls: map[string]*model.ToolCall{}} +} + +func (r *fakeSessionRepo) CreateSession(context.Context, *model.Session) error { + return errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) GetSession(context.Context, string) (*model.Session, error) { + return nil, errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) ListSessions(context.Context, uint64, int, int, *uint64) ([]*model.Session, error) { + return nil, errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) ListByParent(context.Context, string) ([]*model.Session, error) { + return nil, errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) RenameSession(context.Context, string, string) error { return nil } +func (r *fakeSessionRepo) UpdateSessionModel(context.Context, string, string, string) error { + return nil +} +func (r *fakeSessionRepo) CloseSession(context.Context, string) error { + return errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) DeleteSession(context.Context, string) error { + return errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) ctxErr(ctx context.Context) error { + if !r.respectCanceledContext || ctx == nil { + return nil + } + return ctx.Err() +} + +func (r *fakeSessionRepo) AppendMessage(ctx context.Context, m *model.Message) error { + if err := r.ctxErr(ctx); err != nil { + return err + } + r.mu.Lock() + defer r.mu.Unlock() + if r.failAppend { + return errors.New("inject: append") + } + r.nextID++ + if m.ID == "" { + m.ID = "msg-" + strconv.Itoa(r.nextID) + } + cp := *m + r.messages = append(r.messages, &cp) + return nil +} +func (r *fakeSessionRepo) ListMessages(context.Context, string, int) ([]*model.Message, error) { + return nil, errs.ErrNotWiredYet +} +func (r *fakeSessionRepo) CreateToolCall(ctx context.Context, tc *model.ToolCall) error { + if err := r.ctxErr(ctx); err != nil { + return err + } + r.mu.Lock() + defer r.mu.Unlock() + if r.failCreate { + return errors.New("inject: create") + } + r.nextID++ + if tc.ID == "" { + tc.ID = "tc-" + strconv.Itoa(r.nextID) + } + cp := *tc + r.toolCalls[tc.ID] = &cp + return nil +} +func (r *fakeSessionRepo) UpdateToolCallResult(ctx context.Context, id, status string, resultJSON, errStr *string, endedAt time.Time) error { + if err := r.ctxErr(ctx); err != nil { + return err + } + r.mu.Lock() + defer r.mu.Unlock() + if r.failUpdate { + return errors.New("inject: update") + } + tc, ok := r.toolCalls[id] + if !ok { + return errs.ErrNotFound + } + tc.Status = status + tc.ResultJSON = resultJSON + tc.Error = errStr + tc.EndedAt = &endedAt + return nil +} +func (r *fakeSessionRepo) FinalizePendingToolCalls(ctx context.Context, sessionID string, resultJSON, errStr string, endedAt time.Time) (int64, error) { + if err := r.ctxErr(ctx); err != nil { + return 0, err + } + r.mu.Lock() + defer r.mu.Unlock() + if r.failUpdate { + return 0, errors.New("inject: finalize") + } + var n int64 + for _, tc := range r.toolCalls { + if tc.Status != model.StatusPending { + continue + } + tc.Status = model.StatusError + tc.ResultJSON = &resultJSON + tc.Error = &errStr + tc.EndedAt = &endedAt + n++ + } + return n, nil +} +func (r *fakeSessionRepo) SumTokensSince(context.Context, time.Time) (biz.TokenSums, error) { + return biz.TokenSums{}, nil +} + +var _ biz.SessionRepo = (*fakeSessionRepo)(nil) + +func chatModelInfo() *callbacks.RunInfo { + return &callbacks.RunInfo{Name: "ChatModel", Type: "Test", Component: components.ComponentOfChatModel} +} + +func toolInfo(name string) *callbacks.RunInfo { + return &callbacks.RunInfo{Name: name, Type: "Test", Component: components.ComponentOfTool} +} + +func TestPersistenceHandler_NewNilDeps(t *testing.T) { + t.Parallel() + if NewPersistenceHandler(PersistenceDeps{}) != nil { + t.Fatalf("nil session id should yield nil handler") + } + if NewPersistenceHandler(PersistenceDeps{SessionID: "s"}) != nil { + t.Fatalf("nil repo should yield nil handler") + } +} + +func TestPersistenceHandler_AssistantWriteOnEnd(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "sess-1", Repo: repo}) + if h == nil { + t.Fatalf("handler is nil") + } + out := &einomodel.CallbackOutput{ + Message: &schema.Message{Role: schema.Assistant, Content: "hello"}, + TokenUsage: &einomodel.TokenUsage{ + PromptTokens: 5, + CompletionTokens: 3, + TotalTokens: 8, + }, + } + h.OnEnd(context.Background(), chatModelInfo(), out) + if got := h.AssistantWriteCount(); got != 1 { + t.Errorf("assistant writes = %d, want 1", got) + } + repo.mu.Lock() + defer repo.mu.Unlock() + if len(repo.messages) != 1 { + t.Fatalf("messages persisted = %d, want 1", len(repo.messages)) + } + row := repo.messages[0] + if row.SessionID != "sess-1" { + t.Errorf("session_id = %q, want sess-1", row.SessionID) + } + if row.Role != string(schema.Assistant) { + t.Errorf("role = %q, want assistant", row.Role) + } + if row.Content == nil || *row.Content != "hello" { + t.Errorf("content = %v, want hello", row.Content) + } + if row.PromptTokens == nil || *row.PromptTokens != 5 { + t.Errorf("prompt tokens = %v, want 5", row.PromptTokens) + } +} + +func TestPersistenceHandler_ToolStartEndCycle(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "sess-1", Repo: repo}) + ctx := WithToolCallID(WithMessageID(context.Background(), "msg-1"), "call-1") + + // Start + h.OnStart(ctx, toolInfo("query_promql"), &einotool.CallbackInput{ArgumentsInJSON: `{"query":"up"}`}) + + repo.mu.Lock() + if len(repo.toolCalls) != 1 { + repo.mu.Unlock() + t.Fatalf("expected 1 pending tool_call after OnStart, got %d", len(repo.toolCalls)) + } + var tcID string + for k, tc := range repo.toolCalls { + tcID = k + if tc.Status != model.StatusPending { + t.Errorf("tool_call.status = %q, want pending", tc.Status) + } + if tc.ToolName != "query_promql" { + t.Errorf("tool_name = %q, want query_promql", tc.ToolName) + } + if tc.ArgumentsJSON != `{"query":"up"}` { + t.Errorf("arguments_json = %q", tc.ArgumentsJSON) + } + if tc.MessageID != "msg-1" { + t.Errorf("message_id = %q, want msg-1", tc.MessageID) + } + } + repo.mu.Unlock() + _ = tcID + + // End (success) + h.OnEnd(ctx, toolInfo("query_promql"), &einotool.CallbackOutput{Response: `{"ok":true}`}) + + repo.mu.Lock() + defer repo.mu.Unlock() + tc := repo.toolCalls[tcID] + if tc.Status != model.StatusSuccess { + t.Errorf("status after success = %q, want success", tc.Status) + } + if tc.ResultJSON == nil || *tc.ResultJSON != `{"ok":true}` { + t.Errorf("result_json = %v", tc.ResultJSON) + } + // chat_messages role=tool was appended + if len(repo.messages) != 1 { + t.Fatalf("expected 1 message (role=tool) after success, got %d", len(repo.messages)) + } + if repo.messages[0].Role != model.RoleTool { + t.Errorf("appended role = %q, want tool", repo.messages[0].Role) + } +} + +func TestPersistenceHandler_ToolEndPersistsAfterRequestContextCanceled(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + repo.respectCanceledContext = true + h := NewPersistenceHandler(PersistenceDeps{SessionID: "sess-1", Repo: repo}) + ctx := WithToolCallID(WithMessageID(context.Background(), "msg-1"), "call-canceled") + + h.OnStart(ctx, toolInfo("draft_config_change"), &einotool.CallbackInput{ArgumentsInJSON: `{"rule_key":"r1"}`}) + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + h.OnEnd(canceledCtx, toolInfo("draft_config_change"), &einotool.CallbackOutput{Response: `{"kind":"config_draft"}`}) + + repo.mu.Lock() + defer repo.mu.Unlock() + if len(repo.toolCalls) != 1 { + t.Fatalf("toolCalls count = %d, want 1", len(repo.toolCalls)) + } + for _, tc := range repo.toolCalls { + if tc.Status != model.StatusSuccess { + t.Fatalf("tool_call.status = %q, want success", tc.Status) + } + if tc.ResultJSON == nil || *tc.ResultJSON != `{"kind":"config_draft"}` { + t.Fatalf("tool_call.result_json = %v, want config draft response", tc.ResultJSON) + } + } + if len(repo.messages) != 1 { + t.Fatalf("messages = %d, want 1 role=tool message", len(repo.messages)) + } + msg := repo.messages[0] + if msg.Role != model.RoleTool { + t.Fatalf("message.role = %q, want tool", msg.Role) + } + if msg.Content == nil || *msg.Content != `{"kind":"config_draft"}` { + t.Fatalf("message.content = %v, want config draft response", msg.Content) + } +} + +func TestPersistenceHandler_ToolErrorMarksError(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "sess-1", Repo: repo}) + ctx := WithToolCallID(context.Background(), "call-2") + h.OnStart(ctx, toolInfo("flaky"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("flaky"), errors.New("boom")) + + repo.mu.Lock() + defer repo.mu.Unlock() + if len(repo.toolCalls) != 1 { + t.Fatalf("toolCalls count = %d", len(repo.toolCalls)) + } + for _, tc := range repo.toolCalls { + if tc.Status != model.StatusError { + t.Errorf("status = %q, want error", tc.Status) + } + if tc.Error == nil || *tc.Error != "boom" { + t.Errorf("error = %v", tc.Error) + } + } +} + +func TestPersistenceHandler_ToolTimeoutClassified(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s1", Repo: repo}) + ctx := WithToolCallID(context.Background(), "call-3") + h.OnStart(ctx, toolInfo("slow"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("slow"), context.DeadlineExceeded) + repo.mu.Lock() + defer repo.mu.Unlock() + for _, tc := range repo.toolCalls { + if tc.Status != model.StatusTimeout { + t.Errorf("status = %q, want timeout", tc.Status) + } + } +} + +func TestPersistenceHandler_PersistFailureNonFatal(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + repo.failAppend = true + reg := prometheus.NewRegistry() + h := NewPersistenceHandler(PersistenceDeps{ + SessionID: "s1", + Repo: repo, + Registerer: reg, + }) + out := &einomodel.CallbackOutput{Message: &schema.Message{Role: schema.Assistant, Content: "x"}} + // No panic expected; failure is just logged + counted. + h.OnEnd(context.Background(), chatModelInfo(), out) +} + +func TestPersistenceHandler_NeededFiltersComponent(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s", Repo: repo}) + if !h.Needed(context.Background(), chatModelInfo(), callbacks.TimingOnEnd) { + t.Errorf("ChatModel OnEnd should be needed") + } + // ChatModel.OnStart became Needed once flushIncompleteBatch was + // added — it's the cleanest signal that the previous tool batch + // is terminally done, so the autoheal flush hooks there. + if !h.Needed(context.Background(), chatModelInfo(), callbacks.TimingOnStart) { + t.Errorf("ChatModel OnStart should be needed (autoheal flush hook)") + } + if !h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnStart) { + t.Errorf("Tool OnStart should be needed") + } + other := &callbacks.RunInfo{Component: components.Component("Embedding")} + if h.Needed(context.Background(), other, callbacks.TimingOnEnd) { + t.Errorf("non-tool/non-chat component should be skipped") + } +} + +// --- autoheal / batch tracker tests --------------------------------------- + +// assistantEndWithToolCalls drives ChatModel.OnEnd with a synthesised +// assistant message that emits N tool_calls — the same shape ChatModel +// produces when the model wants to fan out to tools. Used to set up +// the batch tracker for autoheal tests. +func assistantEndWithToolCalls(t *testing.T, h *PersistenceHandler, calls ...struct{ ID, Name string }) { + t.Helper() + toolCalls := make([]schema.ToolCall, 0, len(calls)) + for _, c := range calls { + toolCalls = append(toolCalls, schema.ToolCall{ + ID: c.ID, + Type: "function", + Function: schema.FunctionCall{Name: c.Name}, + }) + } + out := &einomodel.CallbackOutput{ + Message: &schema.Message{ + Role: schema.Assistant, + Content: "", + ToolCalls: toolCalls, + }, + } + h.OnEnd(context.Background(), chatModelInfo(), out) +} + +func TestPersistenceHandlerToolCallsReturnsCompletedSnapshot(t *testing.T) { + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s", Repo: repo}) + ctx := WithToolCallID(WithMessageID(context.Background(), "assistant-1"), "call-1") + + h.PersistToolStart(ctx, "query_promql", `{"expr":"up"}`) + h.PersistToolEnd(ctx, "query_promql", `{"status":"success"}`) + + calls := h.ToolCalls() + if len(calls) != 1 { + t.Fatalf("ToolCalls len = %d, want 1", len(calls)) + } + call := calls[0] + if call.ToolName != "query_promql" || call.Status != model.StatusSuccess || call.ResultJSON == nil || *call.ResultJSON != `{"status":"success"}` || call.EndedAt == nil { + t.Fatalf("ToolCalls = %+v, want completed query_promql", call) + } + *call.ResultJSON = "mutated" + if got := h.ToolCalls()[0].ResultJSON; got == nil || *got != `{"status":"success"}` { + t.Fatalf("ToolCalls returned a mutable snapshot: %v", got) + } +} + +func TestAutoheal_NoMissing_NoStubInserted(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s", Repo: repo}) + + // Assistant emits 2 tool_calls and both OnEnds fire normally. + assistantEndWithToolCalls(t, h, + struct{ ID, Name string }{ID: "call_a", Name: "host_bash"}, + struct{ ID, Name string }{ID: "call_b", Name: "host_bash"}, + ) + for _, id := range []string{"call_a", "call_b"} { + ctx := WithToolCallID(context.Background(), id) + h.OnStart(ctx, toolInfo("host_bash"), &einotool.CallbackInput{ArgumentsInJSON: `{}`}) + h.OnEnd(ctx, toolInfo("host_bash"), &einotool.CallbackOutput{Response: `{"ok":true}`}) + } + + // Trigger flush via next ChatModel.OnStart. + h.OnStart(context.Background(), chatModelInfo(), nil) + + repo.mu.Lock() + defer repo.mu.Unlock() + // Expect: 1 assistant + 2 tool messages = 3 total. No autoheal stub. + if got := len(repo.messages); got != 3 { + t.Fatalf("messages = %d, want 3 (1 assistant + 2 tool)", got) + } + for _, m := range repo.messages { + if m.Content != nil && contains(*m.Content, `"autoheal":true`) { + t.Errorf("autoheal stub written when not expected: %s", *m.Content) + } + } +} + +func TestAutoheal_TwoOfFourMissing_StubsInserted(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + reg := prometheus.NewRegistry() + h := NewPersistenceHandler(PersistenceDeps{ + SessionID: "s", + Repo: repo, + Registerer: reg, + }) + + // Assistant emits 4 parallel host_bash tool_calls. + assistantEndWithToolCalls(t, h, + struct{ ID, Name string }{ID: "call_00", Name: "host_bash"}, + struct{ ID, Name string }{ID: "call_01", Name: "host_bash"}, + struct{ ID, Name string }{ID: "call_02", Name: "host_bash"}, + struct{ ID, Name string }{ID: "call_03", Name: "host_bash"}, + ) + // Only 01 and 03's OnStart+OnEnd fire — 00 and 02 silently dropped. + for _, id := range []string{"call_01", "call_03"} { + ctx := WithToolCallID(context.Background(), id) + h.OnStart(ctx, toolInfo("host_bash"), &einotool.CallbackInput{ArgumentsInJSON: `{}`}) + h.OnEnd(ctx, toolInfo("host_bash"), &einotool.CallbackOutput{Response: `{"ok":true}`}) + } + + // Next ChatModel.OnStart triggers flush. + h.OnStart(context.Background(), chatModelInfo(), nil) + + repo.mu.Lock() + defer repo.mu.Unlock() + // Expect 1 assistant + 2 real tool + 2 stub tool = 5. + if got := len(repo.messages); got != 5 { + t.Fatalf("messages = %d, want 5 (1 asst + 2 real tool + 2 stub)", got) + } + stubIDs := map[string]bool{} + for _, m := range repo.messages { + if m.Role != model.RoleTool { + continue + } + if m.Content != nil && contains(*m.Content, `"autoheal":true`) { + if m.ToolCallID == nil { + t.Fatalf("stub row has no ToolCallID") + } + stubIDs[*m.ToolCallID] = true + } + } + if !stubIDs["call_00"] || !stubIDs["call_02"] { + t.Errorf("stub ids = %v, want call_00 + call_02", stubIDs) + } + if stubIDs["call_01"] || stubIDs["call_03"] { + t.Errorf("real responses got autohealed: %v", stubIDs) + } + if v := counterValue(t, reg, "ongrid_chat_tool_response_loss_total", "autoheal_stub", "host_bash"); v != 2 { + t.Errorf("autoheal counter = %v, want 2", v) + } +} + +func TestAutoheal_StartedButMissingEndMarksToolCallError(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s", Repo: repo}) + + assistantEndWithToolCalls(t, h, struct{ ID, Name string }{ID: "call_started", Name: "host_bash"}) + ctx := WithToolCallID(context.Background(), "call_started") + h.OnStart(ctx, toolInfo("host_bash"), &einotool.CallbackInput{ArgumentsInJSON: `{}`}) + + h.FinalizeBatch(context.Background()) + + repo.mu.Lock() + defer repo.mu.Unlock() + var found bool + for _, tc := range repo.toolCalls { + found = true + if tc.Status != model.StatusError { + t.Fatalf("autohealed started tool_call status = %q, want error", tc.Status) + } + if tc.Error == nil || !contains(*tc.Error, "autohealed") { + t.Fatalf("autohealed started tool_call error = %v", tc.Error) + } + } + if !found { + t.Fatalf("expected started tool_call row") + } +} + +func TestFinalizeBatch_MarksPendingToolCallsWhenCallbackStateLost(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s", Repo: repo}) + + if err := repo.CreateToolCall(context.Background(), &model.ToolCall{ + MessageID: "msg-1", + ToolName: "host_du_summary", + ArgumentsJSON: `{}`, + Status: model.StatusPending, + StartedAt: time.Now().UTC(), + CreatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("CreateToolCall: %v", err) + } + + h.FinalizeBatch(context.Background()) + + repo.mu.Lock() + defer repo.mu.Unlock() + for _, tc := range repo.toolCalls { + if tc.Status != model.StatusError { + t.Fatalf("pending tool_call status = %q, want error", tc.Status) + } + if tc.Error == nil || !contains(*tc.Error, "autohealed") { + t.Fatalf("pending tool_call error = %v", tc.Error) + } + } +} + +func TestAutoheal_NoBatch_NoOp(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s", Repo: repo}) + // Very first turn — no prior assistant batch. + h.OnStart(context.Background(), chatModelInfo(), nil) + repo.mu.Lock() + defer repo.mu.Unlock() + if got := len(repo.messages); got != 0 { + t.Fatalf("messages = %d, want 0 (flush should no-op without a batch)", got) + } +} + +func TestAutoheal_SequentialBatches_OnlyOwnFlushed(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s", Repo: repo}) + + // Batch 1: 2 tool_calls, both complete. + assistantEndWithToolCalls(t, h, + struct{ ID, Name string }{ID: "b1_a", Name: "host_bash"}, + struct{ ID, Name string }{ID: "b1_b", Name: "host_bash"}, + ) + for _, id := range []string{"b1_a", "b1_b"} { + ctx := WithToolCallID(context.Background(), id) + h.OnStart(ctx, toolInfo("host_bash"), &einotool.CallbackInput{}) + h.OnEnd(ctx, toolInfo("host_bash"), &einotool.CallbackOutput{Response: `{}`}) + } + // Next ChatModel.OnStart flushes batch 1 (no stubs). + h.OnStart(context.Background(), chatModelInfo(), nil) + // Batch 2: 3 tool_calls, only 1 completes. + assistantEndWithToolCalls(t, h, + struct{ ID, Name string }{ID: "b2_a", Name: "host_bash"}, + struct{ ID, Name string }{ID: "b2_b", Name: "host_bash"}, + struct{ ID, Name string }{ID: "b2_c", Name: "host_bash"}, + ) + ctx := WithToolCallID(context.Background(), "b2_a") + h.OnStart(ctx, toolInfo("host_bash"), &einotool.CallbackInput{}) + h.OnEnd(ctx, toolInfo("host_bash"), &einotool.CallbackOutput{Response: `{}`}) + // FinalizeBatch should flush batch 2's missing b2_b and b2_c only. + h.FinalizeBatch(context.Background()) + + repo.mu.Lock() + defer repo.mu.Unlock() + stubIDs := map[string]bool{} + for _, m := range repo.messages { + if m.Role != model.RoleTool || m.Content == nil { + continue + } + if contains(*m.Content, `"autoheal":true`) && m.ToolCallID != nil { + stubIDs[*m.ToolCallID] = true + } + } + if stubIDs["b1_a"] || stubIDs["b1_b"] { + t.Errorf("batch 1 leaked stubs after its flush: %v", stubIDs) + } + if !stubIDs["b2_b"] || !stubIDs["b2_c"] { + t.Errorf("batch 2 stub ids = %v, want b2_b + b2_c", stubIDs) + } + if stubIDs["b2_a"] { + t.Errorf("completed call b2_a got stubbed: %v", stubIDs) + } +} + +func TestAutoheal_FinalizeBatchIdempotent(t *testing.T) { + t.Parallel() + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "s", Repo: repo}) + assistantEndWithToolCalls(t, h, + struct{ ID, Name string }{ID: "x", Name: "host_bash"}, + ) + h.FinalizeBatch(context.Background()) + h.FinalizeBatch(context.Background()) // second call should be a no-op + repo.mu.Lock() + defer repo.mu.Unlock() + stubCount := 0 + for _, m := range repo.messages { + if m.Content != nil && contains(*m.Content, `"autoheal":true`) { + stubCount++ + } + } + if stubCount != 1 { + t.Errorf("stub count = %d, want 1 (second Finalize must be no-op)", stubCount) + } +} + +// --- helpers -------------------------------------------------------------- + +func contains(s, sub string) bool { + return len(sub) > 0 && len(s) >= len(sub) && indexOf(s, sub) >= 0 +} + +func indexOf(s, sub string) int { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return i + } + } + return -1 +} + +// counterValue gathers the registry and returns the value for a +// label-keyed counter, asserting it exists. Returns -1 when not found. +func counterValue(t *testing.T, reg *prometheus.Registry, name string, labelValues ...string) float64 { + t.Helper() + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + for _, fam := range families { + if fam.GetName() != name { + continue + } + for _, m := range fam.GetMetric() { + labels := m.GetLabel() + if len(labels) != len(labelValues) { + continue + } + match := true + for i, lv := range labelValues { + if labels[i].GetValue() != lv { + match = false + break + } + } + if match { + return m.GetCounter().GetValue() + } + } + } + return -1 +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/reasoning_test.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/reasoning_test.go new file mode 100644 index 000000000..340ea398e --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/reasoning_test.go @@ -0,0 +1,34 @@ +package callbacks + +import ( + "context" + "encoding/json" + "testing" + + einomodel "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + "github.com/stretchr/testify/require" +) + +func TestPersistence_ReasoningStoredButNotExposedAsJSON(t *testing.T) { + for _, reason := range []string{"", "synthetic protocol fixture"} { + t.Run(reason, func(t *testing.T) { + repo := newFakeSessionRepo() + h := NewPersistenceHandler(PersistenceDeps{SessionID: "test", Repo: repo}) + h.OnEnd(context.Background(), chatModelInfo(), &einomodel.CallbackOutput{Message: &schema.Message{Role: schema.Assistant, Content: "visible", ReasoningContent: reason}}) + require.Len(t, repo.messages, 1) + row := repo.messages[0] + if reason == "" { + require.Nil(t, row.ReasoningContent) + } else { + require.NotNil(t, row.ReasoningContent) + require.Equal(t, reason, *row.ReasoningContent) + } + wire, err := json.Marshal(row) + require.NoError(t, err) + require.NotContains(t, string(wire), "ReasoningContent") + require.NotContains(t, string(wire), "synthetic protocol fixture") + require.Equal(t, "visible", *row.Content) + }) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/sse.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/sse.go new file mode 100644 index 000000000..07d1b0251 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/sse.go @@ -0,0 +1,455 @@ +package callbacks + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +// SSEEventType enumerates the streaming events the SSE handler emits. +// — wire shapes mirror the legacy agent.Emit contract used +// in internal/manager/server/aiops/http.go so the SPA round-trips +// without changes after cutover. +type SSEEventType string + +const ( + // SSEEventAssistantStart fires before the ChatModel produces any + // content (— assistant_start frame). + SSEEventAssistantStart SSEEventType = "assistant_start" + // SSEEventAssistantDelta is the new token-level streaming frame + // (— "新增能力"). The delta payload only carries the + // incremental Content chunk so the SPA can append to the bubble. + SSEEventAssistantDelta SSEEventType = "assistant_delta" + // SSEEventAssistantEnd fires after the ChatModel returns. Payload + // matches the legacy `assistant` frame: full content + iteration + + // pending tool count. + SSEEventAssistantEnd SSEEventType = "assistant_end" + // SSEEventToolStart fires when a tool starts. Mirrors the legacy + // `tool_start` frame. + SSEEventToolStart SSEEventType = "tool_start" + // SSEEventToolEnd fires when a tool finishes. Mirrors the legacy + // `tool_end` frame; status carries success / error / timeout. + SSEEventToolEnd SSEEventType = "tool_end" + // SSEEventDone fires once at terminal success. + SSEEventDone SSEEventType = "done" + // SSEEventError fires on any unrecoverable graph error. + SSEEventError SSEEventType = "error" + // SSEEventTaskNotification fires when a background sub-agent worker + // reaches a terminal state The coordinator + // runtime emits this through the same SSE channel as the assistant + // frames so the SPA can render a worker-result tile inline. Payload + // shape lives on TaskNotificationPayload below. + SSEEventTaskNotification SSEEventType = "task_notification" +) + +// SSEEvent is the payload-typed envelope passed to the SSEEmitter. +// The handler stays format-agnostic (it doesn't know about HTTP / +// `event:` lines / JSON encoding); the cutover layer's emitter is +// responsible for encoding into wire frames so the existing http.go +// writeSSE routine stays intact. +type SSEEvent struct { + Type SSEEventType + Iteration int + Assistant *AssistantPayload // for assistant_start / assistant_end + Delta *AssistantDelta // for assistant_delta + Tool *ToolPayload // for tool_start / tool_end + Done *DonePayload // for done + Error *ErrorPayload // for error + Notification *TaskNotificationPayload // for task_notification +} + +// AssistantPayload is the start/end frame body. ContentSoFar is empty +// at start; on end it carries the model's full reply content. +// +// MessageID is the persisted chat_messages row id assigned by the +// PersistenceHandler when it wrote the assistant row. It's filled at +// assistant_end emit time by looking it up via the chain's +// AssistantIDReader (chained ordering: persistence runs before SSE). +// Empty when persistence is disabled or hasn't completed. +type AssistantPayload struct { + Iteration int + MessageID string + Content string + PendingToolCalls int + CreatedAt time.Time +} + +// AssistantDelta is the per-chunk streaming payload. The handler +// emits one for each non-empty Content chunk delivered by eino's +// stream output. +type AssistantDelta struct { + Iteration int + Content string +} + +// ToolPayload is the tool_start / tool_end frame body. +type ToolPayload struct { + ToolCallID string + Name string + ArgsJSON string + Status string // pending | success | error | timeout + StartedAt time.Time + EndedAt *time.Time + DurationMs int64 + ResultJSON string + Error string +} + +// DonePayload is the terminal success frame. +type DonePayload struct { + Iterations int +} + +// ErrorPayload is the terminal failure frame. +type ErrorPayload struct { + Message string + Code string +} + +// TaskNotificationPayload is the body of a task_notification frame +// Emitted by the coordinator runtime when a background +// worker spawned via AgentTool reaches a terminal state. The shape is +// stable wire because the SPA reads each field directly: +// +// event: task_notification +// data: { +// "task_id": "agent-12ab34cd", // worker identifier +// "status": "completed"|"failed"|"killed", +// "summary": "Agent incident-investigator completed", +// "result": "...final assistant content...", // present on completed +// "error": "context canceled", // present on failed/killed +// "usage": { // optional, may be empty +// "duration_ms": 12345 +// } +// } +type TaskNotificationPayload struct { + TaskID string `json:"task_id"` + Status string `json:"status"` + Summary string `json:"summary"` + Result string `json:"result,omitempty"` + Err string `json:"error,omitempty"` + Usage map[string]any `json:"usage,omitempty"` +} + +// SSEEmitter is the seam between the handler and the HTTP writer. +// The cutover layer builds an emitter that wraps the http.Flusher + +// writeSSE helper from internal/manager/server/aiops/http.go so the +// wire format stays byte-compatible. emit MUST NOT block — slow +// consumers must be handled (drop or buffer) inside the implementation +// Streaming non-blocking 约束. +type SSEEmitter func(SSEEvent) + +// SSEHandler translates graph callbacks into SSEEvents. +// +// spec: +// +// OnChatModelStart → assistant_start +// OnChatModelStream(chunk)→ assistant_delta +// OnChatModelEnd → assistant_end +// OnToolStart → tool_start +// OnToolEnd → tool_end +// OnGraphComplete → done +// OnError → error +// +// The graph "complete" event is emitted from OnEnd when info.Component +// is the wrapper graph itself (components.ComponentOfGraph) — eino +// fires the same OnEnd contract on graph-scope as on node-scope. +// +// Concurrency: the emitter is called sequentially within the lifetime +// of a single graph run; tool fan-out (compose ToolsNode parallel +// execution) emits per-tool start/end events that may interleave — +// the iteration counter is atomic so the assistant_start sequence is +// consistent. Cutover-layer emitter implementations should be +// goroutine-safe (the existing http.go writeSSE serializes through +// the response writer, which is already single-goroutine). +type SSEHandler struct { + emit SSEEmitter + + // iterations counts ChatModel turns for the assistant frame's + // `iteration` field. Atomic because tool fan-out callbacks may + // interleave. + iterations atomic.Int64 + + // toolStarts records per-tool start time to compute duration in OnEnd. + toolStartsMu sync.Mutex + toolStarts map[string]toolStart + + // assistantIDRelay (optional) is the cross-handler share with + // PersistenceHandler — see chain.go. NewDefaultHandlers sets it; + // nil when SSE is used standalone (tests). + assistantIDRelay *assistantIDRelay +} + +type toolStart struct { + at time.Time + argsJSON string + name string +} + +// NewSSEHandler builds the handler. Returns nil if emit is nil so the +// cutover layer can opt into SSE on a per-request basis. +func NewSSEHandler(emit SSEEmitter) *SSEHandler { + if emit == nil { + return nil + } + return &SSEHandler{ + emit: emit, + toolStarts: make(map[string]toolStart), + } +} + +// Needed gates which timings the handler observes. +func (h *SSEHandler) Needed(_ context.Context, info *callbacks.RunInfo, timing callbacks.CallbackTiming) bool { + if h == nil || info == nil { + return false + } + switch info.Component { + case components.ComponentOfChatModel: + switch timing { + case callbacks.TimingOnStart, callbacks.TimingOnEnd, callbacks.TimingOnError, callbacks.TimingOnEndWithStreamOutput: + return true + } + case components.ComponentOfTool: + switch timing { + case callbacks.TimingOnStart, callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + default: + // Graph / Workflow / Chain — observe terminal so we can emit + // `done` / `error` frames. + switch timing { + case callbacks.TimingOnEnd, callbacks.TimingOnError: + return true + } + } + return false +} + +// OnStart fires before a component runs. +func (h *SSEHandler) OnStart(ctx context.Context, info *callbacks.RunInfo, input callbacks.CallbackInput) context.Context { + if h == nil || info == nil { + return ctx + } + switch info.Component { + case components.ComponentOfChatModel: + it := h.iterations.Add(1) + h.emit(SSEEvent{ + Type: SSEEventAssistantStart, + Iteration: int(it), + Assistant: &AssistantPayload{Iteration: int(it)}, + }) + case components.ComponentOfTool: + tin := einotool.ConvCallbackInput(input) + args := "" + if tin != nil { + args = tin.ArgumentsInJSON + } + startedAt := time.Now().UTC() + key := toolCallIDFromCtx(ctx, info) + h.toolStartsMu.Lock() + h.toolStarts[key] = toolStart{at: startedAt, argsJSON: args, name: info.Name} + h.toolStartsMu.Unlock() + h.emit(SSEEvent{ + Type: SSEEventToolStart, + Tool: &ToolPayload{ + ToolCallID: key, + Name: info.Name, + ArgsJSON: args, + Status: "pending", + StartedAt: startedAt, + }, + }) + } + return ctx +} + +// OnEnd fires after a component succeeds. +func (h *SSEHandler) OnEnd(ctx context.Context, info *callbacks.RunInfo, output callbacks.CallbackOutput) context.Context { + if h == nil || info == nil { + return ctx + } + switch info.Component { + case components.ComponentOfChatModel: + mo := einomodel.ConvCallbackOutput(output) + if mo == nil { + return ctx + } + var content string + var pending int + if mo.Message != nil { + content = mo.Message.Content + pending = len(mo.Message.ToolCalls) + } + it := int(h.iterations.Load()) + // h.assistantIDRelay carries the row id PersistenceHandler + // just stored — chain.go registers Persistence before SSE so + // by the time we emit here, the AppendMessage for this iter + // is done. Empty string when persistence is disabled + // (e.g. tests that wire only SSE). + h.emit(SSEEvent{ + Type: SSEEventAssistantEnd, + Iteration: it, + Assistant: &AssistantPayload{ + Iteration: it, + MessageID: h.assistantIDRelay.load(), + Content: content, + PendingToolCalls: pending, + CreatedAt: time.Now().UTC(), + }, + }) + case components.ComponentOfTool: + tout := einotool.ConvCallbackOutput(output) + key := toolCallIDFromCtx(ctx, info) + h.toolStartsMu.Lock() + ts, ok := h.toolStarts[key] + delete(h.toolStarts, key) + h.toolStartsMu.Unlock() + endedAt := time.Now().UTC() + var dur int64 + if ok { + dur = endedAt.Sub(ts.at).Milliseconds() + } + body := "" + if tout != nil { + body = tout.Response + } + h.emit(SSEEvent{ + Type: SSEEventToolEnd, + Tool: &ToolPayload{ + ToolCallID: key, + Name: info.Name, + ArgsJSON: ts.argsJSON, + Status: "success", + StartedAt: ts.at, + EndedAt: &endedAt, + DurationMs: dur, + ResultJSON: body, + }, + }) + default: + // Graph-scope OnEnd — emit `done`. + h.emit(SSEEvent{ + Type: SSEEventDone, + Done: &DonePayload{Iterations: int(h.iterations.Load())}, + }) + } + return ctx +} + +// OnError fires when a component returns a non-nil error. +func (h *SSEHandler) OnError(ctx context.Context, info *callbacks.RunInfo, err error) context.Context { + if h == nil || info == nil || err == nil { + return ctx + } + if info.Component == components.ComponentOfTool { + key := toolCallIDFromCtx(ctx, info) + h.toolStartsMu.Lock() + ts, ok := h.toolStarts[key] + delete(h.toolStarts, key) + h.toolStartsMu.Unlock() + endedAt := time.Now().UTC() + var dur int64 + if ok { + dur = endedAt.Sub(ts.at).Milliseconds() + } + status := "error" + if isDeadlineErr(err) { + status = "timeout" + } + h.emit(SSEEvent{ + Type: SSEEventToolEnd, + Tool: &ToolPayload{ + ToolCallID: key, + Name: info.Name, + ArgsJSON: ts.argsJSON, + Status: status, + StartedAt: ts.at, + EndedAt: &endedAt, + DurationMs: dur, + Error: err.Error(), + }, + }) + return ctx + } + // Top-level / ChatModel error — emit terminal `error` frame. + h.emit(SSEEvent{ + Type: SSEEventError, + Error: &ErrorPayload{Message: err.Error()}, + }) + return ctx +} + +// OnStartWithStreamInput is a no-op (we ignore streaming input). +func (h *SSEHandler) OnStartWithStreamInput(ctx context.Context, _ *callbacks.RunInfo, in *schema.StreamReader[callbacks.CallbackInput]) context.Context { + if in != nil { + in.Close() + } + return ctx +} + +// OnEndWithStreamOutput drains the streaming output of a ChatModel and +// fans assistant_delta frames to the SSE emitter as chunks land. +// — assistant_delta 是新增的 token-level 帧。 +// +// We close the receiver-side copy when done so eino can reclaim the +// stream goroutine; the original output stream continues to flow to +// downstream graph nodes. +func (h *SSEHandler) OnEndWithStreamOutput(ctx context.Context, info *callbacks.RunInfo, out *schema.StreamReader[callbacks.CallbackOutput]) context.Context { + if h == nil || info == nil || out == nil { + if out != nil { + out.Close() + } + return ctx + } + if info.Component != components.ComponentOfChatModel { + out.Close() + return ctx + } + go h.drainStream(out) + return ctx +} + +func (h *SSEHandler) drainStream(out *schema.StreamReader[callbacks.CallbackOutput]) { + defer out.Close() + for { + chunk, err := out.Recv() + if err != nil { + return + } + mo := einomodel.ConvCallbackOutput(chunk) + if mo == nil || mo.Message == nil { + continue + } + if mo.Message.Content == "" { + continue + } + it := int(h.iterations.Load()) + h.emit(SSEEvent{ + Type: SSEEventAssistantDelta, + Iteration: it, + Delta: &AssistantDelta{Iteration: it, Content: mo.Message.Content}, + }) + } +} + +// IterationCount returns the number of ChatModel turns the handler has +// observed in this run. Exposed for tests. +func (h *SSEHandler) IterationCount() int { + if h == nil { + return 0 + } + return int(h.iterations.Load()) +} + +// Compile-time check. +var ( + _ callbacks.Handler = (*SSEHandler)(nil) + _ callbacks.TimingChecker = (*SSEHandler)(nil) +) diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/sse_test.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/sse_test.go new file mode 100644 index 000000000..39779c1a0 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/sse_test.go @@ -0,0 +1,222 @@ +package callbacks + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/schema" +) + +// captureSSE collects SSEEvents emitted by the handler. Goroutine-safe +// — the handler is meant to be invoked from a single goroutine in +// production but tool fan-out + stream draining can interleave. +type captureSSE struct { + mu sync.Mutex + events []SSEEvent +} + +func (c *captureSSE) emit(e SSEEvent) { + c.mu.Lock() + c.events = append(c.events, e) + c.mu.Unlock() +} + +func (c *captureSSE) snapshot() []SSEEvent { + c.mu.Lock() + defer c.mu.Unlock() + cp := make([]SSEEvent, len(c.events)) + copy(cp, c.events) + return cp +} + +func (c *captureSSE) hasType(tp SSEEventType) bool { + for _, e := range c.snapshot() { + if e.Type == tp { + return true + } + } + return false +} + +func TestSSEHandler_NilEmitterReturnsNil(t *testing.T) { + t.Parallel() + if NewSSEHandler(nil) != nil { + t.Fatalf("expected nil when emitter is nil") + } +} + +func TestSSEHandler_AssistantStartEnd(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + ctx := context.Background() + h.OnStart(ctx, chatModelInfo(), &einomodel.CallbackInput{}) + h.OnEnd(ctx, chatModelInfo(), &einomodel.CallbackOutput{ + Message: &schema.Message{Role: schema.Assistant, Content: "done"}, + }) + events := cap.snapshot() + if len(events) != 2 { + t.Fatalf("expected 2 events got %d", len(events)) + } + if events[0].Type != SSEEventAssistantStart { + t.Errorf("event[0] = %s, want assistant_start", events[0].Type) + } + if events[0].Iteration != 1 { + t.Errorf("iteration[0] = %d, want 1", events[0].Iteration) + } + if events[1].Type != SSEEventAssistantEnd { + t.Errorf("event[1] = %s, want assistant_end", events[1].Type) + } + if events[1].Assistant == nil || events[1].Assistant.Content != "done" { + t.Errorf("end content payload missing: %+v", events[1].Assistant) + } +} + +func TestSSEHandler_ToolLifecycle(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + ctx := WithToolCallID(context.Background(), "tc-1") + h.OnStart(ctx, toolInfo("query_promql"), &einotool.CallbackInput{ArgumentsInJSON: `{"q":"up"}`}) + h.OnEnd(ctx, toolInfo("query_promql"), &einotool.CallbackOutput{Response: `{"ok":true}`}) + events := cap.snapshot() + if len(events) != 2 { + t.Fatalf("expected 2 events got %d", len(events)) + } + if events[0].Type != SSEEventToolStart { + t.Errorf("event[0] = %s, want tool_start", events[0].Type) + } + if events[0].Tool.ArgsJSON != `{"q":"up"}` { + t.Errorf("args = %q, want %q", events[0].Tool.ArgsJSON, `{"q":"up"}`) + } + if events[1].Type != SSEEventToolEnd { + t.Errorf("event[1] = %s, want tool_end", events[1].Type) + } + if events[1].Tool.Status != "success" { + t.Errorf("status = %s, want success", events[1].Tool.Status) + } + if events[1].Tool.ResultJSON != `{"ok":true}` { + t.Errorf("result = %q", events[1].Tool.ResultJSON) + } +} + +func TestSSEHandler_ToolError(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + ctx := WithToolCallID(context.Background(), "tc-2") + h.OnStart(ctx, toolInfo("flaky"), &einotool.CallbackInput{}) + h.OnError(ctx, toolInfo("flaky"), errors.New("kaboom")) + events := cap.snapshot() + last := events[len(events)-1] + if last.Type != SSEEventToolEnd { + t.Fatalf("last = %s, want tool_end", last.Type) + } + if last.Tool.Status != "error" { + t.Errorf("status = %s, want error", last.Tool.Status) + } + if last.Tool.Error != "kaboom" { + t.Errorf("error = %q", last.Tool.Error) + } +} + +func TestSSEHandler_TopLevelErrorEmitsError(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + graphInfo := &callbacks.RunInfo{Name: "OngridReActAgent", Component: components.Component("Graph")} + h.OnError(context.Background(), graphInfo, errors.New("graph went bad")) + if !cap.hasType(SSEEventError) { + t.Fatalf("expected error event, got %+v", cap.snapshot()) + } +} + +func TestSSEHandler_GraphEndEmitsDone(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + graphInfo := &callbacks.RunInfo{Name: "OngridReActAgent", Component: components.Component("Graph")} + h.OnEnd(context.Background(), graphInfo, nil) + if !cap.hasType(SSEEventDone) { + t.Fatalf("expected done event, got %+v", cap.snapshot()) + } +} + +func TestSSEHandler_NeededGating(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + if !h.Needed(context.Background(), chatModelInfo(), callbacks.TimingOnStart) { + t.Error("ChatModel start should be needed") + } + if !h.Needed(context.Background(), toolInfo("t"), callbacks.TimingOnEnd) { + t.Error("Tool end should be needed") + } + graphInfo := &callbacks.RunInfo{Component: components.Component("Graph")} + if !h.Needed(context.Background(), graphInfo, callbacks.TimingOnEnd) { + t.Error("Graph end should be needed (drives done event)") + } +} + +func TestSSEHandler_IterationCount(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + h := NewSSEHandler(cap.emit) + ctx := context.Background() + for i := 0; i < 3; i++ { + h.OnStart(ctx, chatModelInfo(), &einomodel.CallbackInput{}) + h.OnEnd(ctx, chatModelInfo(), &einomodel.CallbackOutput{Message: &schema.Message{Role: schema.Assistant}}) + } + if h.IterationCount() != 3 { + t.Errorf("IterationCount = %d, want 3", h.IterationCount()) + } +} + +// TestSSEEvent_TaskNotification_Shape pins the wire-shape contract for +// the task_notification frame so the SPA's task_notification +// renderer doesn't drift. The emitter is invoked directly here — the +// runtime is the one that fires this event in production (it bypasses +// the graph callback chain and feeds the SSE channel directly). +func TestSSEEvent_TaskNotification_Shape(t *testing.T) { + t.Parallel() + cap := &captureSSE{} + cap.emit(SSEEvent{ + Type: SSEEventTaskNotification, + Notification: &TaskNotificationPayload{ + TaskID: "agent-12ab34cd", + Status: "completed", + Summary: "Agent incident-investigator completed", + Result: "the answer is 42", + Usage: map[string]any{"duration_ms": int64(1234)}, + }, + }) + events := cap.snapshot() + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + if events[0].Type != SSEEventTaskNotification { + t.Errorf("type = %q, want task_notification", events[0].Type) + } + n := events[0].Notification + if n == nil { + t.Fatalf("Notification nil") + } + if n.TaskID != "agent-12ab34cd" { + t.Errorf("task_id = %q", n.TaskID) + } + if n.Status != "completed" { + t.Errorf("status = %q", n.Status) + } + if n.Result != "the answer is 42" { + t.Errorf("result = %q", n.Result) + } + if got, _ := n.Usage["duration_ms"].(int64); got != 1234 { + t.Errorf("usage.duration_ms = %v", n.Usage["duration_ms"]) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/tracing.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/tracing.go new file mode 100644 index 000000000..09e4d86f2 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/tracing.go @@ -0,0 +1,97 @@ +package callbacks + +import ( + "context" + "errors" + + langfuse "github.com/cloudwego/eino-ext/callbacks/langfuse/v2" + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/schema" + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + + "github.com/ongridio/ongrid/internal/pkg/tracing" +) + +// TraceHandler uses Eino's maintained callback with the existing Tempo exporter. +type TraceHandler struct { + *langfuse.CallbackHandler +} + +func NewTraceHandler(ctx context.Context) (*TraceHandler, error) { + provider, ok := otel.GetTracerProvider().(*sdktrace.TracerProvider) + if !ok { + return nil, nil + } + handler, err := langfuse.NewHandler(ctx, &langfuse.Config{ + TracerProvider: provider, + MaskFunc: tracing.MaskAgentContent, + MaxAttributeValueLength: tracing.AgentContentMaxBytes, + MaxSpanAttributeBytes: 24 * 1024, + }) + if err != nil { + return nil, err + } + return &TraceHandler{CallbackHandler: handler}, nil +} + +func (h *TraceHandler) OnStart(ctx context.Context, info *callbacks.RunInfo, input callbacks.CallbackInput) context.Context { + if !contentComponent(info) { + input = nil + } + ctx = h.CallbackHandler.OnStart(ctx, info, input) + if info != nil && info.Component == components.ComponentOfChatModel { + ctx = tracing.WithModelCallbackSpan(ctx) + } + return ctx +} + +func contentComponent(info *callbacks.RunInfo) bool { + return info != nil && (info.Component == components.ComponentOfChatModel || info.Component == components.ComponentOfTool) +} + +func (h *TraceHandler) OnEnd(ctx context.Context, info *callbacks.RunInfo, output callbacks.CallbackOutput) context.Context { + if !contentComponent(info) { + output = nil + } + return h.CallbackHandler.OnEnd(ctx, info, output) +} + +func (h *TraceHandler) OnEndWithStreamOutput(ctx context.Context, info *callbacks.RunInfo, output *schema.StreamReader[callbacks.CallbackOutput]) context.Context { + if !contentComponent(info) && output != nil { + output = schema.StreamReaderWithConvert(output, func(callbacks.CallbackOutput) (callbacks.CallbackOutput, error) { return nil, nil }) + } + return h.CallbackHandler.OnEndWithStreamOutput(ctx, info, output) +} + +func (h *TraceHandler) OnStartWithStreamInput(ctx context.Context, info *callbacks.RunInfo, input *schema.StreamReader[callbacks.CallbackInput]) context.Context { + if !contentComponent(info) && input != nil { + input = schema.StreamReaderWithConvert(input, func(callbacks.CallbackInput) (callbacks.CallbackInput, error) { return nil, nil }) + } + ctx = h.CallbackHandler.OnStartWithStreamInput(ctx, info, input) + if info != nil && info.Component == components.ComponentOfChatModel { + ctx = tracing.WithModelCallbackSpan(ctx) + } + return ctx +} + +func (h *TraceHandler) OnError(ctx context.Context, info *callbacks.RunInfo, err error) context.Context { + if err != nil { + _, rerun := compose.IsInterruptRerunError(err) + _, interrupted := compose.ExtractInterruptInfo(err) + if !rerun && !interrupted { + switch { + case errors.Is(err, context.Canceled): + err = context.Canceled + case errors.Is(err, context.DeadlineExceeded): + err = context.DeadlineExceeded + default: + // shortcut: snapshot mutable Eino node paths until the upstream callback snapshots errors before its goroutine. + err = errors.New(err.Error()) + } + } + } + return h.CallbackHandler.OnError(ctx, info, err) +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/callbacks/tracing_test.go b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/tracing_test.go new file mode 100644 index 000000000..00fc520fa --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/callbacks/tracing_test.go @@ -0,0 +1,113 @@ +package callbacks + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + + "github.com/ongridio/ongrid/internal/pkg/tracing" +) + +func TestTraceHandlerStreamMasksContentAndSharesProvider(t *testing.T) { + t.Setenv("ONGRID_AGENT_TRACE_CAPTURE_CONTENT", "false") + previous := otel.GetTracerProvider() + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + otel.SetTracerProvider(provider) + defer func() { otel.SetTracerProvider(previous); _ = provider.Shutdown(context.Background()) }() + handler, err := NewTraceHandler(context.Background()) + if err != nil || handler == nil { + t.Fatalf("handler=%v err=%v", handler, err) + } + info := &callbacks.RunInfo{Name: "chat", Component: components.ComponentOfChatModel} + ctx := handler.OnStart(context.Background(), info, &model.CallbackInput{Messages: []*schema.Message{{Role: schema.User, Content: "private-input"}}}) + if _, ok := tracing.ModelCallbackSpan(ctx); !ok { + t.Fatal("model callback ownership missing") + } + childCtx, child := provider.Tracer("test").Start(ctx, "child") + if _, ok := tracing.ModelCallbackSpan(childCtx); ok { + t.Fatal("model ownership leaked into a nested operation") + } + child.End() + handler.OnEndWithStreamOutput(ctx, info, schema.StreamReaderFromArray([]callbacks.CallbackOutput{ + &model.CallbackOutput{Message: &schema.Message{Role: schema.Assistant, Content: "private-output"}, TokenUsage: &model.TokenUsage{PromptTokens: 42, CompletionTokens: 8, TotalTokens: 50}}, + })) + if err := handler.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + spans := recorder.Ended() + if len(spans) != 2 { + t.Fatalf("spans=%d, want model + child", len(spans)) + } + foundUsage := false + for _, span := range spans { + for _, attr := range span.Attributes() { + if strings.Contains(attr.Value.Emit(), "private-") || string(attr.Key) == "langfuse.observation.input" || string(attr.Key) == "langfuse.observation.output" { + t.Errorf("conversation content leaked: %s", attr.Key) + } + if string(attr.Key) == "gen_ai.usage.input_tokens" && attr.Value.AsInt64() == 42 { + foundUsage = true + } + } + } + if !foundUsage { + t.Fatal("stream token usage missing") + } + _, after := provider.Tracer("test").Start(context.Background(), "after callback shutdown") + after.End() + if len(recorder.Ended()) != 3 { + t.Fatal("callback shut down the shared provider") + } +} + +func TestTraceHandlerCapturesBoundedStreamContent(t *testing.T) { + t.Setenv("ONGRID_AGENT_TRACE_CAPTURE_CONTENT", "true") + previous := otel.GetTracerProvider() + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + otel.SetTracerProvider(provider) + defer func() { otel.SetTracerProvider(previous); _ = provider.Shutdown(context.Background()) }() + handler, err := NewTraceHandler(context.Background()) + if err != nil || handler == nil { + t.Fatalf("handler=%v err=%v", handler, err) + } + info := &callbacks.RunInfo{Name: "chat", Component: components.ComponentOfChatModel} + ctx := handler.OnStart(context.Background(), info, &model.CallbackInput{Messages: []*schema.Message{ + {Role: schema.System, Content: strings.Repeat("private-system ", 10000)}, + {Role: schema.User, Content: "hello"}, + }}) + handler.OnEndWithStreamOutput(ctx, info, schema.StreamReaderFromArray([]callbacks.CallbackOutput{ + &model.CallbackOutput{Message: &schema.Message{Role: schema.Assistant, Content: strings.Repeat("输出😀", 10000), ReasoningContent: "private-reasoning"}, TokenUsage: &model.TokenUsage{PromptTokens: 42, CompletionTokens: 8, TotalTokens: 50}}, + })) + if err := handler.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + spans := recorder.Ended() + if len(spans) != 1 { + t.Fatalf("spans=%d", len(spans)) + } + contentCount := 0 + for _, attr := range spans[0].Attributes() { + if strings.Contains(attr.Value.Emit(), "private-") { + t.Fatalf("filtered content was exported: %s", attr.Key) + } + if string(attr.Key) == "langfuse.observation.input" || string(attr.Key) == "langfuse.observation.output" { + contentCount++ + if len(attr.Value.AsString()) > tracing.AgentContentMaxBytes || !json.Valid([]byte(attr.Value.AsString())) { + t.Fatalf("stream payload is not valid bounded JSON: %s", attr.Key) + } + } + } + if contentCount != 2 { + t.Fatalf("input/output attributes = %d", contentCount) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/react.go b/internal/manager/biz/aiops/runtime/v3/graph/react.go new file mode 100644 index 000000000..a1a741d22 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/react.go @@ -0,0 +1,356 @@ +package graph + +import ( + "context" + "errors" + "fmt" + "strings" + + einomodel "github.com/cloudwego/eino/components/model" + einotool "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/flow/agent/react" + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// Node names exposed by the wrapper graph. Stable so audit / SSE +// callbacks can filter by RunInfo.Name. The inner ReAct subgraph names +// are owned by eino (react.GraphName / react.ModelNodeName / +// react.ToolsNodeName) and reused so persistence / SSE handlers see +// "ChatModel" / "Tools" exactly like the canonical eino layout. +const ( + // NodeAssembler is the lambda that turns *Input into []*schema.Message. + NodeAssembler = "MessageAssembler" + + // NodeReact is the wrapped eino react.Agent subgraph. + NodeReact = "ReActSubgraph" + + // NodeProjector is the lambda that turns *schema.Message into *Output. + NodeProjector = "OutputProjector" +) + +// SystemReminderTag wraps the per-turn anti-drift block injected ahead +// of the user message. System-Reminder 周期注入 — kept as +// a constant so audit / red-team grep can locate every injected block. +// +// Block content (see buildSystemReminder below): hardcoded baseline +// constraints + per-turn dynamic hints (consecutive-failure / +// iteration-count / repeated-call heuristics, calcDynamicHints in +// chatruntime/runtime.go) + the persona criticalReminder when a worker +// persona is active. +const SystemReminderTag = "system-reminder" + +// BuildReActGraph constructs the wrapper graph + inner ReAct subgraph +// The returned Runnable accepts an *Input, +// drives the ReAct loop until either a final assistant message lands +// or MaxIterations is reached, and returns an *Output. +// +// Topology: +// +// START +// ↓ +// MessageAssembler (lambda) -- *Input → []*schema.Message +// ↓ +// ReActSubgraph (compose.AnyGraph) -- []*schema.Message → *schema.Message +// ↓ +// OutputProjector (lambda) -- *schema.Message → *Output +// ↓ +// END +// +// The ReActSubgraph is eino's stock react.Agent.ExportGraph() output. +// Internally it expands to ChatModel ↔ Branch(tool_calls?) ↔ ToolsNode +// per the diagram in — we are NOT re-implementing that +// inner shape because eino's flow/agent/react package is already the +// canonical maintained one. Callbacks (PR-6) registered via +// compose.WithCallbacks see the inner ChatModel + ToolsNode invocations +// just as if we had wired them by hand. +// +// Callbacks: handlers are NOT registered at build time. eino's graph +// compile API does not accept a Handler list — handlers ride on the +// Invoke call instead via compose.WithCallbacks(handlers...). The +// cutover layer (NEXT PR) builds the handler chain via +// callbacks.NewDefaultHandlers and threads it into Invoke; this +// function stays callback-agnostic so the same compiled graph can be +// re-used across requests with different per-call handler sets (e.g. +// per-tenant audit sinks). +func BuildReActGraph( + model einomodel.ToolCallingChatModel, + tools []basetool.BaseTool, + cfg Config, +) (compose.Runnable[*Input, *Output], error) { + if model == nil { + return nil, errors.New("graph: BuildReActGraph: model is required") + } + cfg = cfg.applyDefaults() + model = wrapBudgetStopModel(model) + + // 1. Build the inner eino react agent. We pass our adapted tools + // in via ToolsConfig.Tools — eino infers the schema by calling + // each tool's Info(ctx). + einoTools := WrapBaseTools(tools, cfg.ToolPersistence) + baseTools := make([]einotool.BaseTool, 0, len(einoTools)) + for _, t := range einoTools { + baseTools = append(baseTools, t) + } + reactCfg := &react.AgentConfig{ + ToolCallingModel: model, + ToolsConfig: compose.ToolsNodeConfig{ + Tools: baseTools, + // Without this, a single tool call for a name not in this node's + // set — a hallucinated tool, or one filtered out of a worker's + // persona bag — aborts the WHOLE run ("tool X not found in + // toolsNode indexes"). That's what crashed flow Agent nodes. Hand + // the model a recoverable tool-result instead so it picks a real + // tool (or answers directly) and the run continues. + UnknownToolsHandler: func(_ context.Context, name, _ string) (string, error) { + return fmt.Sprintf("Tool %q is not available to you. Use one of the tools provided in this conversation, or answer directly without a tool.", name), nil + }, + }, + // eino's "step" counts every graph node visit — one ReAct + // iteration is ChatModel + ToolsNode = 2 steps. So to give the + // LLM cfg.MaxIterations actual ChatModel turns we double it. + // We also leave headroom (+2) for the framing nodes + // (MessageAssembler / OutputProjector) eino counts toward the + // outer graph budget. Without this fix the LLM was capped at + // ~15 ChatModel turns when MaxIterations=30, surfacing as a + // graph ErrExceededMaxSteps mid-conversation (stream error). + MaxStep: cfg.MaxIterations*2 + 2, + GraphName: "ReActAgent", + ModelNodeName: "ChatModel", + ToolsNodeName: "Tools", + } + reactAgent, err := react.NewAgent(context.Background(), reactCfg) + if err != nil { + // Most common cause: a tool's Info(ctx) returned an unparseable + // JSON Schema (see tool_adapter.go). Wrap so the build site + // gets a clear hint without losing the underlying error. + return nil, fmt.Errorf("graph: build inner ReAct agent: %w", err) + } + + innerGraph, innerNodeOpts := reactAgent.ExportGraph() + + // 2. Build the outer wrapper graph: MessageAssembler → ReAct → OutputProjector. + g := compose.NewGraph[*Input, *Output]() + + assembler := compose.InvokableLambda(func(ctx context.Context, in *Input) ([]*schema.Message, error) { + return assembleMessages(in) + }) + if err := g.AddLambdaNode(NodeAssembler, assembler); err != nil { + return nil, fmt.Errorf("graph: add assembler node: %w", err) + } + + if err := g.AddGraphNode(NodeReact, innerGraph, innerNodeOpts...); err != nil { + return nil, fmt.Errorf("graph: add ReAct subgraph: %w", err) + } + + projector := compose.InvokableLambda(func(ctx context.Context, msg *schema.Message) (*Output, error) { + out := &Output{AssistantMessage: msg} + if msg != nil && msg.ResponseMeta != nil && msg.ResponseMeta.Usage != nil { + out.Usage.PromptTokens = msg.ResponseMeta.Usage.PromptTokens + out.Usage.CompletionTokens = msg.ResponseMeta.Usage.CompletionTokens + out.Usage.TotalTokens = msg.ResponseMeta.Usage.TotalTokens + } + // Iterations is filled in by the metrics / audit handler on the + // caller side (it has the per-graph counter); we leave it 0 here + // because the projector lambda only sees the terminal message. + // — MetricsHandler observes the ChatModel turns. + return out, nil + }) + if err := g.AddLambdaNode(NodeProjector, projector); err != nil { + return nil, fmt.Errorf("graph: add projector node: %w", err) + } + + if err := g.AddEdge(compose.START, NodeAssembler); err != nil { + return nil, err + } + if err := g.AddEdge(NodeAssembler, NodeReact); err != nil { + return nil, err + } + if err := g.AddEdge(NodeReact, NodeProjector); err != nil { + return nil, err + } + if err := g.AddEdge(NodeProjector, compose.END); err != nil { + return nil, err + } + + runnable, err := g.Compile(context.Background(), + compose.WithGraphName("OngridReActAgent"), + // Outer steps = assembler + react-subgraph + projector. The + // inner ReAct's own MaxStep was already applied in reactCfg. + compose.WithMaxRunSteps(cfg.MaxIterations+10), + ) + if err != nil { + return nil, fmt.Errorf("graph: compile wrapper: %w", err) + } + return runnable, nil +} + +// assembleMessages fans Input out into the canonical eino message +// shape: system → history → user-role → user text. +// +// Order matches agent.go's existing for-loop with the per-turn +// system-reminder block extracted into its own user-role message so +// the LLM sees it as a fresh injection on every turn (— +// claude-code's anti-drift mechanism). Specifically: +// +// - system message goes first (skipped if SystemPrompt is empty) +// - history is replayed verbatim (chatruntime strips the trailing +// user row before passing History so the model doesn't see the +// same turn twice) +// - block is its own user-role message just before +// the user text — re-injected per turn so long sessions cannot +// drift past the rules even after the system message scrolls out +// of an attention budget +// - the user text (with mentions inlined as a markdown preamble) +// follows +// +// When UserText is empty the user-role messages are skipped entirely +// (matches the legacy code path where the caller already appended the +// user turn into History). +func assembleMessages(in *Input) ([]*schema.Message, error) { + if in == nil { + return nil, errors.New("graph: assembler: nil input") + } + out := make([]*schema.Message, 0, len(in.History)+3) + // Append the response-language directive to the system prompt (one + // system message — strongest signal). The personas are Chinese, so + // without this the model answers in Chinese even in English mode. + sp := in.SystemPrompt + if dir := languageDirective(in.Locale); dir != "" { + if sp != "" { + sp += "\n\n" + } + sp += dir + } + if sp != "" { + out = append(out, schema.SystemMessage(sp)) + } + out = append(out, in.History...) + + if in.UserText != "" || len(in.UserImages) > 0 { + // 1. Per-turn system-reminder, as its own user-role message. + // — re-injected on every turn so it survives + // long-session attention drift. + if reminder := buildSystemReminder(in); reminder != "" { + out = append(out, schema.UserMessage(reminder)) + } + // 2. The actual user turn with @-mentions inlined as a + // markdown preamble (legacy agent.go format). + userBody := in.UserText + if in.MentionsRendered != "" { + userBody = in.MentionsRendered + "\n\n" + userBody + } + userMessage := schema.UserMessage(userBody) + userMessage.UserInputMultiContent = in.UserImages + out = append(out, userMessage) + } + return out, nil +} + +// buildSystemReminder returns the per-turn anti-drift reminder block +// the MessageAssembler injects ahead of the user message. +// — claude-code's per-turn `` injection adapted for +// ongrid: +// +// - hardcoded baseline rules (always present): device_id digit-id, +// no-loop-on-failing-tool, tools-are-facts, web_search gating +// - persona critical_reminder (Input.AgentReminder) — one line, when +// a worker persona is active for this turn +// - runtime-computed dynamic hints (Input.DynamicHints) — N lines, +// e.g. "tool X failed 2 times in a row", "iteration > 20, summarize +// and respond" +// +// Returns "" when there is literally nothing to inject (currently the +// hardcoded baseline guarantees at least three bullets, so this only +// happens if a future caller drops the baseline; we keep the empty +// branch so the assembler can skip the inline prepend cleanly). +// languageDirective maps a UI locale to an explicit "answer in this +// language" instruction. Empty locale → "" (no directive, back-compat). +// It covers tool-call narration explicitly because that's what drifts +// back to Chinese first when the persona is Chinese. +func languageDirective(locale string) string { + switch normalizeLocale(locale) { + case "en": + return "Respond in English. Everything you write to the user — prose, explanations, headings, and the narration around every tool call — must be in English. Tool descriptions, knowledge-base snippets, persona text, and logs may be in Chinese; render their MEANING in English and never echo raw Chinese to the user. Translate domain terms to their English equivalents (e.g. \"0号病人\" → \"patient zero\", \"根因\" → \"root cause\", \"告警\" → \"alert\", \"巡检\" → \"inspection\"). Leave only proper nouns, identifiers, hostnames, file paths, code, and raw command output verbatim." + case "zh": + return "用中文回复：你的所有叙述、解释、标题，以及每次工具调用前后的说明都必须用中文。工具描述、知识库片段、日志可能是英文，把含义用中文表达即可；标识符、主机名、文件路径、代码、命令原始输出保持原样。" + } + return "" +} + +func reminderLanguageDirective(locale string) string { + switch normalizeLocale(locale) { + case "en": + return "Respond in English; translate Chinese prompt/tool context by meaning, but keep identifiers/paths/commands verbatim." + case "zh": + return "用中文回复；标识符、主机名、路径、代码和命令输出保持原样。" + } + return "" +} + +func normalizeLocale(locale string) string { + l := strings.ToLower(strings.TrimSpace(locale)) + l = strings.ReplaceAll(l, "_", "-") + switch { + case strings.HasPrefix(l, "en"): + return "en" + case strings.HasPrefix(l, "zh"): + return "zh" + default: + return "" + } +} + +func buildSystemReminder(in *Input) string { + if in == nil { + return "" + } + lines := []string{} + if normalizeLocale(in.Locale) == "en" { + lines = append(lines, + "- If the same tool fails twice, change approach instead of repeating it.", + "- device_id / alert_id must be numeric IDs (@-mentions have already been resolved).", + "- For logs or traces on a named device without a numeric device_id, call query_devices once first, then pass device_id to the query tool. Do not guess hostname/instance labels.", + "- For device-scoped logs, use query_logql with device_id plus a stream selector and line filter; do not invent backend labels or DSL. Summarize once matching lines return.", + "- Tool results are facts; do not invent data when evidence is missing.", + "- call_budget_exceeded only applies to the current user turn; a new user message may call tools again.", + ) + } else { + lines = append(lines, + "- 同一工具失败两次后请换思路，不要重复调用", + "- device_id / alert_id 必须是数字 ID（@-mention 已经为你解析）", + "- 按名称查询设备日志或链路时，若没有数字 device_id，先调用一次 query_devices，再把 device_id 传给查询工具；不要猜 hostname/instance 标签", + "- 设备日志用 query_logql 传 device_id，并使用流选择器和行过滤；不要猜后端标签或 DSL，已有命中后直接总结", + "- 工具结果是事实，不要在没有数据时编造", + "- call_budget_exceeded 只限制当前用户消息；新消息可重新调用工具", + ) + } + // Re-assert the response language every turn (system prompt scrolls + // out of attention in long sessions). Prepended so it leads the block. + if dir := reminderLanguageDirective(in.Locale); dir != "" { + lines = append([]string{"- " + dir}, lines...) + } + if !in.WebSearchEnabled { + lines = append(lines, "- web_search 已被关闭，本轮不要调用") + } + if r := strings.TrimSpace(in.AgentReminder); r != "" { + lines = append(lines, "- "+r) + } + for _, h := range in.DynamicHints { + h = strings.TrimSpace(h) + if h == "" { + continue + } + lines = append(lines, "- "+h) + } + if len(lines) == 0 { + return "" + } + parts := make([]string, 0, len(lines)+2) + parts = append(parts, "<"+SystemReminderTag+">") + parts = append(parts, lines...) + parts = append(parts, "") + return strings.Join(parts, "\n") +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/react_test.go b/internal/manager/biz/aiops/runtime/v3/graph/react_test.go new file mode 100644 index 000000000..603c4f30b --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/react_test.go @@ -0,0 +1,670 @@ +package graph + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +func TestAssembleMessagesAddsCurrentTurnImages(t *testing.T) { + data := "aGVsbG8=" + out, err := assembleMessages(&Input{ + UserText: "describe", + UserImages: []schema.MessageInputPart{{ + Type: schema.ChatMessagePartTypeImageURL, + Image: &schema.MessageInputImage{MessagePartCommon: schema.MessagePartCommon{Base64Data: &data, MIMEType: "image/png"}}, + }}, + }) + if err != nil { + t.Fatalf("assembleMessages: %v", err) + } + last := out[len(out)-1] + if last.Content != "describe" || len(last.UserInputMultiContent) != 1 { + t.Fatalf("unexpected current turn: %+v", last) + } +} + +// scriptedState is the shared mutable bookkeeping for scriptedChatModel +// instances. eino's ToolCallingChatModel.WithTools returns a *new* +// instance with tools bound, which means the test must share counters / +// reply pointers between the original and any derived instance — the +// inner ReAct agent calls .WithTools then dispatches to the new copy. +type scriptedState struct { + mu sync.Mutex + replies []*schema.Message + idx int + generateCalls atomic.Int32 + genErr error + inputsLog [][]*schema.Message +} + +// scriptedChatModel is a tool-calling ChatModel test double used by +// the react_test integration tests. It returns one *schema.Message per +// Generate call from the `replies` list, in order; once exhausted, it +// returns the last reply (so loops with MaxIterations work as expected). +// +// Multiple scriptedChatModel instances may share a single *scriptedState +// (eino's WithTools copies the receiver) — counters must therefore be +// stored on the shared state, not the wrapping instance. +type scriptedChatModel struct { + st *scriptedState + + // boundTools is captured by WithTools so the test can assert the + // tool list shape was forwarded. Lives on the per-instance struct + // because each derivation has its own bound tool list. + boundTools []*schema.ToolInfo +} + +func newScriptedChatModel(replies ...*schema.Message) *scriptedChatModel { + return &scriptedChatModel{st: &scriptedState{replies: replies}} +} + +func (s *scriptedChatModel) Generate(_ context.Context, input []*schema.Message, _ ...model.Option) (*schema.Message, error) { + s.st.generateCalls.Add(1) + s.st.mu.Lock() + cp := make([]*schema.Message, len(input)) + copy(cp, input) + s.st.inputsLog = append(s.st.inputsLog, cp) + if s.st.genErr != nil { + err := s.st.genErr + s.st.mu.Unlock() + return nil, err + } + if len(s.st.replies) == 0 { + s.st.mu.Unlock() + return &schema.Message{Role: schema.Assistant, Content: "ok"}, nil + } + if s.st.idx < len(s.st.replies) { + out := s.st.replies[s.st.idx] + s.st.idx++ + s.st.mu.Unlock() + return out, nil + } + out := s.st.replies[len(s.st.replies)-1] + s.st.mu.Unlock() + return out, nil +} + +func (s *scriptedChatModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { + msg, err := s.Generate(ctx, input, opts...) + if err != nil { + return nil, err + } + return schema.StreamReaderFromArray([]*schema.Message{msg}), nil +} + +func (s *scriptedChatModel) BindTools(tools []*schema.ToolInfo) error { + s.boundTools = tools + return nil +} + +// WithTools returns a NEW instance that shares the underlying state +// pointer so Generate counters / reply cursor are observable from the +// original receiver after the inner ReAct agent dispatches through the +// derived copy. +func (s *scriptedChatModel) WithTools(tools []*schema.ToolInfo) (model.ToolCallingChatModel, error) { + return &scriptedChatModel{st: s.st, boundTools: tools}, nil +} + +func (s *scriptedChatModel) generateCalls() int32 { return s.st.generateCalls.Load() } + +func makeAssistantNoTools(content string) *schema.Message { + return &schema.Message{Role: schema.Assistant, Content: content} +} + +func makeAssistantToolCall(content, callID, toolName, args string) *schema.Message { + return &schema.Message{ + Role: schema.Assistant, + Content: content, + ToolCalls: []schema.ToolCall{ + { + ID: callID, + Type: "function", + Function: schema.FunctionCall{Name: toolName, Arguments: args}, + }, + }, + } +} + +func TestBuildReActGraph_FinalReplyOneTurn(t *testing.T) { + t.Parallel() + scripted := newScriptedChatModel(makeAssistantNoTools("hello world")) + tool := &fakeBaseTool{name: "noop", parameters: `{"type":"object","properties":{}}`} + g, err := BuildReActGraph(scripted, []basetool.BaseTool{tool}, Config{}) + if err != nil { + t.Fatalf("BuildReActGraph: %v", err) + } + out, err := g.Invoke(context.Background(), &Input{ + SystemPrompt: "be helpful", + UserText: "hi", + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if out == nil || out.AssistantMessage == nil { + t.Fatalf("expected non-nil output") + } + if out.AssistantMessage.Content != "hello world" { + t.Errorf("content = %q, want hello world", out.AssistantMessage.Content) + } + if scripted.generateCalls() != 1 { + t.Errorf("Generate calls = %d, want 1", scripted.generateCalls()) + } +} + +func TestBuildReActGraph_ToolCallThenFinal(t *testing.T) { + t.Parallel() + scripted := newScriptedChatModel( + makeAssistantToolCall("", "call_1", "echo", `{"a":1}`), + makeAssistantNoTools("done"), + ) + echo := &fakeBaseTool{ + name: "echo", + parameters: `{"type":"object","properties":{"a":{"type":"integer"}}}`, + runResp: `{"echoed":1}`, + } + g, err := BuildReActGraph(scripted, []basetool.BaseTool{echo}, Config{MaxIterations: 5}) + if err != nil { + t.Fatalf("BuildReActGraph: %v", err) + } + out, err := g.Invoke(context.Background(), &Input{UserText: "do it"}) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if out.AssistantMessage.Content != "done" { + t.Errorf("content = %q, want done", out.AssistantMessage.Content) + } + if echo.calls.Load() != 1 { + t.Errorf("tool calls = %d, want 1", echo.calls.Load()) + } + if scripted.generateCalls() != 2 { + t.Errorf("Generate calls = %d, want 2", scripted.generateCalls()) + } +} + +func TestBuildReActGraph_PersistsNestedToolInvocation(t *testing.T) { + t.Parallel() + scripted := newScriptedChatModel( + makeAssistantToolCall("", "call_host_bash", "host_bash", `{"cmd":"docker images"}`), + makeAssistantNoTools("done"), + ) + hostBash := &fakeBaseTool{ + name: "host_bash", + class: "read", + parameters: `{"type":"object","properties":{"cmd":{"type":"string"}}}`, + runResp: `{"ok":true}`, + } + sink := &recordingToolPersistence{} + g, err := BuildReActGraph(scripted, []basetool.BaseTool{hostBash}, Config{MaxIterations: 5, ToolPersistence: sink}) + if err != nil { + t.Fatalf("BuildReActGraph: %v", err) + } + out, err := g.Invoke(context.Background(), &Input{UserText: "run docker images"}) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if out == nil || out.AssistantMessage == nil || out.AssistantMessage.Content != "done" { + t.Fatalf("terminal reply = %+v, want done", out) + } + starts, ends := sink.records() + if len(starts) != 1 || starts[0] != `host_bash:{"cmd":"docker images"}` { + t.Fatalf("start records = %v", starts) + } + if len(ends) != 1 || ends[0] != `host_bash:{"ok":true}` { + t.Fatalf("end records = %v", ends) + } +} + +func TestBuildReActGraph_RecoversFromToolError(t *testing.T) { + t.Parallel() + scripted := newScriptedChatModel( + makeAssistantToolCall("", "call_1", "flaky", `{}`), + makeAssistantNoTools("recovered"), + ) + flaky := &fakeBaseTool{ + name: "flaky", + parameters: `{"type":"object","properties":{}}`, + runErr: errors.New("temporary"), + } + g, err := BuildReActGraph(scripted, []basetool.BaseTool{flaky}, Config{MaxIterations: 5}) + if err != nil { + t.Fatalf("BuildReActGraph: %v", err) + } + out, err := g.Invoke(context.Background(), &Input{UserText: "do it"}) + // einoToolAdapter wraps tool errors as JSON envelopes ( + // invariant: tool failures are facts the LLM consumes, not graph- + // fatal events). ToolsNode therefore returns successfully with the + // envelope in the tool message; ChatModel then runs the second + // scripted reply ("recovered") and the graph completes cleanly. + if err != nil { + t.Fatalf("Invoke should recover from tool error, got %v", err) + } + if out == nil || out.AssistantMessage == nil || !strings.Contains(out.AssistantMessage.Content, "recovered") { + t.Errorf("expected recovered final reply, got %+v", out) + } + if flaky.calls.Load() != 1 { + t.Errorf("tool calls = %d, want 1 (one attempt then ChatModel recovers)", flaky.calls.Load()) + } + if scripted.generateCalls() != 2 { + t.Errorf("ChatModel generate calls = %d, want 2 (initial tool_call turn + recovery turn)", scripted.generateCalls()) + } +} + +func TestBudgetStopModel_FinalizesAfterToolBudget(t *testing.T) { + t.Parallel() + inner := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "现有日志证据表明 api-gateway 错误率升高，但时间窗仍不足以确认根因。", + }) + wrapped := wrapBudgetStopModel(inner) + got, err := wrapped.Generate(context.Background(), []*schema.Message{ + schema.UserMessage("查日志"), + schema.ToolMessage(`{"status":"call_budget_exceeded","tool":"query_logql","final_answer_required":true}`, "call_1", schema.WithToolName("query_logql")), + schema.UserMessage("stop loops"), + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if got.Role != schema.Assistant || len(got.ToolCalls) != 0 { + t.Fatalf("budget stop response = %+v, want assistant without tool calls", got) + } + if !strings.Contains(got.Content, "错误率升高") { + t.Fatalf("budget stop did not use model synthesis: %q", got.Content) + } + if inner.st.generateCalls.Load() != 1 { + t.Fatalf("inner model was called %d time(s), want 1", inner.st.generateCalls.Load()) + } +} + +func TestBudgetStopModel_IgnoresPriorTurnToolBudget(t *testing.T) { + t.Parallel() + inner := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + Content: "new turn answered", + }) + wrapped := wrapBudgetStopModel(inner) + got, err := wrapped.Generate(context.Background(), []*schema.Message{ + schema.UserMessage("查日志"), + schema.ToolMessage(`{"status":"call_budget_exceeded","tool":"query_logql","final_answer_required":true}`, "call_1", schema.WithToolName("query_logql")), + schema.UserMessage("改查最近 10 分钟 api-gateway"), + schema.UserMessage("stop loops"), + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if got.Role != schema.Assistant || got.Content != "new turn answered" { + t.Fatalf("budget stop response = %+v, want inner model response", got) + } + if inner.st.generateCalls.Load() != 1 { + t.Fatalf("inner model was called %d time(s), want 1", inner.st.generateCalls.Load()) + } +} + +func TestBudgetStopModel_PerToolCountResetsAtNewUserTurn(t *testing.T) { + t.Parallel() + inner := newScriptedChatModel(makeAssistantToolCall("继续查询", "call_new_turn", "query_promql", `{}`)) + wrapped := wrapBudgetStopModel(inner) + history := []*schema.Message{schema.UserMessage("上一轮查询")} + for i := 0; i < maxCallsForTool("query_promql"); i++ { + history = append(history, schema.ToolMessage(`{"resultType":"vector","result":[]}`, "call_old_"+string(rune('a'+i)), schema.WithToolName("query_promql"))) + } + history = append(history, schema.UserMessage("新一轮继续查询另一个时间窗")) + + got, err := wrapped.Generate(context.Background(), history) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if len(got.ToolCalls) != 1 || got.ToolCalls[0].Function.Name != "query_promql" { + t.Fatalf("new user turn should have a fresh per-tool budget: %+v", got) + } +} + +func TestBudgetStopModel_PrunesSameBatchRepeatedToolCalls(t *testing.T) { + t.Parallel() + calls := make([]schema.ToolCall, 0, maxCallsForTool("query_k8s_snapshot")+3) + for i := 0; i < maxCallsForTool("query_k8s_snapshot")+3; i++ { + calls = append(calls, schema.ToolCall{ + ID: "call_k8s_" + string(rune('a'+i)), + Function: schema.FunctionCall{Name: "query_k8s_snapshot", Arguments: `{}`}, + }) + } + inner := newScriptedChatModel(&schema.Message{Role: schema.Assistant, ToolCalls: calls}) + wrapped := wrapBudgetStopModel(inner) + got, err := wrapped.Generate(context.Background(), []*schema.Message{ + schema.UserMessage("找 Kubernetes 异常 Pod"), + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if len(got.ToolCalls) != maxCallsForTool("query_k8s_snapshot") { + t.Fatalf("tool calls = %d, want %d", len(got.ToolCalls), maxCallsForTool("query_k8s_snapshot")) + } +} + +func TestBudgetStopModel_DoesNotApplyAggregateToolBudget(t *testing.T) { + t.Parallel() + const distinctTools = 24 + calls := make([]schema.ToolCall, 0, distinctTools) + for i := 0; i < distinctTools; i++ { + calls = append(calls, schema.ToolCall{ + ID: "call_tool_" + string(rune('a'+i)), + Function: schema.FunctionCall{Name: "tool_" + string(rune('a'+i)), Arguments: `{}`}, + }) + } + inner := newScriptedChatModel(&schema.Message{Role: schema.Assistant, ToolCalls: calls}) + wrapped := wrapBudgetStopModel(inner) + got, err := wrapped.Generate(context.Background(), []*schema.Message{ + schema.UserMessage("做一次宽排查"), + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if len(got.ToolCalls) != distinctTools { + t.Fatalf("tool calls = %d, want all %d distinct tools", len(got.ToolCalls), distinctTools) + } +} + +func TestBudgetStopModel_PerToolLimitUsesModelSynthesisWithoutRawJSON(t *testing.T) { + t.Parallel() + inner := newScriptedChatModel( + &schema.Message{Role: schema.Assistant, ToolCalls: []schema.ToolCall{{ + ID: "call_extra", + Function: schema.FunctionCall{Name: "query_promql", Arguments: `{}`}, + }}}, + &schema.Message{Role: schema.Assistant, Content: "本周 CPU 均值证据存在，但 Prometheus 保留期不足，无法可靠计算上周对比。"}, + ) + wrapped := wrapBudgetStopModel(inner) + history := []*schema.Message{schema.UserMessage("对比本周和上周 CPU")} + for i := 0; i < maxCallsForTool("query_promql"); i++ { + history = append(history, schema.ToolMessage(`{"resultType":"matrix","result":[{"values":[[1787212275.995,"0.006107340485348943"]]}]}`, "call_prom_"+string(rune('a'+i)), schema.WithToolName("query_promql"))) + } + + got, err := wrapped.Generate(context.Background(), history) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if len(got.ToolCalls) != 0 { + t.Fatalf("tool calls = %d, want 0", len(got.ToolCalls)) + } + if !strings.Contains(got.Content, "Prometheus 保留期不足") { + t.Fatalf("content missing semantic synthesis: %q", got.Content) + } + if strings.Contains(got.Content, "1787212275") || strings.Contains(got.Content, "resultType") { + t.Fatalf("content leaked raw PromQL evidence: %q", got.Content) + } + if inner.generateCalls() != 2 { + t.Fatalf("model calls = %d, want initial decision plus synthesis", inner.generateCalls()) + } +} + +func TestBudgetStopModel_PerToolLimitAllowsDifferentEvidenceTool(t *testing.T) { + t.Parallel() + inner := newScriptedChatModel( + makeAssistantToolCall("继续查询指标", "call_prom_over", "query_promql", `{}`), + makeAssistantToolCall("改查同期告警", "call_alerts", "query_incidents", `{}`), + ) + wrapped := wrapBudgetStopModel(inner) + history := []*schema.Message{schema.UserMessage("做一次跨信号深度排查")} + for i := 0; i < maxCallsForTool("query_promql"); i++ { + history = append(history, schema.ToolMessage(`{"resultType":"vector","result":[]}`, "call_metric_"+string(rune('a'+i)), schema.WithToolName("query_promql"))) + } + + got, err := wrapped.Generate(context.Background(), history) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if len(got.ToolCalls) != 1 || got.ToolCalls[0].Function.Name != "query_incidents" { + t.Fatalf("different evidence tool should remain available: %+v", got) + } +} + +func TestBudgetStopModel_FallbackIsLocalizedAndNeverLeaksRawJSON(t *testing.T) { + t.Parallel() + retryCall := makeAssistantToolCall("", "call_retry", "query_promql", `{}`) + inner := newScriptedChatModel(retryCall, retryCall) + wrapped := wrapBudgetStopModel(inner) + history := []*schema.Message{ + schema.SystemMessage("Respond in English."), + schema.UserMessage("Compare CPU week over week"), + } + for i := 0; i < maxCallsForTool("query_promql"); i++ { + history = append(history, schema.ToolMessage(`{"resultType":"matrix","result":[{"values":[[1787212275.995,"0.006107340485348943"]]}]}`, "call_en_"+string(rune('a'+i)), schema.WithToolName("query_promql"))) + } + + got, err := wrapped.Generate(context.Background(), history) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if len(got.ToolCalls) != 0 || !strings.Contains(got.Content, "No raw tool output is shown") { + t.Fatalf("fallback = %+v", got) + } + if strings.Contains(got.Content, "1787212275") || strings.Contains(got.Content, "resultType") { + t.Fatalf("fallback leaked raw evidence: %q", got.Content) + } +} + +func TestBudgetStopModel_BlocksHostToolAfterEmptyNamedDeviceLookup(t *testing.T) { + t.Parallel() + inner := newScriptedChatModel(&schema.Message{ + Role: schema.Assistant, + ToolCalls: []schema.ToolCall{{ + ID: "call_processes", + Function: schema.FunctionCall{Name: "get_host_processes", Arguments: `{"device_ids":[1],"top_n":5}`}, + }}, + }) + wrapped := wrapBudgetStopModel(inner) + got, err := wrapped.Generate(context.Background(), []*schema.Message{ + schema.UserMessage("找 edge-001 最占内存的进程"), + schema.ToolMessage(`{"count":0,"items":[]}`, "call_devices", schema.WithToolName("query_devices")), + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if len(got.ToolCalls) != 0 { + t.Fatalf("tool calls = %d, want blocked", len(got.ToolCalls)) + } + if !strings.Contains(got.Content, "edge-001") || !strings.Contains(got.Content, "不会自动改用其他在线设备") { + t.Fatalf("content = %q, want explicit no-fallback clarification", got.Content) + } +} + +func TestBuildReActGraph_NilModelFails(t *testing.T) { + t.Parallel() + if _, err := BuildReActGraph(nil, nil, Config{}); err == nil { + t.Fatalf("expected error for nil model") + } +} + +func TestBuildReActGraph_BadToolSchemaFailsBuild(t *testing.T) { + t.Parallel() + scripted := newScriptedChatModel() + bad := &fakeBaseTool{name: "bad", parameters: `{not json`} + if _, err := BuildReActGraph(scripted, []basetool.BaseTool{bad}, Config{}); err == nil { + t.Fatalf("expected build to fail when tool schema parsing fails") + } +} + +func TestAssembleMessages_MentionsAndReminderInline(t *testing.T) { + t.Parallel() + in := &Input{ + SystemPrompt: "agent rules", + UserText: "find big files on edge-1", + MentionsRendered: "- edge-1: device_id=42 (online)", + WebSearchEnabled: false, + } + out, err := assembleMessages(in) + if err != nil { + t.Fatalf("assembleMessages: %v", err) + } + // system + reminder (user) + user (mentions+text) = 3 + if len(out) != 3 { + t.Fatalf("expected 3 messages, got %d (%+v)", len(out), out) + } + if out[0].Role != schema.System || out[0].Content != "agent rules" { + t.Errorf("first message should be system: got role=%s content=%q", out[0].Role, out[0].Content) + } + reminder := out[1] + if reminder.Role != schema.User { + t.Errorf("reminder role = %s, want user", reminder.Role) + } + if !strings.HasPrefix(reminder.Content, "") || !strings.HasSuffix(reminder.Content, "") { + t.Errorf("reminder message is not a bare block: %q", reminder.Content) + } + if !strings.Contains(reminder.Content, "web_search 已被关闭") { + t.Errorf("reminder missing web_search disabled note: %q", reminder.Content) + } + if !strings.Contains(reminder.Content, "call_budget_exceeded 只限制当前用户消息") { + t.Errorf("reminder missing per-turn budget reset note: %q", reminder.Content) + } + user := out[2] + if user.Role != schema.User { + t.Errorf("last role should be user, got %s", user.Role) + } + if strings.Contains(user.Content, "") { + t.Errorf("user message must NOT inline the reminder block (it is now a separate message): %q", user.Content) + } + if !strings.Contains(user.Content, "edge-1: device_id=42") { + t.Errorf("user content missing mention block: %q", user.Content) + } + if !strings.Contains(user.Content, "find big files on edge-1") { + t.Errorf("user content missing the actual user text: %q", user.Content) + } +} + +func TestAssembleMessages_AgentReminderAndDynamicHints(t *testing.T) { + t.Parallel() + in := &Input{ + SystemPrompt: "agent rules", + UserText: "继续诊断", + WebSearchEnabled: true, + AgentReminder: "结论先于过程，最多列 3 条假设", + DynamicHints: []string{ + "注意: query_logql 已连续失败 2 次，请换工具或要求用户澄清", + "已经跑了 22 轮，如不能在下一轮给出最终回答应当总结当前已知信息回答用户", + }, + } + out, err := assembleMessages(in) + if err != nil { + t.Fatalf("assembleMessages: %v", err) + } + if len(out) != 3 { + t.Fatalf("expected 3 messages got %d", len(out)) + } + rem := out[1].Content + if !strings.Contains(rem, "结论先于过程") { + t.Errorf("reminder missing AgentReminder bullet: %q", rem) + } + if !strings.Contains(rem, "query_logql 已连续失败 2 次") { + t.Errorf("reminder missing dynamic-hint failure bullet: %q", rem) + } + if !strings.Contains(rem, "已经跑了 22 轮") { + t.Errorf("reminder missing dynamic-hint iteration bullet: %q", rem) + } + // web_search ON -> the disabled-note bullet must NOT be present. + if strings.Contains(rem, "web_search 已被关闭") { + t.Errorf("reminder should not mention web_search disabled when WebSearchEnabled=true: %q", rem) + } +} + +func TestAssembleMessages_LocaleDirective(t *testing.T) { + t.Parallel() + // en-US: both the system message and the per-turn reminder must carry + // an explicit English directive (personas are Chinese). + en, err := assembleMessages(&Input{SystemPrompt: "agent rules", UserText: "status?", Locale: "en-US"}) + if err != nil { + t.Fatalf("assembleMessages: %v", err) + } + if !strings.Contains(en[0].Content, "agent rules") || !strings.Contains(en[0].Content, "Respond in English") { + t.Errorf("en system message should append the English directive: %q", en[0].Content) + } + if !strings.Contains(en[1].Content, "Respond in English") { + t.Errorf("en reminder should carry the English directive: %q", en[1].Content) + } + + // zh-CN: Chinese directive instead. + zh, _ := assembleMessages(&Input{SystemPrompt: "agent rules", UserText: "状态？", Locale: "zh-CN"}) + if !strings.Contains(zh[0].Content, "用中文回复") { + t.Errorf("zh system message should append the Chinese directive: %q", zh[0].Content) + } + + // empty locale: no directive at all (back-compat for non-SPA callers + // like the IM bridge — system prompt is left exactly as-is). + none, _ := assembleMessages(&Input{SystemPrompt: "agent rules", UserText: "x"}) + if none[0].Content != "agent rules" { + t.Errorf("empty locale must not alter the system prompt: %q", none[0].Content) + } + if strings.Contains(none[1].Content, "Respond in English") || strings.Contains(none[1].Content, "用中文回复") { + t.Errorf("empty locale reminder must carry no language directive: %q", none[1].Content) + } +} + +func TestBuildSystemReminder_EmptyHintsTrimmed(t *testing.T) { + t.Parallel() + got := buildSystemReminder(&Input{ + WebSearchEnabled: true, + AgentReminder: " ", // all whitespace -> dropped + DynamicHints: []string{"", " ", "real hint"}, + }) + if !strings.Contains(got, "real hint") { + t.Errorf("real hint missing from reminder: %q", got) + } + // AgentReminder was whitespace-only -> no extra empty bullet. + if strings.Contains(got, "- \n") || strings.Contains(got, "- \n") { + t.Errorf("empty bullet leaked: %q", got) + } +} + +func TestAssembleMessages_HistoryReplay(t *testing.T) { + t.Parallel() + in := &Input{ + SystemPrompt: "be cool", + History: []*schema.Message{ + {Role: schema.User, Content: "earlier"}, + {Role: schema.Assistant, Content: "earlier reply"}, + }, + UserText: "follow up", + } + out, err := assembleMessages(in) + if err != nil { + t.Fatalf("assembleMessages: %v", err) + } + // system + 2 history + 1 reminder (user) + 1 user = 5 + if len(out) != 5 { + t.Fatalf("expected 5 messages got %d", len(out)) + } + if out[1].Content != "earlier" { + t.Errorf("history user content lost: %+v", out[1]) + } + if !strings.HasPrefix(out[3].Content, "") { + t.Errorf("expected reminder at index 3: %q", out[3].Content) + } + if out[4].Content != "follow up" { + t.Errorf("user text lost: %q", out[4].Content) + } +} + +func TestAssembleMessages_NilInputFails(t *testing.T) { + t.Parallel() + if _, err := assembleMessages(nil); err == nil { + t.Fatalf("expected error for nil input") + } +} + +func TestConfig_Defaults(t *testing.T) { + t.Parallel() + c := Config{}.applyDefaults() + if c.MaxIterations != 12 { + t.Errorf("MaxIterations default = %d, want 12", c.MaxIterations) + } + if c.ToolTimeout.Seconds() != 15 { + t.Errorf("ToolTimeout default = %v, want 15s", c.ToolTimeout) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/token_budget_test.go b/internal/manager/biz/aiops/runtime/v3/graph/token_budget_test.go new file mode 100644 index 000000000..496b8ecd1 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/token_budget_test.go @@ -0,0 +1,86 @@ +package graph + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +func TestBuildReActGraph_TokenBudgetStopsModelCalls(t *testing.T) { + t.Parallel() + for _, mode := range []string{"invoke", "stream"} { + for _, tc := range []struct { + name string + limit int + toolBudget bool + wantReject bool + }{ + {name: "exhausted", limit: 1, wantReject: true}, + {name: "exhausted_before_synthesis", limit: 1, toolBudget: true, wantReject: true}, + {name: "available", limit: 100000}, + {name: "available_for_synthesis", limit: 100000, toolBudget: true}, + {name: "unlimited", limit: 0}, + } { + t.Run(mode+"/"+tc.name, func(t *testing.T) { + t.Parallel() + inner := newScriptedChatModel(makeAssistantNoTools("diagnosis complete")) + g, err := BuildReActGraph(inner, nil, Config{}) + if err != nil { + t.Fatal(err) + } + handler := llm.NewBudgetCallbackHandler(llm.NewInMemoryBudget(tc.limit), 0) + input := &Input{UserText: "Check the service health and summarize the evidence."} + if tc.toolBudget { + input = &Input{History: []*schema.Message{ + schema.UserMessage(input.UserText), + makeAssistantToolCall("", "call_1", "query_logql", `{}`), + schema.ToolMessage(`{"status":"call_budget_exceeded","tool":"query_logql","final_answer_required":true}`, "call_1", schema.WithToolName("query_logql")), + }} + } + err = runBudgetGraph(context.Background(), g, input, mode, compose.WithCallbacks(handler)) + wantCalls := int32(1) + if tc.wantReject { + wantCalls = 0 + if !errors.Is(err, llm.ErrBudgetExceeded) { + t.Errorf("error = %v, want ErrBudgetExceeded", err) + } + if handler.Stats().Rejects != 1 { + t.Errorf("budget stats = %+v, want one rejection", handler.Stats()) + } + } else if err != nil { + t.Errorf("unexpected error: %v", err) + } + if calls := inner.generateCalls(); calls != wantCalls { + t.Errorf("underlying model calls = %d, want %d", calls, wantCalls) + } + }) + } + } +} + +func runBudgetGraph(ctx context.Context, g compose.Runnable[*Input, *Output], input *Input, mode string, opts ...compose.Option) error { + if mode == "invoke" { + _, err := g.Invoke(ctx, input, opts...) + return err + } + stream, err := g.Stream(ctx, input, opts...) + if err != nil { + return err + } + defer stream.Close() + for { + _, err := stream.Recv() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + } +} diff --git a/internal/manager/biz/aiops/graph/tool_adapter.go b/internal/manager/biz/aiops/runtime/v3/graph/tool_adapter.go similarity index 100% rename from internal/manager/biz/aiops/graph/tool_adapter.go rename to internal/manager/biz/aiops/runtime/v3/graph/tool_adapter.go diff --git a/internal/manager/biz/aiops/runtime/v3/graph/tool_adapter_test.go b/internal/manager/biz/aiops/runtime/v3/graph/tool_adapter_test.go new file mode 100644 index 000000000..33eeccb0a --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/tool_adapter_test.go @@ -0,0 +1,690 @@ +package graph + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// fakeBaseTool is the test double used by tool_adapter + react tests. +// It records every invocation so assertions can inspect args + opts. +type fakeBaseTool struct { + name string + desc string + whenToUse string + parameters string // raw JSON Schema + class string + infoErr error + runErr error + runResp string + calls atomic.Int32 + lastArgs string + lastOptsLen int +} + +type recordingToolPersistence struct { + mu sync.Mutex + starts []string + ends []string +} + +type concurrentCountTool struct { + name string + class string + calls atomic.Int32 +} + +func (t *concurrentCountTool) Info(_ context.Context) (*basetool.ToolInfo, error) { + return &basetool.ToolInfo{ + Name: t.name, + Class: t.class, + Parameters: json.RawMessage(`{"type":"object","properties":{}}`), + }, nil +} + +func (t *concurrentCountTool) InvokableRun(_ context.Context, _ string, _ ...basetool.InvokeOption) (string, error) { + t.calls.Add(1) + return `{"ok":true}`, nil +} + +func (r *recordingToolPersistence) PersistToolStart(_ context.Context, name, args string) { + r.mu.Lock() + defer r.mu.Unlock() + r.starts = append(r.starts, name+":"+args) +} + +func (r *recordingToolPersistence) PersistToolEnd(_ context.Context, name, response string) { + r.mu.Lock() + defer r.mu.Unlock() + r.ends = append(r.ends, name+":"+response) +} + +func (r *recordingToolPersistence) records() (starts, ends []string) { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.starts...), append([]string(nil), r.ends...) +} + +func (f *fakeBaseTool) Info(_ context.Context) (*basetool.ToolInfo, error) { + if f.infoErr != nil { + return nil, f.infoErr + } + return &basetool.ToolInfo{ + Name: f.name, + Description: f.desc, + WhenToUse: f.whenToUse, + Parameters: json.RawMessage(f.parameters), + Class: f.class, + }, nil +} + +func (f *fakeBaseTool) InvokableRun(_ context.Context, argsJSON string, opts ...basetool.InvokeOption) (string, error) { + f.calls.Add(1) + f.lastArgs = argsJSON + f.lastOptsLen = len(opts) + if f.runErr != nil { + return "", f.runErr + } + return f.runResp, nil +} + +func TestWrapBaseTool_NilInner(t *testing.T) { + t.Parallel() + if got := WrapBaseTool(nil); got != nil { + t.Fatalf("WrapBaseTool(nil) = %v, want nil", got) + } +} + +func TestWrapBaseTool_InfoMergesWhenToUse(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{ + name: "search", + desc: "search the web", + whenToUse: "user explicitly asks the public web", + parameters: `{"type":"object","properties":{"q":{"type":"string"}},"required":["q"]}`, + } + wrapped := WrapBaseTool(inner) + info, err := wrapped.Info(context.Background()) + if err != nil { + t.Fatalf("Info: %v", err) + } + if info.Name != "search" { + t.Errorf("Name = %q, want search", info.Name) + } + if !strings.Contains(info.Desc, "search the web") { + t.Errorf("Desc missing description body: %q", info.Desc) + } + if !strings.Contains(info.Desc, "When to use") { + t.Errorf("Desc missing when-to-use header: %q", info.Desc) + } + if !strings.Contains(info.Desc, "user explicitly asks the public web") { + t.Errorf("Desc missing when-to-use body: %q", info.Desc) + } + if info.ParamsOneOf == nil { + t.Errorf("ParamsOneOf is nil; expected populated") + } +} + +func TestWrapBaseTool_InfoNoWhenToUse(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "ping", desc: "ping a host"} + wrapped := WrapBaseTool(inner) + info, err := wrapped.Info(context.Background()) + if err != nil { + t.Fatalf("Info: %v", err) + } + if info.Desc != "ping a host" { + t.Errorf("Desc = %q, want %q", info.Desc, "ping a host") + } +} + +func TestWrapBaseTool_InfoEmptyParameters(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "noargs"} + wrapped := WrapBaseTool(inner) + info, err := wrapped.Info(context.Background()) + if err != nil { + t.Fatalf("Info: %v", err) + } + if info.ParamsOneOf != nil { + t.Errorf("ParamsOneOf should be nil when no parameters declared") + } +} + +func TestWrapBaseTool_InfoBadJSONSchemaFails(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "bad", parameters: `{not valid json`} + wrapped := WrapBaseTool(inner) + if _, err := wrapped.Info(context.Background()); err == nil { + t.Fatalf("expected Info to fail on invalid JSON Schema") + } +} + +func TestWrapBaseTool_InfoBubbleInnerErr(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "x", infoErr: errors.New("boom")} + wrapped := WrapBaseTool(inner) + if _, err := wrapped.Info(context.Background()); err == nil { + t.Fatalf("expected Info to bubble inner error") + } +} + +func TestWrapBaseTool_InvokableRunForwardsArgs(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "echo", runResp: `{"ok":true}`} + wrapped := WrapBaseTool(inner) + out, err := wrapped.InvokableRun(context.Background(), `{"a":1}`) + if err != nil { + t.Fatalf("InvokableRun: %v", err) + } + if out != `{"ok":true}` { + t.Errorf("response = %q, want %q", out, `{"ok":true}`) + } + if inner.calls.Load() != 1 { + t.Errorf("inner call count = %d, want 1", inner.calls.Load()) + } + if inner.lastArgs != `{"a":1}` { + t.Errorf("lastArgs = %q, want %q", inner.lastArgs, `{"a":1}`) + } +} + +func TestWrapBaseTool_PersistsActualInvocationAtAdapterBoundary(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "host_bash", class: "read", runResp: `{"devices":[{"ok":true}]}`} + sink := &recordingToolPersistence{} + wrapped := WrapBaseTool(inner) + out, err := wrapped.InvokableRun(context.Background(), `{"device_ids":[1],"cmd":"docker images"}`, + WithToolInvocationPersistence(sink), + ) + if err != nil { + t.Fatalf("InvokableRun: %v", err) + } + starts, ends := sink.records() + if len(starts) != 1 || starts[0] != `host_bash:{"device_ids":[1],"cmd":"docker images"}` { + t.Fatalf("start records = %v", starts) + } + if len(ends) != 1 || ends[0] != "host_bash:"+out { + t.Fatalf("end records = %v", ends) + } +} + +func TestWrapBaseTool_WithInvokeOpts(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "echo", runResp: `{}`} + wrapped := WrapBaseTool(inner) + _, err := wrapped.InvokableRun(context.Background(), `{}`, + WithInvokeOpts(basetool.WithUserID(42), basetool.WithTenant("acme")), + ) + if err != nil { + t.Fatalf("InvokableRun: %v", err) + } + if inner.lastOptsLen != 2 { + t.Errorf("inner saw %d opts, want 2", inner.lastOptsLen) + } +} + +func TestWrapBaseTool_RunErrIntoEnvelope(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "bad", runErr: errors.New("boom: sandbox path /etc not allowed")} + wrapped := WrapBaseTool(inner) + out, err := wrapped.InvokableRun(context.Background(), `{}`) + // Adapter folds tool errors into a JSON envelope so eino's ToolsNode + // doesn't treat them as graph-fatal. The LLM consumes the envelope + // as a tool result and decides to retry / switch / ask the user. + if err != nil { + t.Fatalf("err should be nil (folded into envelope), got %v", err) + } + if !strings.Contains(out, `"error"`) || !strings.Contains(out, "boom") { + t.Errorf("envelope should carry error text, got %q", out) + } + if !strings.Contains(out, `"status":"failed"`) { + t.Errorf("envelope should mark status=failed, got %q", out) + } +} + +func TestWrapBaseTools_SkipsNil(t *testing.T) { + t.Parallel() + tools := []basetool.BaseTool{nil, &fakeBaseTool{name: "a"}, nil, &fakeBaseTool{name: "b"}} + out := WrapBaseTools(tools) + if len(out) != 2 { + t.Fatalf("WrapBaseTools dropped wrong count: got %d entries, want 2", len(out)) + } +} + +// Per-run memo: an identical read-tool call returns the cached result +// without re-executing; distinct args re-execute. +func TestEinoToolAdapter_MemoizesIdenticalReadCalls(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "query_promql", class: "read", runResp: `{"v":1}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + r1, err1 := a.InvokableRun(ctx, `{"q":"up"}`) + r2, err2 := a.InvokableRun(ctx, `{"q":"up"}`) + if err1 != nil || err2 != nil { + t.Fatalf("errs: %v %v", err1, err2) + } + if r1 != `{"v":1}` || r2 != `{"v":1}` { + t.Fatalf("results = %q, %q; want cached identical", r1, r2) + } + if got := inner.calls.Load(); got != 1 { + t.Errorf("identical read calls should execute once, got %d", got) + } + if _, err := a.InvokableRun(ctx, `{"q":"down"}`); err != nil { + t.Fatalf("distinct call err: %v", err) + } + if got := inner.calls.Load(); got != 2 { + t.Errorf("distinct args should re-execute, got %d", got) + } +} + +// Write/destructive tools are never memoized — the review/mutation flow +// must see every call. +func TestEinoToolAdapter_DoesNotMemoizeWriteTool(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "host_restart_service", class: "destructive", runResp: `{"ok":true}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + _, _ = a.InvokableRun(ctx, `{"svc":"nginx"}`) + _, _ = a.InvokableRun(ctx, `{"svc":"nginx"}`) + if got := inner.calls.Load(); got != 2 { + t.Errorf("destructive tool must NOT be memoized; want 2 executions, got %d", got) + } +} + +// A failed call stays retryable — the error envelope is not cached. +func TestEinoToolAdapter_DoesNotMemoizeErrors(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "query_x", class: "read", runErr: errors.New("boom")} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + _, _ = a.InvokableRun(ctx, `{"q":"a"}`) + _, _ = a.InvokableRun(ctx, `{"q":"a"}`) + if got := inner.calls.Load(); got != 2 { + t.Errorf("errored read calls must stay retryable; want 2 executions, got %d", got) + } +} + +// After maxToolCallsPerRun distinct executions, the tool stops running and +// returns a "synthesize now" directive (catches the varying-args repeat loop +// the memo can't). +func TestEinoToolAdapter_PerToolCallCap(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "query_x", class: "read", runResp: `{"v":1}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + // Distinct args each time so the identical-call memo doesn't short-circuit; + // the cap counts executions. + for i := 0; i < maxToolCallsPerRun; i++ { + out, _ := a.InvokableRun(ctx, fmt.Sprintf(`{"q":"m%d"}`, i)) + if strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("call %d should execute, got budget directive: %s", i, out) + } + } + if got := inner.calls.Load(); got != int32(maxToolCallsPerRun) { + t.Fatalf("expected %d executions, got %d", maxToolCallsPerRun, got) + } + // One past the cap → directive, no execution. + out, _ := a.InvokableRun(ctx, `{"q":"over"}`) + if !strings.Contains(out, "call_budget_exceeded") { + t.Errorf("past the cap should return the budget directive, got %q", out) + } + if got := inner.calls.Load(); got != int32(maxToolCallsPerRun) { + t.Errorf("over-cap call must NOT execute; still want %d, got %d", maxToolCallsPerRun, got) + } +} + +func TestEinoToolAdapter_QueryPromQLUsesStrictCallCap(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "query_promql", class: "read", runResp: `{"v":1}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + limit := maxCallsForTool("query_promql") + + if limit != 10 { + t.Fatalf("query_promql limit = %d, want 10", limit) + } + for i := 0; i < limit; i++ { + out, _ := a.InvokableRun(ctx, fmt.Sprintf(`{"q":"m%d"}`, i)) + if strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("call %d should execute, got budget directive: %s", i, out) + } + } + out, _ := a.InvokableRun(ctx, `{"q":"over"}`) + if !strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("past query_promql cap should return budget directive, got %q", out) + } + if !strings.Contains(out, "aggregated PromQL") { + t.Fatalf("query_promql cap should nudge aggregation, got %q", out) + } + if got := inner.calls.Load(); got != int32(limit) { + t.Fatalf("query_promql executions = %d, want %d", got, limit) + } +} + +func TestEinoToolAdapter_ReadToolsUseGenericCallCap(t *testing.T) { + t.Parallel() + for name, wantLimit := range map[string]int{ + "AgentTool": maxToolCallsPerRun, + "query_logql": maxToolCallsPerRun, + "query_traceql": maxToolCallsPerRun, + "host_bash": maxToolCallsPerRun, + "host_du_summary": maxToolCallsPerRun, + "host_find_large_files": maxToolCallsPerRun, + "query_k8s_snapshot": maxToolCallsPerRun, + "ToolSearch": maxToolCallsPerRun, + } { + name := name + wantLimit := wantLimit + t.Run(name, func(t *testing.T) { + t.Parallel() + limit := maxCallsForTool(name) + if limit != wantLimit { + t.Fatalf("%s limit = %d, want %d", name, limit, wantLimit) + } + inner := &fakeBaseTool{name: name, class: "read", runResp: `{"v":1}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + for i := 0; i < limit; i++ { + out, _ := a.InvokableRun(ctx, fmt.Sprintf(`{"q":"m%d"}`, i)) + if strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("call %d should execute, got budget directive: %s", i, out) + } + } + out, _ := a.InvokableRun(ctx, `{"q":"over"}`) + if !strings.Contains(out, "call_budget_exceeded") || !strings.Contains(out, `"final_answer_required":false`) { + t.Fatalf("past %s cap should block only that tool, got %q", name, out) + } + if got := inner.calls.Load(); got != int32(limit) { + t.Fatalf("%s executions = %d, want %d", name, got, limit) + } + }) + } +} + +func TestEinoToolAdapter_PerToolCallCapReservesBeforeConcurrentExecution(t *testing.T) { + t.Parallel() + inner := &concurrentCountTool{name: "host_bash", class: "read"} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + limit := maxCallsForTool("host_bash") + start := make(chan struct{}) + var wg sync.WaitGroup + outs := make(chan string, limit*3) + for i := 0; i < limit*3; i++ { + i := i + wg.Add(1) + go func() { + defer wg.Done() + <-start + out, _ := a.InvokableRun(ctx, fmt.Sprintf(`{"cmd":"echo %d"}`, i)) + outs <- out + }() + } + close(start) + wg.Wait() + close(outs) + + var budgeted int + for out := range outs { + if strings.Contains(out, "call_budget_exceeded") { + budgeted++ + } + } + if got := inner.calls.Load(); got != int32(limit) { + t.Fatalf("concurrent executions = %d, want exactly limit %d", got, limit) + } + if budgeted == 0 { + t.Fatalf("expected over-limit concurrent calls to receive budget results") + } +} + +func TestEinoToolAdapter_DoesNotCapAggregateToolCalls(t *testing.T) { + t.Parallel() + memo := newToolMemo() + ctx := context.Background() + var executed int32 + const distinctTools = 24 + for i := 0; i < distinctTools; i++ { + inner := &concurrentCountTool{name: fmt.Sprintf("tool_%02d", i), class: "read"} + a := &einoToolAdapter{inner: inner, memo: memo} + out, _ := a.InvokableRun(ctx, `{}`) + if strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("distinct tool %d was blocked by an aggregate cap: %s", i, out) + } + executed += inner.calls.Load() + } + if executed != distinctTools { + t.Fatalf("executed = %d, want all %d distinct tools", executed, distinctTools) + } +} + +func TestEinoToolAdapter_DraftConfigChangeConfirmableDraftCap(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "draft_config_change", class: "read", runResp: `{"kind":"config_draft","draft_hash":"sha256:ok"}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + + out, _ := a.InvokableRun(ctx, `{"rule":"one"}`) + if strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("first draft should execute, got budget directive: %s", out) + } + out, _ = a.InvokableRun(ctx, `{"rule":"two"}`) + if !strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("second draft should be capped, got %q", out) + } + if got := inner.calls.Load(); got != 1 { + t.Fatalf("draft executions = %d, want 1", got) + } +} + +func TestEinoToolAdapter_DraftConfigChangeRequiresMetricCatalogForMetricRules(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "draft_config_change", class: "read", runResp: `{"kind":"config_draft"}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + + out, err := a.InvokableRun(ctx, `{"domain":"alert_rule","action":"create","rule":{"kind":"metric_threshold","conditions":[{"metric":"cpu_pct","operator":">","threshold":85}]}}`) + if err != nil { + t.Fatalf("InvokableRun() error = %v", err) + } + if !strings.Contains(out, "metric_catalog_required") { + t.Fatalf("metric draft without catalog should be blocked, got %q", out) + } + if got := inner.calls.Load(); got != 0 { + t.Fatalf("draft executions = %d, want 0", got) + } +} + +func TestEinoToolAdapter_DraftConfigChangeAllowsAfterMetricCatalog(t *testing.T) { + t.Parallel() + memo := newToolMemo() + catalog := &fakeBaseTool{ + name: "list_metric_catalog", + class: "read", + runResp: `{"status":"ok","metric_count":1,"returned":1,"metrics":[{"name":"node_cpu_seconds_total"}]}`, + } + draft := &fakeBaseTool{name: "draft_config_change", class: "read", runResp: `{"kind":"config_draft","draft_hash":"sha256:ok"}`} + catalogAdapter := &einoToolAdapter{inner: catalog, memo: memo} + draftAdapter := &einoToolAdapter{inner: draft, memo: memo} + ctx := context.Background() + + if out, err := catalogAdapter.InvokableRun(ctx, `{"query":"cpu"}`); err != nil || !strings.Contains(out, `"status":"ok"`) { + t.Fatalf("catalog out=%q err=%v", out, err) + } + out, err := draftAdapter.InvokableRun(ctx, `{"domain":"alert_rule","action":"create","rule":{"kind":"metric_raw","spec":{"metric":"node_cpu_seconds_total","operator":">","threshold":80}}}`) + if err != nil { + t.Fatalf("draft InvokableRun() error = %v", err) + } + if !strings.Contains(out, `"kind":"config_draft"`) { + t.Fatalf("draft output = %q, want config_draft", out) + } + if got := catalog.calls.Load(); got != 1 { + t.Fatalf("catalog executions = %d, want 1", got) + } + if got := draft.calls.Load(); got != 1 { + t.Fatalf("draft executions = %d, want 1", got) + } +} + +func TestEinoToolAdapter_DraftConfigChangeIgnoresCatalogInstructionWhenMetricsExist(t *testing.T) { + t.Parallel() + memo := newToolMemo() + catalog := &fakeBaseTool{ + name: "list_metric_catalog", + class: "read", + runResp: `{"status":"ok","instruction":"The returned metrics do not expose any HTTP status/code label in sample_labels, so they cannot build a 5xx/error-rate/burn-rate SLI. Stop tool use now.","metric_count":1,"returned":1,"metrics":[{"name":"mysql_global_status_slow_queries"}]}`, + } + draft := &fakeBaseTool{name: "draft_config_change", class: "read", runResp: `{"kind":"config_draft","draft_hash":"sha256:ok"}`} + catalogAdapter := &einoToolAdapter{inner: catalog, memo: memo} + draftAdapter := &einoToolAdapter{inner: draft, memo: memo} + ctx := context.Background() + + if _, err := catalogAdapter.InvokableRun(ctx, `{"query":"MySQL slow queries"}`); err != nil { + t.Fatalf("catalog InvokableRun() error = %v", err) + } + out, err := draftAdapter.InvokableRun(ctx, `{"domain":"alert_rule","action":"create","rule":{"kind":"metric_raw","spec":{"expr":"rate(mysql_global_status_slow_queries[5m]) > 0.1"}}}`) + if err != nil { + t.Fatalf("draft InvokableRun() error = %v", err) + } + if !strings.Contains(out, `"kind":"config_draft"`) { + t.Fatalf("draft output = %q, want config_draft", out) + } + if got := draft.calls.Load(); got != 1 { + t.Fatalf("draft executions = %d, want 1", got) + } +} + +func TestEinoToolAdapter_DraftConfigChangeAllowsValidationAfterEmptyMetricCatalog(t *testing.T) { + t.Parallel() + memo := newToolMemo() + catalog := &fakeBaseTool{name: "list_metric_catalog", class: "read", runResp: `{"status":"empty","query":"cpu","metric_count":0,"returned":0,"metrics":[]}`} + draft := &fakeBaseTool{name: "draft_config_change", class: "read", runResp: `{"kind":"config_draft"}`} + catalogAdapter := &einoToolAdapter{inner: catalog, memo: memo} + draftAdapter := &einoToolAdapter{inner: draft, memo: memo} + ctx := context.Background() + + if _, err := catalogAdapter.InvokableRun(ctx, `{"query":"cpu"}`); err != nil { + t.Fatalf("catalog InvokableRun() error = %v", err) + } + out, err := draftAdapter.InvokableRun(ctx, `{"domain":"alert_rule","action":"create","rule":{"kind":"metric_raw","spec":{"metric":"cpu","operator":">","threshold":80}}}`) + if err != nil { + t.Fatalf("draft InvokableRun() error = %v", err) + } + if !strings.Contains(out, `"kind":"config_draft"`) { + t.Fatalf("draft after empty catalog should execute for validation, got %q", out) + } + if got := draft.calls.Load(); got != 1 { + t.Fatalf("draft executions = %d, want 1", got) + } +} + +func TestEinoToolAdapter_DraftConfigChangeAllowsLogRuleWithoutMetricCatalog(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "draft_config_change", class: "read", runResp: `{"kind":"config_draft"}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + + out, err := a.InvokableRun(ctx, `{"domain":"alert_rule","action":"create","rule":{"kind":"log_search","spec":{"keywords":{"include":["error"],"mode":"any"},"window":"5m"}}}`) + if err != nil { + t.Fatalf("InvokableRun() error = %v", err) + } + if !strings.Contains(out, `"kind":"config_draft"`) { + t.Fatalf("log draft should execute without metric catalog, got %q", out) + } + if got := inner.calls.Load(); got != 1 { + t.Fatalf("draft executions = %d, want 1", got) + } +} + +func TestEinoToolAdapter_ListMetricCatalogUsesGenericCallCap(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "list_metric_catalog", class: "read", runResp: `{"status":"ok"}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + + for i := 0; i < maxToolCallsPerRun; i++ { + out, _ := a.InvokableRun(ctx, fmt.Sprintf(`{"query":"mongo metric lookup %d"}`, i)) + if strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("metric catalog lookup %d should execute, got budget directive: %s", i, out) + } + } + out, _ := a.InvokableRun(ctx, `{"query":"mongo all metrics over cap"}`) + if !strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("over-cap metric catalog lookup should be capped, got %q", out) + } + if !strings.Contains(out, `"scope":"current_user_turn"`) || !strings.Contains(out, "expires on the next user message") { + t.Fatalf("budget directive should be explicitly scoped to the current turn, got %q", out) + } + if got := inner.calls.Load(); got != maxToolCallsPerRun { + t.Fatalf("metric catalog executions = %d, want %d", got, maxToolCallsPerRun) + } +} + +func TestEinoToolAdapter_DraftConfigChangeFailureDoesNotConsumeConfirmableDraftCap(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "draft_config_change", class: "read", runErr: errors.New("invalid scope")} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + + out, _ := a.InvokableRun(ctx, `{"rule":"bad"}`) + if !strings.Contains(out, `"status":"failed"`) { + t.Fatalf("failed draft should return failure envelope, got %q", out) + } + inner.runErr = nil + inner.runResp = `{"kind":"config_draft","draft_hash":"sha256:ok"}` + out, _ = a.InvokableRun(ctx, `{"rule":"fixed"}`) + if strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("fixed draft after validation failure should execute, got %q", out) + } + out, _ = a.InvokableRun(ctx, `{"rule":"extra"}`) + if !strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("second successful draft should be capped, got %q", out) + } + if got := inner.calls.Load(); got != 2 { + t.Fatalf("draft executions = %d, want failed+fixed executions", got) + } +} + +func TestEinoToolAdapter_DraftConfigValidationFailedDoesNotConsumeConfirmableDraftCap(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "draft_config_change", class: "read", runResp: `{"kind":"config_validation_failed","validation":{"status":"failed"}}`} + a := &einoToolAdapter{inner: inner, memo: newToolMemo()} + ctx := context.Background() + + out, _ := a.InvokableRun(ctx, `{"rule":"bad"}`) + if !strings.Contains(out, `"kind":"config_validation_failed"`) { + t.Fatalf("validation failure should return structured result, got %q", out) + } + inner.runResp = `{"kind":"config_draft","draft_hash":"sha256:ok"}` + out, _ = a.InvokableRun(ctx, `{"rule":"fixed"}`) + if strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("fixed draft after validation failed result should execute, got %q", out) + } + out, _ = a.InvokableRun(ctx, `{"rule":"extra"}`) + if !strings.Contains(out, "call_budget_exceeded") { + t.Fatalf("second successful draft should be capped, got %q", out) + } + if got := inner.calls.Load(); got != 2 { + t.Fatalf("draft executions = %d, want validation+fixed executions", got) + } +} + +// The single-tool WrapBaseTool path leaves memo nil — no caching. +func TestEinoToolAdapter_NoMemoByDefault(t *testing.T) { + t.Parallel() + inner := &fakeBaseTool{name: "query_x", class: "read", runResp: "ok"} + a := &einoToolAdapter{inner: inner} // memo nil + ctx := context.Background() + _, _ = a.InvokableRun(ctx, `{"q":"a"}`) + _, _ = a.InvokableRun(ctx, `{"q":"a"}`) + if got := inner.calls.Load(); got != 2 { + t.Errorf("memo-less adapter must execute each call; want 2, got %d", got) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/graph/types.go b/internal/manager/biz/aiops/runtime/v3/graph/types.go new file mode 100644 index 000000000..483774abb --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/graph/types.go @@ -0,0 +1,180 @@ +// Package graph is the eino-based agent execution kernel introduced by +// / (Agent compose.Graph 拼 ReAct) + (主参考图 +// — Graph 执行层). It builds a `compose.Runnable[*Input, *Output]` whose +// internal shape is the standard ReAct loop: +// +// START → MessageAssembler → ChatModel → Branch(tool_calls?) → +// yes → ToolsNode → MsgAppend → 回 ChatModel +// no → END +// +// Implementation note (architectural deviation ): the +// inner ReAct subgraph is built via cloudwego/eino's stock +// `flow/agent/react.NewAgent`, which already produces the exact graph +// the spec calls out (ChatModel ↔ Branch ↔ ToolsNode), bound at compile +// time with the user-supplied `model.ToolCallingChatModel` + tools. Our +// own wrapper graph (this package) adds the **MessageAssembler** lambda +// in front so callers pass `*Input` (system prompt + history + user +// text + system-reminder cfg) instead of pre-flattened `[]*schema.Message`, +// and an **OutputProjector** lambda in the back so callers receive +// `*Output` (assistant message + iteration count + usage) instead of +// the bare assistant `*schema.Message`. We are NOT re-implementing the +// branch / tools-node wiring by hand because eino's ReAct implementation +// is the canonical maintained one — re-rolling it would be churn against +// no benefit. See react.go header comment for the layered topology. +// +// PR-6 of scaffolding only. The cutover (replacing agent.go's +// 660-line for-loop with this graph) is the NEXT PR — main.go is not +// touched here. Callers in this PR are tests + internal wiring only. +package graph + +import ( + "time" + + "github.com/cloudwego/eino/schema" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// Input is the structured request for one ReAct-loop run. It mirrors the +// information the legacy agent.go for-loop assembles from the chat +// session before dispatching to the LLM. The MessageAssembler lambda +// (react.go) flattens this into `[]*schema.Message` for the eino graph. +// +// ASCII shows MessageAssembler taking +// `Input{Messages, UserText, SystemPrompt}`; we add WebSearchEnabled and +// MentionsRendered as separate fields so the assembler can decide where +// to inline them (mentions go above the user text; web_search gating is +// enforced by the toolBag, not by message content). +type Input struct { + // SystemPrompt is the base agent instructions. Empty = no system + // message. PR-6: the caller (chatruntime, NEXT PR) stitches together + // base prompt + active skill prompts + agent persona; this layer + // just consumes the final string. + SystemPrompt string + + // History is the prior conversation in eino message form. Includes + // user / assistant / tool messages; ordered chronologically. + // HistoryLimit gating happens upstream — this layer takes the slice + // as-is. + History []*schema.Message + + // UserText is the new user turn (verbatim, post-mention-inlining). + // MentionsRendered, when non-empty, is prepended as a markdown + // preamble per agent.go's existing behaviour. + UserText string + UserImages []schema.MessageInputPart + + // WebSearchEnabled is the per-call gate the SPA's globe toggle + // flips. PR-6 carries this through but enforcement remains at the + // toolBag layer (chatruntime wires the right tool list); the field + // is here so the assembler can echo it into a system-reminder block + // for belt-and-braces (a hijacked tool_call from history replay + // won't slip through). + WebSearchEnabled bool + + // MentionsRendered is the pre-rendered markdown bullet list of + // @-mention context blocks. Empty = no mentions on this turn. + // agent.go's exact format ("用户在消息中引用了以下平台对象 ...") is + // reproduced here for replay parity. + MentionsRendered string + + // AgentReminder is the persona-level critical_reminder string the + // coordinator/runtime resolved for this turn. When + // non-empty it is appended verbatim — without further wrapping — as + // one bullet inside the per-turn block the + // MessageAssembler injects. Empty = no persona reminder for this + // turn (coordinator persona without a critical_reminder, or no + // persona at all). + // + // Static persona prose still lives inside SystemPrompt via + // chatruntime.ComposeSystemPrompt — this field carries only the + // short reminder line that gets re-injected on every turn. + AgentReminder string + // DynamicHints is the runtime-computed list of per-turn hint lines + // (each rendered as one bullet inside the block). + // — examples: "tool X failed N times in a row", "we + // already ran Y iterations". Computed by chatruntime from the + // session's recent message + tool history; the graph is not the + // source of truth and never inspects history to derive its own + // hints. + DynamicHints []string + + // Locale is the UI language the answer should be written in + // ("en-US" / "zh-CN"). The personas are Chinese, so without this the + // model defaults to Chinese even when the SPA is in English mode. The + // assembler turns it into an explicit "respond in " + // directive (system prompt + per-turn reminder). Empty = no directive + // (back-compat; e.g. the IM bridge, which doesn't carry a UI locale). + Locale string +} + +// Output is the terminal result of a graph run. Mirrors the parts of +// agent.Reply this layer can populate without reaching into the +// session repo (which is the persistence callback's job). Iterations +// is best-effort — eino's react agent doesn't expose the step count +// directly, so the OutputProjector counts ChatModel turns on the +// callback channel and stamps the value here. +type Output struct { + // AssistantMessage is the final assistant turn the model produced + // (no tool_calls). Always non-nil on success. + AssistantMessage *schema.Message + + // Iterations is the number of ChatModel calls that ran during this + // turn. Counted via the metrics/audit handler chain; 0 if no + // counter handler was wired. + Iterations int + + // Usage is the aggregated token usage across every ChatModel call + // in this run. Populated by the MetricsHandler (callbacks/metrics.go). + // Zero values when no counter was wired. + Usage llm.Usage +} + +// Config tunes graph behaviour. Defaults match agent.go's existing +// for-loop so the cutover PR sees zero behavioural drift. +// +// 改进点 #5: MaxIterations 内嵌 — eino's ReAct accepts +// MaxStep at construction; we expose it here. +type Config struct { + // ToolPersistence is a per-run sink invoked directly by each tool adapter. + // It is deliberately injected at graph construction because nested Eino + // ToolsNode callbacks are not reliable for durable tool-result writes. + ToolPersistence ToolInvocationPersistence + + // Model is the LLM model id (e.g. "gpt-4o", "claude-sonnet-4-6"). + // Empty = use the underlying ChatModel's default. + Model string + + // Provider is the routing key recognised by RoutingChatModel + // (— provider 路由保留). Empty = use the routing + // model's defaultProvider. + Provider string + + // Temperature controls sampling randomness. 0 retains the legacy + // agent default of 0.1. + Temperature float32 + + // MaxIterations caps the outer ReAct loop. 0 -> 12. Tool-level budgets + // stop repeated evidence gathering earlier; this is the final circuit + // breaker for cross-tool loops. + MaxIterations int + + // ToolTimeout is the per-tool wall clock ceiling. 0 -> 15s + // (agent.go default). Enforced by the BaseTool decorator chain + // (PR-3); this field is kept here so the cutover PR can pass the + // value through to chatruntime when it builds the toolBag. + ToolTimeout time.Duration +} + +// applyDefaults fills in zero-valued Config fields with the same +// defaults agent.go uses. Returned by value so callers don't see +// mutation of their Config struct. +func (c Config) applyDefaults() Config { + if c.MaxIterations <= 0 { + c.MaxIterations = 12 + } + if c.ToolTimeout <= 0 { + c.ToolTimeout = 15 * time.Second + } + return c +} diff --git a/internal/manager/biz/alert/investigator/usecase.go b/internal/manager/biz/alert/investigator/usecase.go index 161a4ed13..8151f61e9 100644 --- a/internal/manager/biz/alert/investigator/usecase.go +++ b/internal/manager/biz/alert/investigator/usecase.go @@ -26,7 +26,7 @@ import ( "sync" "time" - chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" alertmodel "github.com/ongridio/ongrid/internal/manager/model/alert" ) diff --git a/internal/manager/biz/alert/investigator/usecase_test.go b/internal/manager/biz/alert/investigator/usecase_test.go index 6d32ae688..035bcce80 100644 --- a/internal/manager/biz/alert/investigator/usecase_test.go +++ b/internal/manager/biz/alert/investigator/usecase_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" alertmodel "github.com/ongridio/ongrid/internal/manager/model/alert" "github.com/ongridio/ongrid/internal/pkg/errs" ) diff --git a/internal/manager/biz/marketplace/capability_credentials_test.go b/internal/manager/biz/marketplace/capability_credentials_test.go index 5a7317b86..b3387278e 100644 --- a/internal/manager/biz/marketplace/capability_credentials_test.go +++ b/internal/manager/biz/marketplace/capability_credentials_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "testing" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" ) // TestBuildCapabilityDeclaration_BareSkillCredentials guards the terraform-runner diff --git a/internal/manager/biz/marketplace/integration_test.go b/internal/manager/biz/marketplace/integration_test.go index ca042990f..cede49491 100644 --- a/internal/manager/biz/marketplace/integration_test.go +++ b/internal/manager/biz/marketplace/integration_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" ) // TestIntegration_InstallReloadVisibleInRealRegistry exercises the diff --git a/internal/manager/biz/marketplace/usecase.go b/internal/manager/biz/marketplace/usecase.go index b88c2e1c3..8ca0a82de 100644 --- a/internal/manager/biz/marketplace/usecase.go +++ b/internal/manager/biz/marketplace/usecase.go @@ -15,7 +15,7 @@ import ( "strings" "time" - chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" model "github.com/ongridio/ongrid/internal/manager/model/marketplace" "github.com/ongridio/ongrid/internal/pkg/errs" ) diff --git a/internal/manager/biz/marketplace/usecase_test.go b/internal/manager/biz/marketplace/usecase_test.go index 7ef2108c4..ec3e1d56c 100644 --- a/internal/manager/biz/marketplace/usecase_test.go +++ b/internal/manager/biz/marketplace/usecase_test.go @@ -16,7 +16,7 @@ import ( "testing" "time" - chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" model "github.com/ongridio/ongrid/internal/manager/model/marketplace" "github.com/ongridio/ongrid/internal/pkg/errs" ) @@ -125,7 +125,6 @@ func (r *fakeRepo) SetBindings(_ context.Context, tenantID uint64, packID, bindi return errs.ErrNotFound } - // ----- in-memory registries ------------------------------------------- type fakeSkillReg struct { diff --git a/internal/manager/biz/report/generator.go b/internal/manager/biz/report/generator.go index 324ec352c..44c712c67 100644 --- a/internal/manager/biz/report/generator.go +++ b/internal/manager/biz/report/generator.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" model "github.com/ongridio/ongrid/internal/manager/model/report" ) diff --git a/internal/manager/biz/report/generator_test.go b/internal/manager/biz/report/generator_test.go index eae98dbf7..5bc2370a5 100644 --- a/internal/manager/biz/report/generator_test.go +++ b/internal/manager/biz/report/generator_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" model "github.com/ongridio/ongrid/internal/manager/model/report" ) @@ -22,10 +22,10 @@ func (f fakeFacts) Collect(context.Context, Period, Period, Scope) (*ReportFacts // fakeSpawner returns a canned worker (Result / Err). type fakeSpawner struct { - result string - werr string + result string + werr string spawnErr error - gotReq chatruntime.SpawnRequest + gotReq chatruntime.SpawnRequest } func (s *fakeSpawner) SpawnWorker(_ context.Context, req chatruntime.SpawnRequest) (*chatruntime.Worker, error) { @@ -247,10 +247,10 @@ func TestGenerator_NonPendingIsNoOp(t *testing.T) { func TestExtractJSON(t *testing.T) { cases := map[string]string{ - `{"a":1}`: `{"a":1}`, - "```json\n{\"a\":1}\n```": `{"a":1}`, - "```\n{\"a\":1}\n```": `{"a":1}`, - "here you go:\n{\"a\":1}\nthat's it": `{"a":1}`, + `{"a":1}`: `{"a":1}`, + "```json\n{\"a\":1}\n```": `{"a":1}`, + "```\n{\"a\":1}\n```": `{"a":1}`, + "here you go:\n{\"a\":1}\nthat's it": `{"a":1}`, } for in, want := range cases { if got := extractJSON(in); got != want { diff --git a/internal/manager/server/aiops/http.go b/internal/manager/server/aiops/http.go index d66e2b1bf..5bfbd59eb 100644 --- a/internal/manager/server/aiops/http.go +++ b/internal/manager/server/aiops/http.go @@ -31,8 +31,8 @@ import ( biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" "github.com/ongridio/ongrid/internal/manager/biz/aiops/mentions" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" model "github.com/ongridio/ongrid/internal/manager/model/aiops" svc "github.com/ongridio/ongrid/internal/manager/service/aiops" "github.com/ongridio/ongrid/internal/pkg/errs" diff --git a/internal/manager/service/aiops/approval_adapters.go b/internal/manager/service/aiops/approval_adapters.go index 293d0a586..7e38ffcab 100644 --- a/internal/manager/service/aiops/approval_adapters.go +++ b/internal/manager/service/aiops/approval_adapters.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "fmt" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" managerbizapproval "github.com/ongridio/ongrid/internal/manager/biz/approval" managerbizmcp "github.com/ongridio/ongrid/internal/manager/biz/mcp" diff --git a/internal/manager/service/aiops/human_approval.go b/internal/manager/service/aiops/human_approval.go index a47b6578c..40f440b78 100644 --- a/internal/manager/service/aiops/human_approval.go +++ b/internal/manager/service/aiops/human_approval.go @@ -8,7 +8,7 @@ import ( "time" "github.com/google/uuid" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" managerbizapproval "github.com/ongridio/ongrid/internal/manager/biz/approval" ) diff --git a/internal/manager/service/aiops/mcp_approval_runtime_test.go b/internal/manager/service/aiops/mcp_approval_runtime_test.go index 4a802efcc..6eba4e4b5 100644 --- a/internal/manager/service/aiops/mcp_approval_runtime_test.go +++ b/internal/manager/service/aiops/mcp_approval_runtime_test.go @@ -10,8 +10,8 @@ import ( einomodel "github.com/cloudwego/eino/components/model" "github.com/cloudwego/eino/schema" "github.com/glebarez/sqlite" - runtime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/graph" + runtime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" "github.com/ongridio/ongrid/internal/manager/biz/approval" diff --git a/internal/manager/service/aiops/mcp_refresh.go b/internal/manager/service/aiops/mcp_refresh.go index e64cde937..852d01489 100644 --- a/internal/manager/service/aiops/mcp_refresh.go +++ b/internal/manager/service/aiops/mcp_refresh.go @@ -5,7 +5,7 @@ import ( "log/slog" "time" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" diff --git a/internal/manager/service/aiops/mcp_refresh_test.go b/internal/manager/service/aiops/mcp_refresh_test.go index 68d30dff6..9baa27a63 100644 --- a/internal/manager/service/aiops/mcp_refresh_test.go +++ b/internal/manager/service/aiops/mcp_refresh_test.go @@ -3,7 +3,7 @@ package aiops import ( "context" "encoding/json" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" mcpmodel "github.com/ongridio/ongrid/internal/manager/model/mcp" diff --git a/internal/manager/service/aiops/runtime_adapters.go b/internal/manager/service/aiops/runtime_adapters.go index 0f920ec38..643037298 100644 --- a/internal/manager/service/aiops/runtime_adapters.go +++ b/internal/manager/service/aiops/runtime_adapters.go @@ -3,8 +3,8 @@ package aiops import ( "context" "fmt" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" managerbizaiopsmentions "github.com/ongridio/ongrid/internal/manager/biz/aiops/mentions" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" "github.com/ongridio/ongrid/internal/pkg/llm" diff --git a/internal/manager/service/aiops/service.go b/internal/manager/service/aiops/service.go index 63fb6fd96..54dfe9611 100644 --- a/internal/manager/service/aiops/service.go +++ b/internal/manager/service/aiops/service.go @@ -22,7 +22,7 @@ import ( biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" model "github.com/ongridio/ongrid/internal/manager/model/aiops" "github.com/ongridio/ongrid/internal/pkg/errs" "github.com/ongridio/ongrid/internal/pkg/llm" diff --git a/internal/manager/service/aiops/service_kernel_test.go b/internal/manager/service/aiops/service_kernel_test.go index ee6c90aed..3d3e74cfa 100644 --- a/internal/manager/service/aiops/service_kernel_test.go +++ b/internal/manager/service/aiops/service_kernel_test.go @@ -10,7 +10,7 @@ import ( biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" model "github.com/ongridio/ongrid/internal/manager/model/aiops" "github.com/ongridio/ongrid/internal/pkg/errs" ) diff --git a/internal/manager/service/aiops/user_agent.go b/internal/manager/service/aiops/user_agent.go index d01c109b3..60cc22866 100644 --- a/internal/manager/service/aiops/user_agent.go +++ b/internal/manager/service/aiops/user_agent.go @@ -27,7 +27,7 @@ import ( "strings" "time" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" model "github.com/ongridio/ongrid/internal/manager/model/aiops" "github.com/ongridio/ongrid/internal/pkg/errs" ) diff --git a/internal/manager/service/aiops/worker_metrics.go b/internal/manager/service/aiops/worker_metrics.go index 57572e640..df1eaa8e2 100644 --- a/internal/manager/service/aiops/worker_metrics.go +++ b/internal/manager/service/aiops/worker_metrics.go @@ -4,7 +4,7 @@ import ( "context" "time" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" "github.com/ongridio/ongrid/internal/pkg/prom" ) diff --git a/internal/manager/service/flow/runners.go b/internal/manager/service/flow/runners.go index c98783c8d..9d402e181 100644 --- a/internal/manager/service/flow/runners.go +++ b/internal/manager/service/flow/runners.go @@ -6,7 +6,7 @@ import ( "strings" "time" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" managerbizalert "github.com/ongridio/ongrid/internal/manager/biz/alert" managermodelalert "github.com/ongridio/ongrid/internal/manager/model/alert" "github.com/ongridio/ongrid/internal/pkg/llm" diff --git a/internal/manager/service/skill/runtime_catalog.go b/internal/manager/service/skill/runtime_catalog.go index 61308bb0d..526d62242 100644 --- a/internal/manager/service/skill/runtime_catalog.go +++ b/internal/manager/service/skill/runtime_catalog.go @@ -1,7 +1,7 @@ package skill import ( - "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" managerbizskill "github.com/ongridio/ongrid/internal/manager/biz/skill" skillcore "github.com/ongridio/ongrid/internal/skill" ) diff --git a/scripts/test-ci-docs-only.py b/scripts/test-ci-docs-only.py index 9173c53d5..f2456f9bf 100644 --- a/scripts/test-ci-docs-only.py +++ b/scripts/test-ci-docs-only.py @@ -21,7 +21,7 @@ assert docs_only([".github/ISSUE_TEMPLATE/config.yml", ".github/ISSUE_TEMPLATE/bug.yaml"]) for paths in [[], ["main.go"], ["Makefile"], [".github/workflows/ci.yml"], ["internal/manager/biz/knowledge/builtin_vault/guide.md"], - ["internal/manager/biz/aiops/chatruntime/testdata/agent_registry/multi/README.md"], + ["internal/manager/biz/aiops/runtime/v3/chatruntime/testdata/agent_registry/multi/README.md"], ["examples/apm-languages/README.md"], ["agents/reviewer.md"], ["web/src/guide.md"], ["web/public/logo.svg"], ["new-package/README.md"], ["docs/config.yml"], ["docs/assets/app.js"], ["docs/example.go"], From a038e5244cc64f7de8473c11c728762136997b02 Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 23:02:42 +0800 Subject: [PATCH 16/24] fix(agenttrace): preserve runtime instrumentation scope matching --- internal/manager/biz/agenttrace/adapters.go | 2 +- internal/manager/biz/agenttrace/content_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/manager/biz/agenttrace/adapters.go b/internal/manager/biz/agenttrace/adapters.go index f0543c295..b6ff2ffa8 100644 --- a/internal/manager/biz/agenttrace/adapters.go +++ b/internal/manager/biz/agenttrace/adapters.go @@ -50,7 +50,7 @@ func adapt(s *Step) { s.Provider = stringAttr(a, "gen_ai.provider.name", "llm.provider", "gen_ai.system", "llm.system") // LangSmith trace.session_id identifies a project, not a conversation. s.SessionID = stringAttr(a, "gen_ai.conversation.id", "session.id", "langsmith.metadata.thread_id", "langsmith.metadata.session_id", "langsmith.metadata.conversation_id", "traceloop.association.properties.session_id", "traceloop.association.properties.thread_id") - if s.Scope.Name == "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" || s.Scope.Name == "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" { + if s.Scope.Name == "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime" || s.Scope.Name == "github.com/ongridio/ongrid/internal/manager/biz/aiops/agent" { s.ChatSessionID = stringAttr(a, "ongrid.agent.session.id") } if s.Identity.Environment == "" { diff --git a/internal/manager/biz/agenttrace/content_test.go b/internal/manager/biz/agenttrace/content_test.go index 2d82a2235..6fd72ac5f 100644 --- a/internal/manager/biz/agenttrace/content_test.go +++ b/internal/manager/biz/agenttrace/content_test.go @@ -14,7 +14,7 @@ func TestCapturedContentIsDecodedOnlyFromTrace(t *testing.T) { raw, _ := json.Marshal(strings.Repeat("执行快照😀", 10000)) snapshot := tracing.MaskAgentContent(string(raw)) body := testBody(t, testSpan(1, 0, map[string]any{"gen_ai.operation.name": "invoke_agent", "input.value": `"本轮问题"`, "output.value": snapshot, "ongrid.agent.session.id": "deleted-chat", "ongrid.agent.content.capture": true})) - body = bytes.ReplaceAll(body, []byte(`"name":"fixture"`), []byte(`"name":"github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime"`)) + body = bytes.ReplaceAll(body, []byte(`"name":"fixture"`), []byte(`"name":"github.com/ongridio/ongrid/internal/manager/biz/aiops/chatruntime"`)) trace, err := Normalize(fixtureTraceID, body) if err != nil { t.Fatal(err) From 8b1c1db8247e7bf31589f2c1a9c1f43751107614 Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 23:14:05 +0800 Subject: [PATCH 17/24] refactor(aiops): route execution through version-independent contracts --- Makefile | 2 +- cmd/ongrid/bootstrap_automation.go | 7 +- cmd/ongrid/bootstrap_extensions.go | 3 +- cmd/ongrid/bootstrap_imbridge.go | 3 +- cmd/ongrid/bootstrap_runtime.go | 26 +- cmd/ongrid/workers.go | 4 +- docs/rfc/RFC-009-agent-runtime-versions.md | 24 +- internal/manager/biz/aiops/runtime/README.md | 2 +- .../biz/aiops/runtime/contract/context.go | 37 +++ .../biz/aiops/runtime/contract/types.go | 267 ++++++++++++++++++ .../biz/aiops/runtime/isolation_test.go | 4 +- .../manager/biz/aiops/runtime/v3/README.md | 2 +- .../aiops/runtime/v3/chatruntime/runtime.go | 128 ++------- .../aiops/runtime/v3/chatruntime/worker.go | 195 ++----------- .../v3/chatruntime/worker_context_test.go | 6 + .../manager/biz/alert/investigator/usecase.go | 2 +- .../biz/alert/investigator/usecase_test.go | 2 +- internal/manager/biz/report/generator.go | 2 +- internal/manager/biz/report/generator_test.go | 2 +- .../service/aiops/approval_adapters.go | 2 +- .../manager/service/aiops/human_approval.go | 2 +- internal/manager/service/aiops/mcp_refresh.go | 10 +- .../manager/service/aiops/runtime_adapters.go | 19 +- .../manager/service/aiops/runtime_router.go | 153 ++++++++++ .../service/aiops/runtime_router_test.go | 213 ++++++++++++++ internal/manager/service/aiops/service.go | 4 +- .../manager/service/aiops/worker_metrics.go | 3 +- internal/manager/service/flow/runners.go | 7 +- 28 files changed, 798 insertions(+), 333 deletions(-) create mode 100644 internal/manager/biz/aiops/runtime/contract/context.go create mode 100644 internal/manager/biz/aiops/runtime/contract/types.go create mode 100644 internal/manager/service/aiops/runtime_router.go create mode 100644 internal/manager/service/aiops/runtime_router_test.go diff --git a/Makefile b/Makefile index f20532d8a..966e18f3d 100644 --- a/Makefile +++ b/Makefile @@ -914,4 +914,4 @@ test-web-ci-platform: ## Run the serial platform frontend CI tests (requires npm .PHONY: test-agent-runtime test-agent-runtime: ## 统一 Agent runtime 回归（tools 依赖 Linux cmdpolicy） - go test -race ./internal/manager/biz/aiops/... ./internal/manager/biz/approval ./internal/manager/service/aiops ./internal/manager/server/aiops ./internal/manager/biz/imbridge ./internal/manager/service/imbridge ./internal/manager/service/flow ./internal/skill/... ./cmd/ongrid + go test -race ./internal/manager/biz/aiops/... ./internal/manager/biz/approval ./internal/manager/service/aiops ./internal/manager/server/aiops ./internal/manager/biz/imbridge ./internal/manager/service/imbridge ./internal/manager/service/flow ./internal/manager/biz/report ./internal/manager/biz/alert/investigator ./internal/manager/biz/marketplace ./internal/manager/biz/agenttrace ./internal/manager/service/skill ./internal/skill/... ./cmd/ongrid diff --git a/cmd/ongrid/bootstrap_automation.go b/cmd/ongrid/bootstrap_automation.go index 46478b532..8092abaa6 100644 --- a/cmd/ongrid/bootstrap_automation.go +++ b/cmd/ongrid/bootstrap_automation.go @@ -10,7 +10,6 @@ import ( "github.com/google/uuid" aiopsinvestigator "github.com/ongridio/ongrid/internal/manager/biz/aiops/investigator" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" managerbizalert "github.com/ongridio/ongrid/internal/manager/biz/alert" "github.com/ongridio/ongrid/internal/manager/biz/alert/investigator" managerbizflow "github.com/ongridio/ongrid/internal/manager/biz/flow" @@ -81,7 +80,7 @@ func bootstrapAutomation(rootCtx context.Context, cfg *config.Config, log *slog. rcaInvConcrete *investigator.Usecase ) if os.Getenv("ONGRID_INVESTIGATOR_ENABLED") == "true" { - concreteRt, _ := runtime.aiopsRuntime.(*aiopschatruntime.Runtime) + concreteRt := runtime.aiopsRuntime if concreteRt == nil { log.Warn("structured RCA investigator skipped: chatruntime runtime not available") } else { @@ -145,7 +144,7 @@ func bootstrapAutomation(rootCtx context.Context, cfg *config.Config, log *slog. reportRepo := managerreportdata.NewRepo(storage.db) var reportGen managerbizreport.Generator reportSchedulerReady := false - if reportRT, ok := runtime.aiopsRuntime.(*aiopschatruntime.Runtime); ok && reportRT != nil { + if reportRT := runtime.aiopsRuntime; reportRT != nil { // Pass the prom query client for fleet resource trends; a typed-nil // guard mirrors the tools wiring so a missing client stays a clean // untyped nil (collector degrades Resource.Available=false). @@ -195,7 +194,7 @@ func bootstrapAutomation(rootCtx context.Context, cfg *config.Config, log *slog. Notify: managersvcflow.Notifier{Channels: alerts.alertRepo, Router: notifyRouter}, LLM: managersvcflow.LLMRunner{Client: models.llmClient}, } - if flowRT, ok := runtime.aiopsRuntime.(*aiopschatruntime.Runtime); ok && flowRT != nil { + if flowRT := runtime.aiopsRuntime; flowRT != nil { flowExec.Agent = managersvcflow.AgentRunner{Runtime: flowRT} } flowUC := managerbizflow.NewUsecase(flowRepo, flowRunRepo, diff --git a/cmd/ongrid/bootstrap_extensions.go b/cmd/ongrid/bootstrap_extensions.go index 20ad22521..481bc1eba 100644 --- a/cmd/ongrid/bootstrap_extensions.go +++ b/cmd/ongrid/bootstrap_extensions.go @@ -8,7 +8,6 @@ import ( "strings" "time" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" @@ -70,7 +69,7 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. // the next chatruntime construction picks the disk state up. var mpSkillReg managerbizmarketplace.SkillRegistry var mpAgentReg managerbizmarketplace.AgentRegistry - if rt, ok := runtime.aiopsRuntime.(*aiopschatruntime.Runtime); ok && rt != nil { + if rt := runtime.chatRT; rt != nil { if sk := rt.SkillRegistry(); sk != nil { mpSkillReg = sk } diff --git a/cmd/ongrid/bootstrap_imbridge.go b/cmd/ongrid/bootstrap_imbridge.go index c6b7ddb1b..79e4c786b 100644 --- a/cmd/ongrid/bootstrap_imbridge.go +++ b/cmd/ongrid/bootstrap_imbridge.go @@ -6,7 +6,6 @@ import ( "log/slog" managerbizaiopsmentions "github.com/ongridio/ongrid/internal/manager/biz/aiops/mentions" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" managerbizimbridge "github.com/ongridio/ongrid/internal/manager/biz/imbridge" managerbizimbridgedingtalk "github.com/ongridio/ongrid/internal/manager/biz/imbridge/provider/dingtalk" managerbizimbridgefeishu "github.com/ongridio/ongrid/internal/manager/biz/imbridge/provider/feishu" @@ -97,7 +96,7 @@ func bootstrapIMBridge(rootCtx context.Context, log *slog.Logger, storage *stora // Mirror the same wiring on the new graph kernel runtime when it // was successfully constructed. Same searcher, two adapter shapes // — see the type definitions at the bottom of this file. - if rt, ok := runtime.aiopsRuntime.(*aiopschatruntime.Runtime); ok && rt != nil { + if rt := runtime.chatRT; rt != nil { rt.SetMentionResolver(managersvcaiops.ChatMentionResolver{Inner: mentionSearcher}) } diff --git a/cmd/ongrid/bootstrap_runtime.go b/cmd/ongrid/bootstrap_runtime.go index e6f73b308..67a4b751b 100644 --- a/cmd/ongrid/bootstrap_runtime.go +++ b/cmd/ongrid/bootstrap_runtime.go @@ -5,6 +5,7 @@ import ( "log/slog" "time" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" @@ -18,7 +19,7 @@ import ( type runtimeModule struct { bootstrapSkillReg *aiopschatruntime.SkillRegistry bootstrapAgentReg *aiopschatruntime.AgentRegistry - aiopsRuntime managersvcaiops.RuntimeHandler + aiopsRuntime *managersvcaiops.RuntimeRouter chatRT *aiopschatruntime.Runtime humanApprovalBroker *managersvcaiops.HumanApprovalBroker aiopsSvc *managersvcaiops.Service @@ -40,11 +41,9 @@ func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Log bootstrapSkillReg, bootstrapAgentReg := loadBootstrapRegistries(log) var ( - aiopsRuntime managersvcaiops.RuntimeHandler - // chatRT keeps the concrete runtime handle so the ADR-026 - // self-obs sampler ticker (eg.Go in the goroutine wiring below) - // can call CountWorkersByStatus. The interface-typed - // aiopsRuntime is what the chat service consumes. + aiopsRuntime *managersvcaiops.RuntimeRouter + // chatRT owns version-local catalog, persona and credential assembly. + // All external execution and worker metrics use aiopsRuntime. chatRT *aiopschatruntime.Runtime ) humanApprovalBroker := managersvcaiops.NewHumanApprovalBroker() @@ -53,7 +52,12 @@ func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Log if rterr != nil { log.Error("aiops runtime unavailable", slog.Any("err", rterr)) } else { - aiopsRuntime = rt + router, err := managersvcaiops.NewRuntimeRouter(contract.V3, map[contract.Version]managersvcaiops.RuntimeEngine{contract.V3: rt}) + if err != nil { + log.Error("aiops runtime routing unavailable", slog.Any("err", err)) + } else { + aiopsRuntime = router + } chatRT = rt // — wire the coordinator-only AgentTool / // SendMessage / TaskStop trio. Two-step because of the @@ -68,7 +72,7 @@ func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Log // unconditionally via coordinatorOnlyTools (see // chatruntime/worker.go). aiops.toolsReg.SetWorkerSpawner( - managersvcaiops.WorkerSpawner{Runtime: rt}, + managersvcaiops.WorkerSpawner{Runtime: aiopsRuntime}, managersvcaiops.AgentRegistry{Inner: rt.AgentRegistry()}, ) // SendMessage / TaskStop are control-plane micro-ops; 15s @@ -93,9 +97,9 @@ func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Log HumanApproval: humanApprovalBroker, } wrappedCoord := []aiopstoolsbase.BaseTool{ - aiopstoolsdec.Wrap(aiopstools.NewAgentTool(managersvcaiops.WorkerSpawner{Runtime: rt}, managersvcaiops.AgentRegistry{Inner: rt.AgentRegistry()}, log), coordDepsDispatch), - aiopstoolsdec.Wrap(aiopstools.NewSendMessageTool(managersvcaiops.WorkerSpawner{Runtime: rt}, log), coordDepsFast), - aiopstoolsdec.Wrap(aiopstools.NewTaskStopTool(managersvcaiops.WorkerSpawner{Runtime: rt}, log), coordDepsFast), + aiopstoolsdec.Wrap(aiopstools.NewAgentTool(managersvcaiops.WorkerSpawner{Runtime: aiopsRuntime}, managersvcaiops.AgentRegistry{Inner: rt.AgentRegistry()}, log), coordDepsDispatch), + aiopstoolsdec.Wrap(aiopstools.NewSendMessageTool(managersvcaiops.WorkerSpawner{Runtime: aiopsRuntime}, log), coordDepsFast), + aiopstoolsdec.Wrap(aiopstools.NewTaskStopTool(managersvcaiops.WorkerSpawner{Runtime: aiopsRuntime}, log), coordDepsFast), } if err := rt.AppendToolBag(wrappedCoord); err != nil { log.Error("append coordinator tools failed", slog.Any("err", err)) diff --git a/cmd/ongrid/workers.go b/cmd/ongrid/workers.go index c63b0692c..46829b314 100644 --- a/cmd/ongrid/workers.go +++ b/cmd/ongrid/workers.go @@ -54,8 +54,8 @@ func runWorkers(rootCtx context.Context, cfg *config.Config, log *slog.Logger, n // would have lit up here at running > 10 for hours. Interval is 15s // because workers are short-lived and an orphan is unusual; faster // polling would just thrash the mutex. - if runtime.chatRT != nil { - eg.Go(func() error { return managersvcaiops.RunWorkerMetrics(egCtx, runtime.chatRT) }) + if runtime.aiopsRuntime != nil { + eg.Go(func() error { return managersvcaiops.RunWorkerMetrics(egCtx, runtime.aiopsRuntime) }) } // Packet captures outlive the HTTP/SSE turn that created them. Register diff --git a/docs/rfc/RFC-009-agent-runtime-versions.md b/docs/rfc/RFC-009-agent-runtime-versions.md index 866f82034..d92ec50aa 100644 --- a/docs/rfc/RFC-009-agent-runtime-versions.md +++ b/docs/rfc/RFC-009-agent-runtime-versions.md @@ -73,4 +73,26 @@ v1 固定为 GitHub 最早正式 release `v0.7.158`（2026-05-27，`04b279631978 已完成：`runtime/v1/{chatruntime,graph}` 提取首个 release；`runtime/v2/{chatruntime,graph}` 提取 v0.18.2；当前实现及测试迁移至 `runtime/v3/{chatruntime,graph}`，现有调用方指向 v3。v1/v2 保留源提交、文件 blob 清单和兼容改动说明。三版本均参与标准测试，禁止直接跨版本 import。 -本次仅完成源码隔离。版本契约、生产路由、会话版本字段、版本化工具装配及管理员设置仍未实现。外部 persona/skills、业务工具和依赖仍共享，不能将源码提取理解为完整复现旧 release，v1 接入前必须补齐现行安全与审批验收。 +本次仅完成源码隔离。版本契约和单版本生产路由已接入（见下节）；会话版本字段、历史版本适配、版本化工具装配及管理员设置仍未实现。外部 persona/skills、业务工具和依赖仍共享，不能将源码提取理解为完整复现旧 release，v1 接入前必须补齐现行安全与审批验收。 + +## 统一契约与路由：能力保留检查（2026-10-16） + +已抽取 `runtime/contract` 的完整请求、回复、流事件、Worker 快照与上下文载体。v3 使用类型别名保持调用兼容，Worker 锁和 cancel 仍留在 v3 私有状态，v1/v2 冻结源码不变。Service 侧 `RuntimeRouter` 接管聊天与外部 Worker 调用；当前只注册 v3，默认版本和注册表不可变，未开放用户切换，也没有会话版本迁移。未知或未装配版本明确拒绝，不自动降级。 + +Worker 查询、继续、停止按实例实际归属分派，覆盖内部 reviewer 创建的 Worker；重名导致归属不明确时拒绝控制。后台 Worker 保留版本和权限快照，不继承请求取消和 SSE callback 状态。此处的版本上下文是进程内执行归属，不替代未来的会话/审批持久化版本。 + +| 必须保留的能力 | 迁移措施与回归 | +| --- | --- | +| 普通聊天、模型/语言选择、附件、mentions、联网开关 | 原样透传完整 Request；Router 请求/结果回归、AssistantChat E2E | +| SSE、工具起止、审批卡片、后台完成事件 | 共享完整 Event 协议，路由不重写 payload；完整事件序列回归 | +| Worker 同步/后台、查询、继续、停止、指标 | 保留原状态机，路由查询归属；并发控制/错误分派/快照回归 | +| 报告、告警调查、工作流 Agent 节点 | 移除对具体 v3 Runtime 的类型断言，改依赖契约；相关包纳入标准 Agent race 目标 | +| IM、mention 解析 | Service 使用路由，hydrator 仍注入实际 v3 实例；IM 回归 | +| Skills、marketplace、persona 热加载 | 保留实际 v3 注册器的装配与热更新，避免路由包装后断言失败导致能力静默关闭 | +| MCP、shell、安装 Skill、页面产物、消息工具 | 原装配和装饰器保留；MCP 刷新仅依赖消费方的窄目录接口；MCP/Skills/Flow E2E | +| 人工审批及 reviewer | 保留独立审批域和执行器；事件上下文改用共享契约，reviewer 仍由所属实例执行 | +| Trace 会话关联 | 保持原 instrumentation scope；agenttrace 纳入标准 Agent race 目标 | + +验收：扩展后的 `make test-agent-runtime`（Linux，含 race）、架构检查及 11 个边界样例通过；真实 Manager + 临时 MySQL 的 AssistantChat、MCP CRUD、Skills registry 和 Flow MCP 节点 E2E 通过（39.771s）；额外的 RCA 调查、工作流页面产物和 Edge 概览 E2E 通过（32.953s）。模型和 MCP endpoint 为测试替身，不代表真实模型验收。 + +下一阶段在不改变上述能力的前提下增加持久化会话版本、旧版本适配及独立目录装配，之后才开放管理员切换。回滚本阶段只需回退代码；本阶段没有 API/schema/配置格式变更。 diff --git a/internal/manager/biz/aiops/runtime/README.md b/internal/manager/biz/aiops/runtime/README.md index 7d478aea1..67aff0462 100644 --- a/internal/manager/biz/aiops/runtime/README.md +++ b/internal/manager/biz/aiops/runtime/README.md @@ -10,4 +10,4 @@ 每个版本独立持有 `chatruntime`（上下文、prompt、Worker、目录策略）和 `graph`（Eino 编排、工具适配、回调）。禁止版本间 import，`isolation_test.go` 在标准 Go 测试中检查该边界，并验证三版本指标注册共存。业务执行能力与基础设施继续共享；尚未声称三版本已可切换。 -验证：Linux 环境执行 `make test-agent-runtime`。不新增 schema/API/设置，不改变当前运行时选择；需要撤销本次隔离时回退该源码迁移提交即可。版本路由及管理设置的后续方案见 [RFC-009](../../../../../docs/rfc/RFC-009-agent-runtime-versions.md)。 +验证：Linux 环境执行 `make test-agent-runtime`。不新增 schema/API/设置，不改变当前运行时选择；需要撤销本次隔离时回退该源码迁移提交即可。执行入口已通过统一契约和路由，当前只注册 v3。会话持久化版本、旧版本装配及管理设置的后续方案见 [RFC-009](../../../../../docs/rfc/RFC-009-agent-runtime-versions.md)。 diff --git a/internal/manager/biz/aiops/runtime/contract/context.go b/internal/manager/biz/aiops/runtime/contract/context.go new file mode 100644 index 000000000..d41be8710 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/contract/context.go @@ -0,0 +1,37 @@ +package contract + +import "context" + +// Version identifies execution policy, independently of the product release. +type Version string + +const ( + V1 Version = "v1" + V2 Version = "v2" + V3 Version = "v3" +) + +type emitKey struct{} +type versionKey struct{} + +func WithEmit(ctx context.Context, emit Emit) context.Context { + if emit == nil { + return ctx + } + return context.WithValue(ctx, emitKey{}, emit) +} + +func EmitFromContext(ctx context.Context) Emit { + emit, _ := ctx.Value(emitKey{}).(Emit) + return emit +} + +// WithVersion pins nested execution to its originating runtime. +func WithVersion(ctx context.Context, version Version) context.Context { + return context.WithValue(ctx, versionKey{}, version) +} + +func VersionFromContext(ctx context.Context) Version { + version, _ := ctx.Value(versionKey{}).(Version) + return version +} diff --git a/internal/manager/biz/aiops/runtime/contract/types.go b/internal/manager/biz/aiops/runtime/contract/types.go new file mode 100644 index 000000000..9f2deb0fd --- /dev/null +++ b/internal/manager/biz/aiops/runtime/contract/types.go @@ -0,0 +1,267 @@ +// Package contract defines version-independent runtime inputs, results and events. +// Execution policy and mutable worker state remain inside each version. +package contract + +import ( + "context" + "time" + + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// Mention 是运行时的 mention 输入；Service 从 chatprotocol 转换。 +type Mention struct { + Type string + ID string + Label string +} + +// MentionResolver hydrates each @-mention into one bullet line. nil disables inlining. +type MentionResolver interface { + Resolve(ctx context.Context, mentions []Mention) []string +} + +// EventType enumerates streaming events emitted by the runtime. The +// values are byte-equal to the legacy agent.EventType strings so the +// SSE adapter at the service layer can route both kernels through +// the same wire format. New event names introduced in PR-6 +// (assistant_start / assistant_delta / assistant_end) are NOT +// surfaced through this enum yet — the cutover keeps the legacy +// frame names so the SPA round-trips without changes. +type EventType string + +const ( + // EventAssistant fires after every assistant turn is persisted. + EventAssistant EventType = "assistant" + // EventToolStart fires after a tool_call row is persisted as + // pending, before the tool actually runs. + EventToolStart EventType = "tool_start" + // EventToolEnd fires once the tool finishes (success/error/timeout). + EventToolEnd EventType = "tool_end" + // EventDone fires once at terminal success with the final Reply. + EventDone EventType = "done" + // EventError fires on terminal failure. + EventError EventType = "error" +) + +// AssistantEvent describes one assistant turn's persisted row. Same +// shape as legacy agent.AssistantEvent. +type AssistantEvent struct { + Iteration int + MessageID string + Content string + CreatedAt time.Time + PendingToolCalls int +} + +// ToolEvent describes one tool_call lifecycle event. Same shape as +// legacy agent.ToolEvent. +type ToolEvent struct { + ToolCallID string + Name string + DeviceID *uint64 + Status string + StartedAt time.Time + EndedAt *time.Time + DurationMs int64 + Error string + ArgsJSON string + ResultJSON string +} + +// Event is the streaming envelope. Mirrors legacy agent.Event so the +// service-layer adapter can copy fields one-to-one without a switch. +// +// Notification carries the task_notification payload — set when +// Type == EventTaskNotification and a background worker reaches a +// terminal state. The SPA / SSE adapter renders it as a separate frame +// type next to assistant / tool / done. +type Event struct { + Type EventType + Assistant *AssistantEvent + Tool *ToolEvent + Done *Reply + Error string + Notification *TaskNotification + // Approval carries the approval_pending payload — set when + // Type == EventApprovalPending and a blocking tool (cloud_bash) has + // queued a proposal and is awaiting the human decision (HLD-021). + Approval *ApprovalPending +} + +// Emit is the streaming callback. nil-safe. +type Emit func(Event) + +// Request bundles every per-call input the runtime needs. +type Request struct { + SessionID string + UserID uint64 + // Role is the caller's system role (admin | user | viewer). + // gates the toolbag against this — viewer drops Class!=read so the + // LLM cannot reach mutating tools no matter what the persona allows. + Role string + UserText string + Attachments []aiopsmodel.Attachment + Mentions []Mention + Provider string + Model string + WebSearchEnabled bool + // Locale is the UI language ("en-US"/"zh-CN") the reply should use; + // threaded into graph.Input so the assembler adds a language directive. + Locale string + // Emit is the streaming sink. nil = no streaming (blocking call). + Emit Emit +} + +// Reply is the terminal result returned by Handle. Mirrors the +// legacy agent.Reply shape so the service layer can fan it back out +// through the same DTO without translation losses. +type Reply struct { + Message *aiopsmodel.Message + Usage llm.Usage + Iterations int + ToolCalls []*aiopsmodel.ToolCall +} + +type WorkerStatus string + +const ( + // WorkerStatusPending — accepted but not yet started. Observable + // only in tests that intercept between SpawnWorker accept and goroutine + // scheduling; in production this state is essentially transient. + WorkerStatusPending WorkerStatus = "pending" + // WorkerStatusRunning — graph.Invoke is in flight. + WorkerStatusRunning WorkerStatus = "running" + // WorkerStatusCompleted — graph.Invoke returned a final assistant + // message with no error. + WorkerStatusCompleted WorkerStatus = "completed" + // WorkerStatusFailed — graph.Invoke returned an error, or the agent + // name was unknown. + WorkerStatusFailed WorkerStatus = "failed" + // WorkerStatusKilled — explicitly stopped via StopWorker. + WorkerStatusKilled WorkerStatus = "killed" +) + +// Worker tracks a spawned sub-agent. Unique by ID within a Runtime +// instance. +type Worker struct { + // ID is "agent-<8 hex>" — unique per Runtime, stable for the + // worker's lifetime. + ID string + // AgentName is the persona name (frontmatter `name` field). + AgentName string + // SessionID is the worker's own session id (distinct from the + // coordinator's session). For PR-1 of this is a synthetic + // in-memory id; the chat_sessions row + parent_session_id column + // migration is follow-up. + SessionID string + // ParentSessionID is the coordinator session that requested the + // spawn. Tracked here so a future audit query can rebuild the parent + // → worker tree without a schema change. + ParentSessionID string + // Prompt is the initial user message handed to the worker. + Prompt string + // Status is the lifecycle state. Read-only outside the runtime; + // callers should always pull through GetWorker for a snapshot. + Status WorkerStatus + // Background indicates whether this is a fire-and-forget spawn. + Background bool + // StartedAt is the wall time the worker entered "running". + StartedAt time.Time + // EndedAt is the wall time of any terminal transition. nil while + // running. + EndedAt *time.Time + // Result is the worker's final assistant content (when terminal = + // completed). Empty for failed / killed. + Result string + // Err is the failure reason (when terminal = failed / killed). + Err string +} + +// SpawnRequest describes a worker to spawn. See AgentTool.InvokableRun +// for the LLM-facing argument shape that produces this struct. +type SpawnRequest struct { + // AgentName is the persona name (frontmatter `name`). Must be + // resolvable through the Runtime's AgentRegistry. + AgentName string + // Prompt is the initial user message for the worker. + Prompt string + // Background = true: fire-and-forget; SpawnWorker returns immediately + // with status=running. Background = false: SpawnWorker blocks until + // the worker reaches a terminal state. + Background bool + // ParentSession is the coordinator's chat session id. + ParentSession string + // ParentEmit is the SSE emitter the coordinator runtime threads + // through. Used to deliver the task_notification frame when the + // worker (background=true) finishes. nil = no notification (sync + // callers don't need it). + ParentEmit Emit + // SessionKind overrides the persisted chat_sessions.kind for the + // worker session. Empty = "work". The investigator usecase + // sets "investigation" so auto-spawned RCA transcripts stay out of + // the /chat list. + SessionKind string + // OwnerUserID overrides the persisted chat_sessions.user_id when + // the worker has no ParentSession to inherit from. The investigator + // usecase passes 0 to mark these rows as "system-owned" — they + // don't belong to any operator and the internal audience prevents + // them from appearing in the primary chat list. + OwnerUserID uint64 + // Locale is the UI language ("en", "zh-CN", ...) the worker's reply + // should use. Threaded from the parent coordinator's request so a + // sub-agent answers in the same language the user is typing in + // (otherwise GLM defaults to zh on English questions). Empty = no + // directive, back-compat with investigator/auto-spawned workers. + Locale string + // Provider + Model are the coordinator's resolved LLM choice. Threaded + // into runWorker's g.Invoke as chatModelOpts so the sub-agent runs + // against the same LLM the user picked for the coordinator — without + // this, runWorker falls through to the RoutingChatModel default and + // installs without an OpenAI key see specialists fail with + // `provider "openai" not configured`. Empty = no override; the worker + // uses whatever the routing default is. + Provider string + Model string +} + +// TaskNotification is the payload of a task_notification streaming +// frame. Mirrors the schema laid out in — coordinator's +// SSE listener fans this back through the legacy SSE envelope so the +// SPA only needs to learn one new event type. +type TaskNotification struct { + TaskID string `json:"task_id"` + Status WorkerStatus `json:"status"` + Summary string `json:"summary"` + Result string `json:"result,omitempty"` + Err string `json:"error,omitempty"` + Usage map[string]any `json:"usage,omitempty"` +} + +// EventTaskNotification is the new streaming event type the runtime +// emits when a background worker reaches a terminal state. The SPA +// listens for this in addition to the existing assistant / tool / +// done frames. +const EventTaskNotification EventType = "task_notification" + +// EventApprovalPending fires when a synchronous-blocking tool (HLD-021, +// e.g. cloud_bash) has queued a human-approval proposal and is about to +// block waiting for the decision. The frame surfaces the inline approve/ +// reject card LIVE — the tool no longer returns a pending_approval result +// blob (it blocks, then returns the real executor result), so the card +// must be driven by this frame instead. ToolCallID ties the card to the +// tool call's existing streaming card so the UI shows a single card. +const EventApprovalPending EventType = "approval_pending" + +// ApprovalPending is the EventApprovalPending payload. Emitted by the +// proposer shim (cmd/main.go) via EmitFromContext while a blocking tool +// awaits the human decision. +type ApprovalPending struct { + ApprovalID string + ToolCallID string + Kind string + ToolName string + Command string + Credentials []string +} diff --git a/internal/manager/biz/aiops/runtime/isolation_test.go b/internal/manager/biz/aiops/runtime/isolation_test.go index 60b1a07a2..1999116a5 100644 --- a/internal/manager/biz/aiops/runtime/isolation_test.go +++ b/internal/manager/biz/aiops/runtime/isolation_test.go @@ -18,7 +18,7 @@ import ( // Versioned policies must never silently delegate to another generation. func TestRuntimeVersionsDoNotImportEachOther(t *testing.T) { const base = "github.com/ongridio/ongrid/internal/manager/biz/aiops/" - for _, version := range []string{"v1", "v2", "v3"} { + for _, version := range []string{"v1", "v2", "v3", "contract"} { t.Run(version, func(t *testing.T) { err := filepath.WalkDir(version, func(path string, entry fs.DirEntry, err error) error { if err != nil { @@ -36,7 +36,7 @@ func TestRuntimeVersionsDoNotImportEachOther(t *testing.T) { if err != nil { return err } - if strings.HasPrefix(imported, base+"runtime/") && !strings.HasPrefix(imported, base+"runtime/"+version+"/") { + if strings.HasPrefix(imported, base+"runtime/") && imported != base+"runtime/contract" && !strings.HasPrefix(imported, base+"runtime/"+version+"/") { t.Errorf("%s imports another runtime: %s", path, imported) } if imported == base+"graph" || strings.HasPrefix(imported, base+"graph/") || imported == base+"chatruntime" { diff --git a/internal/manager/biz/aiops/runtime/v3/README.md b/internal/manager/biz/aiops/runtime/v3/README.md index 04d20ecae..0fdab2d79 100644 --- a/internal/manager/biz/aiops/runtime/v3/README.md +++ b/internal/manager/biz/aiops/runtime/v3/README.md @@ -2,4 +2,4 @@ 从草稿 PR #503 的 `cc25f29f` 迁移当前 `chatruntime/`、`graph/` 及其测试到本目录。只调整 import 与测试夹具路径，保留 tracing scope 名称和现有行为。 -当前生产装配调用本版本。后续策略变更在本目录实施，不修改 v1/v2 的历史行为基线。业务工具、领域模型、存储和外部 agents/skills 资源仍然共享；跨版本路由与版本化装配尚待实施。 +当前生产装配调用本版本。后续策略变更在本目录实施，不修改 v1/v2 的历史行为基线。业务工具、领域模型、存储和外部 agents/skills 资源仍然共享；统一契约和执行路由已接入；会话版本持久化、历史版本适配和版本化装配尚待实施。 diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go index cf7b853b1..5040a2dff 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go @@ -22,6 +22,7 @@ import ( "go.opentelemetry.io/otel/attribute" biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph" "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph/callbacks" "github.com/ongridio/ongrid/internal/manager/biz/aiops/toolreplay" @@ -32,120 +33,25 @@ import ( "github.com/ongridio/ongrid/internal/pkg/tracing" ) -// Mention 是运行时的 mention 输入；Service 从 chatprotocol 转换。 -type Mention struct { - Type string - ID string - Label string -} - -// MentionResolver hydrates each @-mention into one bullet line. nil disables inlining. -type MentionResolver interface { - Resolve(ctx context.Context, mentions []Mention) []string -} - -// EventType enumerates streaming events emitted by the runtime. The -// values are byte-equal to the legacy agent.EventType strings so the -// SSE adapter at the service layer can route both kernels through -// the same wire format. New event names introduced in PR-6 -// (assistant_start / assistant_delta / assistant_end) are NOT -// surfaced through this enum yet — the cutover keeps the legacy -// frame names so the SPA round-trips without changes. -type EventType string +// Public protocol aliases preserve existing callers while ownership moves to contract. +type Mention = contract.Mention +type MentionResolver = contract.MentionResolver +type EventType = contract.EventType +type AssistantEvent = contract.AssistantEvent +type ToolEvent = contract.ToolEvent +type Event = contract.Event +type Emit = contract.Emit +type Request = contract.Request +type Reply = contract.Reply const ( - // EventAssistant fires after every assistant turn is persisted. - EventAssistant EventType = "assistant" - // EventToolStart fires after a tool_call row is persisted as - // pending, before the tool actually runs. - EventToolStart EventType = "tool_start" - // EventToolEnd fires once the tool finishes (success/error/timeout). - EventToolEnd EventType = "tool_end" - // EventDone fires once at terminal success with the final Reply. - EventDone EventType = "done" - // EventError fires on terminal failure. - EventError EventType = "error" + EventAssistant = contract.EventAssistant + EventToolStart = contract.EventToolStart + EventToolEnd = contract.EventToolEnd + EventDone = contract.EventDone + EventError = contract.EventError ) -// AssistantEvent describes one assistant turn's persisted row. Same -// shape as legacy agent.AssistantEvent. -type AssistantEvent struct { - Iteration int - MessageID string - Content string - CreatedAt time.Time - PendingToolCalls int -} - -// ToolEvent describes one tool_call lifecycle event. Same shape as -// legacy agent.ToolEvent. -type ToolEvent struct { - ToolCallID string - Name string - DeviceID *uint64 - Status string - StartedAt time.Time - EndedAt *time.Time - DurationMs int64 - Error string - ArgsJSON string - ResultJSON string -} - -// Event is the streaming envelope. Mirrors legacy agent.Event so the -// service-layer adapter can copy fields one-to-one without a switch. -// -// Notification carries the task_notification payload — set when -// Type == EventTaskNotification and a background worker reaches a -// terminal state. The SPA / SSE adapter renders it as a separate frame -// type next to assistant / tool / done. -type Event struct { - Type EventType - Assistant *AssistantEvent - Tool *ToolEvent - Done *Reply - Error string - Notification *TaskNotification - // Approval carries the approval_pending payload — set when - // Type == EventApprovalPending and a blocking tool (cloud_bash) has - // queued a proposal and is awaiting the human decision (HLD-021). - Approval *ApprovalPending -} - -// Emit is the streaming callback. nil-safe. -type Emit func(Event) - -// Request bundles every per-call input the runtime needs. -type Request struct { - SessionID string - UserID uint64 - // Role is the caller's system role (admin | user | viewer). - // gates the toolbag against this — viewer drops Class!=read so the - // LLM cannot reach mutating tools no matter what the persona allows. - Role string - UserText string - Attachments []aiopsmodel.Attachment - Mentions []Mention - Provider string - Model string - WebSearchEnabled bool - // Locale is the UI language ("en-US"/"zh-CN") the reply should use; - // threaded into graph.Input so the assembler adds a language directive. - Locale string - // Emit is the streaming sink. nil = no streaming (blocking call). - Emit Emit -} - -// Reply is the terminal result returned by Handle. Mirrors the -// legacy agent.Reply shape so the service layer can fan it back out -// through the same DTO without translation losses. -type Reply struct { - Message *aiopsmodel.Message - Usage llm.Usage - Iterations int - ToolCalls []*aiopsmodel.ToolCall -} - // Config tunes runtime behaviour. ToolBag is the wrapped (decorated) // tool list; the runtime does NOT wrap tools itself — that's the // caller's job at construction time so the decorator deps (audit @@ -261,7 +167,7 @@ type Runtime struct { log *slog.Logger workersMu sync.Mutex - workers map[string]*Worker + workers map[string]*workerState // toolMu protects cfg.ToolBag, which is updated when an MCP server is // verified, edited, disabled, or deleted while chats are in flight. diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go index a7fb64b21..a5cd03885 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go @@ -53,6 +53,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph" "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/graph/callbacks" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" @@ -63,180 +64,30 @@ import ( // WorkerStatus enumerates the worker state machine. See package header // for the transition diagram. -type WorkerStatus string +type WorkerStatus = contract.WorkerStatus +type Worker = contract.Worker +type SpawnRequest = contract.SpawnRequest +type TaskNotification = contract.TaskNotification +type ApprovalPending = contract.ApprovalPending const ( - // WorkerStatusPending — accepted but not yet started. Observable - // only in tests that intercept between SpawnWorker accept and goroutine - // scheduling; in production this state is essentially transient. - WorkerStatusPending WorkerStatus = "pending" - // WorkerStatusRunning — graph.Invoke is in flight. - WorkerStatusRunning WorkerStatus = "running" - // WorkerStatusCompleted — graph.Invoke returned a final assistant - // message with no error. - WorkerStatusCompleted WorkerStatus = "completed" - // WorkerStatusFailed — graph.Invoke returned an error, or the agent - // name was unknown. - WorkerStatusFailed WorkerStatus = "failed" - // WorkerStatusKilled — explicitly stopped via StopWorker. - WorkerStatusKilled WorkerStatus = "killed" + WorkerStatusPending = contract.WorkerStatusPending + WorkerStatusRunning = contract.WorkerStatusRunning + WorkerStatusCompleted = contract.WorkerStatusCompleted + WorkerStatusFailed = contract.WorkerStatusFailed + WorkerStatusKilled = contract.WorkerStatusKilled + EventTaskNotification = contract.EventTaskNotification + EventApprovalPending = contract.EventApprovalPending ) -// Worker tracks a spawned sub-agent. Unique by ID within a Runtime -// instance. -type Worker struct { - // ID is "agent-<8 hex>" — unique per Runtime, stable for the - // worker's lifetime. - ID string - // AgentName is the persona name (frontmatter `name` field). - AgentName string - // SessionID is the worker's own session id (distinct from the - // coordinator's session). For PR-1 of this is a synthetic - // in-memory id; the chat_sessions row + parent_session_id column - // migration is follow-up. - SessionID string - // ParentSessionID is the coordinator session that requested the - // spawn. Tracked here so a future audit query can rebuild the parent - // → worker tree without a schema change. - ParentSessionID string - // Prompt is the initial user message handed to the worker. - Prompt string - // Status is the lifecycle state. Read-only outside the runtime; - // callers should always pull through GetWorker for a snapshot. - Status WorkerStatus - // Background indicates whether this is a fire-and-forget spawn. - Background bool - // StartedAt is the wall time the worker entered "running". - StartedAt time.Time - // EndedAt is the wall time of any terminal transition. nil while - // running. - EndedAt *time.Time - // Result is the worker's final assistant content (when terminal = - // completed). Empty for failed / killed. - Result string - // Err is the failure reason (when terminal = failed / killed). - Err string - - // Internal fields — not surfaced to callers via copies. +type workerState struct { + Worker cancel context.CancelFunc mu sync.Mutex } -// SpawnRequest describes a worker to spawn. See AgentTool.InvokableRun -// for the LLM-facing argument shape that produces this struct. -type SpawnRequest struct { - // AgentName is the persona name (frontmatter `name`). Must be - // resolvable through the Runtime's AgentRegistry. - AgentName string - // Prompt is the initial user message for the worker. - Prompt string - // Background = true: fire-and-forget; SpawnWorker returns immediately - // with status=running. Background = false: SpawnWorker blocks until - // the worker reaches a terminal state. - Background bool - // ParentSession is the coordinator's chat session id. - ParentSession string - // ParentEmit is the SSE emitter the coordinator runtime threads - // through. Used to deliver the task_notification frame when the - // worker (background=true) finishes. nil = no notification (sync - // callers don't need it). - ParentEmit Emit - // SessionKind overrides the persisted chat_sessions.kind for the - // worker session. Empty = "work". The investigator usecase - // sets "investigation" so auto-spawned RCA transcripts stay out of - // the /chat list. - SessionKind string - // OwnerUserID overrides the persisted chat_sessions.user_id when - // the worker has no ParentSession to inherit from. The investigator - // usecase passes 0 to mark these rows as "system-owned" — they - // don't belong to any operator and the internal audience prevents - // them from appearing in the primary chat list. - OwnerUserID uint64 - // Locale is the UI language ("en", "zh-CN", ...) the worker's reply - // should use. Threaded from the parent coordinator's request so a - // sub-agent answers in the same language the user is typing in - // (otherwise GLM defaults to zh on English questions). Empty = no - // directive, back-compat with investigator/auto-spawned workers. - Locale string - // Provider + Model are the coordinator's resolved LLM choice. Threaded - // into runWorker's g.Invoke as chatModelOpts so the sub-agent runs - // against the same LLM the user picked for the coordinator — without - // this, runWorker falls through to the RoutingChatModel default and - // installs without an OpenAI key see specialists fail with - // `provider "openai" not configured`. Empty = no override; the worker - // uses whatever the routing default is. - Provider string - Model string -} - -// TaskNotification is the payload of a task_notification streaming -// frame. Mirrors the schema laid out in — coordinator's -// SSE listener fans this back through the legacy SSE envelope so the -// SPA only needs to learn one new event type. -type TaskNotification struct { - TaskID string `json:"task_id"` - Status WorkerStatus `json:"status"` - Summary string `json:"summary"` - Result string `json:"result,omitempty"` - Err string `json:"error,omitempty"` - Usage map[string]any `json:"usage,omitempty"` -} - -// EventTaskNotification is the new streaming event type the runtime -// emits when a background worker reaches a terminal state. The SPA -// listens for this in addition to the existing assistant / tool / -// done frames. -const EventTaskNotification EventType = "task_notification" - -// EventApprovalPending fires when a synchronous-blocking tool (HLD-021, -// e.g. cloud_bash) has queued a human-approval proposal and is about to -// block waiting for the decision. The frame surfaces the inline approve/ -// reject card LIVE — the tool no longer returns a pending_approval result -// blob (it blocks, then returns the real executor result), so the card -// must be driven by this frame instead. ToolCallID ties the card to the -// tool call's existing streaming card so the UI shows a single card. -const EventApprovalPending EventType = "approval_pending" - -// ApprovalPending is the EventApprovalPending payload. Emitted by the -// proposer shim (cmd/main.go) via EmitFromContext while a blocking tool -// awaits the human decision. -type ApprovalPending struct { - ApprovalID string - ToolCallID string - Kind string - ToolName string - Command string - Credentials []string -} - -// emitCtxKey is the unexported context key that threads the active -// per-request Emit through the graph layer down into AgentTool's -// InvokableRun via the WorkerSpawner shim. The shim reads it through -// EmitFromContext to populate SpawnRequest.ParentEmit so a background -// worker can fire its task_notification through the SSE channel that -// owns the user's chat session. -type emitCtxKeyT struct{} - -var emitCtxKey = emitCtxKeyT{} - -// withEmit returns ctx augmented with emit. Internal — Handle threads -// the request emitter in before invoking the graph. -func withEmit(ctx context.Context, emit Emit) context.Context { - if emit == nil { - return ctx - } - return context.WithValue(ctx, emitCtxKey, emit) -} - -// EmitFromContext retrieves the active streaming Emit from ctx, if any. -// Returns nil when no emitter was attached (e.g. blocking call without -// SSE). The wiring shim at the cmd/main.go boundary uses this to thread -// task_notification frames through the same SSE channel that owns the -// coordinator's chat session. -func EmitFromContext(ctx context.Context) Emit { - v, _ := ctx.Value(emitCtxKey).(Emit) - return v -} +func withEmit(ctx context.Context, emit Emit) context.Context { return contract.WithEmit(ctx, emit) } +func EmitFromContext(ctx context.Context) Emit { return contract.EmitFromContext(ctx) } // Locale ctx propagation lives in the tools package (next to AgentTool, // the consumer). Runtime calls tools.WithLocale before g.Invoke; the @@ -352,7 +203,7 @@ func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, } workerCtx, cancel := context.WithCancel(parent) - w := &Worker{ + w := &workerState{Worker: Worker{ ID: id, AgentName: agentDef.Name, SessionID: sessID, @@ -360,12 +211,13 @@ func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, Prompt: req.Prompt, Status: WorkerStatusPending, Background: req.Background, - cancel: cancel, + }, + cancel: cancel, } rt.workersMu.Lock() if rt.workers == nil { - rt.workers = map[string]*Worker{} + rt.workers = map[string]*workerState{} } rt.workers[id] = w rt.workersMu.Unlock() @@ -765,7 +617,7 @@ func scopedWorkerToolCallID(workerID, toolCallID string) string { // notificationFor produces a task_notification Event for the given // terminal worker. Field shape mirrors -func (rt *Runtime) notificationFor(w *Worker) Event { +func (rt *Runtime) notificationFor(w *workerState) Event { w.mu.Lock() defer w.mu.Unlock() summary := fmt.Sprintf("Agent %s %s", w.AgentName, w.Status) @@ -918,7 +770,7 @@ func matchesAny(name string, patterns []string) bool { // snapshotWorker returns a copy of w safe to publish to a caller. The // internal lock + cancel func are not copied — the returned *Worker is // strictly read-only. -func (rt *Runtime) snapshotWorker(w *Worker) *Worker { +func (rt *Runtime) snapshotWorker(w *workerState) *Worker { if w == nil { return nil } @@ -1084,6 +936,7 @@ func workerChatModelOpts(ctx context.Context) []model.Option { // request cancellation, callback handlers, or mutable request slots. func detachedWorkerContext(ctx context.Context) context.Context { parent := trace.ContextWithSpanContext(context.Background(), trace.SpanContextFromContext(ctx)) + parent = contract.WithVersion(parent, contract.VersionFromContext(ctx)) parent = basetool.WithWebSearchAllowed(parent, basetool.WebSearchAllowed(ctx)) parent = basetool.WithAgentWriteAllowed(parent, basetool.AgentWriteAllowedFromContext(ctx)) if tenant, ok := tenantctx.From(ctx); ok { diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/worker_context_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker_context_test.go index 752eb2fd0..a80da7abd 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/worker_context_test.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker_context_test.go @@ -2,6 +2,7 @@ package chatruntime import ( "context" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" "github.com/ongridio/ongrid/internal/pkg/tenantctx" "go.opentelemetry.io/otel/trace" @@ -13,6 +14,8 @@ func TestDetachedWorkerContextPreservesAuthorizationWithoutRequestState(t *testi for _, allowed := range []bool{false, true} { ctx, cancel := context.WithCancel(context.Background()) ctx = context.WithValue(ctx, requestKey{}, "callback state") + ctx = contract.WithVersion(ctx, contract.V3) + ctx = contract.WithEmit(ctx, func(contract.Event) {}) span := trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{2}, TraceFlags: trace.FlagsSampled}) ctx = trace.ContextWithSpanContext(ctx, span) ctx = basetool.WithWebSearchAllowed(ctx, allowed) @@ -26,6 +29,9 @@ func TestDetachedWorkerContextPreservesAuthorizationWithoutRequestState(t *testi if worker.Err() != nil || worker.Done() != nil { t.Fatal("worker inherited request cancellation") } + if contract.VersionFromContext(worker) != contract.V3 || contract.EmitFromContext(worker) != nil { + t.Fatal("worker version lost or request emitter retained") + } if worker.Value(requestKey{}) != nil { t.Fatal("worker retained request-only state") } diff --git a/internal/manager/biz/alert/investigator/usecase.go b/internal/manager/biz/alert/investigator/usecase.go index 8151f61e9..5b87b4a96 100644 --- a/internal/manager/biz/alert/investigator/usecase.go +++ b/internal/manager/biz/alert/investigator/usecase.go @@ -26,7 +26,7 @@ import ( "sync" "time" - chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" + chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" alertmodel "github.com/ongridio/ongrid/internal/manager/model/alert" ) diff --git a/internal/manager/biz/alert/investigator/usecase_test.go b/internal/manager/biz/alert/investigator/usecase_test.go index 035bcce80..f495caa06 100644 --- a/internal/manager/biz/alert/investigator/usecase_test.go +++ b/internal/manager/biz/alert/investigator/usecase_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" + chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" alertmodel "github.com/ongridio/ongrid/internal/manager/model/alert" "github.com/ongridio/ongrid/internal/pkg/errs" ) diff --git a/internal/manager/biz/report/generator.go b/internal/manager/biz/report/generator.go index 44c712c67..6719a32d0 100644 --- a/internal/manager/biz/report/generator.go +++ b/internal/manager/biz/report/generator.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" + chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" model "github.com/ongridio/ongrid/internal/manager/model/report" ) diff --git a/internal/manager/biz/report/generator_test.go b/internal/manager/biz/report/generator_test.go index 5bc2370a5..53be7a85f 100644 --- a/internal/manager/biz/report/generator_test.go +++ b/internal/manager/biz/report/generator_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" + chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" model "github.com/ongridio/ongrid/internal/manager/model/report" ) diff --git a/internal/manager/service/aiops/approval_adapters.go b/internal/manager/service/aiops/approval_adapters.go index 7e38ffcab..1e29a6487 100644 --- a/internal/manager/service/aiops/approval_adapters.go +++ b/internal/manager/service/aiops/approval_adapters.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "fmt" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" managerbizapproval "github.com/ongridio/ongrid/internal/manager/biz/approval" managerbizmcp "github.com/ongridio/ongrid/internal/manager/biz/mcp" diff --git a/internal/manager/service/aiops/human_approval.go b/internal/manager/service/aiops/human_approval.go index 40f440b78..1e3ff690d 100644 --- a/internal/manager/service/aiops/human_approval.go +++ b/internal/manager/service/aiops/human_approval.go @@ -8,7 +8,7 @@ import ( "time" "github.com/google/uuid" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" managerbizapproval "github.com/ongridio/ongrid/internal/manager/biz/approval" ) diff --git a/internal/manager/service/aiops/mcp_refresh.go b/internal/manager/service/aiops/mcp_refresh.go index 852d01489..952cb2f38 100644 --- a/internal/manager/service/aiops/mcp_refresh.go +++ b/internal/manager/service/aiops/mcp_refresh.go @@ -5,7 +5,6 @@ import ( "log/slog" "time" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" @@ -14,16 +13,21 @@ import ( "github.com/ongridio/ongrid/internal/pkg/mcpclient" ) +type toolSourceRuntime interface { + ReplaceToolSource(string, []aiopstoolsbase.BaseTool, []aiopstoolsbase.BaseTool) error + ToolCount() int +} + // MCPToolRefresher keeps runtime tools and the unredacted search catalog in sync. type MCPToolRefresher struct { - runtime *aiopschatruntime.Runtime + runtime toolSourceRuntime deps aiopstoolsdec.Deps caller MCPCaller proposer MCPProposer log *slog.Logger } -func NewMCPToolRefresher(runtime *aiopschatruntime.Runtime, deps aiopstoolsdec.Deps, caller MCPCaller, proposer MCPProposer, log *slog.Logger) *MCPToolRefresher { +func NewMCPToolRefresher(runtime toolSourceRuntime, deps aiopstoolsdec.Deps, caller MCPCaller, proposer MCPProposer, log *slog.Logger) *MCPToolRefresher { return &MCPToolRefresher{runtime: runtime, deps: deps, caller: caller, proposer: proposer, log: log} } diff --git a/internal/manager/service/aiops/runtime_adapters.go b/internal/manager/service/aiops/runtime_adapters.go index 643037298..abda3176e 100644 --- a/internal/manager/service/aiops/runtime_adapters.go +++ b/internal/manager/service/aiops/runtime_adapters.go @@ -4,6 +4,7 @@ import ( "context" "fmt" managerbizaiopsmentions "github.com/ongridio/ongrid/internal/manager/biz/aiops/mentions" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" @@ -16,7 +17,7 @@ type ChatMentionResolver struct { Inner *managerbizaiopsmentions.Searcher } -func (a ChatMentionResolver) Resolve(ctx context.Context, in []aiopschatruntime.Mention) []string { +func (a ChatMentionResolver) Resolve(ctx context.Context, in []contract.Mention) []string { if a.Inner == nil || len(in) == 0 { return nil } @@ -43,16 +44,16 @@ func (a ChatMentionResolver) Resolve(ctx context.Context, in []aiopschatruntime. // emitter from ctx so a background worker's task_notification frame // lands on the user's SSE channel. type WorkerSpawner struct { - Runtime *aiopschatruntime.Runtime + Runtime WorkerRuntime } func (s WorkerSpawner) SpawnWorker(ctx context.Context, req aiopstools.SpawnWorkerRequest) (*aiopstools.WorkerHandle, error) { - w, err := s.Runtime.SpawnWorker(ctx, aiopschatruntime.SpawnRequest{ + w, err := s.Runtime.SpawnWorker(ctx, contract.SpawnRequest{ AgentName: req.AgentName, Prompt: req.Prompt, Background: req.Background, ParentSession: req.ParentSession, - ParentEmit: aiopschatruntime.EmitFromContext(ctx), + ParentEmit: contract.EmitFromContext(ctx), Locale: req.Locale, Provider: req.Provider, Model: req.Model, @@ -84,10 +85,10 @@ func (s WorkerSpawner) GetWorker(workerID string) (*aiopstools.WorkerHandle, boo // NewRuntime returns; until then mutating tools fail closed. type ReviewSpawner struct { mu sync.RWMutex - rt *aiopschatruntime.Runtime + rt WorkerRuntime } -func (s *ReviewSpawner) SetRuntime(rt *aiopschatruntime.Runtime) { +func (s *ReviewSpawner) SetRuntime(rt WorkerRuntime) { s.mu.Lock() defer s.mu.Unlock() s.rt = rt @@ -100,11 +101,11 @@ func (s *ReviewSpawner) SpawnReviewer(ctx context.Context, req aiopstoolsdec.Rev if rt == nil { return nil, fmt.Errorf("reviewer runtime not wired") } - w, err := rt.SpawnWorker(ctx, aiopschatruntime.SpawnRequest{ + w, err := rt.SpawnWorker(ctx, contract.SpawnRequest{ AgentName: req.AgentName, Prompt: req.Prompt, Background: false, - ParentEmit: aiopschatruntime.EmitFromContext(ctx), + ParentEmit: contract.EmitFromContext(ctx), }) if err != nil { return nil, err @@ -119,7 +120,7 @@ func (s *ReviewSpawner) SpawnReviewer(ctx context.Context, req aiopstoolsdec.Rev // workerToHandle copies the chatruntime.Worker fields the tools layer // cares about into the seam-side shape. Duration is computed from // StartedAt + EndedAt when both are set; zero otherwise. -func workerToHandle(w *aiopschatruntime.Worker) *aiopstools.WorkerHandle { +func workerToHandle(w *contract.Worker) *aiopstools.WorkerHandle { if w == nil { return nil } diff --git a/internal/manager/service/aiops/runtime_router.go b/internal/manager/service/aiops/runtime_router.go new file mode 100644 index 000000000..7682f11f4 --- /dev/null +++ b/internal/manager/service/aiops/runtime_router.go @@ -0,0 +1,153 @@ +package aiops + +import ( + "context" + "fmt" + "reflect" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +// WorkerRuntime is the execution contract shared by all worker consumers. +// Persona and tool catalog mutation stay with version-specific assembly. +type WorkerRuntime interface { + SpawnWorker(context.Context, contract.SpawnRequest) (*contract.Worker, error) + SendToWorker(context.Context, string, string) error + StopWorker(context.Context, string) error + GetWorker(string) (*contract.Worker, bool) +} + +type RuntimeEngine interface { + RuntimeHandler + WorkerRuntime + CountWorkersByStatus() (running, pending int) +} + +// RuntimeRouter owns execution dispatch, never tool or persona policy. +// The engine registry and default are immutable. A live default-setting change +// will require durable session binding before it can be exposed to operators. +type RuntimeRouter struct { + engines map[contract.Version]RuntimeEngine + defaultVersion contract.Version +} + +func NewRuntimeRouter(defaultVersion contract.Version, engines map[contract.Version]RuntimeEngine) (*RuntimeRouter, error) { + copy := make(map[contract.Version]RuntimeEngine, len(engines)) + for version, engine := range engines { + if version != contract.V1 && version != contract.V2 && version != contract.V3 { + return nil, fmt.Errorf("%w: unknown runtime version %q", errs.ErrInvalid, version) + } + if engine == nil || (reflect.ValueOf(engine).Kind() == reflect.Ptr && reflect.ValueOf(engine).IsNil()) { + return nil, fmt.Errorf("%w: runtime %s is nil", errs.ErrNotWiredYet, version) + } + copy[version] = engine + } + if copy[defaultVersion] == nil { + return nil, fmt.Errorf("%w: default runtime %q unavailable", errs.ErrNotWiredYet, defaultVersion) + } + return &RuntimeRouter{engines: copy, defaultVersion: defaultVersion}, nil +} + +func (r *RuntimeRouter) resolve(ctx context.Context) (context.Context, RuntimeEngine, contract.Version, error) { + if r == nil { + return ctx, nil, "", errs.ErrNotWiredYet + } + version := contract.VersionFromContext(ctx) + if version == "" { + version = r.defaultVersion + } + engine := r.engines[version] + if engine == nil { + return ctx, nil, version, fmt.Errorf("%w: runtime %q unavailable", errs.ErrNotWiredYet, version) + } + return contract.WithVersion(ctx, version), engine, version, nil +} + +func (r *RuntimeRouter) Handle(ctx context.Context, req *contract.Request) (*contract.Reply, error) { + if req == nil { + return nil, fmt.Errorf("%w: request required", errs.ErrInvalid) + } + ctx, engine, _, err := r.resolve(ctx) + if err != nil { + return nil, err + } + return engine.Handle(ctx, req) +} + +func (r *RuntimeRouter) SpawnWorker(ctx context.Context, req contract.SpawnRequest) (*contract.Worker, error) { + ctx, engine, version, err := r.resolve(ctx) + if err != nil { + return nil, err + } + worker, err := engine.SpawnWorker(ctx, req) + if err != nil { + return nil, err + } + if worker == nil || worker.ID == "" { + return nil, fmt.Errorf("runtime %s returned no worker identity", version) + } + return worker, nil +} + +func (r *RuntimeRouter) workerEngine(workerID string) (RuntimeEngine, contract.Version, bool) { + if r == nil { + return nil, "", false + } + var owner RuntimeEngine + var version contract.Version + // Resolve against the owning runtime, including workers created by its + // reviewer path. Do not duplicate its mutable lifecycle registry here. + for candidate, engine := range r.engines { + if _, found := engine.GetWorker(workerID); !found { + continue + } + if owner != nil { + return nil, "", false + } // ambiguous ID: never pick a version arbitrarily + owner, version = engine, candidate + } + return owner, version, owner != nil +} + +func (r *RuntimeRouter) SendToWorker(ctx context.Context, workerID, message string) error { + engine, version, ok := r.workerEngine(workerID) + if !ok { + return fmt.Errorf("%w: worker %q", errs.ErrNotFound, workerID) + } + if requested := contract.VersionFromContext(ctx); requested != "" && requested != version { + return fmt.Errorf("%w: worker runtime mismatch", errs.ErrInvalid) + } + return engine.SendToWorker(contract.WithVersion(ctx, version), workerID, message) +} + +func (r *RuntimeRouter) StopWorker(ctx context.Context, workerID string) error { + engine, version, ok := r.workerEngine(workerID) + if !ok { + return fmt.Errorf("%w: worker %q", errs.ErrNotFound, workerID) + } + if requested := contract.VersionFromContext(ctx); requested != "" && requested != version { + return fmt.Errorf("%w: worker runtime mismatch", errs.ErrInvalid) + } + return engine.StopWorker(contract.WithVersion(ctx, version), workerID) +} + +func (r *RuntimeRouter) GetWorker(workerID string) (*contract.Worker, bool) { + engine, _, ok := r.workerEngine(workerID) + if !ok { + return nil, false + } + return engine.GetWorker(workerID) +} + +func (r *RuntimeRouter) CountWorkersByStatus() (running, pending int) { + if r == nil { + return 0, 0 + } + for _, engine := range r.engines { + active, waiting := engine.CountWorkersByStatus() + running += active + pending += waiting + } + return +} diff --git a/internal/manager/service/aiops/runtime_router_test.go b/internal/manager/service/aiops/runtime_router_test.go new file mode 100644 index 000000000..bb520eab8 --- /dev/null +++ b/internal/manager/service/aiops/runtime_router_test.go @@ -0,0 +1,213 @@ +package aiops + +import ( + "context" + "errors" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +type routerEngine struct { + handle func(context.Context, *contract.Request) (*contract.Reply, error) + spawn func(context.Context, contract.SpawnRequest) (*contract.Worker, error) + send func(context.Context, string, string) error + stop func(context.Context, string) error + worker *contract.Worker +} + +func (e *routerEngine) Handle(ctx context.Context, req *contract.Request) (*contract.Reply, error) { + return e.handle(ctx, req) +} +func (e *routerEngine) SpawnWorker(ctx context.Context, req contract.SpawnRequest) (*contract.Worker, error) { + return e.spawn(ctx, req) +} +func (e *routerEngine) SendToWorker(ctx context.Context, id, message string) error { + return e.send(ctx, id, message) +} +func (e *routerEngine) StopWorker(ctx context.Context, id string) error { return e.stop(ctx, id) } +func (e *routerEngine) GetWorker(id string) (*contract.Worker, bool) { + return e.worker, e.worker != nil && e.worker.ID == id +} +func (e *routerEngine) CountWorkersByStatus() (int, int) { return 1, 2 } + +func TestRuntimeRouterPreservesRequestEventsAndResult(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + type identityKey struct{} + ctx = context.WithValue(ctx, identityKey{}, "identity") + reply := &contract.Reply{Message: &model.Message{ID: "final"}, Iterations: 7, ToolCalls: []*model.ToolCall{{ID: "tool"}}} + events := []contract.Event{ + {Type: contract.EventAssistant, Assistant: &contract.AssistantEvent{MessageID: "a", Content: "answer"}}, + {Type: contract.EventToolStart, Tool: &contract.ToolEvent{ToolCallID: "t", Name: "web_search", ArgsJSON: `{"q":"test"}`}}, + {Type: contract.EventToolEnd, Tool: &contract.ToolEvent{ToolCallID: "t", ResultJSON: `{"result":"ok"}`}}, + {Type: contract.EventApprovalPending, Approval: &contract.ApprovalPending{ApprovalID: "approval", Credentials: []string{"credential"}}}, + {Type: contract.EventTaskNotification, Notification: &contract.TaskNotification{TaskID: "worker", Status: contract.WorkerStatusCompleted}}, + {Type: contract.EventError, Error: "provider error"}, + {Type: contract.EventDone, Done: reply}, + } + var got []contract.Event + req := &contract.Request{SessionID: "session", UserID: 42, Role: "admin", UserText: "prompt", Attachments: []model.Attachment{{ID: "image"}}, Mentions: []contract.Mention{{Type: "device", ID: "12"}}, Provider: "provider", Model: "model", Locale: "en", WebSearchEnabled: true, Emit: func(e contract.Event) { got = append(got, e) }} + engine := &routerEngine{handle: func(inner context.Context, in *contract.Request) (*contract.Reply, error) { + if in != req { + t.Fatal("router replaced request") + } + if inner.Value(identityKey{}) != "identity" || inner.Done() != ctx.Done() || contract.VersionFromContext(inner) != contract.V3 { + t.Fatal("context lost at routing boundary") + } + for _, event := range events { + in.Emit(event) + } + return reply, nil + }} + router, err := NewRuntimeRouter(contract.V3, map[contract.Version]RuntimeEngine{contract.V3: engine}) + if err != nil { + t.Fatal(err) + } + result, err := router.Handle(ctx, req) + if err != nil || result != reply || !reflect.DeepEqual(got, events) { + t.Fatalf("payload loss: result=%+v, err=%v, events=%+v", result, err, got) + } +} + +func TestRuntimeRouterRejectsUnavailableVersionWithoutFallback(t *testing.T) { + var calls atomic.Int32 + engine := &routerEngine{handle: func(context.Context, *contract.Request) (*contract.Reply, error) { calls.Add(1); return nil, nil }} + registry := map[contract.Version]RuntimeEngine{contract.V3: engine} + router, err := NewRuntimeRouter(contract.V3, registry) + if err != nil { + t.Fatal(err) + } + delete(registry, contract.V3) // the caller cannot mutate the router registry + for _, version := range []contract.Version{contract.V1, contract.V2, "v99"} { + if _, err := router.Handle(contract.WithVersion(t.Context(), version), &contract.Request{}); !errors.Is(err, errs.ErrNotWiredYet) { + t.Fatalf("version %s: %v", version, err) + } + } + if calls.Load() != 0 { + t.Fatal("unsupported version silently fell back") + } + if _, err := router.Handle(t.Context(), &contract.Request{}); err != nil { + t.Fatal(err) + } + if _, err := router.Handle(t.Context(), nil); !errors.Is(err, errs.ErrInvalid) { + t.Fatal(err) + } + for _, registry := range []map[contract.Version]RuntimeEngine{nil, {contract.V3: nil}, {contract.V3: (*routerEngine)(nil)}, {"v99": engine}} { + if _, err := NewRuntimeRouter(contract.V3, registry); err == nil { + t.Fatal("invalid registry accepted") + } + } +} + +func TestRuntimeRouterRoutesWorkerControlToOwner(t *testing.T) { + // Workers created directly by version-local review paths must remain visible. + first := &routerEngine{worker: &contract.Worker{ID: "v2-worker", Status: contract.WorkerStatusRunning}} + second := &routerEngine{worker: &contract.Worker{ID: "v3-worker", Status: contract.WorkerStatusRunning}} + var sends, stops atomic.Int32 + first.send = func(ctx context.Context, id, message string) error { + if contract.VersionFromContext(ctx) != contract.V2 || id != "v2-worker" || message != "follow up" { + t.Error("incorrect worker route") + } + sends.Add(1) + return nil + } + first.stop = func(ctx context.Context, id string) error { + if contract.VersionFromContext(ctx) != contract.V2 || id != "v2-worker" { + t.Error("incorrect stop route") + } + stops.Add(1) + return nil + } + router, err := NewRuntimeRouter(contract.V3, map[contract.Version]RuntimeEngine{contract.V2: first, contract.V3: second}) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if err := router.SendToWorker(t.Context(), "v2-worker", "follow up"); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + if err := router.StopWorker(t.Context(), "v2-worker"); err != nil { + t.Fatal(err) + } + if sends.Load() != 8 || stops.Load() != 1 { + t.Fatal("worker controls lost") + } + if err := router.StopWorker(contract.WithVersion(t.Context(), contract.V3), "v2-worker"); !errors.Is(err, errs.ErrInvalid) { + t.Fatal("cross-version control accepted") + } + if _, ok := router.GetWorker("missing"); ok { + t.Fatal("missing worker found") + } + if err := router.SendToWorker(t.Context(), "missing", "x"); !errors.Is(err, errs.ErrNotFound) { + t.Fatal(err) + } + if w, ok := router.GetWorker("v2-worker"); !ok || w.ID != "v2-worker" { + t.Fatal("worker lookup lost") + } + if running, pending := router.CountWorkersByStatus(); running != 2 || pending != 4 { + t.Fatalf("worker metrics lost: %d %d", running, pending) + } +} + +func TestRuntimeRouterPreservesWorkerSpawnAndErrors(t *testing.T) { + sentinel := errors.New("execution failed") + now := time.Now() + want := &contract.Worker{ID: "worker", AgentName: "expert", SessionID: "session", ParentSessionID: "parent", Prompt: "prompt", Status: contract.WorkerStatusCompleted, Background: true, StartedAt: now, EndedAt: &now, Result: "result", Err: "details"} + var notified bool + req := contract.SpawnRequest{AgentName: "expert", Prompt: "prompt", Background: true, ParentSession: "parent", SessionKind: "flow", OwnerUserID: 42, Locale: "en", Provider: "provider", Model: "model", ParentEmit: func(contract.Event) { notified = true }} + engine := &routerEngine{spawn: func(ctx context.Context, in contract.SpawnRequest) (*contract.Worker, error) { + if contract.VersionFromContext(ctx) != contract.V3 { + t.Fatal("worker version lost") + } + if in.ParentEmit == nil { + t.Fatal("worker event sink lost") + } + in.ParentEmit(contract.Event{}) + in.ParentEmit = nil + copy := req + copy.ParentEmit = nil + if !reflect.DeepEqual(in, copy) { + t.Fatal("worker input lost") + } + return want, nil + }, handle: func(context.Context, *contract.Request) (*contract.Reply, error) { return nil, sentinel }} + router, err := NewRuntimeRouter(contract.V3, map[contract.Version]RuntimeEngine{contract.V3: engine}) + if err != nil { + t.Fatal(err) + } + got, err := router.SpawnWorker(t.Context(), req) + if err != nil || got != want || !notified { + t.Fatal("worker result or event lost") + } + if _, err := router.Handle(t.Context(), &contract.Request{}); !errors.Is(err, sentinel) { + t.Fatal("runtime error lost") + } +} + +func TestRuntimeRouterRejectsAmbiguousWorkerIdentity(t *testing.T) { + engine := func() *routerEngine { return &routerEngine{worker: &contract.Worker{ID: "collision"}} } + router, err := NewRuntimeRouter(contract.V3, map[contract.Version]RuntimeEngine{contract.V2: engine(), contract.V3: engine()}) + if err != nil { + t.Fatal(err) + } + if _, ok := router.GetWorker("collision"); ok { + t.Fatal("ambiguous worker selected") + } + if err := router.StopWorker(t.Context(), "collision"); err == nil { + t.Fatal("ambiguous worker controlled") + } +} diff --git a/internal/manager/service/aiops/service.go b/internal/manager/service/aiops/service.go index 54dfe9611..e03081673 100644 --- a/internal/manager/service/aiops/service.go +++ b/internal/manager/service/aiops/service.go @@ -22,7 +22,7 @@ import ( biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" "github.com/ongridio/ongrid/internal/manager/biz/aiops/chatprotocol" - "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" + chatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" model "github.com/ongridio/ongrid/internal/manager/model/aiops" "github.com/ongridio/ongrid/internal/pkg/errs" "github.com/ongridio/ongrid/internal/pkg/llm" @@ -63,7 +63,7 @@ type Service struct { } // RuntimeHandler is the narrow contract the service depends on for the -// graph kernel. *chatruntime.Runtime satisfies it by structural typing. +// runtime router. Version implementations satisfy it by structural typing. // The seam is a service-private interface so unit tests can inject a // fake without standing up the full graph + tool decorator chain. type RuntimeHandler interface { diff --git a/internal/manager/service/aiops/worker_metrics.go b/internal/manager/service/aiops/worker_metrics.go index df1eaa8e2..f5f9106fd 100644 --- a/internal/manager/service/aiops/worker_metrics.go +++ b/internal/manager/service/aiops/worker_metrics.go @@ -4,12 +4,11 @@ import ( "context" "time" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" "github.com/ongridio/ongrid/internal/pkg/prom" ) // RunWorkerMetrics samples active worker sessions until shutdown. -func RunWorkerMetrics(ctx context.Context, runtime *aiopschatruntime.Runtime) error { +func RunWorkerMetrics(ctx context.Context, runtime interface{ CountWorkersByStatus() (running, pending int) }) error { t := time.NewTicker(15 * time.Second) defer t.Stop() for { diff --git a/internal/manager/service/flow/runners.go b/internal/manager/service/flow/runners.go index 9d402e181..f3b79020f 100644 --- a/internal/manager/service/flow/runners.go +++ b/internal/manager/service/flow/runners.go @@ -6,7 +6,7 @@ import ( "strings" "time" - aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" managerbizalert "github.com/ongridio/ongrid/internal/manager/biz/alert" managermodelalert "github.com/ongridio/ongrid/internal/manager/model/alert" "github.com/ongridio/ongrid/internal/pkg/llm" @@ -20,7 +20,10 @@ type channelGetter interface { // AgentRunner implements bizflow.AgentRunner over the chatruntime — // one synchronous worker per agent node (mirrors the RCA investigator's // WorkerSpawner usage). -type AgentRunner struct{ Runtime *aiopschatruntime.Runtime } +type agentWorkerSpawner interface { + SpawnWorker(context.Context, aiopschatruntime.SpawnRequest) (*aiopschatruntime.Worker, error) +} +type AgentRunner struct{ Runtime agentWorkerSpawner } func (s AgentRunner) RunAgent(ctx context.Context, persona, prompt string) (string, error) { w, err := s.Runtime.SpawnWorker(ctx, aiopschatruntime.SpawnRequest{ From d3a6601b2bb7dcfd6e3f18aea3389dcb66c7a055 Mon Sep 17 00:00:00 2001 From: singchia Date: Fri, 16 Oct 2026 23:52:38 +0800 Subject: [PATCH 18/24] fix(aiops): remove implicit tool execution timeout Remove the blanket 15-second deadline from chat and workflow tools. Preserve explicit operation budgets and caller cancellation, and cover execution beyond the former cutoff with race tests. Refs #523 --- cmd/ongrid/runtime_builder.go | 6 +-- .../biz/aiops/runtime/v3/graph/react_test.go | 4 +- .../biz/aiops/runtime/v3/graph/types.go | 10 ++-- .../biz/aiops/tools/decorators/chain.go | 9 ++-- .../aiops/tools/decorators/decorators_test.go | 47 +++++++++++++++++-- .../biz/aiops/tools/decorators/timeout.go | 11 ++--- internal/manager/service/flow/tools.go | 2 - 7 files changed, 56 insertions(+), 33 deletions(-) diff --git a/cmd/ongrid/runtime_builder.go b/cmd/ongrid/runtime_builder.go index 4f7c966c5..b63a915c2 100644 --- a/cmd/ongrid/runtime_builder.go +++ b/cmd/ongrid/runtime_builder.go @@ -214,7 +214,6 @@ func buildAIOpsRuntime( reviewSpawner := &managersvcaiops.ReviewSpawner{} reviewSink := managersvcaiops.NewMutatingProposalSink(mutatingProposals) deps := aiopstoolsdec.Deps{ - Timeout: 15 * time.Second, Limiter: aiopstoolsdec.NewTokenBucketLimiter(0), Registerer: reg, ReviewSpawner: reviewSpawner, @@ -226,8 +225,8 @@ func buildAIOpsRuntime( toolDeps := deps if info, err := t.Info(context.Background()); err == nil && info != nil && info.Name == aiopstools.ToolNameCapturePCAP { // A short capture includes capture duration plus parser work. Keep - // the normal 15s ceiling for every other tool, but let this bounded - // evidence collection finish in the same chat turn. + // this operation-specific ceiling; other tools own their execution + // budgets without a blanket runtime deadline. toolDeps.Timeout = 45 * time.Second } wrapped = append(wrapped, aiopstoolsdec.Wrap(t, toolDeps)) @@ -311,7 +310,6 @@ func buildAIOpsRuntime( Model: cfg.OpenAI.Model, Temperature: 0.1, MaxIterations: 30, - ToolTimeout: 15 * time.Second, }, CallbackDeps: cbDeps, Logger: log.With(slog.String("comp", "chatruntime")), diff --git a/internal/manager/biz/aiops/runtime/v3/graph/react_test.go b/internal/manager/biz/aiops/runtime/v3/graph/react_test.go index 603c4f30b..3dfa4d86f 100644 --- a/internal/manager/biz/aiops/runtime/v3/graph/react_test.go +++ b/internal/manager/biz/aiops/runtime/v3/graph/react_test.go @@ -664,7 +664,7 @@ func TestConfig_Defaults(t *testing.T) { if c.MaxIterations != 12 { t.Errorf("MaxIterations default = %d, want 12", c.MaxIterations) } - if c.ToolTimeout.Seconds() != 15 { - t.Errorf("ToolTimeout default = %v, want 15s", c.ToolTimeout) + if c.ToolTimeout != 0 { + t.Errorf("ToolTimeout default = %v, want no implicit deadline", c.ToolTimeout) } } diff --git a/internal/manager/biz/aiops/runtime/v3/graph/types.go b/internal/manager/biz/aiops/runtime/v3/graph/types.go index 483774abb..620552de8 100644 --- a/internal/manager/biz/aiops/runtime/v3/graph/types.go +++ b/internal/manager/biz/aiops/runtime/v3/graph/types.go @@ -159,10 +159,9 @@ type Config struct { // breaker for cross-tool loops. MaxIterations int - // ToolTimeout is the per-tool wall clock ceiling. 0 -> 15s - // (agent.go default). Enforced by the BaseTool decorator chain - // (PR-3); this field is kept here so the cutover PR can pass the - // value through to chatruntime when it builds the toolBag. + // ToolTimeout is retained for source compatibility. Graph execution does + // not consume it; explicit operation budgets belong to the tool chain. + // Deprecated: configure the tool itself instead. ToolTimeout time.Duration } @@ -173,8 +172,5 @@ func (c Config) applyDefaults() Config { if c.MaxIterations <= 0 { c.MaxIterations = 12 } - if c.ToolTimeout <= 0 { - c.ToolTimeout = 15 * time.Second - } return c } diff --git a/internal/manager/biz/aiops/tools/decorators/chain.go b/internal/manager/biz/aiops/tools/decorators/chain.go index 720cbd3f5..80818d30e 100644 --- a/internal/manager/biz/aiops/tools/decorators/chain.go +++ b/internal/manager/biz/aiops/tools/decorators/chain.go @@ -9,14 +9,14 @@ import ( ) // Deps bundles the cross-cutting dependencies a decorator chain needs. -// Zero values fall back to safe defaults (DefaultTimeout, no audit, +// Zero values fall back to safe defaults (no added deadline, no audit, // NoopLimiter, prometheus.DefaultRegisterer) so a minimal Wrap call // still produces a well-behaved tool. // // ASCII diagram — these are the inputs to the standard // decorator stack. type Deps struct { - // Timeout is the per-call ceiling. Zero -> DefaultTimeout (15s). + // Timeout is an optional per-call ceiling. Non-positive adds no deadline. Timeout time.Duration // Audit is the seam to chat_tool_calls. Nil -> audit disabled. @@ -74,11 +74,10 @@ type Deps struct { // user_id on the proposal). // - review_gate OUTSIDE timeout because the reviewer worker is // itself a graph.Invoke against an LLM with its own multi-turn -// budget; constraining it to the inner tool's 15s timeout is +// budget; constraining it to an optional inner-tool timeout is // unrealistic. ReviewGate carries its own independent Timeout // (default 60s) for the reviewer round-trip; once approve fires, -// the inner tool runs under the timeout decorator's 15s budget -// normally. +// the inner tool runs under an explicitly configured timeout, if any. // - review_gate OUTSIDE audit because a rejected proposal has no // execution to audit — chat_tool_calls is the **execution** log, // chat_mutating_proposals is the **decision** log. Putting diff --git a/internal/manager/biz/aiops/tools/decorators/decorators_test.go b/internal/manager/biz/aiops/tools/decorators/decorators_test.go index 12977db00..ba5d3e61d 100644 --- a/internal/manager/biz/aiops/tools/decorators/decorators_test.go +++ b/internal/manager/biz/aiops/tools/decorators/decorators_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "strings" "sync" "sync/atomic" @@ -96,11 +97,47 @@ func TestTimeout_PassesThroughOnFastTool(t *testing.T) { } } -func TestTimeout_DefaultWhenZero(t *testing.T) { - inner := &fakeTool{name: "x"} - tt := WithTimeout(inner, 0).(*TimeoutTool) - if tt.timeout != DefaultTimeout { - t.Errorf("got %v, want %v", tt.timeout, DefaultTimeout) +func TestTimeout_UnsetAddsNoDeadline(t *testing.T) { + for _, d := range []time.Duration{0, -time.Second} { + inner := &fakeTool{name: "x"} + if got := WithTimeout(inner, d); got != inner { + t.Fatalf("timeout %s unexpectedly wrapped tool", d) + } + } +} + +// A real execution beyond the previous 15s cutoff guards the default chain, +// including callers that do not explicitly populate Deps.Timeout. +func TestWrap_LongToolWithoutImplicitDeadline(t *testing.T) { + t.Parallel() + inner := &fakeTool{name: "long_scan", delay: 16 * time.Second, respectCtx: true, result: "scan complete"} + wrapped := Wrap(inner, Deps{Registerer: prometheus.NewRegistry()}) + out, err := wrapped.InvokableRun(context.Background(), "{}") + if err != nil || out != "scan complete" { + t.Fatalf("long scan = %q, %v", out, err) + } +} + +func TestWrap_UnsetTimeoutPreservesCallerCancellation(t *testing.T) { + for _, deadline := range []bool{false, true} { + t.Run(fmt.Sprint("deadline=", deadline), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + want := context.Canceled + if deadline { + cancel() + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Millisecond) + want = context.DeadlineExceeded + } else { + cancel() + } + defer cancel() + inner := &fakeTool{name: "cancel_scan", delay: time.Hour, respectCtx: true} + wrapped := Wrap(inner, Deps{Registerer: prometheus.NewRegistry()}) + _, err := wrapped.InvokableRun(ctx, "{}") + if !errors.Is(err, want) || errors.Is(err, ErrToolTimeout) { + t.Fatalf("caller cancellation = %v, want %v", err, want) + } + }) } } diff --git a/internal/manager/biz/aiops/tools/decorators/timeout.go b/internal/manager/biz/aiops/tools/decorators/timeout.go index a071d0e33..6c38ae1d9 100644 --- a/internal/manager/biz/aiops/tools/decorators/timeout.go +++ b/internal/manager/biz/aiops/tools/decorators/timeout.go @@ -19,12 +19,6 @@ import ( "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" ) -// DefaultTimeout is the per-call timeout applied when WithTimeout is -// constructed without an explicit duration. cites 15s -// ("TimeoutTool 15s per-call ToolTimeout"); tools that legitimately -// need more (e.g. correlate_incident) override at registration. -const DefaultTimeout = 15 * time.Second - // ErrToolTimeout wraps the inner tool's error when the parent context's // deadline (set by this decorator) fires. Callers can errors.Is() it to // distinguish a decorator-imposed timeout from a tool-level error. @@ -38,10 +32,11 @@ type TimeoutTool struct { } // WithTimeout returns inner wrapped so InvokableRun is bounded by d. -// Zero or negative d falls back to DefaultTimeout. +// Zero or negative d adds no deadline; the caller and tool retain ownership +// of cancellation and operation-specific budgets. func WithTimeout(inner basetool.BaseTool, d time.Duration) basetool.BaseTool { if d <= 0 { - d = DefaultTimeout + return inner } return &TimeoutTool{inner: inner, timeout: d} } diff --git a/internal/manager/service/flow/tools.go b/internal/manager/service/flow/tools.go index 664d596fd..c75b457b0 100644 --- a/internal/manager/service/flow/tools.go +++ b/internal/manager/service/flow/tools.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "strings" - "time" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" @@ -28,7 +27,6 @@ func NewToolInvoker(reg *aiopstools.Registry, registerer prometheus.Registerer) inv := &ToolInvoker{ Registry: reg, deps: aiopstoolsdec.Deps{ - Timeout: 15 * time.Second, Limiter: aiopstoolsdec.NewTokenBucketLimiter(0), Registerer: registerer, }, From 250e5e8d843a564399a849d8b3add1a6f11ddc75 Mon Sep 17 00:00:00 2001 From: singchia Date: Sat, 17 Oct 2026 00:00:57 +0800 Subject: [PATCH 19/24] fix(aiops): extend built-in tool execution budgets Align filesystem and shell deadlines across Manager and Edge, allow longer cloud commands and diagnostic skills, and preserve caller cancellation and explicit shorter deadlines. Cover long shell execution and deadline propagation with race tests. Refs #523 --- internal/edgeagent/bash/handlers.go | 2 +- internal/edgeagent/bash/handlers_test.go | 24 ++++++++ internal/edgeagent/cmdpolicy/policy.go | 2 +- internal/edgeagent/host_files/handlers.go | 17 ++---- .../manager/biz/aiops/tools/bash_basetool.go | 8 +-- .../biz/aiops/tools/describe_k8s_resource.go | 2 +- .../biz/aiops/tools/execution_budget_test.go | 57 +++++++++++++++++++ .../aiops/tools/expand_topology_basetool.go | 2 +- .../biz/aiops/tools/find_outlier_edges.go | 2 +- .../tools/find_topology_node_basetool.go | 2 +- .../biz/aiops/tools/get_edge_summary.go | 2 +- .../aiops/tools/get_edge_summary_basetool.go | 2 +- .../biz/aiops/tools/get_incident_detail.go | 2 +- .../manager/biz/aiops/tools/get_topology.go | 2 +- .../biz/aiops/tools/host_files_basetool.go | 8 +-- internal/manager/biz/aiops/tools/host_load.go | 2 +- .../manager/biz/aiops/tools/host_processes.go | 2 +- .../aiops/tools/network_inventory_basetool.go | 2 +- .../biz/aiops/tools/query_alert_rules.go | 2 +- .../manager/biz/aiops/tools/query_edges.go | 2 +- .../biz/aiops/tools/query_incidents.go | 2 +- .../manager/biz/aiops/tools/query_k8s_logs.go | 2 +- .../biz/aiops/tools/query_k8s_snapshot.go | 2 +- .../manager/biz/aiops/tools/query_logql.go | 2 +- .../manager/biz/aiops/tools/query_promql.go | 2 +- .../manager/biz/aiops/tools/query_traceql.go | 2 +- .../manager/biz/aiops/tools/rank_edges.go | 2 +- .../service/aiops/approval_executors.go | 3 +- internal/pkg/tunnel/bash.go | 2 +- internal/pkg/tunnel/host_files.go | 9 +++ internal/skill/builtin/host_netns_inspect.go | 2 +- internal/skill/builtin/read_journal.go | 4 +- internal/skill/builtin/web_search.go | 2 +- internal/skill/loader.go | 2 +- internal/skill/subprocess.go | 5 +- 35 files changed, 133 insertions(+), 54 deletions(-) create mode 100644 internal/manager/biz/aiops/tools/execution_budget_test.go diff --git a/internal/edgeagent/bash/handlers.go b/internal/edgeagent/bash/handlers.go index a20fe11a1..dca261b0f 100644 --- a/internal/edgeagent/bash/handlers.go +++ b/internal/edgeagent/bash/handlers.go @@ -115,7 +115,7 @@ func makeHandler(sandbox *cmdpolicy.Sandbox, log *slog.Logger) tunnel.Handler { // Optional per-call timeout override. We bound the override at // 5 min just so a runaway agent call can't park a tunnel slot - // indefinitely; the policy default (30s) is the more useful + // indefinitely; the policy default (300s) is the more useful // number. callCtx := ctx if req.Timeout > 0 { diff --git a/internal/edgeagent/bash/handlers_test.go b/internal/edgeagent/bash/handlers_test.go index a2e82111e..2d6bf5fa1 100644 --- a/internal/edgeagent/bash/handlers_test.go +++ b/internal/edgeagent/bash/handlers_test.go @@ -147,3 +147,27 @@ func TestHandler_RejectsForbiddenSyntax(t *testing.T) { } } } + +// Exercise the real shell path past the old sandbox ceiling; no fake clock +// can establish that the command survives all handler/sandbox deadlines. +func TestHandler_DefaultAllowsCommandBeyondThirtySeconds(t *testing.T) { + t.Parallel() + sb := &cmdpolicy.Sandbox{Policy: cmdpolicy.DefaultReadOnly()} + resp := invokeHandler(t, makeHandler(sb, nil), tunnel.BashExecRequest{ + Cmd: "sleep 31; printf completed", Unrestricted: true, + }) + if resp.ExitCode != 0 || resp.Stdout != "completed" { + t.Fatalf("long command failed: %+v", resp) + } +} + +func TestHandler_ExplicitShortTimeoutStillStopsCommand(t *testing.T) { + t.Parallel() + sb := &cmdpolicy.Sandbox{Policy: cmdpolicy.DefaultReadOnly()} + resp := invokeHandler(t, makeHandler(sb, nil), tunnel.BashExecRequest{ + Cmd: "exec sleep 10", Unrestricted: true, Timeout: 1, + }) + if resp.ExitCode == 0 { + t.Fatalf("explicit timeout did not stop command: %+v", resp) + } +} diff --git a/internal/edgeagent/cmdpolicy/policy.go b/internal/edgeagent/cmdpolicy/policy.go index f12dc090a..4a5be4e8c 100644 --- a/internal/edgeagent/cmdpolicy/policy.go +++ b/internal/edgeagent/cmdpolicy/policy.go @@ -28,7 +28,7 @@ import ( const ( defaultStdoutCap = 64 * 1024 defaultStderrCap = 16 * 1024 - defaultTimeout = 30 * time.Second + defaultTimeout = 5 * time.Minute defaultMaxArgs = 32 ) diff --git a/internal/edgeagent/host_files/handlers.go b/internal/edgeagent/host_files/handlers.go index 044499196..4b1cc32bd 100644 --- a/internal/edgeagent/host_files/handlers.go +++ b/internal/edgeagent/host_files/handlers.go @@ -55,18 +55,9 @@ import ( "github.com/ongridio/ongrid/internal/pkg/tunnel" ) -// hostFilesPerPathTimeout caps a single path's handler work (one find, -// one du, one stat). 30 s is generous for stat / du -d 1 but means a -// `find /` with a deep tree may be cut short — that is intentional. The -// manager BaseTool uses a 60 s tunnel timeout (matching -// hostFilesBatchTimeout below) so this 30 s budget keeps any single -// path from monopolising the tunnel slot. -const hostFilesPerPathTimeout = 30 * time.Second - -// hostFilesBatchTimeout caps the whole batch (all paths together). The -// manager-side BaseTool uses the same value for its tunnel.Caller call -// timeout so the LLM-visible upper bound is uniform. -const hostFilesBatchTimeout = 60 * time.Second +// Scans may traverse large trees; keep budgets aligned with the caller. +const hostFilesPerPathTimeout = tunnel.HostFilesPerPathTimeout +const hostFilesBatchTimeout = tunnel.HostFilesBatchTimeout // hostFilesBatchConcurrency caps how many paths run in parallel inside // a single batch. Set deliberately low because each find/du subprocess @@ -449,7 +440,7 @@ func collectFilesystems(ctx context.Context, sb *SandboxConfig, paths []string) seen[clean] = struct{}{} wanted = append(wanted, clean) } - cctx, cancel := context.WithTimeout(ctx, hostFilesPerPathTimeout) + cctx, cancel := context.WithTimeout(ctx, tunnel.HostFilesMetadataTimeout) defer cancel() out := make([]tunnel.HostFilesystem, 0, 2) seenMount := map[string]struct{}{} diff --git a/internal/manager/biz/aiops/tools/bash_basetool.go b/internal/manager/biz/aiops/tools/bash_basetool.go index a2ede8038..ad8700564 100644 --- a/internal/manager/biz/aiops/tools/bash_basetool.go +++ b/internal/manager/biz/aiops/tools/bash_basetool.go @@ -128,7 +128,7 @@ var BashSchema = json.RawMessage(`{ "description": "兼容旧参数：单台设备 id。优先使用 device_ids。" }, "cmd": {"type": "string", "description": "**单条** shell-like 命令，跑在每个 device_id 上。读命令直接走只读 sandbox；写命令只在用户明确要求时触发内置确认卡，批准后才执行。支持 pipes (|)；读路径不支持 redirects / ; && || $() <() heredocs / backticks。如果想跑两条不同命令，分两次调用。"}, - "timeout_seconds": {"type": "integer", "minimum": 1, "maximum": 300, "description": "Optional per-device timeout override. Default 30s; max 300s. 共享给所有 device。"} + "timeout_seconds": {"type": "integer", "minimum": 1, "maximum": 300, "description": "Optional per-device timeout override. Default 300s; max 300s. 共享给所有 device。"} }, "required": ["cmd"] }`) @@ -136,11 +136,11 @@ var BashSchema = json.RawMessage(`{ // bashCallTimeout caps the manager → edge round-trip. The edge // enforces its own (shorter) timeout from the policy; this is the // outer ceiling per inner call so a hung tunnel can't park a slot. -const bashCallTimeout = 60 * time.Second +const bashCallTimeout = 5*time.Minute + 30*time.Second // bashBatchTimeout caps the whole batched run. Wider than the per-id -// ceiling so up to batchConcurrency calls can finish. -const bashBatchTimeout = 120 * time.Second +// ceiling so all four waves of a 16-device batch can finish. +const bashBatchTimeout = ((batchMaxIDs+batchConcurrency-1)/batchConcurrency)*bashCallTimeout + 30*time.Second // bashBatchArgs is the typed form of BashSchema. type bashBatchArgs struct { diff --git a/internal/manager/biz/aiops/tools/describe_k8s_resource.go b/internal/manager/biz/aiops/tools/describe_k8s_resource.go index c22af386a..a4746df44 100644 --- a/internal/manager/biz/aiops/tools/describe_k8s_resource.go +++ b/internal/manager/biz/aiops/tools/describe_k8s_resource.go @@ -59,7 +59,7 @@ const describeK8sResourceWhenToUse = "When the user explicitly asks to describe "or needs current Events for one named object. NOT for counts/lists/overview (use query_k8s_snapshot). " + "Read-only only; does not execute kubectl, exec, logs, scale, restart, or delete." -const describeK8sResourceCallTimeout = 15 * time.Second +const describeK8sResourceCallTimeout = 60 * time.Second type DescribeK8sResourceTool struct { caller Caller diff --git a/internal/manager/biz/aiops/tools/execution_budget_test.go b/internal/manager/biz/aiops/tools/execution_budget_test.go new file mode 100644 index 000000000..672791c14 --- /dev/null +++ b/internal/manager/biz/aiops/tools/execution_budget_test.go @@ -0,0 +1,57 @@ +package tools + +import ( + "context" + "testing" + "time" + + "github.com/ongridio/ongrid/internal/pkg/tunnel" +) + +type budgetCaller struct { + t *testing.T + min time.Duration + parentDeadline time.Time +} + +func (c budgetCaller) Call(ctx context.Context, _ uint64, method string, _ []byte) ([]byte, error) { + c.t.Helper() + deadline, ok := ctx.Deadline() + if !ok { + c.t.Fatal("missing execution deadline") + } + if !c.parentDeadline.IsZero() { + if !deadline.Equal(c.parentDeadline) { + c.t.Fatalf("caller deadline changed: %v", deadline) + } + } else if time.Until(deadline) < c.min { + c.t.Fatalf("execution budget too short: %s", time.Until(deadline)) + } + if method == tunnel.MethodBashExec { + return []byte(`{"allowed":true}`), nil + } + return []byte(`{"results":[]}`), nil +} + +func TestExecutionBudgets_AllowLongWorkAndPreserveCallerDeadline(t *testing.T) { + for _, short := range []bool{false, true} { + ctx := context.Background() + caller := budgetCaller{t: t} + if short { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Second) + defer cancel() + caller.parentDeadline, _ = ctx.Deadline() + } + // A full batch has four waves, then df metadata and response delivery. + caller.min = 20*time.Minute + 30*time.Second + if _, err := dispatchEdgeCall(ctx, caller, 7, tunnel.MethodDuSummary, struct{}{}, "du_summary"); err != nil { + t.Fatal(err) + } + caller.min = 5 * time.Minute + tool := &BashTool{resolver: &fakeHostFilesResolver{mapping: map[uint64]uint64{1: 7}}, caller: caller} + if _, err := tool.InvokableRun(ctx, `{"device_ids":[1],"cmd":"du /var","timeout_seconds":300}`); err != nil { + t.Fatal(err) + } + } +} diff --git a/internal/manager/biz/aiops/tools/expand_topology_basetool.go b/internal/manager/biz/aiops/tools/expand_topology_basetool.go index 90488f740..a79acf35c 100644 --- a/internal/manager/biz/aiops/tools/expand_topology_basetool.go +++ b/internal/manager/biz/aiops/tools/expand_topology_basetool.go @@ -23,7 +23,7 @@ const ( // something silly. expandTopologyDefaultDepth = 2 expandTopologyMaxDepth = 5 - expandTopologyCallTimeout = 8 * time.Second + expandTopologyCallTimeout = 60 * time.Second ) const expandTopologyDescription = "Walk the business topology graph outward from a starting node (or a device's node). " + diff --git a/internal/manager/biz/aiops/tools/find_outlier_edges.go b/internal/manager/biz/aiops/tools/find_outlier_edges.go index 0326c6ac4..ac506b217 100644 --- a/internal/manager/biz/aiops/tools/find_outlier_edges.go +++ b/internal/manager/biz/aiops/tools/find_outlier_edges.go @@ -50,7 +50,7 @@ type OutlierEdgeRow struct { Metric string `json:"metric"` } -const outlierCallTimeout = 30 * time.Second +const outlierCallTimeout = 60 * time.Second // executeFindOutlierEdges builds a z-score PromQL of the shape // diff --git a/internal/manager/biz/aiops/tools/find_topology_node_basetool.go b/internal/manager/biz/aiops/tools/find_topology_node_basetool.go index 41bdf09e3..cbac3450f 100644 --- a/internal/manager/biz/aiops/tools/find_topology_node_basetool.go +++ b/internal/manager/biz/aiops/tools/find_topology_node_basetool.go @@ -19,7 +19,7 @@ const ( // one shot — anything more is noise that crowds out reasoning. findTopologyNodeMaxLimit = 50 findTopologyNodeDefaultLimit = 20 - findTopologyNodeCallTimeout = 5 * time.Second + findTopologyNodeCallTimeout = 60 * time.Second ) const findTopologyNodeDescription = "Search the business topology by name substring (case-insensitive). " + diff --git a/internal/manager/biz/aiops/tools/get_edge_summary.go b/internal/manager/biz/aiops/tools/get_edge_summary.go index c99c3078a..db9454fb3 100644 --- a/internal/manager/biz/aiops/tools/get_edge_summary.go +++ b/internal/manager/biz/aiops/tools/get_edge_summary.go @@ -62,7 +62,7 @@ type GetEdgeSummaryArgs struct { // edgeSummaryCallTimeout caps the whole multi-step round trip. It must // be larger than hostLoadCallTimeout so a slow edge doesn't blow the // outer ctx before the incident query runs. -const edgeSummaryCallTimeout = 30 * time.Second +const edgeSummaryCallTimeout = 60 * time.Second // EdgeSummaryIncidentRow is the trimmed incident envelope embedded in // the summary. Mirrors fields the LLM most likely needs to talk about diff --git a/internal/manager/biz/aiops/tools/get_edge_summary_basetool.go b/internal/manager/biz/aiops/tools/get_edge_summary_basetool.go index a58793450..b951a99fd 100644 --- a/internal/manager/biz/aiops/tools/get_edge_summary_basetool.go +++ b/internal/manager/biz/aiops/tools/get_edge_summary_basetool.go @@ -33,7 +33,7 @@ import ( // edgeSummaryBatchTimeout caps the whole batched run. Wider than the // per-id ceiling so up to batchConcurrency stitches can finish. -const edgeSummaryBatchTimeout = 90 * time.Second +const edgeSummaryBatchTimeout = ((batchMaxIDs+batchConcurrency-1)/batchConcurrency)*edgeSummaryCallTimeout + 30*time.Second // GetEdgeSummaryTool is the BaseTool form of get_edge_summary. type GetEdgeSummaryTool struct { diff --git a/internal/manager/biz/aiops/tools/get_incident_detail.go b/internal/manager/biz/aiops/tools/get_incident_detail.go index 8367ddeaa..5c0f8d8ef 100644 --- a/internal/manager/biz/aiops/tools/get_incident_detail.go +++ b/internal/manager/biz/aiops/tools/get_incident_detail.go @@ -48,7 +48,7 @@ type IncidentEventRow struct { OccurredAt time.Time `json:"occurred_at"` } -const incidentDetailCallTimeout = 10 * time.Second +const incidentDetailCallTimeout = 60 * time.Second func (r *Registry) executeGetIncidentDetail(ctx context.Context, args json.RawMessage) (ExecuteResult, error) { if r.alertUC == nil { diff --git a/internal/manager/biz/aiops/tools/get_topology.go b/internal/manager/biz/aiops/tools/get_topology.go index 70d8bea24..e03082ecf 100644 --- a/internal/manager/biz/aiops/tools/get_topology.go +++ b/internal/manager/biz/aiops/tools/get_topology.go @@ -48,7 +48,7 @@ type TopologyInfo struct { ChannelCounter func(ctx context.Context) (int, error) } -const topologyCallTimeout = 10 * time.Second +const topologyCallTimeout = 60 * time.Second func (r *Registry) executeGetTopology(ctx context.Context, _ json.RawMessage) (ExecuteResult, error) { callCtx, cancel := context.WithTimeout(ctx, topologyCallTimeout) diff --git a/internal/manager/biz/aiops/tools/host_files_basetool.go b/internal/manager/biz/aiops/tools/host_files_basetool.go index a5cd6ec13..6b4187605 100644 --- a/internal/manager/biz/aiops/tools/host_files_basetool.go +++ b/internal/manager/biz/aiops/tools/host_files_basetool.go @@ -6,7 +6,6 @@ import ( "fmt" "log/slog" "path/filepath" - "time" "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" devicebiz "github.com/ongridio/ongrid/internal/manager/biz/device" @@ -35,11 +34,8 @@ import ( // closure-style legacy registry path (registry.go::Tool) is NOT used — // these tools are BaseTool-native from day one (改进点 #1). -// hostFilesCallTimeout caps a single tunnel round-trip. The edge runs -// up to 4 paths concurrently with a 30 s per-path budget and a 60 s -// whole-batch ceiling, so we mirror 60 s here. Stat is always cheap; -// find/du can saturate this budget on a large tree. -const hostFilesCallTimeout = 60 * time.Second +// Allow all scan waves, metadata collection and response delivery to finish. +const hostFilesCallTimeout = tunnel.HostFilesCallTimeout // hostFilesMaxBatchPaths is the manager-side hard upper bound on // len(paths). Mirrored at the edge in internal/edgeagent/host_files diff --git a/internal/manager/biz/aiops/tools/host_load.go b/internal/manager/biz/aiops/tools/host_load.go index 201f04c1c..a8232dfd4 100644 --- a/internal/manager/biz/aiops/tools/host_load.go +++ b/internal/manager/biz/aiops/tools/host_load.go @@ -37,7 +37,7 @@ type GetHostLoadArgs struct { // hostLoadCallTimeout caps how long a single tool dispatch may wait on // the frontier round-trip. We derive a child ctx with this deadline so // long-running edge calls cannot wedge the agent loop. -const hostLoadCallTimeout = 15 * time.Second +const hostLoadCallTimeout = 60 * time.Second // executeGetHostLoad resolves edge_name -> edge.ID via manager/biz/edge and // dispatches a get_host_load reverse call through the frontier. diff --git a/internal/manager/biz/aiops/tools/host_processes.go b/internal/manager/biz/aiops/tools/host_processes.go index 7defb10ae..189bbcb26 100644 --- a/internal/manager/biz/aiops/tools/host_processes.go +++ b/internal/manager/biz/aiops/tools/host_processes.go @@ -48,7 +48,7 @@ type GetProcessListArgs struct { // processListCallTimeout caps how long a single dispatch may wait. Same // rationale as hostLoadCallTimeout. -const processListCallTimeout = 15 * time.Second +const processListCallTimeout = 60 * time.Second // executeGetProcessList resolves edge_name -> edge.ID and dispatches a // get_process_list reverse call through the frontier. TopN defaults to diff --git a/internal/manager/biz/aiops/tools/network_inventory_basetool.go b/internal/manager/biz/aiops/tools/network_inventory_basetool.go index 9e50e540e..ef3267b93 100644 --- a/internal/manager/biz/aiops/tools/network_inventory_basetool.go +++ b/internal/manager/biz/aiops/tools/network_inventory_basetool.go @@ -19,7 +19,7 @@ const ( ToolNameQueryNetworkDevices = "query_network_devices" ToolNameGetNetworkNeighbors = "get_network_neighbors" ToolNameQueryNetworkInterfaces = "query_network_interfaces" - networkInventoryCallTimeout = 8 * time.Second + networkInventoryCallTimeout = 60 * time.Second networkInventoryDefaultLimit = 50 networkInventoryMaxLimit = 100 ) diff --git a/internal/manager/biz/aiops/tools/query_alert_rules.go b/internal/manager/biz/aiops/tools/query_alert_rules.go index 46683426f..6f3fc2a6d 100644 --- a/internal/manager/biz/aiops/tools/query_alert_rules.go +++ b/internal/manager/biz/aiops/tools/query_alert_rules.go @@ -65,7 +65,7 @@ type AlertRuleRow struct { UpdatedAt time.Time `json:"updated_at"` } -const queryAlertRulesCallTimeout = 10 * time.Second +const queryAlertRulesCallTimeout = 60 * time.Second func (r *Registry) executeQueryAlertRules(ctx context.Context, args json.RawMessage) (ExecuteResult, error) { if r.alertUC == nil { diff --git a/internal/manager/biz/aiops/tools/query_edges.go b/internal/manager/biz/aiops/tools/query_edges.go index a9d398ade..8820b1507 100644 --- a/internal/manager/biz/aiops/tools/query_edges.go +++ b/internal/manager/biz/aiops/tools/query_edges.go @@ -90,7 +90,7 @@ type EdgeRow struct { } // queryEdgesCallTimeout caps the biz call. -const queryEdgesCallTimeout = 10 * time.Second +const queryEdgesCallTimeout = 60 * time.Second // executeQueryEdges resolves the filter and lists matching devices. func (r *Registry) executeQueryEdges(ctx context.Context, args json.RawMessage) (ExecuteResult, error) { diff --git a/internal/manager/biz/aiops/tools/query_incidents.go b/internal/manager/biz/aiops/tools/query_incidents.go index e2bf2e4c3..5f4176c90 100644 --- a/internal/manager/biz/aiops/tools/query_incidents.go +++ b/internal/manager/biz/aiops/tools/query_incidents.go @@ -85,7 +85,7 @@ type IncidentRow struct { ResolvedAt *time.Time `json:"resolved_at,omitempty"` } -const queryIncidentsCallTimeout = 15 * time.Second +const queryIncidentsCallTimeout = 60 * time.Second func (r *Registry) executeQueryIncidents(ctx context.Context, args json.RawMessage) (ExecuteResult, error) { if r.alertUC == nil { diff --git a/internal/manager/biz/aiops/tools/query_k8s_logs.go b/internal/manager/biz/aiops/tools/query_k8s_logs.go index 068e04b57..119cc385e 100644 --- a/internal/manager/biz/aiops/tools/query_k8s_logs.go +++ b/internal/manager/biz/aiops/tools/query_k8s_logs.go @@ -81,7 +81,7 @@ const queryK8sLogsWhenToUse = "Use when the user asks for recent Pod logs, Crash "This is a bounded live Kubernetes pods/log fallback; prefer query_logql for production log search and query_k8s_snapshot for counts/lists. " + "Never use this for kubectl exec, Secret reads, arbitrary files, or high-volume log export." -const queryK8sLogsCallTimeout = 20 * time.Second +const queryK8sLogsCallTimeout = 60 * time.Second type QueryK8sLogsTool struct { caller Caller diff --git a/internal/manager/biz/aiops/tools/query_k8s_snapshot.go b/internal/manager/biz/aiops/tools/query_k8s_snapshot.go index a0ff4c439..d75b1df1b 100644 --- a/internal/manager/biz/aiops/tools/query_k8s_snapshot.go +++ b/internal/manager/biz/aiops/tools/query_k8s_snapshot.go @@ -98,7 +98,7 @@ const queryK8sSnapshotWhenToUse = "When the user asks Kubernetes inventory quest "collect workloads, pods, and warning Events once, then use describe_k8s_resource only for ambiguous Pods and query_k8s_logs only for containers that actually start or restart. " + "This reads the manager DB snapshot by design. NOT for live describe/logs/exec/restart/scale/delete; use the live K8s/controller tools when those exist." -const queryK8sSnapshotCallTimeout = 10 * time.Second +const queryK8sSnapshotCallTimeout = 60 * time.Second type K8sSnapshotReader interface { ListClusters(ctx context.Context, f k8sbiz.ListClustersFilter) ([]*k8smodel.Cluster, error) diff --git a/internal/manager/biz/aiops/tools/query_logql.go b/internal/manager/biz/aiops/tools/query_logql.go index db11b7414..6ab0a7f48 100644 --- a/internal/manager/biz/aiops/tools/query_logql.go +++ b/internal/manager/biz/aiops/tools/query_logql.go @@ -152,7 +152,7 @@ func logQLStreamSelectorEnd(query string) (int, bool) { // queryLogqlCallTimeout caps how long a single dispatch may wait. Mirrors // query_promql for symmetry across signal types. -const queryLogqlCallTimeout = 30 * time.Second +const queryLogqlCallTimeout = 60 * time.Second // executeQueryLogQL runs the query against the selected backend and hands its // compact backend-specific result shape back to the LLM via ResultJSON. diff --git a/internal/manager/biz/aiops/tools/query_promql.go b/internal/manager/biz/aiops/tools/query_promql.go index 1c50df8f7..d7f62510d 100644 --- a/internal/manager/biz/aiops/tools/query_promql.go +++ b/internal/manager/biz/aiops/tools/query_promql.go @@ -46,7 +46,7 @@ type QueryPromQLArgs struct { // queryPromqlCallTimeout caps how long a single dispatch may wait. Same // rationale as the other tool timeouts. -const queryPromqlCallTimeout = 30 * time.Second +const queryPromqlCallTimeout = 60 * time.Second const maxQueryPromQLLookbackSeconds = 7 * 24 * 3600 diff --git a/internal/manager/biz/aiops/tools/query_traceql.go b/internal/manager/biz/aiops/tools/query_traceql.go index a96801c2d..4b725779e 100644 --- a/internal/manager/biz/aiops/tools/query_traceql.go +++ b/internal/manager/biz/aiops/tools/query_traceql.go @@ -192,7 +192,7 @@ func traceQLSearchFilters(in QueryTraceQLArgs) (string, map[string]string, error // queryTraceqlCallTimeout caps how long a single dispatch may wait. Mirrors // query_promql for symmetry across signal types. -const queryTraceqlCallTimeout = 30 * time.Second +const queryTraceqlCallTimeout = 60 * time.Second // executeQueryTraceQL runs the TraceQL search and hands the raw Tempo // response back to the LLM via ResultJSON. diff --git a/internal/manager/biz/aiops/tools/rank_edges.go b/internal/manager/biz/aiops/tools/rank_edges.go index 61f46501f..773e5159c 100644 --- a/internal/manager/biz/aiops/tools/rank_edges.go +++ b/internal/manager/biz/aiops/tools/rank_edges.go @@ -61,7 +61,7 @@ type RankEdgeRow struct { Metric string `json:"metric"` } -const rankEdgesCallTimeout = 30 * time.Second +const rankEdgesCallTimeout = 60 * time.Second // rankMetricExpr returns the PromQL fragment that yields a per-edge // scalar for the requested rank-by name. The closed set must stay in diff --git a/internal/manager/service/aiops/approval_executors.go b/internal/manager/service/aiops/approval_executors.go index 8a409171b..e6175cb8e 100644 --- a/internal/manager/service/aiops/approval_executors.go +++ b/internal/manager/service/aiops/approval_executors.go @@ -7,6 +7,7 @@ import ( "log/slog" "os" "strings" + "time" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" managerbizapproval "github.com/ongridio/ongrid/internal/manager/biz/approval" @@ -94,7 +95,7 @@ func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { if workdir != "" { env["HOME"] = workdir } - res, err := e.Runner.Run(ctx, runner.Spec{Script: p.Command, Env: env, Workdir: workdir}) + res, err := e.Runner.Run(ctx, runner.Spec{Script: p.Command, Env: env, Workdir: workdir, Timeout: 10 * time.Minute}) if err != nil { return "", err } diff --git a/internal/pkg/tunnel/bash.go b/internal/pkg/tunnel/bash.go index 6ab27eaa8..18ed1b88e 100644 --- a/internal/pkg/tunnel/bash.go +++ b/internal/pkg/tunnel/bash.go @@ -27,7 +27,7 @@ type BashExecRequest struct { Cmd string `json:"cmd"` // Timeout overrides the policy default per-call ceiling. 0 → use - // the sandbox default (30s). Negative is treated as 0. + // the sandbox default (300s). Negative is treated as 0. Timeout int `json:"timeout_seconds,omitempty"` // Unrestricted, when true, tells the edge to BYPASS cmdpolicy entirely diff --git a/internal/pkg/tunnel/host_files.go b/internal/pkg/tunnel/host_files.go index 801e1ab46..a7ff1c153 100644 --- a/internal/pkg/tunnel/host_files.go +++ b/internal/pkg/tunnel/host_files.go @@ -2,6 +2,15 @@ package tunnel import "time" +// Filesystem scan budgets are shared by Manager and Edge so transport waits +// outlast all four waves of a maximum-size batch plus the df snapshot. +const ( + HostFilesPerPathTimeout = 5 * time.Minute + HostFilesBatchTimeout = 4 * HostFilesPerPathTimeout + HostFilesMetadataTimeout = 30 * time.Second + HostFilesCallTimeout = HostFilesBatchTimeout + HostFilesMetadataTimeout + 30*time.Second +) + // host_files RPC method names. These are the wire constants the manager // (Caller) and edge (RegisterHandler) use to address the three filesystem // inspection tools. The LLM calls these via the BaseTool diff --git a/internal/skill/builtin/host_netns_inspect.go b/internal/skill/builtin/host_netns_inspect.go index 140bda266..5761b0fbf 100644 --- a/internal/skill/builtin/host_netns_inspect.go +++ b/internal/skill/builtin/host_netns_inspect.go @@ -156,7 +156,7 @@ func (HostNetnsInspect) Execute(ctx context.Context, params json.RawMessage) (js includeRoutes = *p.IncludeRoutes } - cctx, cancel := context.WithTimeout(ctx, 15*time.Second) + cctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() var names []string diff --git a/internal/skill/builtin/read_journal.go b/internal/skill/builtin/read_journal.go index 2dc9def76..bf7a80abc 100644 --- a/internal/skill/builtin/read_journal.go +++ b/internal/skill/builtin/read_journal.go @@ -60,7 +60,7 @@ type readJournalResult struct { } // Execute runs journalctl with the configured filters and returns its -// stdout split by newline. Hard 30s timeout protects the dispatcher from +// stdout split by newline. Hard 2m timeout protects the dispatcher from // a hanging journal. func (ReadJournal) Execute(ctx context.Context, params json.RawMessage) (json.RawMessage, error) { var p readJournalParams @@ -83,7 +83,7 @@ func (ReadJournal) Execute(ctx context.Context, params json.RawMessage) (json.Ra return json.Marshal(res) } - cctx, cancel := context.WithTimeout(ctx, 30*time.Second) + cctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() args := []string{"--no-pager", "--output=short-iso", "-n", strconv.Itoa(p.Lines)} diff --git a/internal/skill/builtin/web_search.go b/internal/skill/builtin/web_search.go index d62e55b63..59d4f0cef 100644 --- a/internal/skill/builtin/web_search.go +++ b/internal/skill/builtin/web_search.go @@ -240,7 +240,7 @@ func (s *webSearchSkill) Execute(ctx context.Context, params json.RawMessage) (j braveURL = "https://api.search.brave.com/res/v1/web/search" } if client == nil { - client = &http.Client{Timeout: 30 * time.Second} + client = &http.Client{Timeout: 60 * time.Second} } var p webSearchParams diff --git a/internal/skill/loader.go b/internal/skill/loader.go index 9f801099c..b2c2808c3 100644 --- a/internal/skill/loader.go +++ b/internal/skill/loader.go @@ -40,7 +40,7 @@ type SkillManifest struct { EnvAllow []string `json:"env_allow,omitempty"` // TimeoutSeconds caps the subprocess runtime. Zero falls back to - // DefaultSubprocessTimeout (30s). + // DefaultSubprocessTimeout (5m). TimeoutSeconds int `json:"timeout_seconds,omitempty"` // Class follows the same {safe, mutating, dangerous} taxonomy as diff --git a/internal/skill/subprocess.go b/internal/skill/subprocess.go index 517d47bce..614a9cc63 100644 --- a/internal/skill/subprocess.go +++ b/internal/skill/subprocess.go @@ -24,8 +24,9 @@ const MaxSubprocessStdout = 16 * 1024 * 1024 const MaxSubprocessStderrTail = 4 * 1024 // DefaultSubprocessTimeout is the deadline applied when a manifest does -// not specify timeout_seconds. 30s mirrors most LLM tool deadlines. -const DefaultSubprocessTimeout = 30 * time.Second +// not specify timeout_seconds. Allow longer diagnostic subprocesses while +// retaining explicit manifest deadlines and caller cancellation. +const DefaultSubprocessTimeout = 5 * time.Minute // SubprocessSkill runs an external executable per invocation. The skill // body lives in a separate process: ongrid pipes a JSON args blob to its From a912e6e450c06da9394940bbfaf31370bf371fe3 Mon Sep 17 00:00:00 2001 From: singchia Date: Sat, 17 Oct 2026 00:12:35 +0800 Subject: [PATCH 20/24] fix(skills): allow five minutes for journal reads Refs #523 --- internal/skill/builtin/read_journal.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/skill/builtin/read_journal.go b/internal/skill/builtin/read_journal.go index bf7a80abc..83fe4771d 100644 --- a/internal/skill/builtin/read_journal.go +++ b/internal/skill/builtin/read_journal.go @@ -60,7 +60,7 @@ type readJournalResult struct { } // Execute runs journalctl with the configured filters and returns its -// stdout split by newline. Hard 2m timeout protects the dispatcher from +// stdout split by newline. Hard 5m timeout protects the dispatcher from // a hanging journal. func (ReadJournal) Execute(ctx context.Context, params json.RawMessage) (json.RawMessage, error) { var p readJournalParams @@ -83,7 +83,7 @@ func (ReadJournal) Execute(ctx context.Context, params json.RawMessage) (json.Ra return json.Marshal(res) } - cctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + cctx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() args := []string{"--no-pager", "--output=short-iso", "-n", strconv.Itoa(p.Lines)} From 6f9fc4e80d3b909e9481e6ea8dc62b7152a194d5 Mon Sep 17 00:00:00 2001 From: singchia Date: Sat, 17 Oct 2026 00:12:35 +0800 Subject: [PATCH 21/24] feat(aiops): persist runtime ownership for new sessions Pin new conversations and v3 worker sessions to their execution version. Route persisted versions without fallback and reject conflicting contexts. Preserve unclassified historical sessions until provenance migration and old-runtime adapters are ready. Validate additive migration, routing and worker boundaries with race tests and real Manager/MySQL chat, RCA and flow E2E. Refs #503. --- api/manager/aiops/v1/aiops.proto | 2 + cmd/ongrid/bootstrap_runtime.go | 2 +- .../PRD-009-agent-runtime-selection.md | 2 +- docs/rfc/RFC-009-agent-runtime-versions.md | 16 +++- .../aiops/runtime/v3/chatruntime/runtime.go | 7 ++ .../v3/chatruntime/runtime_worker_test.go | 16 ++++ .../aiops/runtime/v3/chatruntime/worker.go | 8 ++ .../data/aiops/store/session_model_test.go | 36 ++++++++ internal/manager/model/aiops/model.go | 4 + internal/manager/server/aiops/http.go | 2 + internal/manager/server/aiops/http_test.go | 9 ++ .../manager/service/aiops/runtime_router.go | 55 ++++++++++- .../service/aiops/runtime_router_test.go | 91 +++++++++++++++++-- internal/manager/service/aiops/service.go | 13 +++ tests/e2e/assistant_chat_test.go | 3 + web/src/api/chat.ts | 1 + 16 files changed, 253 insertions(+), 14 deletions(-) diff --git a/api/manager/aiops/v1/aiops.proto b/api/manager/aiops/v1/aiops.proto index 407fa2edc..5c8156bc0 100644 --- a/api/manager/aiops/v1/aiops.proto +++ b/api/manager/aiops/v1/aiops.proto @@ -56,6 +56,8 @@ message ChatSession { string provider = 8 [json_name = "provider"]; string model = 9 [json_name = "model"]; APMSourceScope apm_source = 10 [json_name = "apm_source"]; + // Server-assigned execution version. Empty for unclassified historical sessions. + string runtime_version = 11 [json_name = "runtime_version"]; } // ToolCall 一次 LLM 发起的工具调用（OpenAI tool_calls 的投影）。 diff --git a/cmd/ongrid/bootstrap_runtime.go b/cmd/ongrid/bootstrap_runtime.go index 67a4b751b..cbc64afc6 100644 --- a/cmd/ongrid/bootstrap_runtime.go +++ b/cmd/ongrid/bootstrap_runtime.go @@ -52,7 +52,7 @@ func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Log if rterr != nil { log.Error("aiops runtime unavailable", slog.Any("err", rterr)) } else { - router, err := managersvcaiops.NewRuntimeRouter(contract.V3, map[contract.Version]managersvcaiops.RuntimeEngine{contract.V3: rt}) + router, err := managersvcaiops.NewRuntimeRouter(aiops.aiopsRepo, contract.V3, map[contract.Version]managersvcaiops.RuntimeEngine{contract.V3: rt}) if err != nil { log.Error("aiops runtime routing unavailable", slog.Any("err", err)) } else { diff --git a/docs/requirements/PRD-009-agent-runtime-selection.md b/docs/requirements/PRD-009-agent-runtime-selection.md index 364caa22c..aec78c67a 100644 --- a/docs/requirements/PRD-009-agent-runtime-selection.md +++ b/docs/requirements/PRD-009-agent-runtime-selection.md @@ -1,6 +1,6 @@ # PRD-009：Agent Runtime 版本选择 -- 状态：提议，尚未实现 +- 状态：部分基础设施已实现（源码隔离、路由、新会话版本持久化）；管理员版本选择尚未实现 - 日期：2026-10-16 - 优先级：P1 - 关联：RFC-009、PR #503 diff --git a/docs/rfc/RFC-009-agent-runtime-versions.md b/docs/rfc/RFC-009-agent-runtime-versions.md index d92ec50aa..a8eea78b4 100644 --- a/docs/rfc/RFC-009-agent-runtime-versions.md +++ b/docs/rfc/RFC-009-agent-runtime-versions.md @@ -1,6 +1,6 @@ # RFC-009：Agent Runtime v1/v2/v3 隔离与显式切换 -- 状态：提议；用户已确认仅管理员设置入口、release 默认最新稳定 Runtime，源码隔离已实现，版本切换尚未实现 +- 状态：提议；用户已确认仅管理员设置入口、release 默认最新稳定 Runtime，源码隔离、新会话版本持久化已实现，版本切换尚未实现 - 日期：2026-10-16 - 关联：PR #503、RFC-008、ADR-035；PRD-009 - 基线：正式发布标签 v0.18.2（`3b41874cae792329fd61cfe0450e57335efa3f1d`）的 Graph；不把已退役的 legacy for-loop 当作 v2 @@ -73,7 +73,7 @@ v1 固定为 GitHub 最早正式 release `v0.7.158`（2026-05-27，`04b279631978 已完成：`runtime/v1/{chatruntime,graph}` 提取首个 release；`runtime/v2/{chatruntime,graph}` 提取 v0.18.2；当前实现及测试迁移至 `runtime/v3/{chatruntime,graph}`，现有调用方指向 v3。v1/v2 保留源提交、文件 blob 清单和兼容改动说明。三版本均参与标准测试，禁止直接跨版本 import。 -本次仅完成源码隔离。版本契约和单版本生产路由已接入（见下节）；会话版本字段、历史版本适配、版本化工具装配及管理员设置仍未实现。外部 persona/skills、业务工具和依赖仍共享，不能将源码提取理解为完整复现旧 release，v1 接入前必须补齐现行安全与审批验收。 +源码隔离后，版本契约、单版本生产路由及新会话版本持久化已接入（见下节）；历史版本适配、版本化工具装配及管理员设置仍未实现。外部 persona/skills、业务工具和依赖仍共享，不能将源码提取理解为完整复现旧 release，v1 接入前必须补齐现行安全与审批验收。 ## 统一契约与路由：能力保留检查（2026-10-16） @@ -96,3 +96,15 @@ Worker 查询、继续、停止按实例实际归属分派，覆盖内部 review 验收：扩展后的 `make test-agent-runtime`（Linux，含 race）、架构检查及 11 个边界样例通过；真实 Manager + 临时 MySQL 的 AssistantChat、MCP CRUD、Skills registry 和 Flow MCP 节点 E2E 通过（39.771s）；额外的 RCA 调查、工作流页面产物和 Edge 概览 E2E 通过（32.953s）。模型和 MCP endpoint 为测试替身，不代表真实模型验收。 下一阶段在不改变上述能力的前提下增加持久化会话版本、旧版本适配及独立目录装配，之后才开放管理员切换。回滚本阶段只需回退代码；本阶段没有 API/schema/配置格式变更。 + +## 新会话版本持久化（2026-10-17） + +已增加可空 `chat_sessions.runtime_version` 字段，通过项目现有 AutoMigrate 做加法升级。新会话创建时读取路由器的默认版本并保存；当前仅装配 v3。v3 Worker（包括内部 reviewer、独立系统任务）写入 v3，后台 context 继续保留版本。HTTP 会话响应和 proto 增加只读可选 `runtime_version`，前端 API 类型同步，不增加聊天页选择器。 + +生产路由器读取会话/父会话的持久化版本，显式 context 与记录冲突时拒绝执行；未装配或未知版本不自动降级。v3 入口也校验版本，防止内部直接调用绕过路由。缺失的父会话在路由入口明确失败，避免把有归属的请求变成无父任务；无父会话的系统任务继续使用默认版本。 + +历史空值暂不回填、不推断。当前仍只有 v3，空值沿用现有执行方式；这不是完成历史迁移，也不能据此开放默认版本切换。正式发布历史会话与试验环境历史会话的来源分类、旧版本适配、审批来源版本及独立可变目录仍是开放切换前的必需项。 + +验证：SQLite 模拟缺列升级并连续迁移两次，历史空值保持不变；新记录重读、改标题/模型后版本仍保留。路由回归覆盖固定版本优先、context 冲突、未知/未装配版本、父任务继承和缺失会话。真实 Manager + 临时 MySQL 的聊天、RCA 和工作流 E2E 通过（35.118s；模型为测试替身）。 + +回滚：恢复前一代码/镜像，保留新增可空列即可，不需要删除列或还原聊天数据。旧代码不会执行版本约束；在多版本能力开放后不能把这种镜像回滚视为等价版本切换。本阶段未部署或推送。 diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go index 5040a2dff..2813b1575 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go @@ -368,6 +368,13 @@ func (rt *Runtime) Handle(ctx context.Context, req *Request) (result *Reply, ret // ErrForbidden) so non-owners can't fingerprint sessions. return nil, errs.ErrNotFound } + if version := contract.VersionFromContext(ctx); version != "" && version != contract.V3 { + return nil, fmt.Errorf("%w: request runtime mismatch", errs.ErrInvalid) + } + if sess.RuntimeVersion != "" && sess.RuntimeVersion != string(contract.V3) { + return nil, fmt.Errorf("%w: session runtime mismatch", errs.ErrInvalid) + } + ctx = contract.WithVersion(ctx, contract.V3) ctx, span := startAgentSpan(ctx, sess, "default") tracing.RecordAgentContent(span, "input.value", req.UserText) var runErr error diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_worker_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_worker_test.go index e7c32d24a..ed2541120 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_worker_test.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime_worker_test.go @@ -60,6 +60,9 @@ func TestSpawnWorker_Sync(t *testing.T) { if err != nil || row == nil { t.Fatalf("worker chat_sessions row missing: %v", err) } + if row.RuntimeVersion != "v3" { + t.Errorf("worker runtime = %q, want v3", row.RuntimeVersion) + } if row.AgentID == nil || *row.AgentID != "incident-investigator" { t.Errorf("row.AgentID = %v, want incident-investigator", row.AgentID) } @@ -631,3 +634,16 @@ func newSeededRuntimeStore(t *testing.T, userID uint64) (*memSessions, string) { store := newMemSessions(&model.Session{ID: parentID, UserID: userID, Title: "parent", RootSessionID: &rootID, OwnerAgentID: &owner}) return store, parentID } + +func TestSpawnWorker_RejectsParentFromAnotherRuntime(t *testing.T) { + store := newMemSessions(&model.Session{ID: "parent", UserID: 7, RuntimeVersion: "v2"}) + rt := newRuntimeWithStore(t, "incident-investigator", newScriptedChatModel(&schema.Message{Role: schema.Assistant, Content: "must not run"}), nil, store) + _, err := rt.SpawnWorker(context.Background(), SpawnRequest{AgentName: "incident-investigator", Prompt: "work", ParentSession: "parent"}) + if err == nil || !strings.Contains(err.Error(), "runtime mismatch") { + t.Fatalf("cross-runtime spawn = %v", err) + } + children, listErr := store.ListByParent(context.Background(), "parent") + if listErr != nil || len(children) != 0 { + t.Fatalf("rejected spawn persisted children: %+v %v", children, listErr) + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go index a5cd03885..0208543d9 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go @@ -105,6 +105,10 @@ func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, if rt == nil { return nil, errors.New("chatruntime: nil runtime") } + if version := contract.VersionFromContext(ctx); version != "" && version != contract.V3 { + return nil, fmt.Errorf("chatruntime: worker runtime mismatch") + } + ctx = contract.WithVersion(ctx, contract.V3) if rt.cfg.ChatModel == nil { return nil, errors.New("chatruntime: ChatModel is required") } @@ -133,6 +137,9 @@ func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, if req.ParentSession != "" && rt.cfg.Sessions != nil { parentSess, err := rt.cfg.Sessions.GetSession(ctx, req.ParentSession) if err == nil && parentSess != nil { + if parentSess.RuntimeVersion != "" && parentSess.RuntimeVersion != string(contract.V3) { + return nil, fmt.Errorf("chatruntime: parent runtime mismatch") + } if parentSess.ParentSessionID != nil && *parentSess.ParentSessionID != "" { return nil, fmt.Errorf("chatruntime: delegation depth exceeded for parent session %q", req.ParentSession) } @@ -174,6 +181,7 @@ func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, effectiveOwner = req.OwnerUserID } row := &aiopsmodel.Session{ + RuntimeVersion: string(contract.V3), ID: sessID, APMSource: sourceScope, UserID: effectiveOwner, diff --git a/internal/manager/data/aiops/store/session_model_test.go b/internal/manager/data/aiops/store/session_model_test.go index 28612b281..9aa3ed294 100644 --- a/internal/manager/data/aiops/store/session_model_test.go +++ b/internal/manager/data/aiops/store/session_model_test.go @@ -52,3 +52,39 @@ func TestAPMSourceScopeSurvivesReload(t *testing.T) { t.Fatalf("reload: %+v %v", got, err) } } + +func TestSessionRuntimeVersion_PersistsWithoutChangingHistoricalRows(t *testing.T) { + repo := newTestRepo(t) + ctx := context.Background() + legacy := &model.Session{UserID: 1, Title: "historical"} + if err := repo.CreateSession(ctx, legacy); err != nil { + t.Fatal(err) + } + // Simulate a pre-feature database, then run the real migration twice. + if err := repo.db.Migrator().DropColumn(&model.Session{}, "RuntimeVersion"); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err := Migrate(repo.db); err != nil { + t.Fatal(err) + } + } + old, err := repo.GetSession(ctx, legacy.ID) + if err != nil || old.RuntimeVersion != "" { + t.Fatalf("historical version fabricated: %+v %v", old, err) + } + pinned := &model.Session{UserID: 1, Title: "new", RuntimeVersion: "v3"} + if err := repo.CreateSession(ctx, pinned); err != nil { + t.Fatal(err) + } + if err := repo.UpdateSessionModel(ctx, pinned.ID, "custom", "model"); err != nil { + t.Fatal(err) + } + if err := repo.RenameSession(ctx, pinned.ID, "renamed"); err != nil { + t.Fatal(err) + } + got, err := NewSessionRepoWithAttachmentRoot(repo.db, t.TempDir()).GetSession(ctx, pinned.ID) + if err != nil || got.RuntimeVersion != "v3" { + t.Fatalf("runtime pin lost: %+v %v", got, err) + } +} diff --git a/internal/manager/model/aiops/model.go b/internal/manager/model/aiops/model.go index 8f214e879..24d4b1f21 100644 --- a/internal/manager/model/aiops/model.go +++ b/internal/manager/model/aiops/model.go @@ -68,6 +68,10 @@ type APMSourceScope struct { } type Session struct { + // RuntimeVersion pins execution policy. Empty means historical provenance + // has not been classified; migrations must not guess v2 versus v3. + RuntimeVersion string `gorm:"size:16;column:runtime_version"` + APMSource *APMSourceScope `gorm:"serializer:json;type:text;column:apm_source"` ID string `gorm:"primaryKey;type:char(36);column:id"` diff --git a/internal/manager/server/aiops/http.go b/internal/manager/server/aiops/http.go index 5bfbd59eb..eff6421bc 100644 --- a/internal/manager/server/aiops/http.go +++ b/internal/manager/server/aiops/http.go @@ -327,6 +327,7 @@ type createSessionReq struct { } type sessionDTO struct { + RuntimeVersion string `json:"runtime_version,omitempty"` APMSource *model.APMSourceScope `json:"apm_source,omitempty"` ID string `json:"id"` UserID uint64 `json:"user_id"` @@ -1246,6 +1247,7 @@ func (h *Handler) listModels(w http.ResponseWriter, r *http.Request) { func toSessionDTO(s *model.Session) sessionDTO { out := sessionDTO{ + RuntimeVersion: s.RuntimeVersion, ID: s.ID, UserID: s.UserID, Title: s.Title, diff --git a/internal/manager/server/aiops/http_test.go b/internal/manager/server/aiops/http_test.go index ffb2c6d6a..af1b1fb52 100644 --- a/internal/manager/server/aiops/http_test.go +++ b/internal/manager/server/aiops/http_test.go @@ -522,3 +522,12 @@ func TestValidateUploadedImageWebPDimensions(t *testing.T) { // string reaching the handler is forwarded to the service layer (which // returns ErrNotFound on miss). The previous test relied on numeric // "0" being a sentinel invalid id, which no longer makes sense. + +func TestSessionDTO_RuntimeVersion(t *testing.T) { + for _, version := range []string{"", "v2", "v3"} { + got := toSessionDTO(&model.Session{ID: "s", RuntimeVersion: version}) + if got.RuntimeVersion != version { + t.Fatalf("runtime=%q, want %q", got.RuntimeVersion, version) + } + } +} diff --git a/internal/manager/service/aiops/runtime_router.go b/internal/manager/service/aiops/runtime_router.go index 7682f11f4..174ac2e10 100644 --- a/internal/manager/service/aiops/runtime_router.go +++ b/internal/manager/service/aiops/runtime_router.go @@ -6,6 +6,7 @@ import ( "reflect" "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" "github.com/ongridio/ongrid/internal/pkg/errs" ) @@ -24,15 +25,21 @@ type RuntimeEngine interface { CountWorkersByStatus() (running, pending int) } +// RuntimeSessionReader supplies durable execution ownership to the router. +type RuntimeSessionReader interface { + GetSession(context.Context, string) (*model.Session, error) +} + // RuntimeRouter owns execution dispatch, never tool or persona policy. -// The engine registry and default are immutable. A live default-setting change -// will require durable session binding before it can be exposed to operators. +// Its registry and default are immutable; historical-session classification +// is still required before exposing a live default-version setting. type RuntimeRouter struct { + sessions RuntimeSessionReader engines map[contract.Version]RuntimeEngine defaultVersion contract.Version } -func NewRuntimeRouter(defaultVersion contract.Version, engines map[contract.Version]RuntimeEngine) (*RuntimeRouter, error) { +func NewRuntimeRouter(sessions RuntimeSessionReader, defaultVersion contract.Version, engines map[contract.Version]RuntimeEngine) (*RuntimeRouter, error) { copy := make(map[contract.Version]RuntimeEngine, len(engines)) for version, engine := range engines { if version != contract.V1 && version != contract.V2 && version != contract.V3 { @@ -46,7 +53,7 @@ func NewRuntimeRouter(defaultVersion contract.Version, engines map[contract.Vers if copy[defaultVersion] == nil { return nil, fmt.Errorf("%w: default runtime %q unavailable", errs.ErrNotWiredYet, defaultVersion) } - return &RuntimeRouter{engines: copy, defaultVersion: defaultVersion}, nil + return &RuntimeRouter{sessions: sessions, engines: copy, defaultVersion: defaultVersion}, nil } func (r *RuntimeRouter) resolve(ctx context.Context) (context.Context, RuntimeEngine, contract.Version, error) { @@ -64,10 +71,46 @@ func (r *RuntimeRouter) resolve(ctx context.Context) (context.Context, RuntimeEn return contract.WithVersion(ctx, version), engine, version, nil } +// SessionVersion resolves new-session policy once, before persistence. +func (r *RuntimeRouter) SessionVersion(ctx context.Context) (contract.Version, error) { + _, _, version, err := r.resolve(ctx) + return version, err +} + +// sessionContext honors durable ownership. Empty historical values retain +// the current single-version behavior until a provenance migration is ready. +func (r *RuntimeRouter) sessionContext(ctx context.Context, id string) (context.Context, error) { + if r == nil { + return ctx, errs.ErrNotWiredYet + } + if r.sessions == nil || id == "" { + return ctx, nil + } + session, err := r.sessions.GetSession(ctx, id) + if err != nil { + return ctx, fmt.Errorf("runtime session: %w", err) + } + if session == nil { + return ctx, errs.ErrNotFound + } + if session.RuntimeVersion == "" { + return ctx, nil + } + version := contract.Version(session.RuntimeVersion) + if requested := contract.VersionFromContext(ctx); requested != "" && requested != version { + return ctx, fmt.Errorf("%w: session runtime mismatch", errs.ErrInvalid) + } + return contract.WithVersion(ctx, version), nil +} + func (r *RuntimeRouter) Handle(ctx context.Context, req *contract.Request) (*contract.Reply, error) { if req == nil { return nil, fmt.Errorf("%w: request required", errs.ErrInvalid) } + ctx, err := r.sessionContext(ctx, req.SessionID) + if err != nil { + return nil, err + } ctx, engine, _, err := r.resolve(ctx) if err != nil { return nil, err @@ -76,6 +119,10 @@ func (r *RuntimeRouter) Handle(ctx context.Context, req *contract.Request) (*con } func (r *RuntimeRouter) SpawnWorker(ctx context.Context, req contract.SpawnRequest) (*contract.Worker, error) { + ctx, err := r.sessionContext(ctx, req.ParentSession) + if err != nil { + return nil, err + } ctx, engine, version, err := r.resolve(ctx) if err != nil { return nil, err diff --git a/internal/manager/service/aiops/runtime_router_test.go b/internal/manager/service/aiops/runtime_router_test.go index bb520eab8..919ee51af 100644 --- a/internal/manager/service/aiops/runtime_router_test.go +++ b/internal/manager/service/aiops/runtime_router_test.go @@ -66,7 +66,7 @@ func TestRuntimeRouterPreservesRequestEventsAndResult(t *testing.T) { } return reply, nil }} - router, err := NewRuntimeRouter(contract.V3, map[contract.Version]RuntimeEngine{contract.V3: engine}) + router, err := NewRuntimeRouter(nil, contract.V3, map[contract.Version]RuntimeEngine{contract.V3: engine}) if err != nil { t.Fatal(err) } @@ -80,7 +80,7 @@ func TestRuntimeRouterRejectsUnavailableVersionWithoutFallback(t *testing.T) { var calls atomic.Int32 engine := &routerEngine{handle: func(context.Context, *contract.Request) (*contract.Reply, error) { calls.Add(1); return nil, nil }} registry := map[contract.Version]RuntimeEngine{contract.V3: engine} - router, err := NewRuntimeRouter(contract.V3, registry) + router, err := NewRuntimeRouter(nil, contract.V3, registry) if err != nil { t.Fatal(err) } @@ -100,7 +100,7 @@ func TestRuntimeRouterRejectsUnavailableVersionWithoutFallback(t *testing.T) { t.Fatal(err) } for _, registry := range []map[contract.Version]RuntimeEngine{nil, {contract.V3: nil}, {contract.V3: (*routerEngine)(nil)}, {"v99": engine}} { - if _, err := NewRuntimeRouter(contract.V3, registry); err == nil { + if _, err := NewRuntimeRouter(nil, contract.V3, registry); err == nil { t.Fatal("invalid registry accepted") } } @@ -125,7 +125,7 @@ func TestRuntimeRouterRoutesWorkerControlToOwner(t *testing.T) { stops.Add(1) return nil } - router, err := NewRuntimeRouter(contract.V3, map[contract.Version]RuntimeEngine{contract.V2: first, contract.V3: second}) + router, err := NewRuntimeRouter(nil, contract.V3, map[contract.Version]RuntimeEngine{contract.V2: first, contract.V3: second}) if err != nil { t.Fatal(err) } @@ -185,7 +185,7 @@ func TestRuntimeRouterPreservesWorkerSpawnAndErrors(t *testing.T) { } return want, nil }, handle: func(context.Context, *contract.Request) (*contract.Reply, error) { return nil, sentinel }} - router, err := NewRuntimeRouter(contract.V3, map[contract.Version]RuntimeEngine{contract.V3: engine}) + router, err := NewRuntimeRouter(nil, contract.V3, map[contract.Version]RuntimeEngine{contract.V3: engine}) if err != nil { t.Fatal(err) } @@ -200,7 +200,7 @@ func TestRuntimeRouterPreservesWorkerSpawnAndErrors(t *testing.T) { func TestRuntimeRouterRejectsAmbiguousWorkerIdentity(t *testing.T) { engine := func() *routerEngine { return &routerEngine{worker: &contract.Worker{ID: "collision"}} } - router, err := NewRuntimeRouter(contract.V3, map[contract.Version]RuntimeEngine{contract.V2: engine(), contract.V3: engine()}) + router, err := NewRuntimeRouter(nil, contract.V3, map[contract.Version]RuntimeEngine{contract.V2: engine(), contract.V3: engine()}) if err != nil { t.Fatal(err) } @@ -211,3 +211,82 @@ func TestRuntimeRouterRejectsAmbiguousWorkerIdentity(t *testing.T) { t.Fatal("ambiguous worker controlled") } } + +func TestRuntimeRouter_DurableSessionVersion(t *testing.T) { + for _, tc := range []struct { + name, stored string + requested contract.Version + want contract.Version + fail bool + }{ + {name: "pinned overrides default", stored: "v2", want: contract.V2}, + {name: "historical retains current default", want: contract.V3}, + {name: "matching context", stored: "v2", requested: contract.V2, want: contract.V2}, + {name: "conflict rejected", stored: "v2", requested: contract.V3, fail: true}, + {name: "unavailable rejected", stored: "v1", fail: true}, + {name: "unknown rejected", stored: "v99", fail: true}, + } { + t.Run(tc.name, func(t *testing.T) { + repo := &memSessions{sessions: map[string]*model.Session{"s": {ID: "s", RuntimeVersion: tc.stored}}} + calls := 0 + engine := func(version contract.Version) *routerEngine { + return &routerEngine{ + handle: func(ctx context.Context, _ *contract.Request) (*contract.Reply, error) { + calls++ + if version != tc.want || contract.VersionFromContext(ctx) != version { + t.Fatalf("wrong engine: %s", version) + } + return &contract.Reply{}, nil + }, + spawn: func(ctx context.Context, _ contract.SpawnRequest) (*contract.Worker, error) { + calls++ + if version != tc.want || contract.VersionFromContext(ctx) != version { + t.Fatalf("wrong worker engine: %s", version) + } + return &contract.Worker{ID: "w"}, nil + }, + } + } + router, err := NewRuntimeRouter(repo, contract.V3, map[contract.Version]RuntimeEngine{contract.V2: engine(contract.V2), contract.V3: engine(contract.V3)}) + if err != nil { + t.Fatal(err) + } + ctx := contract.WithVersion(context.Background(), tc.requested) + _, err = router.Handle(ctx, &contract.Request{SessionID: "s"}) + if (err != nil) != tc.fail { + t.Fatalf("Handle error=%v", err) + } + _, err = router.SpawnWorker(ctx, contract.SpawnRequest{ParentSession: "s"}) + if (err != nil) != tc.fail { + t.Fatalf("SpawnWorker error=%v", err) + } + if tc.fail && calls != 0 { + t.Fatal("rejected request executed") + } + if !tc.fail && calls != 2 { + t.Fatalf("calls=%d", calls) + } + _, err = router.Handle(ctx, &contract.Request{SessionID: "missing"}) + if !errors.Is(err, errs.ErrNotFound) { + t.Fatalf("missing session=%v", err) + } + }) + } +} + +func TestCreateSession_PinsValidatedRuntimeDefault(t *testing.T) { + repo := &memSessions{sessions: map[string]*model.Session{}} + router, err := NewRuntimeRouter(repo, contract.V2, map[contract.Version]RuntimeEngine{contract.V2: &routerEngine{}}) + if err != nil { + t.Fatal(err) + } + svc := &Service{sessions: repo, runtime: router} + row, err := svc.CreateSession(context.Background(), Caller{UserID: 7}, CreateSessionInput{Title: "pinned"}) + if err != nil || row.RuntimeVersion != "v2" { + t.Fatalf("create=%+v %v", row, err) + } + _, err = svc.CreateSession(contract.WithVersion(context.Background(), contract.V3), Caller{UserID: 7}, CreateSessionInput{}) + if !errors.Is(err, errs.ErrNotWiredYet) { + t.Fatalf("unavailable version=%v", err) + } +} diff --git a/internal/manager/service/aiops/service.go b/internal/manager/service/aiops/service.go index e03081673..34dbc7f11 100644 --- a/internal/manager/service/aiops/service.go +++ b/internal/manager/service/aiops/service.go @@ -201,7 +201,20 @@ func (s *Service) CreateSession(ctx context.Context, caller Caller, in CreateSes if (provider == "") != (selectedModel == "") { return nil, fmt.Errorf("%w: provider and model must be supplied together", errs.ErrInvalid) } + // The current direct-runtime compatibility path is v3. Production uses + // the router's validated default; no caller-facing version override yet. + version := chatruntime.V3 + if selector, ok := s.runtime.(interface { + SessionVersion(context.Context) (chatruntime.Version, error) + }); ok { + var err error + version, err = selector.SessionVersion(ctx) + if err != nil { + return nil, err + } + } sess := &model.Session{ + RuntimeVersion: string(version), UserID: caller.UserID, Title: title, RelatedIncidentID: in.RelatedIncidentID, diff --git a/tests/e2e/assistant_chat_test.go b/tests/e2e/assistant_chat_test.go index 8ae37fbed..8efaec914 100644 --- a/tests/e2e/assistant_chat_test.go +++ b/tests/e2e/assistant_chat_test.go @@ -1074,6 +1074,9 @@ func createChatSession(t *testing.T, env *testenv.Env, token, title string) stri if err != nil || status != http.StatusCreated { t.Fatalf("create chat session: status=%d body=%v err=%v", status, body, err) } + if body["runtime_version"] != "v3" { + t.Fatalf("new session runtime = %v, want v3", body["runtime_version"]) + } id, _ := body["id"].(string) if id == "" { t.Fatalf("create chat session returned empty id: %v", body) diff --git a/web/src/api/chat.ts b/web/src/api/chat.ts index 9e38ba969..913dbeb7c 100644 --- a/web/src/api/chat.ts +++ b/web/src/api/chat.ts @@ -49,6 +49,7 @@ export type ChatMessage = { }; export type ChatSession = { + runtime_version?: string; apm_source?: { trace_id: string; repo_id: string; source_directory: string; revision: string; commit_sha: string; error?: string }; id: string; user_id: number; From b138a6b68e9ec84a68ff230225eb467fb3664a6f Mon Sep 17 00:00:00 2001 From: singchia Date: Sat, 17 Oct 2026 00:21:30 +0800 Subject: [PATCH 22/24] feat(aiops): bind approvals to their originating runtime Persist runtime ownership and dispatch approved actions by exact kind/version. Restore the originating context and reject mismatched or lost single-use tool capabilities without fallback. Preserve unversioned records without guessing historical provenance. Validate migration, concurrent registration, exact routing, MCP lifecycle and approval HTTP E2E. Refs #503. --- api/manager/approval/v1/approval.proto | 26 +++++++ cmd/ongrid/bootstrap_extensions.go | 2 + docs/rfc/RFC-009-agent-runtime-versions.md | 14 +++- internal/manager/biz/approval/usecase.go | 49 +++++++++----- internal/manager/biz/approval/usecase_test.go | 64 ++++++++++++++++++ .../approval/store/runtime_version_test.go | 55 +++++++++++++++ internal/manager/model/approval/model.go | 4 +- .../service/aiops/approval_adapters.go | 67 ++++++++++--------- .../service/aiops/approval_executors.go | 26 +++++-- .../manager/service/aiops/human_approval.go | 35 ++++++---- .../service/aiops/human_approval_test.go | 37 +++++++++- .../aiops/mcp_approval_runtime_test.go | 2 +- .../service/aiops/mcp_approval_test.go | 13 ++-- tests/e2e/assistant_chat_test.go | 6 +- web/src/api/approvals.ts | 1 + 15 files changed, 328 insertions(+), 73 deletions(-) create mode 100644 api/manager/approval/v1/approval.proto create mode 100644 internal/manager/data/approval/store/runtime_version_test.go diff --git a/api/manager/approval/v1/approval.proto b/api/manager/approval/v1/approval.proto new file mode 100644 index 000000000..f903afefd --- /dev/null +++ b/api/manager/approval/v1/approval.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; +package ongrid.manager.approval.v1; +import "google/protobuf/timestamp.proto"; +option go_package = "github.com/ongridio/ongrid/api/gen/manager/approval/v1;approvalv1"; + +// Approval is the read-only record returned by the existing approval inbox. +// Runtime ownership is assigned by the producer, never by the approver. +message Approval { + string id = 1; + string kind = 2; + string title = 3; + string summary = 4; + string payload = 5; + string source = 6; + string session_id = 7; + string status = 8; + uint64 proposed_by = 9; + optional uint64 approved_by = 10; + optional string reason = 11; + optional string result = 12; + google.protobuf.Timestamp created_at = 13; + google.protobuf.Timestamp decided_at = 14; + google.protobuf.Timestamp executed_at = 15; + // Empty for historical or non-runtime proposals; never inferred at approval. + string runtime_version = 16; +} diff --git a/cmd/ongrid/bootstrap_extensions.go b/cmd/ongrid/bootstrap_extensions.go index 481bc1eba..971c479ed 100644 --- a/cmd/ongrid/bootstrap_extensions.go +++ b/cmd/ongrid/bootstrap_extensions.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" @@ -138,6 +139,7 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. approvalUC := managerbizapproval.NewUsecase(managerapprovaldata.NewRepo(storage.db), log.With(slog.String("comp", "approval"))) runtime.humanApprovalBroker.SetUsecase(approvalUC) approvalUC.RegisterExecutor(managersvcaiops.GenericToolApprovalKind, runtime.humanApprovalBroker.Execute) + managersvcaiops.RegisterRuntimeApprovalExecutor(approvalUC, managersvcaiops.GenericToolApprovalKind, contract.V3, runtime.humanApprovalBroker.Execute) approvalHandler := managerserverapproval.NewHandler(approvalUC) // HLD-017 cloud_bash producer: register the execute-on-approve executor // (resolve the bound credential → inject into the Runner sandbox → run) diff --git a/docs/rfc/RFC-009-agent-runtime-versions.md b/docs/rfc/RFC-009-agent-runtime-versions.md index a8eea78b4..01e02c337 100644 --- a/docs/rfc/RFC-009-agent-runtime-versions.md +++ b/docs/rfc/RFC-009-agent-runtime-versions.md @@ -103,8 +103,20 @@ Worker 查询、继续、停止按实例实际归属分派，覆盖内部 review 生产路由器读取会话/父会话的持久化版本，显式 context 与记录冲突时拒绝执行；未装配或未知版本不自动降级。v3 入口也校验版本，防止内部直接调用绕过路由。缺失的父会话在路由入口明确失败，避免把有归属的请求变成无父任务；无父会话的系统任务继续使用默认版本。 -历史空值暂不回填、不推断。当前仍只有 v3，空值沿用现有执行方式；这不是完成历史迁移，也不能据此开放默认版本切换。正式发布历史会话与试验环境历史会话的来源分类、旧版本适配、审批来源版本及独立可变目录仍是开放切换前的必需项。 +历史空值暂不回填、不推断。当前仍只有 v3，空值沿用现有执行方式；这不是完成历史迁移，也不能据此开放默认版本切换。正式发布历史会话与试验环境历史会话的来源分类、旧版本适配及独立可变目录仍是开放切换前的必需项；审批来源版本已在后续阶段接入，见下节。 验证：SQLite 模拟缺列升级并连续迁移两次，历史空值保持不变；新记录重读、改标题/模型后版本仍保留。路由回归覆盖固定版本优先、context 冲突、未知/未装配版本、父任务继承和缺失会话。真实 Manager + 临时 MySQL 的聊天、RCA 和工作流 E2E 通过（35.118s；模型为测试替身）。 回滚：恢复前一代码/镜像，保留新增可空列即可，不需要删除列或还原聊天数据。旧代码不会执行版本约束；在多版本能力开放后不能把这种镜像回滚视为等价版本切换。本阶段未部署或推送。 + +## 审批来源版本与执行归属（2026-10-17） + +`approvals.runtime_version` 记录提案产生时的执行版本。cloud_bash、host_bash、MCP、安装 Skill 和通用工具审批均从原始执行 context 写入；审批 HTTP 响应及前端 API 类型回显该字段，批准请求不能改写归属。补充审批实体 proto 契约，不改变现有接口路径。 + +审批用例按 `(kind, runtime_version)` 精确选择执行器，不读取当前默认版本，也不回退到其他版本或无版本执行器。Service 装配恢复来源版本到执行 context。执行器注册表使用读写锁，执行期间不持锁。通用工具的内存单次能力另带版本校验，拒绝跨版本执行和重复使用；重启后能力丢失仍明确失败，不自动重建、重放或恢复。 + +当前仅注册 v3 的版本化执行器。现有无版本 Graph 审批/非 Runtime 用例继续通过独立空版本键执行，旧数据不回填；已退役的 legacy K8s 执行器没有恢复。v1/v2 尚未装配，标记这些版本的提案会在缺少执行器时明确 failed，不能借通用执行器绕过。 + +验证：持久化后重建审批用例，原版本执行器仍被选中；原版本缺失时其他执行器零调用；并发注册/批准、重复批准及单次能力的版本匹配/拒绝通过 race 测试。MCP 批准请求携带不同版本 context 时仍恢复提案来源。SQLite 缺列升级与重复迁移保留旧记录状态及空版本。真实 Manager + 临时 MySQL 聊天和抓包审批拒绝 E2E 通过（25.958s，模型为替身），审批 API 回显 v3。 + +回滚限制：新增列可保留，不做破坏性迁移；但旧代码不识别审批版本约束。多版本启用后，回滚到无版本路由的旧镜像前必须先停止接收新提案并处理待审批记录，不能声称仅换镜像就保留版本隔离。此阶段未部署、推送或合并。 diff --git a/internal/manager/biz/approval/usecase.go b/internal/manager/biz/approval/usecase.go index 87a8da8e6..9b93ba104 100644 --- a/internal/manager/biz/approval/usecase.go +++ b/internal/manager/biz/approval/usecase.go @@ -13,6 +13,7 @@ import ( "fmt" "log/slog" "strings" + "sync" "time" model "github.com/ongridio/ongrid/internal/manager/model/approval" @@ -36,11 +37,14 @@ type Repo interface { // approval records never imply that an unavailable action was executed. type Executor func(ctx context.Context, payloadJSON string) (resultJSON string, err error) +type executorKey struct{ kind, runtimeVersion string } + // Usecase is the inbox facade. type Usecase struct { - repo Repo - log *slog.Logger - executors map[string]Executor + executorMu sync.RWMutex + repo Repo + log *slog.Logger + executors map[executorKey]Executor } // NewUsecase wires the repo. @@ -48,24 +52,33 @@ func NewUsecase(repo Repo, log *slog.Logger) *Usecase { if log == nil { log = slog.Default() } - return &Usecase{repo: repo, log: log, executors: map[string]Executor{}} + return &Usecase{repo: repo, log: log, executors: map[executorKey]Executor{}} } // RegisterExecutor wires the execute-on-approve handler for a Kind. Called // at boot by the producer subsystem (e.g. cloud-shell). Idempotent. func (u *Usecase) RegisterExecutor(kind string, fn Executor) { - u.executors[kind] = fn + u.RegisterRuntimeExecutor(kind, "", fn) +} + +// RegisterRuntimeExecutor registers an exact (kind, runtime) execution route. +// Versioned proposals never fall back to an unversioned or newer executor. +func (u *Usecase) RegisterRuntimeExecutor(kind, runtimeVersion string, fn Executor) { + u.executorMu.Lock() + defer u.executorMu.Unlock() + u.executors[executorKey{kind: kind, runtimeVersion: runtimeVersion}] = fn } // ProposeInput is what a producer queues. type ProposeInput struct { - Kind string - Title string - Summary string - Payload any // marshaled to PayloadJSON - Source string // SourceAgent / SourceFlow - SessionID string - ProposedBy uint64 + RuntimeVersion string + Kind string + Title string + Summary string + Payload any // marshaled to PayloadJSON + Source string // SourceAgent / SourceFlow + SessionID string + ProposedBy uint64 } // Propose records a pending action. Producer-facing (not admin-gated — the @@ -74,6 +87,9 @@ func (u *Usecase) Propose(ctx context.Context, in ProposeInput) (*model.Approval if strings.TrimSpace(in.Kind) == "" || strings.TrimSpace(in.Title) == "" { return nil, fmt.Errorf("%w: kind + title required", errs.ErrInvalid) } + if len(in.RuntimeVersion) > 16 { + return nil, fmt.Errorf("%w: runtime version too long", errs.ErrInvalid) + } payload, err := json.Marshal(in.Payload) if err != nil { return nil, err @@ -83,7 +99,8 @@ func (u *Usecase) Propose(ctx context.Context, in ProposeInput) (*model.Approval src = model.SourceAgent } a := &model.Approval{ - Kind: in.Kind, Title: in.Title, Summary: in.Summary, + RuntimeVersion: in.RuntimeVersion, + Kind: in.Kind, Title: in.Title, Summary: in.Summary, PayloadJSON: string(payload), Source: src, SessionID: in.SessionID, Status: model.StatusPending, ProposedBy: in.ProposedBy, } @@ -120,11 +137,13 @@ func (u *Usecase) Approve(ctx context.Context, approverID uint64, id string) (*m if err != nil { return nil, err } - exec := u.executors[a.Kind] + u.executorMu.RLock() + exec := u.executors[executorKey{kind: a.Kind, runtimeVersion: a.RuntimeVersion}] + u.executorMu.RUnlock() var res string var runErr error if exec == nil { - runErr = fmt.Errorf("approval executor for %q is unavailable; submit a new request", a.Kind) + runErr = fmt.Errorf("approval executor for %q runtime %q is unavailable; submit a new request", a.Kind, a.RuntimeVersion) } else { res, runErr = exec(ctx, a.PayloadJSON) } diff --git a/internal/manager/biz/approval/usecase_test.go b/internal/manager/biz/approval/usecase_test.go index 867bc7ec7..4a498ff29 100644 --- a/internal/manager/biz/approval/usecase_test.go +++ b/internal/manager/biz/approval/usecase_test.go @@ -3,6 +3,7 @@ package approval_test import ( "context" "errors" + "fmt" "path/filepath" "strings" "sync" @@ -147,3 +148,66 @@ func TestInvalidationCannotOverwriteDecisionOrPermitLaterExecution(t *testing.T) }) } } + +func TestApproval_RuntimeOwnershipSurvivesReloadAndNeverFallsBack(t *testing.T) { + for _, available := range []bool{true, false} { + t.Run(fmt.Sprint("origin_available=", available), func(t *testing.T) { + repo := approvalRepo(t) + producer := approval.NewUsecase(repo, nil) + row, err := producer.Propose(context.Background(), approval.ProposeInput{Kind: "action", Title: "pinned", Payload: struct{}{}, RuntimeVersion: "v2"}) + if err != nil { + t.Fatal(err) + } + // Reconstruct the usecase to prove routing is based on the stored record. + consumer := approval.NewUsecase(repo, nil) + var originCalls, otherCalls atomic.Int32 + wrong := func(context.Context, string) (string, error) { otherCalls.Add(1); return `{}`, nil } + consumer.RegisterExecutor("action", wrong) + consumer.RegisterRuntimeExecutor("action", "v3", wrong) + if available { + consumer.RegisterRuntimeExecutor("action", "v2", func(context.Context, string) (string, error) { originCalls.Add(1); return `{"origin":"v2"}`, nil }) + } + result, err := consumer.Approve(context.Background(), 9, row.ID) + if err != nil { + t.Fatal(err) + } + if result.RuntimeVersion != "v2" || otherCalls.Load() != 0 { + t.Fatalf("runtime leaked: %+v, other=%d", result, otherCalls.Load()) + } + if available { + if result.Status != model.StatusExecuted || originCalls.Load() != 1 { + t.Fatalf("origin failed: %+v", result) + } + } else if result.Status != model.StatusFailed { + t.Fatalf("missing origin did not fail: %+v", result) + } + if _, err = consumer.Approve(context.Background(), 9, row.ID); err == nil { + t.Fatal("terminal approval executed again") + } + }) + } +} + +func TestApproval_RuntimeRegistrationConcurrentWithExecution(t *testing.T) { + uc := approval.NewUsecase(approvalRepo(t), nil) + row, err := uc.Propose(context.Background(), approval.ProposeInput{Kind: "action", Title: "concurrent", Payload: struct{}{}, RuntimeVersion: "v3"}) + if err != nil { + t.Fatal(err) + } + var calls atomic.Int32 + execute := func(context.Context, string) (string, error) { calls.Add(1); return `{}`, nil } + uc.RegisterRuntimeExecutor("action", "v3", execute) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 100; i++ { + uc.RegisterRuntimeExecutor("action", "v3", execute) + } + }() + result, err := uc.Approve(context.Background(), 9, row.ID) + wg.Wait() + if err != nil || result.Status != model.StatusExecuted || calls.Load() != 1 { + t.Fatalf("execution=%+v %v calls=%d", result, err, calls.Load()) + } +} diff --git a/internal/manager/data/approval/store/runtime_version_test.go b/internal/manager/data/approval/store/runtime_version_test.go new file mode 100644 index 000000000..9ff5cb3ea --- /dev/null +++ b/internal/manager/data/approval/store/runtime_version_test.go @@ -0,0 +1,55 @@ +package store + +import ( + "context" + "path/filepath" + "testing" + + "github.com/glebarez/sqlite" + model "github.com/ongridio/ongrid/internal/manager/model/approval" + "gorm.io/gorm" +) + +func TestRuntimeVersionMigrationKeepsHistoricalApprovalsUnclassified(t *testing.T) { + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "approvals.db")), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := sqlDB.Close(); err != nil { + t.Error(err) + } + }) + if err := Migrate(db); err != nil { + t.Fatal(err) + } + repo := NewRepo(db) + old := &model.Approval{Kind: "action", Title: "historical", PayloadJSON: "{}"} + if err := repo.Create(context.Background(), old); err != nil { + t.Fatal(err) + } + if err := db.Migrator().DropColumn(&model.Approval{}, "RuntimeVersion"); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err := Migrate(db); err != nil { + t.Fatal(err) + } + } + got, err := repo.Get(context.Background(), old.ID) + if err != nil || got.RuntimeVersion != "" || got.Status != model.StatusPending { + t.Fatalf("historical row changed: %+v %v", got, err) + } + pinned := &model.Approval{Kind: "action", Title: "new", PayloadJSON: "{}", RuntimeVersion: "v3"} + if err := repo.Create(context.Background(), pinned); err != nil { + t.Fatal(err) + } + got, err = repo.Get(context.Background(), pinned.ID) + if err != nil || got.RuntimeVersion != "v3" { + t.Fatalf("version not durable: %+v %v", got, err) + } +} diff --git a/internal/manager/model/approval/model.go b/internal/manager/model/approval/model.go index 68795bf09..21797cd08 100644 --- a/internal/manager/model/approval/model.go +++ b/internal/manager/model/approval/model.go @@ -17,7 +17,9 @@ import ( // Approval is one pending/decided dangerous action. type Approval struct { - ID string `gorm:"primaryKey;type:char(36);column:id" json:"id"` + // RuntimeVersion is immutable execution ownership, independent of the current default. + RuntimeVersion string `gorm:"size:16;column:runtime_version" json:"runtime_version,omitempty"` + ID string `gorm:"primaryKey;type:char(36);column:id" json:"id"` // Kind routes execution on approve (biz registers an executor per kind), // e.g. "shell_command", "restart_service". Also the UI category. diff --git a/internal/manager/service/aiops/approval_adapters.go b/internal/manager/service/aiops/approval_adapters.go index 1e29a6487..a89115de0 100644 --- a/internal/manager/service/aiops/approval_adapters.go +++ b/internal/manager/service/aiops/approval_adapters.go @@ -4,15 +4,16 @@ import ( "context" "encoding/json" "fmt" + "os" + "path/filepath" + "strings" + "time" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" managerbizapproval "github.com/ongridio/ongrid/internal/manager/biz/approval" managerbizmcp "github.com/ongridio/ongrid/internal/manager/biz/mcp" managerbizsecret "github.com/ongridio/ongrid/internal/manager/biz/secret" - "os" - "path/filepath" - "strings" - "time" ) // cloudBashPayload is the approval payload for a queued cloud_bash command. @@ -135,13 +136,14 @@ func (s CloudBashProposer) ProposeAndAwait(ctx context.Context, command string, title = title[:100] + "…" } a, err := s.Usecase.Propose(ctx, managerbizapproval.ProposeInput{ - Kind: "cloud_bash", - Title: title, - Summary: strings.Join(credentials, ", "), // plain names; card shows them - Payload: cloudBashPayload{Command: command, Credentials: credentials, SessionID: sessionID}, - Source: "agent", - SessionID: sessionID, - ProposedBy: userID, + RuntimeVersion: string(aiopschatruntime.VersionFromContext(ctx)), + Kind: "cloud_bash", + Title: title, + Summary: strings.Join(credentials, ", "), // plain names; card shows them + Payload: cloudBashPayload{Command: command, Credentials: credentials, SessionID: sessionID}, + Source: "agent", + SessionID: sessionID, + ProposedBy: userID, }) if err != nil { return "", err @@ -179,13 +181,14 @@ func (s HostBashProposer) ProposeAndAwait(ctx context.Context, deviceIDs []uint6 title = title[:100] + "…" } a, err := s.Usecase.Propose(ctx, managerbizapproval.ProposeInput{ - Kind: "host_bash", - Title: title, - Summary: fmt.Sprintf("device_ids=%v", deviceIDs), - Payload: hostBashPayload{DeviceIDs: deviceIDs, Command: command, TimeoutSeconds: timeoutSeconds}, - Source: "agent", - SessionID: sessionID, - ProposedBy: userID, + RuntimeVersion: string(aiopschatruntime.VersionFromContext(ctx)), + Kind: "host_bash", + Title: title, + Summary: fmt.Sprintf("device_ids=%v", deviceIDs), + Payload: hostBashPayload{DeviceIDs: deviceIDs, Command: command, TimeoutSeconds: timeoutSeconds}, + Source: "agent", + SessionID: sessionID, + ProposedBy: userID, }) if err != nil { return "", err @@ -265,13 +268,14 @@ func (s InstallSkillProposer) ProposeInstall(ctx context.Context, url, sourceTyp title = title[:120] + "…" } a, err := s.Usecase.Propose(ctx, managerbizapproval.ProposeInput{ - Kind: "install_skill", - Title: title, - Summary: sourceType, - Payload: installSkillPayload{URL: url, Type: sourceType, Ref: ref, UserID: userID}, - Source: "agent", - SessionID: sessionID, - ProposedBy: userID, + RuntimeVersion: string(aiopschatruntime.VersionFromContext(ctx)), + Kind: "install_skill", + Title: title, + Summary: sourceType, + Payload: installSkillPayload{URL: url, Type: sourceType, Ref: ref, UserID: userID}, + Source: "agent", + SessionID: sessionID, + ProposedBy: userID, }) if err != nil { return "", err @@ -310,12 +314,13 @@ func (s MCPProposer) ProposeMCPCallAndAwait(ctx context.Context, server, tool st cmd = cmd[:200] + "…" } a, err := s.Usecase.Propose(ctx, managerbizapproval.ProposeInput{ - Kind: "mcp_call", - Title: cmd, - Payload: mcpCallPayload{Server: server, Tool: tool, Arguments: args, Command: cmd}, - Source: "agent", - SessionID: sessionID, - ProposedBy: userID, + RuntimeVersion: string(aiopschatruntime.VersionFromContext(ctx)), + Kind: "mcp_call", + Title: cmd, + Payload: mcpCallPayload{Server: server, Tool: tool, Arguments: args, Command: cmd}, + Source: "agent", + SessionID: sessionID, + ProposedBy: userID, }) if err != nil { return "", err diff --git a/internal/manager/service/aiops/approval_executors.go b/internal/manager/service/aiops/approval_executors.go index e6175cb8e..ee15c1327 100644 --- a/internal/manager/service/aiops/approval_executors.go +++ b/internal/manager/service/aiops/approval_executors.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" managerbizapproval "github.com/ongridio/ongrid/internal/manager/biz/approval" managerbizdevice "github.com/ongridio/ongrid/internal/manager/biz/device" @@ -35,8 +36,25 @@ type ApprovalExecutors struct { Log *slog.Logger } +// RegisterRuntimeApprovalExecutor restores the recorded origin at execution. +// It never reads the administrator's current default runtime. +func RegisterRuntimeApprovalExecutor(uc *managerbizapproval.Usecase, kind string, version contract.Version, execute managerbizapproval.Executor) { + if execute == nil { + uc.RegisterRuntimeExecutor(kind, string(version), nil) + return + } + uc.RegisterRuntimeExecutor(kind, string(version), func(ctx context.Context, payload string) (string, error) { + return execute(contract.WithVersion(ctx, version), payload) + }) +} + func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { - approvalUC.RegisterExecutor("cloud_bash", func(ctx context.Context, payloadJSON string) (string, error) { + register := func(kind string, execute managerbizapproval.Executor) { + // Preserve the existing unversioned route for historical/non-runtime proposals. + approvalUC.RegisterExecutor(kind, execute) + RegisterRuntimeApprovalExecutor(approvalUC, kind, contract.V3, execute) + } + register("cloud_bash", func(ctx context.Context, payloadJSON string) (string, error) { var p cloudBashPayload if err := json.Unmarshal([]byte(payloadJSON), &p); err != nil { return "", err @@ -105,7 +123,7 @@ func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { }) return string(out), nil }) - approvalUC.RegisterExecutor("host_bash", func(ctx context.Context, payloadJSON string) (string, error) { + register("host_bash", func(ctx context.Context, payloadJSON string) (string, error) { var p hostBashPayload if err := json.Unmarshal([]byte(payloadJSON), &p); err != nil { return "", err @@ -115,7 +133,7 @@ func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { }) // HLD-018 P2: mcp_call executor — on approve, connect the server and run // the tool. Trusted servers skip this and run synchronously in the tool. - approvalUC.RegisterExecutor("mcp_call", func(ctx context.Context, payloadJSON string) (string, error) { + register("mcp_call", func(ctx context.Context, payloadJSON string) (string, error) { var p mcpCallPayload if err := json.Unmarshal([]byte(payloadJSON), &p); err != nil { return "", err @@ -133,7 +151,7 @@ func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { // Class=destructive — a skill can ship a binary cloud_bash later runs, so // this only runs after a human approves. The approval IS the authorization, // so the install runs with admin authority; installed_by = proposing user. - approvalUC.RegisterExecutor("install_skill", func(ctx context.Context, payloadJSON string) (string, error) { + register("install_skill", func(ctx context.Context, payloadJSON string) (string, error) { var p installSkillPayload if err := json.Unmarshal([]byte(payloadJSON), &p); err != nil { return "", err diff --git a/internal/manager/service/aiops/human_approval.go b/internal/manager/service/aiops/human_approval.go index 1e3ff690d..b198c5611 100644 --- a/internal/manager/service/aiops/human_approval.go +++ b/internal/manager/service/aiops/human_approval.go @@ -27,14 +27,19 @@ type genericAgentToolApprovalPayload struct { // an in-memory, single-use executor that closes over the exact wrapped tool, // arguments and request context. A manager restart loses that capability and // therefore fails closed; it can never execute an unapproved reconstruction. +type pendingToolExecution struct { + version aiopschatruntime.Version + execute aiopstoolsdec.HumanApprovalExecutor +} + type HumanApprovalBroker struct { mu sync.Mutex uc *managerbizapproval.Usecase - pending map[string]aiopstoolsdec.HumanApprovalExecutor + pending map[string]pendingToolExecution } func NewHumanApprovalBroker() *HumanApprovalBroker { - return &HumanApprovalBroker{pending: make(map[string]aiopstoolsdec.HumanApprovalExecutor)} + return &HumanApprovalBroker{pending: make(map[string]pendingToolExecution)} } func (b *HumanApprovalBroker) SetUsecase(uc *managerbizapproval.Usecase) { @@ -51,7 +56,7 @@ func (b *HumanApprovalBroker) ProposeAndAwait(ctx context.Context, req aiopstool b.mu.Lock() uc := b.uc if uc != nil { - b.pending[token] = execute + b.pending[token] = pendingToolExecution{version: aiopschatruntime.VersionFromContext(ctx), execute: execute} } b.mu.Unlock() if uc == nil { @@ -63,13 +68,14 @@ func (b *HumanApprovalBroker) ProposeAndAwait(ctx context.Context, req aiopstool b.mu.Unlock() } a, err := uc.Propose(ctx, managerbizapproval.ProposeInput{ - Kind: GenericToolApprovalKind, - Title: truncateApprovalTitle(req.ToolName + " confirmation"), - Summary: req.Summary, - Payload: genericAgentToolApprovalPayload{ExecutionToken: token, ToolName: req.ToolName, ArgsJSON: req.ArgsJSON, Summary: req.Summary}, - Source: "agent", - SessionID: req.SessionID, - ProposedBy: req.UserID, + RuntimeVersion: string(aiopschatruntime.VersionFromContext(ctx)), + Kind: GenericToolApprovalKind, + Title: truncateApprovalTitle(req.ToolName + " confirmation"), + Summary: req.Summary, + Payload: genericAgentToolApprovalPayload{ExecutionToken: token, ToolName: req.ToolName, ArgsJSON: req.ArgsJSON, Summary: req.Summary}, + Source: "agent", + SessionID: req.SessionID, + ProposedBy: req.UserID, }) if err != nil { cleanup() @@ -132,13 +138,16 @@ func (b *HumanApprovalBroker) Execute(ctx context.Context, payloadJSON string) ( return "", fmt.Errorf("decode generic tool approval: %w", err) } b.mu.Lock() - execute := b.pending[payload.ExecutionToken] + pending := b.pending[payload.ExecutionToken] delete(b.pending, payload.ExecutionToken) b.mu.Unlock() - if execute == nil { + if pending.execute == nil { return "", fmt.Errorf("approved tool execution is no longer available; retry the request") } - return execute(ctx) + if pending.version != aiopschatruntime.VersionFromContext(ctx) { + return "", fmt.Errorf("approved tool runtime mismatch; submit a new request") + } + return pending.execute(ctx) } func truncateApprovalTitle(title string) string { diff --git a/internal/manager/service/aiops/human_approval_test.go b/internal/manager/service/aiops/human_approval_test.go index 35259fa5d..367f8b529 100644 --- a/internal/manager/service/aiops/human_approval_test.go +++ b/internal/manager/service/aiops/human_approval_test.go @@ -5,16 +5,18 @@ import ( "sync" "sync/atomic" "testing" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" ) func TestHumanApprovalExecutorIsSingleUseAndFailsClosedAfterRestart(t *testing.T) { const payload = `{"execution_token":"approved-token","tool_name":"test"}` broker := NewHumanApprovalBroker() var calls, successes atomic.Int32 - broker.pending["approved-token"] = func(context.Context) (string, error) { + broker.pending["approved-token"] = pendingToolExecution{execute: func(context.Context) (string, error) { calls.Add(1) return `{"status":"executed"}`, nil - } + }} var wg sync.WaitGroup for i := 0; i < 8; i++ { wg.Add(1) @@ -33,3 +35,34 @@ func TestHumanApprovalExecutorIsSingleUseAndFailsClosedAfterRestart(t *testing.T t.Fatal("a new broker reconstructed a lost execution capability") } } + +func TestHumanApprovalExecutorRejectsWrongRuntime(t *testing.T) { + broker := NewHumanApprovalBroker() + calls := 0 + broker.pending["pinned"] = pendingToolExecution{version: contract.V3, execute: func(context.Context) (string, error) { calls++; return `{}`, nil }} + _, err := broker.Execute(contract.WithVersion(context.Background(), contract.V2), `{"execution_token":"pinned"}`) + if err == nil || calls != 0 { + t.Fatalf("cross-runtime executor ran: calls=%d err=%v", calls, err) + } + if _, err := broker.Execute(contract.WithVersion(context.Background(), contract.V3), `{"execution_token":"pinned"}`); err == nil { + t.Fatal("rejected capability reused") + } +} + +func TestHumanApprovalExecutorAcceptsOriginRuntimeOnce(t *testing.T) { + broker := NewHumanApprovalBroker() + broker.pending["pinned"] = pendingToolExecution{version: contract.V3, execute: func(ctx context.Context) (string, error) { + if contract.VersionFromContext(ctx) != contract.V3 { + t.Fatal("origin lost") + } + return `{"ok":true}`, nil + }} + ctx := contract.WithVersion(context.Background(), contract.V3) + out, err := broker.Execute(ctx, `{"execution_token":"pinned"}`) + if err != nil || out != `{"ok":true}` { + t.Fatalf("execution=%q %v", out, err) + } + if _, err := broker.Execute(ctx, `{"execution_token":"pinned"}`); err == nil { + t.Fatal("capability reused") + } +} diff --git a/internal/manager/service/aiops/mcp_approval_runtime_test.go b/internal/manager/service/aiops/mcp_approval_runtime_test.go index 6eba4e4b5..b143193a4 100644 --- a/internal/manager/service/aiops/mcp_approval_runtime_test.go +++ b/internal/manager/service/aiops/mcp_approval_runtime_test.go @@ -75,7 +75,7 @@ func TestMCPApprovalRuntimeResumesWithResult(t *testing.T) { t.Fatal(err) } uc := approval.NewUsecase(approvalstore.NewRepo(db), nil) - uc.RegisterExecutor("mcp_call", func(context.Context, string) (string, error) { return `{"clusters":["test"]}`, nil }) + uc.RegisterRuntimeExecutor("mcp_call", "v3", func(context.Context, string) (string, error) { return `{"clusters":["test"]}`, nil }) repo := chatstore.NewSessionRepoWithAttachmentRoot(db, t.TempDir()) sess := &chatmodel.Session{ID: "session-445", UserID: 7} if err := repo.CreateSession(context.Background(), sess); err != nil { diff --git a/internal/manager/service/aiops/mcp_approval_test.go b/internal/manager/service/aiops/mcp_approval_test.go index fb4ef46a2..a21d88b1a 100644 --- a/internal/manager/service/aiops/mcp_approval_test.go +++ b/internal/manager/service/aiops/mcp_approval_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/glebarez/sqlite" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" "github.com/ongridio/ongrid/internal/manager/biz/approval" "github.com/ongridio/ongrid/internal/manager/data/approval/store" "gorm.io/gorm" @@ -39,7 +40,10 @@ func TestMCPApprovalLifecycle(t *testing.T) { } uc := approval.NewUsecase(store.NewRepo(db), nil) var executions atomic.Int32 - uc.RegisterExecutor("mcp_call", func(_ context.Context, payload string) (string, error) { + RegisterRuntimeApprovalExecutor(uc, "mcp_call", contract.V3, func(execCtx context.Context, payload string) (string, error) { + if contract.VersionFromContext(execCtx) != contract.V3 { + return "", errors.New("approval execution lost origin") + } executions.Add(1) var p mcpCallPayload if err := json.Unmarshal([]byte(payload), &p); err != nil { @@ -55,6 +59,7 @@ func TestMCPApprovalLifecycle(t *testing.T) { }) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + ctx = contract.WithVersion(ctx, contract.V3) type outcome struct { result string err error @@ -71,7 +76,7 @@ func TestMCPApprovalLifecycle(t *testing.T) { t.Fatal(err) } if len(rows) > 0 { - if rows[0].SessionID != "session-445" || rows[0].ProposedBy != 7 { + if rows[0].RuntimeVersion != "v3" || rows[0].SessionID != "session-445" || rows[0].ProposedBy != 7 { t.Fatalf("proposal lost session/user: %+v", rows[0]) } id = rows[0].ID @@ -101,10 +106,10 @@ func TestMCPApprovalLifecycle(t *testing.T) { cancel() want = `"status":"cancelled"` default: - if _, err := uc.Approve(ctx, 7, id); err != nil { + if _, err := uc.Approve(contract.WithVersion(ctx, contract.V2), 7, id); err != nil { t.Fatal(err) } - if _, err := uc.Approve(ctx, 7, id); err == nil { + if _, err := uc.Approve(contract.WithVersion(ctx, contract.V2), 7, id); err == nil { t.Fatal("duplicate approval accepted") } if decision == "failure" { diff --git a/tests/e2e/assistant_chat_test.go b/tests/e2e/assistant_chat_test.go index 8efaec914..b11be4c78 100644 --- a/tests/e2e/assistant_chat_test.go +++ b/tests/e2e/assistant_chat_test.go @@ -1219,7 +1219,11 @@ func TestAssistantChat_CaptureRequiresHumanApproval(t *testing.T) { if proposal.ApprovalID == "" || proposal.ToolName != "capture_pcap" { t.Fatalf("unexpected approval: %s", line) } - status, _, err := env.DoJSON("POST", "/api/v1/approvals/"+proposal.ApprovalID+"/reject", map[string]string{"reason": "E2E denies capture"}, admin.AccessToken) + status, row, err := env.DoJSON("GET", "/api/v1/approvals/"+proposal.ApprovalID, nil, admin.AccessToken) + if err != nil || status != http.StatusOK || row["runtime_version"] != "v3" { + t.Fatalf("approval origin lost: status=%d row=%v err=%v", status, row, err) + } + status, _, err = env.DoJSON("POST", "/api/v1/approvals/"+proposal.ApprovalID+"/reject", map[string]string{"reason": "E2E denies capture"}, admin.AccessToken) if err != nil || status != http.StatusOK { t.Fatalf("reject status=%d err=%v", status, err) } diff --git a/web/src/api/approvals.ts b/web/src/api/approvals.ts index 219cd682c..05451d97e 100644 --- a/web/src/api/approvals.ts +++ b/web/src/api/approvals.ts @@ -5,6 +5,7 @@ import { request } from './client'; // here for a human to approve (→ executes) or reject. Admin-only. export interface Approval { + runtime_version?: string; id: string; kind: string; title: string; From b8551807c34a5c060a4c4d300cbeae9d2ecb6134 Mon Sep 17 00:00:00 2001 From: singchia Date: Sat, 17 Oct 2026 12:42:42 +0800 Subject: [PATCH 23/24] feat(aiops): complete v3 dynamic capabilities and reliable search Add runtime selection, on-demand skill snapshots and MCP lifecycle refresh with approval revision checks. Validate search providers atomically and preserve explicit fallback and execution budgets. Verify full isolated backend E2E with race and 16 browser settings scenarios. Record test deployment, rollback and known container full-suite failures. --- api/manager/setting/v1/setting.proto | 31 +++ cmd/ongrid/bootstrap_extensions.go | 13 +- cmd/ongrid/bootstrap_runtime.go | 8 +- cmd/ongrid/main.go | 2 +- deploy/searxng/README.md | 11 + deploy/searxng/settings.cn.yml | 13 + .../PRD-010-reliable-web-search.md | 39 +++ docs/rfc/RFC-009-agent-runtime-versions.md | 22 ++ docs/test/v3-phase123-20261017.md | 43 ++++ .../aiops/runtime/v3/chatruntime/runtime.go | 22 +- .../runtime/v3/chatruntime/skill_assets.go | 217 ++++++++++++++++ .../runtime/v3/chatruntime/skill_catalog.go | 205 ++++++++++++++++ .../v3/chatruntime/skill_catalog_test.go | 124 ++++++++++ .../runtime/v3/chatruntime/skill_registry.go | 12 +- .../aiops/runtime/v3/chatruntime/worker.go | 32 ++- .../biz/aiops/tools/basetool/skill_context.go | 24 ++ internal/manager/biz/aiops/tools/toolbag.go | 1 + internal/manager/biz/mcp/lifecycle_test.go | 147 +++++++++++ internal/manager/biz/mcp/usecase.go | 165 +++++++++++-- internal/manager/biz/setting/agent.go | 26 ++ internal/manager/biz/setting/probe.go | 168 ++++++++++--- internal/manager/biz/setting/service.go | 2 + internal/manager/biz/setting/service_test.go | 42 ++++ internal/manager/biz/setting/websearch.go | 57 +++++ .../biz/setting/websearch_policy_test.go | 96 ++++++++ internal/manager/model/setting/model.go | 14 +- internal/manager/server/integration/http.go | 30 +++ .../manager/server/integration/http_test.go | 26 ++ .../service/aiops/approval_adapters.go | 27 +- .../service/aiops/approval_executors.go | 58 ++++- internal/manager/service/aiops/mcp_refresh.go | 10 +- internal/manager/service/aiops/mcp_watch.go | 130 ++++++++++ .../manager/service/aiops/runtime_router.go | 44 +++- .../service/aiops/runtime_router_test.go | 43 ++++ internal/pkg/mcpclient/client.go | 71 ++++-- internal/pkg/mcpclient/notifications.go | 93 +++++++ internal/pkg/mcpclient/notifications_test.go | 85 +++++++ internal/skill/builtin/web_search.go | 85 ++++--- internal/skill/builtin/web_search_policy.go | 231 ++++++++++++++++++ .../skill/builtin/web_search_policy_test.go | 159 ++++++++++++ web/e2e/agent-runtime-settings.spec.ts | 49 ++++ web/e2e/web-search-settings.spec.ts | 67 +++++ web/src/api/settings.ts | 8 + web/src/pages/SettingsLayout.tsx | 2 +- web/src/pages/settings/Agent.test.tsx | 37 +++ web/src/pages/settings/Agent.tsx | 70 ++++++ web/src/pages/settings/Integrations.tsx | 50 +++- 47 files changed, 2745 insertions(+), 166 deletions(-) create mode 100644 deploy/searxng/README.md create mode 100644 deploy/searxng/settings.cn.yml create mode 100644 docs/requirements/PRD-010-reliable-web-search.md create mode 100644 docs/test/v3-phase123-20261017.md create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/skill_assets.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/skill_catalog.go create mode 100644 internal/manager/biz/aiops/runtime/v3/chatruntime/skill_catalog_test.go create mode 100644 internal/manager/biz/aiops/tools/basetool/skill_context.go create mode 100644 internal/manager/biz/mcp/lifecycle_test.go create mode 100644 internal/manager/biz/setting/websearch_policy_test.go create mode 100644 internal/manager/service/aiops/mcp_watch.go create mode 100644 internal/pkg/mcpclient/notifications.go create mode 100644 internal/pkg/mcpclient/notifications_test.go create mode 100644 internal/skill/builtin/web_search_policy.go create mode 100644 internal/skill/builtin/web_search_policy_test.go create mode 100644 web/e2e/agent-runtime-settings.spec.ts create mode 100644 web/e2e/web-search-settings.spec.ts diff --git a/api/manager/setting/v1/setting.proto b/api/manager/setting/v1/setting.proto index debc8b672..f13cb5c50 100644 --- a/api/manager/setting/v1/setting.proto +++ b/api/manager/setting/v1/setting.proto @@ -5,6 +5,10 @@ package ongrid.manager.setting.v1; option go_package = "github.com/ongridio/ongrid/api/gen/manager/setting/v1;settingv1"; // SettingService 管理管理员可编辑的运行时配置。 +// agent.runtime_version accepts "release" (follow this release default) or +// "v3" (pin new sessions to v3). v1/v2 are not wired and are rejected. +// Existing sessions retain their runtime; legacy unversioned sessions stay v3. +// Read/write via the existing system-settings category/key API. // Legacy platform.auto_apm_enabled values are retained but no longer affect discovery. service SettingService { // ApplyBuiltInObservabilityLimits saves storage limits and recreates only @@ -28,6 +32,9 @@ service SettingService { // ValidateWebSearchConfiguration 使用未落库的 provider 配置执行最小查询。 // HTTP 实现：POST /api/v1/integrations/websearch/test。 rpc ValidateWebSearchConfiguration(ValidateWebSearchConfigurationRequest) returns (ValidateWebSearchConfigurationResponse); + // POST /api/v1/integrations/websearch/validate-and-save; admin only. + // Validates primary and backup, then atomically saves the complete draft. + rpc ValidateAndSaveWebSearchConfiguration(ValidateAndSaveWebSearchConfigurationRequest) returns (ValidateAndSaveWebSearchConfigurationResponse); // ValidateLLMConfiguration 使用未落库的草稿配置发起最小模型请求， // 返回可机器识别的失败原因。HTTP 实现： // POST /api/v1/integrations/llm/test。 @@ -96,11 +103,18 @@ message ValidateTempoConfigurationResponse { string status = 1 [json_name = "status"]; } +// websearch settings also accept fallback_provider (empty disables fallback), +// searxng_engines (comma-separated names), request_timeout_seconds (2..60), +// and total_timeout_seconds (5..120). Fallback requires explicit admin selection. message ValidateWebSearchConfigurationRequest { string provider = 1 [json_name = "provider"]; string searxng_url = 2 [json_name = "searxng_url"]; string tavily_api_key = 3 [json_name = "tavily_api_key"]; string brave_api_key = 4 [json_name = "brave_api_key"]; + string fallback_provider = 5 [json_name = "fallback_provider"]; + string searxng_engines = 6 [json_name = "searxng_engines"]; + string request_timeout_seconds = 7 [json_name = "request_timeout_seconds"]; + string total_timeout_seconds = 8 [json_name = "total_timeout_seconds"]; } message ValidateWebSearchConfigurationResponse { @@ -109,6 +123,23 @@ message ValidateWebSearchConfigurationResponse { string sample = 3 [json_name = "sample"]; } +message ValidateAndSaveWebSearchConfigurationRequest { + string provider = 1 [json_name = "provider"]; + string searxng_url = 2 [json_name = "searxng_url"]; + string tavily_api_key = 3 [json_name = "tavily_api_key"]; + string brave_api_key = 4 [json_name = "brave_api_key"]; + string fallback_provider = 5 [json_name = "fallback_provider"]; + string searxng_engines = 6 [json_name = "searxng_engines"]; + string request_timeout_seconds = 7 [json_name = "request_timeout_seconds"]; + string total_timeout_seconds = 8 [json_name = "total_timeout_seconds"]; +} + +message ValidateAndSaveWebSearchConfigurationResponse { + string status = 1 [json_name = "status"]; + string provider = 2 [json_name = "provider"]; + string sample = 3 [json_name = "sample"]; +} + message ValidateLLMConfigurationRequest { // provider 是稳定提供商 ID：openai / anthropic / zhipu / gemini / // deepseek / kimi / custom。 diff --git a/cmd/ongrid/bootstrap_extensions.go b/cmd/ongrid/bootstrap_extensions.go index 971c479ed..a22dcfb34 100644 --- a/cmd/ongrid/bootstrap_extensions.go +++ b/cmd/ongrid/bootstrap_extensions.go @@ -69,10 +69,12 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. // still works for List/Install but the hot-reload is a no-op until // the next chatruntime construction picks the disk state up. var mpSkillReg managerbizmarketplace.SkillRegistry + var skillAssetLeaser managersvcaiops.SkillAssetLeaser var mpAgentReg managerbizmarketplace.AgentRegistry if rt := runtime.chatRT; rt != nil { if sk := rt.SkillRegistry(); sk != nil { mpSkillReg = sk + skillAssetLeaser = sk } if ag := rt.AgentRegistry(); ag != nil { mpAgentReg = ag @@ -159,6 +161,7 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. Runner: cloudBashRunner, Workspace: wsMgr, SkillsRoot: marketplaceSkillsRoot, + SkillAssets: skillAssetLeaser, Frontier: tunnel.fbClient, Edges: devices.edgeUC, Devices: devices.deviceUC, @@ -261,7 +264,7 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. HumanApproval: runtime.humanApprovalBroker, } mcpCaller := managersvcaiops.MCPCaller{Usecase: mcpUC} - mcpProposer := managersvcaiops.MCPProposer{Usecase: approvalUC} + mcpProposer := managersvcaiops.MCPProposer{Usecase: approvalUC, MCP: mcpUC} // Keep the graph-facing decorators and ToolSearch's unredacted catalog // in sync. Registration changes call this with one server's fresh test // result, so an admin save never reconnects every MCP endpoint. @@ -271,6 +274,14 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. // Chat path: load enabled servers at boot; later changes use the same // per-server refresh hook. The FLOW path uses LIVE managersvcflow.MCPSource. mcpTools.LoadEnabled(rootCtx, mcpUC) + go func() { + defer func() { + if recovered := recover(); recovered != nil { + log.Error("MCP refresh supervisor panic", slog.Any("panic", recovered)) + } + }() + mcpTools.RunRefresh(rootCtx, mcpUC, time.Minute) + }() } if secretbox.KeyIsWeak() { diff --git a/cmd/ongrid/bootstrap_runtime.go b/cmd/ongrid/bootstrap_runtime.go index cbc64afc6..b01188c0d 100644 --- a/cmd/ongrid/bootstrap_runtime.go +++ b/cmd/ongrid/bootstrap_runtime.go @@ -26,7 +26,7 @@ type runtimeModule struct { aiopsHandler *managerserveraiops.Handler } -func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Logger, reg *prometheus.Registry, tracing *tracingModule, models *modelsModule, devices *devicesModule, telemetry *telemetryModule, tunnel *tunnelModule, aiops *aiopsModule) *runtimeModule { +func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Logger, reg *prometheus.Registry, tracing *tracingModule, models *modelsModule, devices *devicesModule, telemetry *telemetryModule, tunnel *tunnelModule, aiops *aiopsModule, settings *settingsModule) *runtimeModule { // AgentRegistry + SkillRegistry are loaded UNCONDITIONALLY from // ./agents + configured skill roots + the marketplace root. These are // YAML+markdown metadata, so the /v1/agents @@ -52,7 +52,11 @@ func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Log if rterr != nil { log.Error("aiops runtime unavailable", slog.Any("err", rterr)) } else { - router, err := managersvcaiops.NewRuntimeRouter(aiops.aiopsRepo, contract.V3, map[contract.Version]managersvcaiops.RuntimeEngine{contract.V3: rt}) + router, err := managersvcaiops.NewRuntimeRouter(aiops.aiopsRepo, contract.V3, map[contract.Version]managersvcaiops.RuntimeEngine{contract.V3: rt}, + managersvcaiops.WithRuntimeDefaultProvider(func(ctx context.Context) (contract.Version, error) { + version, err := settings.settingSvc.AgentRuntimeVersion(ctx) + return contract.Version(version), err + })) if err != nil { log.Error("aiops runtime routing unavailable", slog.Any("err", err)) } else { diff --git a/cmd/ongrid/main.go b/cmd/ongrid/main.go index f96862d0f..8dc19d194 100644 --- a/cmd/ongrid/main.go +++ b/cmd/ongrid/main.go @@ -85,7 +85,7 @@ func run() (runErr error) { } aiops := bootstrapAIOps(cfg, log, notifyRouter, storage, settings, models, devices, alerts, telemetry, tunnel) knowledge := bootstrapKnowledge(rootCtx, cfg, log, storage, settings, telemetry, aiops) - runtime := bootstrapRuntime(rootCtx, cfg, log, reg, tracing, models, devices, telemetry, tunnel, aiops) + runtime := bootstrapRuntime(rootCtx, cfg, log, reg, tracing, models, devices, telemetry, tunnel, aiops, settings) imbridge := bootstrapIMBridge(rootCtx, log, storage, models, devices, alerts, telemetry, aiops, runtime) automation := bootstrapAutomation(rootCtx, cfg, log, reg, notifyRouter, &cleanup, storage, settings, models, alerts, telemetry, aiops, runtime) system := bootstrapSystem(cfg, storage, iam, settings, models, telemetry, aiops, knowledge, automation) diff --git a/deploy/searxng/README.md b/deploy/searxng/README.md new file mode 100644 index 000000000..34f96cd84 --- /dev/null +++ b/deploy/searxng/README.md @@ -0,0 +1,11 @@ +# 搜索引擎运维 + +先从 Manager 所在环境执行实际 `/search?format=json&q=Kubernetes` 查询，确认至少一个带标题的 HTTP(S) 结果。HTTP 200 或首页可访问不足以证明搜索可用。同时检查 `unresponsive_engines`；timeout、CAPTCHA 和空结果是不同问题。 + +默认 settings.yml 保留通用部署行为。中国网络环境可参考 settings.cn.yml，将其 `use_default_settings` 和 `engines` 两段合并到部署配置，保留原 server/security 配置。不要用该片段直接覆盖整个配置。它显式限定 360search 和 sogou，不再继承其它默认引擎。抓取源可能随时触发验证码；2026-10-17 测试环境 360search 返回官方文档，搜狗部分请求 CAPTCHA，并非持续可用性保证。 + +修改前备份实际挂载文件；只重启 SearXNG 容器。修改后分别验证直接查询、设置页实际搜索测试、临时 Agent 会话及其返回链接。失败则恢复备份并重启。不要直接改用户会话或修改历史工具结果。 + +管理员在「设置 → 集成 → 联网搜索」选择主备来源和 SearXNG 引擎。备用默认关闭；确认允许发送查询内容并配置必要凭证后启用。正式 API 来源必须在该部署环境实际验证，不能仅依赖供应商名称或 API key 存在。 + +单来源预算和总预算不同于 SearXNG 内部引擎预算；工具层延长等待不会解除上游 CAPTCHA、封禁或网络不可达。连续失败后的冷却为单 Manager 进程内状态，下次查询触发恢复探测，不进行后台自动重放。设置页测试直接探测来源，不以缓存健康状态代替真实查询。 diff --git a/deploy/searxng/settings.cn.yml b/deploy/searxng/settings.cn.yml new file mode 100644 index 000000000..431a4c467 --- /dev/null +++ b/deploy/searxng/settings.cn.yml @@ -0,0 +1,13 @@ +# Optional profile for environments where the global engines are unreachable. +# Overlay these keys onto settings.yml; retain server/security configuration. +# Availability must be verified from the actual deployment before use. +use_default_settings: + engines: + keep_only: + - 360search + - sogou +engines: + - name: 360search + disabled: false + - name: sogou + disabled: false diff --git a/docs/requirements/PRD-010-reliable-web-search.md b/docs/requirements/PRD-010-reliable-web-search.md new file mode 100644 index 000000000..488645c6a --- /dev/null +++ b/docs/requirements/PRD-010-reliable-web-search.md @@ -0,0 +1,39 @@ +# PRD-010 联网搜索可靠性 + +状态：已确认，实施中。2026-10-17 用户确认主备来源、错误区分和真实搜索验证方案。 + +## 问题与目标 + +测试环境询问 VKE 时 Agent 已正常调用 web_search，但 SearXNG 多个引擎超时。工具丢弃 unresponsive_engines，只返回空结果。Bing 返回 HTTP 200 的空搜索页；百度或搜狗可能触发验证码。仅提高 HTTP 超时无法解决这些问题。 + +目标：搜索失败不能伪装为没有相关信息；管理员可以配置经验证的主备来源；来源故障不会无限拖住会话；已有工具能力、调用接口和来源引用保留。 + +## 行为 + +- 保留 SearXNG、Tavily、Brave 三种来源。默认不新增第三方查询；备用来源必须由管理员显式选择，已有 API key 不等于允许备用查询。 +- 搜索结果增加 success、partial、no_results、failed 状态；保留结果、引擎失败和尝试来源。HTTP/解码异常沿用工具错误通道，不转换成正常空结果。返回可用链接时保留结果，不因为部分引擎失败而抛弃。 +- 主来源失败或正常空结果时最多尝试一个配置的备用来源；不自动改写查询或向更多供应商广播。模型根据上下文消歧义，引用返回链接；原文读取和内容真实性验证不能仅靠搜索摘要保证。 +- 连接/TLS 建立预算 5 秒；单来源默认 20 秒、可配 2–60；总预算默认 45 秒、可配 5–120。上层取消优先。SearXNG 引擎自身的请求预算仍由该服务配置管理。 +- 每个进程的来源连续失败 3 次后冷却 1 分钟，下一次需求触发单次恢复探测；没有后台无限重试。成功或正常空结果复位失败计数，来源地址/凭证变化重新探测。进程重启清除健康状态。 +- 设置页配置主备、SearXNG 引擎、预算；展示最近一次手动测试成功样例或失败原因。测试查询为 Kubernetes，必须返回有标题的 HTTP(S) 链接。分别验证主备后原子保存完整配置；失败不写入。 +- 去除重复链接、拒绝非 HTTP(S) 或缺少标题的结果，最多返回请求数量。不虚构来源或保证稀有查询一定有结果。 + +## 验收 + +覆盖全部引擎失败、部分成功、真实空结果、去重、验证码反馈、备用显式授权、请求超时切换、上层取消、冷却恢复、配置变化复位、无密钥备用拒绝保存、管理员权限、失败不部分写入。真实测试环境通过新建临时 Agent 会话验证搜索和官方来源引用，清理验证会话。 + +浏览器验收覆盖 light/dark、中英文、390/768/1280/1920 宽度；检查新增字段、备用密钥展开、保存和失败展示。 + +## 边界与后续 + +免费抓取引擎无法保证长期可用。正式 API 主备仍需管理员提供凭证并从部署环境验收；本次不购买服务、不创建第三方账号。当前健康状态为进程内故障冷却与手动查询探测，不是跨实例持久化健康看板或定时巡检。来源正确性与原文核验另需读取网页能力，本次不把摘要当作原文验证。 + +## 回滚 + +代码回退可保留新增 system_settings 行，旧代码忽略新策略；关闭备用恢复单来源行为。测试机 SearXNG 配置修改前备份到 `/opt/ongrid/searxng/settings.yml.before-search-recovery-20261017112118`；恢复该文件为 settings.yml 并重启 ongrid-searxng 即可回滚，但会重新出现原引擎不可用的问题。Manager、Edge、前端镜像不随配置恢复而自动更新。 + +## 本阶段验证记录 + +2026-10-17：测试环境仅更新 SearXNG 配置并重启该容器，真实模型新会话两次搜索成功且返回官方链接；临时会话已关闭。新的策略与设置页仍在本地分支，未推送、未合并、未部署 Manager/前端。该环境尚未配置 Tavily/Brave API key，因此未声称完成真实第三方主备验收。 + +相关 Go 包（builtin、setting、integration、cmd/ongrid）race 测试、前端 14 项设置回归、8 项 Chrome 深浅主题与四宽度交互检查通过；proto 生成、源码快照架构检查通过。新增 Go 测试覆盖管理员保存权限、主备整组校验和失败不写入。 diff --git a/docs/rfc/RFC-009-agent-runtime-versions.md b/docs/rfc/RFC-009-agent-runtime-versions.md index 01e02c337..1b4b6c9a9 100644 --- a/docs/rfc/RFC-009-agent-runtime-versions.md +++ b/docs/rfc/RFC-009-agent-runtime-versions.md @@ -120,3 +120,25 @@ Worker 查询、继续、停止按实例实际归属分派，覆盖内部 review 验证：持久化后重建审批用例，原版本执行器仍被选中；原版本缺失时其他执行器零调用；并发注册/批准、重复批准及单次能力的版本匹配/拒绝通过 race 测试。MCP 批准请求携带不同版本 context 时仍恢复提案来源。SQLite 缺列升级与重复迁移保留旧记录状态及空版本。真实 Manager + 临时 MySQL 聊天和抓包审批拒绝 E2E 通过（25.958s，模型为替身），审批 API 回显 v3。 回滚限制：新增列可保留，不做破坏性迁移；但旧代码不识别审批版本约束。多版本启用后，回滚到无版本路由的旧镜像前必须先停止接收新提案并处理待审批记录，不能声称仅换镜像就保留版本隔离。此阶段未部署、推送或合并。 + + +### 管理员运行时选择入口（2026-10-17，本地实现） + +「设置 → 助理」第二张配置卡增加运行时下拉框。`agent.runtime_version` 通过既有系统设置 API 保存：`release` 跟随发行版默认，`v3` 固定当前已装配引擎。当前两者均执行 v3；v1/v2 仅显示为禁用选项，后端单项及批量写入也拒绝这些值。本阶段不代表完成多版本切换。 + +新会话和无父会话的系统任务读取持久化选择；已绑定会话与子任务沿用其归属，未分类历史会话固定使用当前装配默认（v3），不查询新会话设置。设置读取失败或数据库存在不支持的值时，新会话失败而不是静默切换引擎。历史来源分类与 v1/v2 适配、独立目录和审批执行器仍须完成，之后才能开放对应选项。 + +回滚：恢复前一代码即可，新增设置行可以保留（旧版不读取）；也可以删除 `agent/runtime_version` 恢复发行版默认。没有本阶段新增 schema。尚未推送或部署。 + +### v3 动态能力收口（2026-10-17，未推送、未合并） + +保持 Eino Graph、Worker、Approval/Operation 既有状态流转；不将 `TurnPlan` 的阶段枚举误当成完整持久化执行状态机，也不在本次重写 runtime。 + +- Skill：每轮提供名称/说明目录和 `load_skill`；新增显式 `on_demand`，保留默认/`always` 和 `keyword` 自动加载。Worker 使用自己的任务解析 Skill，并按实际加载的 Skill 重新解析凭据绑定，不继承协调者的整组凭据。 +- `load_skill` 输出版本、资源指纹、资源目录和正文，进入现有工具消息持久化链路；自动加载的版本写入 runtime span。资源采用回合副本，包含所属插件的相对脚本/资源；审批执行另外取得引用，聊天取消不能删除已开始执行的命令正在使用的资源。最后一个引用释放后清理临时副本。 +- 卸载会撤销后续资源获取；正常 registry 热更新保留已有任务副本。当前 marketplace 没有原地升级 API，重复安装返回冲突；卸载再装属于撤销和新安装，不视为平滑升级。重启后的旧内存副本不可恢复，相关未完成审批必须重新发起，不能悄悄换用当前版本。 +- MCP：手动连接测试、启动发现和后台发现走同一发布路径；并发探测按开始顺序确定最新结果，修改/禁用/删除使旧探测失效。临时网络失败保留最后验证的工具；后台刷新不替换内容未变的执行句柄。 +- MCP 新审批保存配置/凭据/工具目录指纹，执行前核对，禁止旧批准跳转到新端点或新凭据。禁用服务在执行入口直接拒绝。v3 中不带配置指纹的旧待审批操作要求重新发起；非 runtime 的历史路由仍检查启用状态。 +- MCP 按 [`tools/list` 分页与变更通知](https://modelcontextprotocol.io/specification/2025-03-26/server/tools)发现全部工具；支持 [`Streamable HTTP GET SSE`](https://modelcontextprotocol.io/specification/2025-03-26/basic/transports) 的服务可主动通知刷新，每分钟发现作为无通知服务及断线后的兜底。每个服务一条顺序刷新链路，关闭时取消并等待退出。 + +回滚：测试部署保留旧 Manager/前端镜像、Edge 二进制、Compose 配置和数据库备份。新增 Skill 加载模式只用于明确配置的 Skill；旧 runtime 不识别 `on_demand`，回滚前应还原这类新配置。临时资源副本不是持久化发行版目录，不承诺跨进程续跑。测试部署与现场验证结果单独记录，不以本地测试通过代替现场验收。 diff --git a/docs/test/v3-phase123-20261017.md b/docs/test/v3-phase123-20261017.md new file mode 100644 index 000000000..6edcd8e54 --- /dev/null +++ b/docs/test/v3-phase123-20261017.md @@ -0,0 +1,43 @@ +# v3 第 1–3 项测试记录（2026-10-17） + +范围：测试环境收口、Skill 按需加载与资源保留、MCP 动态生命周期。保留现有 Eino Graph、Worker、审批及 Operation 状态机；第 4 项内部拆分未实施。分支 `refactor/unified-agent-runtime`；按用户要求提交并推送，保持草稿 PR，未合并。 + +## 实现与边界 + +- 主助理与 Worker 都有本轮 Skill 目录和 `load_skill`。保留默认/`always`、`keyword`，新增 `on_demand`。Worker 按自身加载情况解析凭据；加载说明不扩大工具权限。 +- 资源副本保留所属插件的公共脚本与相对路径。回合和正在执行的审批各持有引用，最后一个引用释放后清理；卸载撤销后续获取。凭据删除/绑定撤销在审批执行时检查。 +- MCP 支持工具分页、GET SSE 的列表变更通知及每分钟兜底发现；旧探测不能覆盖更新的探测或配置。临时连接失败保留验证过的工具目录，未变化的后台刷新保留执行句柄。 +- 新 v3 MCP 审批固定配置、凭据和工具目录指纹，执行时再检查；旧 v3 无指纹审批需要重新发起。服务禁用后拒绝执行。 +- 现有市场安装接口没有原地升级；卸载重装属于撤销。资源副本不跨 Manager 重启续跑。v1/v2 仍未接入执行，设置下拉框禁用这两个选项，避免误导。 + +## 验证 + +- Linux `make test-agent-runtime`（race）通过。 +- 聊天/MCP/Skill/Flow/RCA/附件相关 E2E（race）通过。 +- 新增并发覆盖：MCP 探测期间禁用/删除/替换/更新探测、审批期间端点/凭据/权限/Schema 变化、工具分页及通知取消。 +- 新增 Skill 覆盖：按需/兼容激活、卸载拒绝加载、旧资源保留、执行中引用保留、最终清理、嵌套 Skill 的插件公共脚本。 +- 相关 Go 包 race 检查通过；源码快照依赖分层检查通过；`git diff --check` 通过。 +- 设置页 19 项单元测试、16 项 Chrome UI 测试通过。覆盖 390/768/1280/1920、light/dark、中英文。前端构建通过；已实看截图。 + +## 测试环境 + +目标：`https://101.34.63.91/`。 + +- Manager：`ongrid:v0.18.2-v3-test.20261017.3`。 +- 前端：`ongrid-web:v0.18.2-v3-test.20261017.2`。 +- 本机 Edge：`v0.18.2-v3-test.20261017.2`。 +- 数据库、原 Compose/nginx 配置、Edge 二进制及回滚脚本：`/opt/ongrid/test-deploy-20261017-v3-phase123/`。该目录含敏感备份，不复制进仓库。 +- 回滚入口：在测试机执行该目录的 `rollback.sh`，恢复原 Manager/前端和 Edge。新增数据库列保持兼容；数据库备份仅供故障恢复，不在应用回滚时覆盖新数据。 +- 首次部署脚本误用了 `/api/v1/healthz`，触发自动回滚；纠正为实际的 `/healthz`、`/readyz` 后部署成功。不是服务健康故障。 +- 现场新会话记录为 `v3`；真实 web_search 成功返回 Kubernetes 官方链接；现有 MCP `approval_demo` 状态正常；两台设备在线。 +- 临时安装 `v3-smoke-20261017`，实际 `load_skill` 返回版本 `1.0.0`、revision、资源目录和正文，工具消息持久化成功。随后卸载测试包并清理临时聊天；不改用户原聊天。 +- 本机真实 `host_bash` 执行 `/var` 一级目录 `du` 成功，约 14.7 秒；最终 `.3` 包执行根目录一级扫描成功，约 8.7 秒。实际扫描未超过 15 秒，不能据此声称现场覆盖了超 15 秒执行；超时策略另有代码与回归验证。 +- 现场 Chrome 检查：设置第二张卡片显示运行时下拉框，v1/v2 禁用；原聊天页面可打开，无页面脚本错误。 + +完整现场结果保存在上述部署目录的 `smoke-result.json`、`du-smoke-result.json`、`long-scan-smoke-result.json`；浏览器结果保存在本机 `/private/tmp/ongrid-v3-live-ui-result.json`。日志读取的 5 分钟预算和审批取消等路径由回归覆盖，不声称已在现场耗时运行至全部预算上限。 + +## 提交前 E2E 复验 + +- 全量后端：`go test -race -tags=e2e -count=1 -timeout=20m -json ./tests/e2e/...` 通过；26 个顶层测试，含子测试共 83 项通过，无失败或跳过，耗时约 76.6 秒。使用临时 MySQL、真实 Manager 和模拟外部服务，不代表全部真实供应商联调。 +- Chrome：`agent-runtime-settings.spec.ts` 和 `web-search-settings.spec.ts` 共 16 项通过，耗时 42.2 秒。 +- 补跑 Linux 全仓 `go test ./...`：复现此前记录的容器 AutoAPM worker 资源探测、worktree Git 元数据解析、不可访问文件错误断言三处失败，未将全仓检查标为通过；最终结果同时记录在 PR Validation。 diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go index 2813b1575..b1c832728 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/runtime.go @@ -443,10 +443,16 @@ func (rt *Runtime) Handle(ctx context.Context, req *Request) (result *Reply, ret // 5. Resolve active skills + compose the system prompt. policy := Policy{AllowedClasses: []string{"*"}} - var activeSkills []*Skill - if rt.cfg.SkillRegistry != nil { - activeSkills = rt.cfg.SkillRegistry.Resolve(req.UserText, policy) + activeSkills, skillLoader, skillCatalog := rt.cfg.SkillRegistry.catalog(req.UserText, policy) + if skillLoader != nil { + defer skillLoader.close() + for _, sk := range activeSkills { + if _, err := skillLoader.load(ctx, sk); err != nil { + return nil, err + } + } } + // 5·HLD-017: attach the active skills' design-time bound credentials to // ctx so cloud_bash queues them for injection at approve time. The // binding was chosen at install time; nothing is decided at run time. @@ -468,6 +474,9 @@ func (rt *Runtime) Handle(ctx context.Context, req *Request) (result *Reply, ret // to the global default with a log line — never crash a session. basePrompt := rt.cfg.BasePrompt runtimeToolBag := basetool.FilterWebSearch(ctx, rt.toolBagSnapshot()) + if skillLoader != nil { + runtimeToolBag = append(runtimeToolBag, skillLoader) + } sessionToolBag := runtimeToolBag agentReminderForPersona := "" // Coordinator iff session is pinned to "default" or has no @@ -544,7 +553,11 @@ func (rt *Runtime) Handle(ctx context.Context, req *Request) (result *Reply, ret if digest := buildToolCapabilityDigest(sessionToolBag); digest != "" { basePrompt = basePrompt + "\n\n" + digest } - systemPrompt := ComposeSystemPrompt(basePrompt, activeSkills, nil) + if skillLoader != nil && toolBagContains(sessionToolBag, "load_skill") { + basePrompt += "\n\n" + skillCatalog + } + systemPrompt := ComposeSystemPrompt(basePrompt, activeSkills, nil) + skillLoader.activePrompt() + span.SetAttributes(attribute.String("ongrid.skills.active", skillLoader.activePrompt())) // 6. Build the eino history slice. Convert persisted rows into // schema.Message so the graph assembler can replay them. @@ -585,6 +598,7 @@ func (rt *Runtime) Handle(ctx context.Context, req *Request) (result *Reply, ret handlers := callbacks.NewDefaultHandlers(deps) toolPersistence := callbacks.EnableSynchronousToolPersistence(handlers) graphCfg.ToolPersistence = toolPersistence + sessionToolBag = skillLoader.bind(sessionToolBag, rt.cfg.CredentialBinder) g, err := graph.BuildReActGraph(rt.cfg.ChatModel, sessionToolBag, graphCfg) if err != nil { return nil, fmt.Errorf("chatruntime: build graph: %w", err) diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_assets.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_assets.go new file mode 100644 index 000000000..cc59ed635 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_assets.go @@ -0,0 +1,217 @@ +package chatruntime + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/google/uuid" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +type skillAssetLease struct { + ref basetool.SkillAssetRef + source, root string + users int + revoked bool +} + +// snapshotSkill preserves scripts and relative references for the duration of +// a turn and any already-running approved commands. No source file is edited. +func (r *SkillRegistry) snapshotSkill(ctx context.Context, sk *Skill) (basetool.SkillAssetRef, error) { + if sk.Dir == "" { + return r.registerSkillAsset(sk, "", basetool.SkillAssetRef{ID: uuid.NewString(), Name: sk.Name, Version: sk.Version, Revision: skillRevision(sk)}) + } + source, err := filepath.EvalSymlinks(sk.Dir) + if err != nil { + return basetool.SkillAssetRef{}, fmt.Errorf("resolve skill assets: %w", err) + } + // Include the containing plugin so ../scripts and shared bin/ keep working. + root := source + for dir := source; dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + kind, _, _ := DetectContainer(dir) + if kind == ContainerClaude || kind == ContainerOpenclaw { + root = dir + break + } + } + dst, err := os.MkdirTemp("", "ongrid-skill-") + if err != nil { + return basetool.SkillAssetRef{}, fmt.Errorf("create skill snapshot: %w", err) + } + cleanup := func() { + if err := os.RemoveAll(dst); err != nil { /* private temp snapshot; caller receives the original error */ + } + } + digest := sha256.New() + var total int64 + err = filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == ".git" { + return filepath.SkipDir + } + return os.MkdirAll(filepath.Join(dst, rel), 0o700) + } + info, err := os.Stat(path) + if err != nil { + return err + } + actual, err := filepath.EvalSymlinks(path) + if err != nil { + return err + } + if !pathHasPrefix(actual, root) { + return fmt.Errorf("skill asset escapes package: %s", rel) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("unsupported skill asset: %s", rel) + } + total += info.Size() + if total > 256<<20 { + return fmt.Errorf("skill package exceeds 256 MiB snapshot budget") + } + src, err := os.Open(actual) + if err != nil { + return err + } + defer src.Close() + out, err := os.OpenFile(filepath.Join(dst, rel), os.O_CREATE|os.O_EXCL|os.O_WRONLY, info.Mode().Perm()&0o555) + if err != nil { + return err + } + fmt.Fprintf(digest, "%s\x00%d\x00", rel, info.Mode().Perm()) + _, copyErr := io.Copy(io.MultiWriter(out, digest), io.LimitReader(src, (256<<20)+1)) + closeErr := out.Close() + if copyErr != nil { + return copyErr + } + return closeErr + }) + if err != nil { + cleanup() + return basetool.SkillAssetRef{}, fmt.Errorf("snapshot skill %s: %w", sk.Name, err) + } + rel, err := filepath.Rel(root, source) + if err == nil { + manifest := filepath.Join(dst, rel, "SKILL.md") + if _, statErr := os.Stat(manifest); statErr == nil { + captured, _, parseErr := ParseSkillMd(manifest) + if parseErr != nil || skillRevision(captured) != skillRevision(sk) { + cleanup() + return basetool.SkillAssetRef{}, fmt.Errorf("skill %q changed while loading; retry the turn", sk.Name) + } + } + } + if err != nil { + cleanup() + return basetool.SkillAssetRef{}, err + } + ref := basetool.SkillAssetRef{ID: uuid.NewString(), Name: sk.Name, Version: sk.Version, Revision: hex.EncodeToString(digest.Sum(nil)), Directory: filepath.Join(dst, rel), PackageDirectory: dst} + registered, err := r.registerSkillAsset(sk, dst, ref) + if err != nil { + cleanup() + } + return registered, err +} + +func (r *SkillRegistry) registerSkillAsset(sk *Skill, root string, ref basetool.SkillAssetRef) (basetool.SkillAssetRef, error) { + r.mu.Lock() + defer r.mu.Unlock() + present := false + for _, current := range r.skills { + if current.Name == sk.Name && current.Dir == sk.Dir { + present = true + break + } + } + if !present { + return basetool.SkillAssetRef{}, fmt.Errorf("skill %q was removed during snapshot", sk.Name) + } + if r.assets == nil { + r.assets = make(map[string]*skillAssetLease) + } + r.assets[ref.ID] = &skillAssetLease{ref: ref, source: sk.Dir, root: root, users: 1} + return ref, nil +} + +// AcquireSkillAssets validates origin references after approval and holds them +// until command completion, including when the initiating chat is cancelled. +func (r *SkillRegistry) AcquireSkillAssets(ctx context.Context, refs []basetool.SkillAssetRef) (func(), error) { + if err := ctx.Err(); err != nil { + return nil, err + } + r.mu.Lock() + for _, ref := range refs { + if ref.ID == "" { + continue + } + lease := r.assets[ref.ID] + if lease == nil || lease.revoked || lease.ref != ref { + r.mu.Unlock() + return nil, fmt.Errorf("skill %q snapshot expired or was revoked; load it again", ref.Name) + } + } + for _, ref := range refs { + if ref.ID != "" { + r.assets[ref.ID].users++ + } + } + r.mu.Unlock() + var once sync.Once + return func() { once.Do(func() { r.releaseSkillAssets(refs) }) }, nil +} + +func (r *SkillRegistry) releaseSkillAssets(refs []basetool.SkillAssetRef) { + var roots []string + r.mu.Lock() + for _, ref := range refs { + if lease := r.assets[ref.ID]; lease != nil { + lease.users-- + if lease.users == 0 { + if lease.root != "" { + roots = append(roots, lease.root) + } + delete(r.assets, ref.ID) + } + } + } + r.mu.Unlock() + for _, root := range roots { + if err := os.RemoveAll(root); err != nil { /* temp cleanup cannot change a completed task's result */ + } + } +} + +// Called under registry lock after replacement. A normal upgrade keeps the +// same registered name/path; removing it permanently revokes existing leases. +func (r *SkillRegistry) revokeRemovedAssets() { + for _, lease := range r.assets { + found := false + for _, sk := range r.skills { + if sk.Name == lease.ref.Name && strings.TrimSpace(sk.Dir) == lease.source { + found = true + break + } + } + if !found { + lease.revoked = true + } + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_catalog.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_catalog.go new file mode 100644 index 000000000..120db1755 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_catalog.go @@ -0,0 +1,205 @@ +package chatruntime + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// skillCatalogTool captures a turn's instructions. Reload changes the next +// turn's catalog, never the body already advertised to the current graph. +type skillCatalogTool struct { + registry *SkillRegistry + skills []*Skill + mu sync.Mutex + loaded map[string]basetool.SkillAssetRef +} + +func skillRevision(s *Skill) string { + return fmt.Sprintf("%x", sha256.Sum256([]byte(s.Name+"\x00"+s.Version+"\x00"+s.PromptBody))) +} + +func (t *skillCatalogTool) Info(context.Context) (*basetool.ToolInfo, error) { + return &basetool.ToolInfo{Name: "load_skill", Class: "read", Origin: basetool.OriginSkill, + Description: "按名称加载本轮 Skill 说明，返回确切版本、内容指纹和资源目录。加载说明不会授予额外工具或凭据权限。", + Parameters: json.RawMessage(`{"type":"object","properties":{"name":{"type":"string"}},"required":["name"],"additionalProperties":false}`)}, nil +} + +func (t *skillCatalogTool) InvokableRun(ctx context.Context, raw string, _ ...basetool.InvokeOption) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + var args struct { + Name string `json:"name"` + } + if err := json.Unmarshal([]byte(raw), &args); err != nil { + return "", fmt.Errorf("decode load_skill: %w", err) + } + for _, sk := range t.skills { + if sk.Name != args.Name { + continue + } + // Uninstall revokes future loads even for a graph built before the change. + present := false + for _, current := range t.registry.All() { + if current.Name == sk.Name && current.Dir == sk.Dir { + present = true + break + } + } + if !present { + return "", fmt.Errorf("skill %q was removed; refresh the task", args.Name) + } + ref, err := t.load(ctx, sk) + if err != nil { + return "", err + } + out, err := json.Marshal(struct { + Name string `json:"name"` + Version string `json:"version"` + Revision string `json:"revision"` + Directory string `json:"directory,omitempty"` + Body string `json:"body"` + }{sk.Name, sk.Version, ref.Revision, ref.Directory, sk.PromptBody}) + if err != nil { + return "", fmt.Errorf("encode skill: %w", err) + } + return string(out), nil + } + return "", fmt.Errorf("skill %q is not in this turn's catalog", args.Name) +} + +func (r *SkillRegistry) catalog(query string, policy Policy) ([]*Skill, *skillCatalogTool, string) { + if r == nil { + return nil, nil, "" + } + all := r.All() + var visible, active []*Skill + var b strings.Builder + b.WriteString("## 可用 Skill\n按任务匹配名称和说明，调用 load_skill 读取详细步骤。Skill 是操作说明，不能扩大本轮工具权限。\n") + for _, sk := range all { + filtered := filterToolsByPolicy(sk.Tools, policy) + if len(sk.Tools) > 0 && len(filtered) == 0 { + continue + } + sk.Tools = filtered + visible = append(visible, sk) + fmt.Fprintf(&b, "- %s: %s", sk.Name, strings.Join(strings.Fields(sk.Description), " ")) + if sk.Version != "" { + fmt.Fprintf(&b, " (version=%s)", sk.Version) + } + b.WriteByte('\n') + if activationMatches(sk.Activation, strings.ToLower(query)) { + active = append(active, sk) + } + } + if len(visible) == 0 { + return nil, nil, "" + } + return active, &skillCatalogTool{registry: r, skills: visible}, b.String() +} + +func toolBagContains(bag []basetool.BaseTool, name string) bool { + for _, t := range bag { + info, err := t.Info(context.Background()) + if err == nil && info != nil && info.Name == name { + return true + } + } + return false +} + +func (t *skillCatalogTool) load(ctx context.Context, sk *Skill) (basetool.SkillAssetRef, error) { + t.mu.Lock() + if ref, ok := t.loaded[sk.Name]; ok { + t.mu.Unlock() + return ref, nil + } + t.mu.Unlock() + ref, err := t.registry.snapshotSkill(ctx, sk) + if err != nil { + return ref, err + } + t.mu.Lock() + if previous, ok := t.loaded[sk.Name]; ok { + t.mu.Unlock() + t.registry.releaseSkillAssets([]basetool.SkillAssetRef{ref}) + return previous, nil + } + if t.loaded == nil { + t.loaded = make(map[string]basetool.SkillAssetRef) + } + t.loaded[sk.Name] = ref + t.mu.Unlock() + return ref, nil +} +func (t *skillCatalogTool) refs() []basetool.SkillAssetRef { + t.mu.Lock() + defer t.mu.Unlock() + refs := make([]basetool.SkillAssetRef, 0, len(t.loaded)) + for _, ref := range t.loaded { + refs = append(refs, ref) + } + sort.Slice(refs, func(i, j int) bool { return refs[i].Name < refs[j].Name }) + return refs +} +func (t *skillCatalogTool) close() { t.registry.releaseSkillAssets(t.refs()) } + +type skillBoundTool struct { + basetool.BaseTool + catalog *skillCatalogTool + binder CredentialBinder +} + +func (t skillBoundTool) InvokableRun(ctx context.Context, raw string, opts ...basetool.InvokeOption) (string, error) { + refs := t.catalog.refs() + info, err := t.BaseTool.Info(ctx) + if err != nil { + return "", err + } + if info != nil && info.Name == "cloud_bash" { + release, err := t.catalog.registry.AcquireSkillAssets(ctx, refs) + if err != nil { + return "", err + } + defer release() + } + names := make([]string, 0, len(refs)) + for _, ref := range refs { + names = append(names, ref.Name) + } + var creds []string + if t.binder != nil { + creds = t.binder.BoundCredentialNamesForSkills(ctx, names) + } + ctx = basetool.WithBoundCredentials(ctx, creds) + ctx = basetool.WithSkillAssets(ctx, refs) + return t.BaseTool.InvokableRun(ctx, raw, opts...) +} +func (t *skillCatalogTool) bind(bag []basetool.BaseTool, binder CredentialBinder) []basetool.BaseTool { + if t == nil { + return bag + } + out := make([]basetool.BaseTool, len(bag)) + for i, tool := range bag { + out[i] = skillBoundTool{BaseTool: tool, catalog: t, binder: binder} + } + return out +} + +func (t *skillCatalogTool) activePrompt() string { + if t == nil { + return "" + } + var b strings.Builder + for _, ref := range t.refs() { + fmt.Fprintf(&b, "\nSkill %s version=%s revision=%s; resource_directory=%s. Use this directory for relative scripts and assets.\n", ref.Name, ref.Version, ref.Revision, ref.Directory) + } + return b.String() +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_catalog_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_catalog_test.go new file mode 100644 index 000000000..32858a9c9 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_catalog_test.go @@ -0,0 +1,124 @@ +package chatruntime + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +func TestSkillCatalogActivationAndSnapshot(t *testing.T) { + reg := NewSkillRegistry() + reg.Add(&Skill{Name: "manual", Description: "manual help", PromptBody: "original", Activation: Activation{Mode: "on_demand"}}) + reg.Add(&Skill{Name: "legacy", Description: "legacy help", PromptBody: "always"}) + active, loader, catalog := reg.catalog("hello", Policy{AllowedClasses: []string{"*"}}) + defer loader.close() + if len(active) != 1 || active[0].Name != "legacy" || !strings.Contains(catalog, "manual help") { + t.Fatalf("active=%v catalog=%s", active, catalog) + } + reg.All()[0].PromptBody = "modified by caller" + out, err := loader.InvokableRun(context.Background(), `{"name":"manual"}`) + if err != nil || !strings.Contains(out, "original") { + t.Fatalf("load=%s err=%v", out, err) + } + if len(loader.refs()) != 1 || loader.refs()[0].Name != "manual" { + t.Fatal("load did not record actual skill") + } + if err := reg.Reload(""); err != nil { + t.Fatal(err) + } + if _, err := loader.InvokableRun(context.Background(), `{"name":"manual"}`); err == nil { + t.Fatal("removed skill still loads") + } +} + +func TestSkillAssetUpgradeRetentionAndUninstall(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "demo") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + manifest := "---\nname: demo\ndescription: demo\nversion: v1\n---\nold instructions" + if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte(manifest), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "script.sh"), []byte("old script"), 0o700); err != nil { + t.Fatal(err) + } + reg := NewSkillRegistry() + if err := reg.Load(root); err != nil { + t.Fatal(err) + } + _, loader, _ := reg.catalog("hello", Policy{AllowedClasses: []string{"*"}}) + sk := reg.All()[0] + ref, err := loader.load(context.Background(), sk) + if err != nil { + t.Fatal(err) + } + release, err := reg.AcquireSkillAssets(context.Background(), []basetool.SkillAssetRef{ref}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "script.sh"), []byte("new script"), 0o700); err != nil { + t.Fatal(err) + } + if err := reg.Reload(root); err != nil { + t.Fatal(err) + } + old, err := os.ReadFile(filepath.Join(ref.Directory, "script.sh")) + if err != nil || string(old) != "old script" { + t.Fatalf("old=%s err=%v", old, err) + } + if err := reg.Reload(""); err != nil { + t.Fatal(err) + } + if _, err := reg.AcquireSkillAssets(context.Background(), []basetool.SkillAssetRef{ref}); err == nil { + t.Fatal("revoked snapshot acquired") + } + loader.close() + if _, err := os.Stat(ref.Directory); err != nil { + t.Fatal("running command lost assets", err) + } + release() + if _, err := os.Stat(ref.Directory); !os.IsNotExist(err) { + t.Fatalf("snapshot leaked: %v", err) + } +} + +func TestSkillSnapshotIncludesSharedPluginAssets(t *testing.T) { + root := t.TempDir() + files := map[string]string{ + "openclaw.plugin.json": `{"id":"shared-assets","name":"shared-assets","version":"1","skills":["skills"]}`, + "skills/demo/SKILL.md": "---\nname: demo\ndescription: demo\n---\nUse ../../bin/shared.sh", + "bin/shared.sh": "echo shared", + } + for name, data := range files { + path := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(data), 0o700); err != nil { + t.Fatal(err) + } + } + reg := NewSkillRegistry() + if err := reg.Load(root); err != nil { + t.Fatal(err) + } + _, loader, _ := reg.catalog("test", Policy{AllowedClasses: []string{"*"}}) + defer loader.close() + ref, err := loader.load(context.Background(), reg.All()[0]) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(ref.Directory, "../../bin/shared.sh")) + if err != nil || string(data) != "echo shared" { + t.Fatalf("shared asset=%s err=%v", data, err) + } + if ref.PackageDirectory == ref.Directory { + t.Fatal("nested skill must retain its package root") + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_registry.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_registry.go index 75618033e..49a1a910c 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_registry.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/skill_registry.go @@ -24,6 +24,7 @@ type SkillRegistry struct { mu sync.RWMutex skills []*Skill warnings []LoadWarning + assets map[string]*skillAssetLease } // NewSkillRegistry returns an empty registry. @@ -52,6 +53,7 @@ func (r *SkillRegistry) Load(skillsRoot string) error { r.mu.Lock() r.skills = append([]*Skill(nil), res.Skills...) r.warnings = append([]LoadWarning(nil), res.Warnings...) + r.revokeRemovedAssets() r.mu.Unlock() return nil } @@ -85,6 +87,7 @@ func (r *SkillRegistry) Reload(skillsRoot string, extras ...string) error { r.mu.Lock() r.skills = newSkills r.warnings = newWarnings + r.revokeRemovedAssets() r.mu.Unlock() return nil } @@ -95,7 +98,12 @@ func (r *SkillRegistry) All() []*Skill { r.mu.RLock() defer r.mu.RUnlock() out := make([]*Skill, len(r.skills)) - copy(out, r.skills) + for i, sk := range r.skills { + clone := *sk + clone.Tools = append([]ToolDecl(nil), sk.Tools...) + clone.Activation.Keywords = append([]string(nil), sk.Activation.Keywords...) + out[i] = &clone + } return out } @@ -181,6 +189,8 @@ func activationMatches(a Activation, queryLower string) bool { switch mode { case "", "always": return true + case "on_demand": + return false case "keyword": for _, kw := range a.Keywords { kw = strings.ToLower(strings.TrimSpace(kw)) diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go index 0208543d9..2c283a671 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/worker.go @@ -462,15 +462,43 @@ func (rt *Runtime) runWorker(ctx context.Context, agentDef *Agent, sessID, userT // Workers are not an authorization bypass. The coordinator stamps the // resolved gate on context; absent wiring fails closed so a direct spawn // cannot expose mutating tools unexpectedly. - workerTools := filterToolsForAgentRole(basetool.FilterWebSearch(ctx, rt.toolBagSnapshot()), agentDef, false, !basetool.AgentWriteAllowedFromContext(ctx)) + workerBag := basetool.FilterWebSearch(ctx, rt.toolBagSnapshot()) + activeSkills, skillLoader, skillCatalog := rt.cfg.SkillRegistry.catalog(userText, Policy{AllowedClasses: []string{"*"}}) + ctx = basetool.WithBoundCredentials(ctx, nil) + ctx = basetool.WithSkillAssets(ctx, nil) + if skillLoader != nil { + defer skillLoader.close() + for _, sk := range activeSkills { + if _, err := skillLoader.load(ctx, sk); err != nil { + return "", err + } + } + } + + if skillLoader != nil { + workerBag = append(workerBag, skillLoader) + } + workerTools := filterToolsForAgentRole(workerBag, agentDef, false, !basetool.AgentWriteAllowedFromContext(ctx)) + if rt.cfg.CredentialBinder != nil && len(activeSkills) > 0 { + names := make([]string, 0, len(activeSkills)) + for _, sk := range activeSkills { + names = append(names, sk.Name) + } + ctx = basetool.WithBoundCredentials(ctx, rt.cfg.CredentialBinder.BoundCredentialNamesForSkills(ctx, names)) + } // Thread the persona-filtered tool view onto ctx so ToolSearch // (which runs inside the worker graph) only returns tools the // worker persona is allowed to see. ctx is per-request (each // worker has its own), so concurrent workers don't race. + workerTools = skillLoader.bind(workerTools, rt.cfg.CredentialBinder) ctx = basetool.WithFilteredTools(ctx, workerTools) - systemPrompt := ComposeSystemPrompt(rt.cfg.BasePrompt, nil, agentDef) + systemPrompt := ComposeSystemPrompt(rt.cfg.BasePrompt, activeSkills, agentDef) + skillLoader.activePrompt() + span.SetAttributes(attribute.String("ongrid.skills.active", skillLoader.activePrompt())) + if skillLoader != nil && toolBagContains(workerTools, "load_skill") { + systemPrompt += "\n\n" + skillCatalog + } if digest := buildToolCapabilityDigest(workerTools); digest != "" { systemPrompt = systemPrompt + "\n\n" + digest } diff --git a/internal/manager/biz/aiops/tools/basetool/skill_context.go b/internal/manager/biz/aiops/tools/basetool/skill_context.go new file mode 100644 index 000000000..8db41a899 --- /dev/null +++ b/internal/manager/biz/aiops/tools/basetool/skill_context.go @@ -0,0 +1,24 @@ +package basetool + +import "context" + +// SkillAssetRef identifies a runtime-owned immutable package snapshot. It never +// grants command execution: cloud_bash still requires the normal approval. +type SkillAssetRef struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version,omitempty"` + Revision string `json:"revision"` + Directory string `json:"directory"` + PackageDirectory string `json:"package_directory,omitempty"` +} + +type skillAssetsKey struct{} + +func WithSkillAssets(ctx context.Context, refs []SkillAssetRef) context.Context { + return context.WithValue(ctx, skillAssetsKey{}, append([]SkillAssetRef(nil), refs...)) +} +func SkillAssetsFromContext(ctx context.Context) []SkillAssetRef { + refs, _ := ctx.Value(skillAssetsKey{}).([]SkillAssetRef) + return append([]SkillAssetRef(nil), refs...) +} diff --git a/internal/manager/biz/aiops/tools/toolbag.go b/internal/manager/biz/aiops/tools/toolbag.go index d61a8d7d2..08b8688d6 100644 --- a/internal/manager/biz/aiops/tools/toolbag.go +++ b/internal/manager/biz/aiops/tools/toolbag.go @@ -55,6 +55,7 @@ import ( // ToolSearch sits in the same flat slice as everything else. var tierByName = map[string]string{ // core (always full schema). + "load_skill": "core", "get_host_load": "core", "get_host_processes": "core", "query_promql": "core", diff --git a/internal/manager/biz/mcp/lifecycle_test.go b/internal/manager/biz/mcp/lifecycle_test.go new file mode 100644 index 000000000..f9d6ea128 --- /dev/null +++ b/internal/manager/biz/mcp/lifecycle_test.go @@ -0,0 +1,147 @@ +package mcp + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + model "github.com/ongridio/ongrid/internal/manager/model/mcp" + "github.com/ongridio/ongrid/internal/pkg/errs" + "github.com/ongridio/ongrid/internal/pkg/mcpclient" +) + +func TestDiscoveryCannotRestoreDisabledOrDeletedServer(t *testing.T) { + for _, action := range []string{"disable", "delete", "replace", "refresh"} { + t.Run(action, func(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + first := true // httptest requests are sequential until the first tools/list blocks. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + ID int `json:"id"` + Method string `json:"method"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + if req.Method == "notifications/initialized" { + w.WriteHeader(http.StatusAccepted) + return + } + result := `{"protocolVersion":"` + mcpclient.ProtocolVersion + `"}` + if req.Method == "tools/list" { + if first { + first = false + close(entered) + <-release + } + result = `{"tools":[{"name":"search","inputSchema":{"type":"object"}}]}` + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": json.RawMessage(result)}); err != nil { + t.Error(err) + } + })) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + uc := NewUsecase(newFakeRepo(), fakeSecrets{}, nil) + row, err := uc.Create(ctx, &model.Server{Name: "docs", Endpoint: server.URL, Enabled: true}) + if err != nil { + t.Fatal(err) + } + published := 0 + uc.SetToolChangeHook(func(_ context.Context, _ string, s *model.Server, _ []mcpclient.Tool) { + if s != nil { + published++ + } + }) + done := make(chan error, 1) + go func() { _, err := uc.TestConnection(ctx, row.ID); done <- err }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + switch action { + case "delete": + err = uc.Delete(ctx, row.ID) + case "refresh": + _, err = uc.TestConnection(ctx, row.ID) + default: + patch := *row + if action == "disable" { + patch.Enabled = false + } else { + patch.Endpoint = server.URL + "/new" + } + err = uc.Update(ctx, row.ID, &patch) + } + close(release) + if err != nil { + t.Fatal(err) + } + if err := <-done; !errors.Is(err, errs.ErrConflict) { + t.Fatalf("stale discovery: %v", err) + } + want := 0 + if action == "refresh" { + want = 1 + } + if published != want { + t.Fatalf("published %d catalogs, want %d", published, want) + } + }) + } +} + +func TestCallToolRejectsDisabledServerBeforeConnecting(t *testing.T) { + uc := NewUsecase(newFakeRepo(), fakeSecrets{}, nil) + if _, err := uc.Create(context.Background(), &model.Server{Name: "disabled", Endpoint: "http://127.0.0.1:1"}); err != nil { + t.Fatal(err) + } + if _, err := uc.CallTool(context.Background(), "disabled", "write", nil); !errors.Is(err, errs.ErrForbidden) { + t.Fatalf("got %v", err) + } +} + +func TestApprovalRevisionRejectsChangedEndpointAndCredential(t *testing.T) { + for _, change := range []string{"endpoint", "credential", "schema", "permission"} { + t.Run(change, func(t *testing.T) { + ctx := context.Background() + repo := newFakeRepo() + secrets := fakeSecrets{fields: map[string]string{"token": "first"}} + uc := NewUsecase(repo, secrets, nil) + row, err := uc.Create(ctx, &model.Server{Name: "test", Endpoint: "http://127.0.0.1:1", Credential: "test", Enabled: true}) + if err != nil { + t.Fatal(err) + } + revision, err := uc.Revision(ctx, row.Name) + if err != nil { + t.Fatal(err) + } + switch change { + case "endpoint": + row.Endpoint += "/replacement" + err = uc.Update(ctx, row.ID, row) + case "permission": + row.Trusted = true + err = uc.Update(ctx, row.ID, row) + case "schema": + err = repo.SetToolsCache(ctx, row.ID, `[{"name":"different"}]`) + case "credential": + secrets.fields["token"] = "rotated" + } + if err != nil { + t.Fatal(err) + } + if _, err := uc.CallToolRevision(ctx, row.Name, "write", nil, revision); !errors.Is(err, errs.ErrConflict) { + t.Fatalf("expected revision conflict before connection, got %v", err) + } + }) + } +} diff --git a/internal/manager/biz/mcp/usecase.go b/internal/manager/biz/mcp/usecase.go index 58793fe81..5fdf1f1ff 100644 --- a/internal/manager/biz/mcp/usecase.go +++ b/internal/manager/biz/mcp/usecase.go @@ -7,6 +7,7 @@ package mcp import ( "context" + "crypto/sha256" "encoding/json" "fmt" "log/slog" @@ -53,6 +54,11 @@ type Usecase struct { secrets SecretResolver log *slog.Logger onToolsChange ToolChangeHook + // commitGate serializes registration changes and discovery publication. + // Remote MCP requests never hold it; waiting respects request cancellation. + commitGate chan struct{} + probeSequence uint64 + latestProbe map[uint64]uint64 } // SetToolChangeHook wires the optional runtime catalog updater. It is set @@ -74,9 +80,20 @@ func NewUsecase(repo Repo, secrets SecretResolver, log *slog.Logger) *Usecase { if log == nil { log = slog.Default() } - return &Usecase{repo: repo, secrets: secrets, log: log} + return &Usecase{repo: repo, secrets: secrets, log: log, commitGate: make(chan struct{}, 1), latestProbe: make(map[uint64]uint64)} } +func (u *Usecase) enterCommit(ctx context.Context) error { + select { + case u.commitGate <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (u *Usecase) leaveCommit() { <-u.commitGate } + // Create validates and stores a new server registration. func (u *Usecase) Create(ctx context.Context, s *model.Server) (*model.Server, error) { if s == nil { @@ -99,6 +116,10 @@ func (u *Usecase) Update(ctx context.Context, id uint64, patch *model.Server) er if err := normalizeAndValidate(patch); err != nil { return err } + if err := u.enterCommit(ctx); err != nil { + return err + } + defer u.leaveCommit() previous, err := u.repo.Get(ctx, id) if err != nil { return err @@ -106,6 +127,8 @@ func (u *Usecase) Update(ctx context.Context, id uint64, patch *model.Server) er if err := u.repo.Update(ctx, id, patch); err != nil { return err } + // Invalidate discoveries started before this change, including disable/re-enable. + delete(u.latestProbe, id) updated, err := u.repo.Get(ctx, id) if err != nil { return err @@ -134,6 +157,10 @@ func (u *Usecase) Update(ctx context.Context, id uint64, patch *model.Server) er // Delete removes a server registration. func (u *Usecase) Delete(ctx context.Context, id uint64) error { + if err := u.enterCommit(ctx); err != nil { + return err + } + defer u.leaveCommit() server, err := u.repo.Get(ctx, id) if err != nil { return err @@ -141,6 +168,7 @@ func (u *Usecase) Delete(ctx context.Context, id uint64) error { if err := u.repo.Delete(ctx, id); err != nil { return err } + delete(u.latestProbe, id) u.notifyToolsChanged(ctx, server.Name, nil, nil) return nil } @@ -174,10 +202,28 @@ func (u *Usecase) ListEnabled(ctx context.Context) ([]*model.Server, error) { // the mcp_call approval executor (post-approval). Errors when the server is // unknown, unreachable, or the tool reports isError. func (u *Usecase) CallTool(ctx context.Context, serverName, toolName string, args map[string]any) (string, error) { + return u.CallToolRevision(ctx, serverName, toolName, args, "") +} + +// CallToolRevision prevents a pending approval from following a replaced endpoint, +// credential or tool schema. Empty revision is reserved for existing direct callers. +func (u *Usecase) CallToolRevision(ctx context.Context, serverName, toolName string, args map[string]any, revision string) (string, error) { s, err := u.repo.GetByName(ctx, serverName) if err != nil { return "", err } + if !s.Enabled { + return "", fmt.Errorf("%w: MCP server is disabled", errs.ErrForbidden) + } + if revision != "" { + current, err := u.serverRevision(ctx, s) + if err != nil { + return "", err + } + if current != revision { + return "", fmt.Errorf("%w: MCP registration changed; request a new approval", errs.ErrConflict) + } + } cli, err := u.BuildClient(ctx, s) if err != nil { return "", err @@ -195,6 +241,44 @@ func (u *Usecase) CallTool(ctx context.Context, serverName, toolName string, arg return res.TextContent(), nil } +// Revision returns an opaque identity without exposing resolved credentials. +func (u *Usecase) Revision(ctx context.Context, name string) (string, error) { + s, err := u.repo.GetByName(ctx, name) + if err != nil { + return "", err + } + if !s.Enabled { + return "", fmt.Errorf("%w: MCP server is disabled", errs.ErrForbidden) + } + return u.serverRevision(ctx, s) +} + +func (u *Usecase) serverRevision(ctx context.Context, s *model.Server) (string, error) { + var fields map[string]string + if s.Credential != "" { + if u.secrets == nil { + return "", fmt.Errorf("MCP credential resolver unavailable") + } + var err error + fields, err = u.secrets.ResolveFields(ctx, s.Credential) + if err != nil { + return "", fmt.Errorf("resolve MCP revision credential: %w", err) + } + } + // Exclude health/status timestamps: a healthy refresh must not invalidate + // an approval whose configuration and schema remain identical. + data, err := json.Marshal(struct { + ID uint64 + Name, Transport, Endpoint, Credential, Headers, Tools string + Enabled, Trusted bool + Fields map[string]string + }{s.ID, s.Name, s.Transport, s.Endpoint, s.Credential, s.HeaderTemplateJSON, s.ToolsCacheJSON, s.Enabled, s.Trusted, fields}) + if err != nil { + return "", fmt.Errorf("encode MCP revision: %w", err) + } + return fmt.Sprintf("%x", sha256.Sum256(data)), nil +} + // BuildClient constructs an MCP client for s, resolving the referenced // credential (if any) and expanding {{field}} placeholders in the header // template into concrete HTTP headers. stdio transport is not supported yet. @@ -234,46 +318,85 @@ func (u *Usecase) BuildClient(ctx context.Context, s *model.Server) (*mcpclient. // outcome. On success it stores the tools snapshot + status "ok"; on failure // it records status "error" with the error message and returns the error. func (u *Usecase) TestConnection(ctx context.Context, id uint64) ([]mcpclient.Tool, error) { + return u.refreshConnection(ctx, id, true) +} + +// RefreshConnection is the background path; unchanged schemas keep their live +// tool handles so polling does not interrupt an otherwise valid running turn. +func (u *Usecase) RefreshConnection(ctx context.Context, id uint64) ([]mcpclient.Tool, error) { + return u.refreshConnection(ctx, id, false) +} +func (u *Usecase) refreshConnection(ctx context.Context, id uint64, forcePublish bool) ([]mcpclient.Tool, error) { + if err := u.enterCommit(ctx); err != nil { + return nil, err + } s, err := u.repo.Get(ctx, id) if err != nil { + u.leaveCommit() return nil, err } - cli, err := u.BuildClient(ctx, s) + startedRevision, err := u.serverRevision(ctx, s) if err != nil { - if setErr := u.repo.SetStatus(ctx, id, "error", err.Error()); setErr != nil { - u.log.Warn("set mcp error status", slog.Uint64("server_id", id), slog.Any("err", setErr)) - } + u.leaveCommit() return nil, err } - if err := cli.Initialize(ctx); err != nil { - if setErr := u.repo.SetStatus(ctx, id, "error", err.Error()); setErr != nil { - u.log.Warn("set mcp error status", slog.Uint64("server_id", id), slog.Any("err", setErr)) - } + u.probeSequence++ + ticket := u.probeSequence + u.latestProbe[id] = ticket + u.leaveCommit() + + // Network discovery is intentionally outside the publication gate. An admin + // can disable/delete a server immediately even when its endpoint is hung. + tools, probeErr := u.discover(ctx, s) + if err := u.enterCommit(ctx); err != nil { return nil, err } - tools, err := cli.ListTools(ctx) + defer u.leaveCommit() + if u.latestProbe[id] != ticket { + return nil, fmt.Errorf("%w: MCP discovery superseded by a configuration change or newer refresh", errs.ErrConflict) + } + delete(u.latestProbe, id) + currentRevision, err := u.serverRevision(ctx, s) if err != nil { - if setErr := u.repo.SetStatus(ctx, id, "error", err.Error()); setErr != nil { - u.log.Warn("set mcp error status", slog.Uint64("server_id", id), slog.Any("err", setErr)) - } return nil, err } - if b, mErr := json.Marshal(tools); mErr == nil { - if setErr := u.repo.SetToolsCache(ctx, id, string(b)); setErr != nil { - u.log.Warn("cache mcp tools", slog.Uint64("server_id", id), slog.Any("err", setErr)) + if currentRevision != startedRevision { + return nil, fmt.Errorf("%w: MCP credentials changed during discovery", errs.ErrConflict) + } + if probeErr != nil { + if err := u.repo.SetStatus(ctx, id, "error", probeErr.Error()); err != nil { + return nil, fmt.Errorf("record MCP discovery failure: %w", err) } - } else { - u.log.Warn("marshal mcp tools", slog.Uint64("server_id", id), slog.Any("err", mErr)) + // Temporary unavailability does not revoke the last verified catalog. + return nil, probeErr + } + b, err := json.Marshal(tools) + if err != nil { + return nil, fmt.Errorf("marshal MCP tools: %w", err) } - if setErr := u.repo.SetStatus(ctx, id, "ok", ""); setErr != nil { - u.log.Warn("set mcp ok status", slog.Uint64("server_id", id), slog.Any("err", setErr)) + if err := u.repo.SetToolsCache(ctx, id, string(b)); err != nil { + return nil, fmt.Errorf("cache MCP tools: %w", err) } - if s.Enabled { + if err := u.repo.SetStatus(ctx, id, "ok", ""); err != nil { + return nil, fmt.Errorf("record MCP discovery: %w", err) + } + if s.Enabled && (forcePublish || s.ToolsCacheJSON != string(b)) { u.notifyToolsChanged(ctx, s.Name, s, tools) } return tools, nil } +func (u *Usecase) discover(ctx context.Context, s *model.Server) ([]mcpclient.Tool, error) { + cli, err := u.BuildClient(ctx, s) + if err != nil { + return nil, err + } + if err := cli.Initialize(ctx); err != nil { + return nil, err + } + return cli.ListTools(ctx) +} + func cachedTools(raw string) ([]mcpclient.Tool, error) { if strings.TrimSpace(raw) == "" { return nil, nil diff --git a/internal/manager/biz/setting/agent.go b/internal/manager/biz/setting/agent.go index 43b583480..4e29d80d9 100644 --- a/internal/manager/biz/setting/agent.go +++ b/internal/manager/biz/setting/agent.go @@ -2,6 +2,7 @@ package setting import ( "context" + "errors" "fmt" "log/slog" "strconv" @@ -68,10 +69,35 @@ func (s *Service) AgentOutputLocale(ctx context.Context) (string, bool, error) { return v, true, nil } +// AgentRuntimeVersion reads new-session policy without a cache: selection is +// infrequent and must observe the latest persisted administrator choice. +// Empty means follow the release default. Errors must not silently reroute work. +func (s *Service) AgentRuntimeVersion(ctx context.Context) (string, error) { + row, err := s.repo.Get(ctx, model.CategoryAgent, model.KeyAgentRuntimeVersion) + if errors.Is(err, errs.ErrNotFound) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("read agent runtime version: %w", err) + } + if err := validateAgentSetting(model.KeyAgentRuntimeVersion, row.Value); err != nil { + return "", err + } + if row.Value == "release" { + return "", nil + } + return row.Value, nil +} + // validateAgentSetting validates typed settings before persistence. Other // agent keys remain backwards-compatible with the generic setting store. func validateAgentSetting(key, value string) error { switch key { + case model.KeyAgentRuntimeVersion: + if value != "release" && value != "v3" { + return fmt.Errorf("%w: runtime_version must be release or v3; v1/v2 are not available", errs.ErrInvalid) + } + return nil case model.KeyAgentOutputLocale: locale := strings.ToLower(strings.TrimSpace(value)) if locale != "" && locale != "zh" && locale != "en" { diff --git a/internal/manager/biz/setting/probe.go b/internal/manager/biz/setting/probe.go index f2c60bf22..98c2b5201 100644 --- a/internal/manager/biz/setting/probe.go +++ b/internal/manager/biz/setting/probe.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "strconv" "strings" "time" @@ -144,29 +145,33 @@ func (p *TempoReadinessProbe) Probe(ctx context.Context) error { // here rather than going through the skill registry — the registry // path adds the agent-loop audit pipeline + JSON envelope that we // don't want for an admin probe. The provider URLs / keys are read -// from the same WebSearchResolver the skill uses, so a successful -// probe means the skill itself will work. +// from the same WebSearchResolver the skill uses. A successful probe +// confirms usable results at that moment, not future availability. type WebSearchProbe struct { resolver *WebSearchResolver timeout time.Duration } type WebSearchProbeInput struct { - Provider string `json:"provider"` - SearxngURL string `json:"searxng_url"` - TavilyAPIKey string `json:"tavily_api_key"` - BraveAPIKey string `json:"brave_api_key"` + Provider string `json:"provider"` + SearxngURL string `json:"searxng_url"` + TavilyAPIKey string `json:"tavily_api_key"` + BraveAPIKey string `json:"brave_api_key"` + FallbackProvider string `json:"fallback_provider"` + SearxngEngines string `json:"searxng_engines"` + RequestTimeoutSeconds string `json:"request_timeout_seconds"` + TotalTimeoutSeconds string `json:"total_timeout_seconds"` } // NewWebSearchProbe builds the probe. resolver must be non-nil; the // probe Probe() returns an error otherwise. func NewWebSearchProbe(r *WebSearchResolver) *WebSearchProbe { - return &WebSearchProbe{resolver: r, timeout: 8 * time.Second} + return &WebSearchProbe{resolver: r, timeout: 20 * time.Second} } // Probe runs a 1-result query against the selected provider. Returns // (provider, sample-title, nil) on success; sample is the first -// result's title if any (empty when the provider returned zero hits). +// result's title; zero usable results is a failed health probe. func (p *WebSearchProbe) Probe(ctx context.Context) (string, string, error) { if p == nil || p.resolver == nil { return "", "", fmt.Errorf("web_search probe not wired") @@ -174,6 +179,7 @@ func (p *WebSearchProbe) Probe(ctx context.Context) (string, string, error) { return p.ProbeConfiguration(ctx, WebSearchProbeInput{ Provider: p.resolver.Provider(ctx), SearxngURL: p.resolver.SearxngURL(ctx), TavilyAPIKey: p.resolver.TavilyAPIKey(ctx), BraveAPIKey: p.resolver.BraveAPIKey(ctx), + FallbackProvider: p.resolver.FallbackProvider(ctx), SearxngEngines: p.resolver.SearxngEngines(ctx), }) } @@ -181,22 +187,53 @@ func (p *WebSearchProbe) ProbeConfiguration(ctx context.Context, in WebSearchPro if p == nil { return "", "", fmt.Errorf("web_search probe not wired") } - provider := strings.ToLower(strings.TrimSpace(in.Provider)) - cctx, cancel := context.WithTimeout(ctx, p.timeout) + for key, value := range map[string]string{ + model.KeyWebSearchProvider: in.Provider, model.KeyWebSearchFallbackProvider: in.FallbackProvider, + model.KeySearxngEngines: in.SearxngEngines, model.KeyWebSearchRequestTimeout: in.RequestTimeoutSeconds, model.KeyWebSearchTotalTimeout: in.TotalTimeoutSeconds, + } { + if err := validateWebSearchSetting(key, value); err != nil { + return "", "", err + } + } + total := 45 * time.Second + if n, err := strconv.Atoi(in.TotalTimeoutSeconds); err == nil { + total = time.Duration(n) * time.Second + } + ctx, cancel := context.WithTimeout(ctx, total) defer cancel() - switch provider { - case model.ProviderSearxng: - return p.probeSearxng(cctx, in.SearxngURL) - case model.ProviderTavily: - return p.probeTavily(cctx, in.TavilyAPIKey) - case model.ProviderBrave: - return p.probeBrave(cctx, in.BraveAPIKey) - default: - return "", "", fmt.Errorf("unsupported web search provider %q", in.Provider) + per := p.timeout + if n, err := strconv.Atoi(in.RequestTimeoutSeconds); err == nil { + per = time.Duration(n) * time.Second } + provider := strings.ToLower(strings.TrimSpace(in.Provider)) + probe := func(name string) (string, string, error) { + cctx, stop := context.WithTimeout(ctx, per) + defer stop() + switch name { + case model.ProviderSearxng: + return p.probeSearxng(cctx, in.SearxngURL, in.SearxngEngines) + case model.ProviderTavily: + return p.probeTavily(cctx, in.TavilyAPIKey) + case model.ProviderBrave: + return p.probeBrave(cctx, in.BraveAPIKey) + default: + return "", "", fmt.Errorf("unsupported web search provider %q", name) + } + } + name, sample, err := probe(provider) + if err != nil { + return name, sample, err + } + if in.FallbackProvider != "" && in.FallbackProvider != provider { + _, _, err = probe(in.FallbackProvider) + if err != nil { + return name, "", fmt.Errorf("backup %s: %w", in.FallbackProvider, err) + } + } + return name, sample, nil } -func (p *WebSearchProbe) probeSearxng(ctx context.Context, rawURL string) (string, string, error) { +func (p *WebSearchProbe) probeSearxng(ctx context.Context, rawURL string, engines ...string) (string, string, error) { base := strings.TrimRight(strings.TrimSpace(rawURL), "/") if base == "" { base = model.DefaultSearxngURL @@ -205,10 +242,13 @@ func (p *WebSearchProbe) probeSearxng(ctx context.Context, rawURL string) (strin return model.ProviderSearxng, "", fmt.Errorf("searxng url: %w", err) } q := url.Values{} - q.Set("q", "ongrid web search probe") + q.Set("q", "Kubernetes") q.Set("format", "json") q.Set("safesearch", "1") q.Set("pageno", "1") + if len(engines) > 0 && engines[0] != "" { + q.Set("engines", engines[0]) + } full := base + "/search?" + q.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, full, nil) @@ -217,18 +257,19 @@ func (p *WebSearchProbe) probeSearxng(ctx context.Context, rawURL string) (strin } req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "ongrid-web-search-probe/1.0") - resp, err := newHTTPClient(p.timeout, false).Do(req) + resp, err := newHTTPClient(0, false).Do(req) if err != nil { return model.ProviderSearxng, "", fmt.Errorf("dial %s: %w", base, err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - body, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) - return model.ProviderSearxng, "", fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + return model.ProviderSearxng, "", fmt.Errorf("search provider returned HTTP %d", resp.StatusCode) } var out struct { - Results []struct { + UnresponsiveEngines [][]string `json:"unresponsive_engines"` + Results []struct { Title string `json:"title"` + URL string `json:"url"` } `json:"results"` } body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) @@ -236,8 +277,14 @@ func (p *WebSearchProbe) probeSearxng(ctx context.Context, rawURL string) (strin return model.ProviderSearxng, "", fmt.Errorf("decode: %w", err) } sample := "" - if len(out.Results) > 0 { - sample = out.Results[0].Title + for _, result := range out.Results { + if validProbeResult(result.Title, result.URL) { + sample = result.Title + break + } + } + if sample == "" { + return "", "", fmt.Errorf("search probe returned no usable links; engine failures: %.300s", fmt.Sprint(out.UnresponsiveEngines)) } return model.ProviderSearxng, sample, nil } @@ -249,7 +296,7 @@ func (p *WebSearchProbe) probeTavily(ctx context.Context, rawAPIKey string) (str } body, _ := json.Marshal(map[string]any{ "api_key": apiKey, - "query": "ongrid web search probe", + "query": "Kubernetes", "max_results": 1, }) req, err := http.NewRequestWithContext(ctx, http.MethodPost, @@ -259,18 +306,18 @@ func (p *WebSearchProbe) probeTavily(ctx context.Context, rawAPIKey string) (str } req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - resp, err := newHTTPClient(p.timeout, false).Do(req) + resp, err := newHTTPClient(0, false).Do(req) if err != nil { return model.ProviderTavily, "", err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - buf, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) - return model.ProviderTavily, "", fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(buf))) + return model.ProviderTavily, "", fmt.Errorf("search provider returned HTTP %d", resp.StatusCode) } var out struct { Results []struct { Title string `json:"title"` + URL string `json:"url"` } `json:"results"` } buf, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) @@ -278,8 +325,14 @@ func (p *WebSearchProbe) probeTavily(ctx context.Context, rawAPIKey string) (str return model.ProviderTavily, "", err } sample := "" - if len(out.Results) > 0 { - sample = out.Results[0].Title + for _, result := range out.Results { + if validProbeResult(result.Title, result.URL) { + sample = result.Title + break + } + } + if sample == "" { + return "", "", fmt.Errorf("search probe returned no usable links; check engine failures, CAPTCHA or network access") } return model.ProviderTavily, sample, nil } @@ -290,7 +343,7 @@ func (p *WebSearchProbe) probeBrave(ctx context.Context, rawAPIKey string) (stri return model.ProviderBrave, "", fmt.Errorf("brave api key not configured") } q := url.Values{} - q.Set("q", "ongrid web search probe") + q.Set("q", "Kubernetes") q.Set("count", "1") full := "https://api.search.brave.com/res/v1/web/search?" + q.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, full, nil) @@ -299,19 +352,19 @@ func (p *WebSearchProbe) probeBrave(ctx context.Context, rawAPIKey string) (stri } req.Header.Set("Accept", "application/json") req.Header.Set("X-Subscription-Token", apiKey) - resp, err := newHTTPClient(p.timeout, false).Do(req) + resp, err := newHTTPClient(0, false).Do(req) if err != nil { return model.ProviderBrave, "", err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - buf, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) - return model.ProviderBrave, "", fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(buf))) + return model.ProviderBrave, "", fmt.Errorf("search provider returned HTTP %d", resp.StatusCode) } var out struct { Web struct { Results []struct { Title string `json:"title"` + URL string `json:"url"` } `json:"results"` } `json:"web"` } @@ -320,8 +373,14 @@ func (p *WebSearchProbe) probeBrave(ctx context.Context, rawAPIKey string) (stri return model.ProviderBrave, "", err } sample := "" - if len(out.Web.Results) > 0 { - sample = out.Web.Results[0].Title + for _, result := range out.Web.Results { + if validProbeResult(result.Title, result.URL) { + sample = result.Title + break + } + } + if sample == "" { + return model.ProviderBrave, "", fmt.Errorf("search probe returned no usable links") } return model.ProviderBrave, sample, nil } @@ -394,3 +453,34 @@ func probeOTLPHTTPTraceEndpoint(ctx context.Context, fullURL, user, pass string, body, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) return fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) } + +func validProbeResult(title, rawURL string) bool { + u, err := url.Parse(rawURL) + return err == nil && strings.TrimSpace(title) != "" && u.Host != "" && (u.Scheme == "http" || u.Scheme == "https") +} + +// ValidateAndSaveConfiguration prevents a partially saved provider/credential +// tuple from rerouting live queries. Failed probes leave all settings intact. +func (p *WebSearchProbe) ValidateAndSaveConfiguration(ctx context.Context, in WebSearchProbeInput) (string, string, error) { + if p == nil || p.resolver == nil || p.resolver.svc == nil { + return "", "", fmt.Errorf("search settings store not wired") + } + provider, sample, err := p.ProbeConfiguration(ctx, in) + if err != nil { + return provider, sample, err + } + rows := []model.Setting{ + {Category: model.CategoryWebSearch, Key: model.KeyWebSearchProvider, Value: in.Provider}, + {Category: model.CategoryWebSearch, Key: model.KeyWebSearchFallbackProvider, Value: in.FallbackProvider}, + {Category: model.CategoryWebSearch, Key: model.KeySearxngURL, Value: in.SearxngURL}, + {Category: model.CategoryWebSearch, Key: model.KeySearxngEngines, Value: in.SearxngEngines}, + {Category: model.CategoryWebSearch, Key: model.KeyWebSearchRequestTimeout, Value: in.RequestTimeoutSeconds}, + {Category: model.CategoryWebSearch, Key: model.KeyWebSearchTotalTimeout, Value: in.TotalTimeoutSeconds}, + {Category: model.CategoryWebSearch, Key: model.KeyTavilyAPIKey, Value: in.TavilyAPIKey, Sensitive: true}, + {Category: model.CategoryWebSearch, Key: model.KeyBraveAPIKey, Value: in.BraveAPIKey, Sensitive: true}, + } + if err := p.resolver.svc.SetBatch(ctx, rows); err != nil { + return provider, "", fmt.Errorf("save search configuration: %w", err) + } + return provider, sample, nil +} diff --git a/internal/manager/biz/setting/service.go b/internal/manager/biz/setting/service.go index 376bb3049..8aff13c7e 100644 --- a/internal/manager/biz/setting/service.go +++ b/internal/manager/biz/setting/service.go @@ -167,6 +167,8 @@ func validateSetting(category, key, value string) error { return fmt.Errorf("%w: auto_apm_enabled must be true or false", errs.ErrInvalid) } return nil + case model.CategoryWebSearch: + return validateWebSearchSetting(key, value) case model.CategoryAgent: return validateAgentSetting(key, value) case model.CategoryObservability: diff --git a/internal/manager/biz/setting/service_test.go b/internal/manager/biz/setting/service_test.go index 5418a8764..cbd6c0e38 100644 --- a/internal/manager/biz/setting/service_test.go +++ b/internal/manager/biz/setting/service_test.go @@ -374,3 +374,45 @@ func TestServiceDeleteInvalidatesCache(t *testing.T) { t.Fatalf("post-Delete Get = (%q,%v); want (\"\", false)", v, ok) } } + +func TestAgentRuntimeVersionSelection(t *testing.T) { + repo := newFakeRepo() + svc := New(repo, nil) + ctx := t.Context() + if got, err := svc.AgentRuntimeVersion(ctx); err != nil || got != "" { + t.Fatalf("default: %q %v", got, err) + } + for _, value := range []string{"v3", "release", "v3"} { + if err := svc.Set(ctx, model.CategoryAgent, model.KeyAgentRuntimeVersion, value, false); err != nil { + t.Fatal(err) + } + want := value + if want == "release" { + want = "" + } + if got, err := svc.AgentRuntimeVersion(ctx); err != nil || got != want { + t.Fatalf("saved: %q %v", got, err) + } + } + for _, value := range []string{"v1", "v2", "v99", "", "V3"} { + if err := svc.Set(ctx, model.CategoryAgent, model.KeyAgentRuntimeVersion, value, false); !errors.Is(err, errs.ErrInvalid) { + t.Fatalf("accepted %q: %v", value, err) + } + if err := svc.SetBatch(ctx, []model.Setting{{Category: model.CategoryAgent, Key: model.KeyAgentRuntimeVersion, Value: value}}); !errors.Is(err, errs.ErrInvalid) { + t.Fatalf("batch accepted %q: %v", value, err) + } + } + if err := svc.Delete(ctx, model.CategoryAgent, model.KeyAgentRuntimeVersion); err != nil { + t.Fatal(err) + } + if got, err := svc.AgentRuntimeVersion(ctx); err != nil || got != "" { + t.Fatalf("deleted: %q %v", got, err) + } + // Unsupported persisted values cannot silently fall back to v3. + if _, err := repo.Set(ctx, model.CategoryAgent, model.KeyAgentRuntimeVersion, "v2", false); err != nil { + t.Fatal(err) + } + if _, err := svc.AgentRuntimeVersion(ctx); !errors.Is(err, errs.ErrInvalid) { + t.Fatalf("invalid persisted version: %v", err) + } +} diff --git a/internal/manager/biz/setting/websearch.go b/internal/manager/biz/setting/websearch.go index 8395372ff..f06405d8a 100644 --- a/internal/manager/biz/setting/websearch.go +++ b/internal/manager/biz/setting/websearch.go @@ -2,7 +2,12 @@ package setting import ( "context" + "fmt" + "github.com/ongridio/ongrid/internal/pkg/errs" + "regexp" + "strconv" "strings" + "time" model "github.com/ongridio/ongrid/internal/manager/model/setting" ) @@ -91,3 +96,55 @@ func (r *WebSearchResolver) TavilyAPIKey(ctx context.Context) string { func (r *WebSearchResolver) BraveAPIKey(ctx context.Context) string { return r.get(ctx, model.KeyBraveAPIKey) } + +// Fallback is opt-in; merely storing an API key does not authorize queries. +func (r *WebSearchResolver) FallbackProvider(ctx context.Context) string { + v := r.get(ctx, model.KeyWebSearchFallbackProvider) + if v == model.ProviderSearxng || v == model.ProviderTavily || v == model.ProviderBrave { + return v + } + return "" +} +func (r *WebSearchResolver) SearxngEngines(ctx context.Context) string { + return r.get(ctx, model.KeySearxngEngines) +} +func (r *WebSearchResolver) SearchTimeouts(ctx context.Context) (time.Duration, time.Duration) { + per, total := 20, 45 + if v, err := strconv.Atoi(r.get(ctx, model.KeyWebSearchRequestTimeout)); err == nil && v >= 2 && v <= 60 { + per = v + } + if v, err := strconv.Atoi(r.get(ctx, model.KeyWebSearchTotalTimeout)); err == nil && v >= 5 && v <= 120 { + total = v + } + return time.Duration(per) * time.Second, time.Duration(total) * time.Second +} +func validateWebSearchSetting(key, value string) error { + switch key { + case model.KeyWebSearchProvider, model.KeyWebSearchFallbackProvider: + if value != "" && value != model.ProviderSearxng && value != model.ProviderTavily && value != model.ProviderBrave { + return fmt.Errorf("%w: unsupported search provider", errs.ErrInvalid) + } + case model.KeySearxngEngines: + if len(value) > 200 { + return fmt.Errorf("%w: engine list too long", errs.ErrInvalid) + } + for _, name := range strings.Split(value, ",") { + if name != "" && !regexp.MustCompile(`^[a-zA-Z0-9 _-]+$`).MatchString(name) { + return fmt.Errorf("%w: invalid engine name", errs.ErrInvalid) + } + } + case model.KeyWebSearchRequestTimeout, model.KeyWebSearchTotalTimeout: + if value == "" { + return nil + } + min, max := 2, 60 + if key == model.KeyWebSearchTotalTimeout { + min, max = 5, 120 + } + n, err := strconv.Atoi(value) + if err != nil || n < min || n > max { + return fmt.Errorf("%w: %s must be %d..%d seconds", errs.ErrInvalid, key, min, max) + } + } + return nil +} diff --git a/internal/manager/biz/setting/websearch_policy_test.go b/internal/manager/biz/setting/websearch_policy_test.go new file mode 100644 index 000000000..6502588bd --- /dev/null +++ b/internal/manager/biz/setting/websearch_policy_test.go @@ -0,0 +1,96 @@ +package setting + +import ( + "fmt" + model "github.com/ongridio/ongrid/internal/manager/model/setting" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestSearchProbeRequiresUsableResults(t *testing.T) { + for _, tc := range []struct { + body string + ok bool + }{ + {`{"results":[],"unresponsive_engines":[["bing","timeout"]]}`, false}, + {`{"results":[]}`, false}, + {`{"results":[{"title":"answer","url":"javascript:alert(1)"}]}`, false}, + {`{"results":[{"title":"Kubernetes","url":"https://kubernetes.io"}]}`, true}, + } { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("engines") != "360search" || r.URL.Query().Get("q") != "Kubernetes" { + t.Error("probe ignored configuration") + } + fmt.Fprint(w, tc.body) + })) + probe := NewWebSearchProbe(nil) + _, sample, err := probe.ProbeConfiguration(t.Context(), WebSearchProbeInput{Provider: "searxng", SearxngURL: srv.URL, SearxngEngines: "360search"}) + srv.Close() + if (err == nil) != tc.ok { + t.Fatalf("body=%s err=%v", tc.body, err) + } + if tc.ok && sample != "Kubernetes" { + t.Fatal("missing sample") + } + if strings.Contains(tc.body, "timeout") && !strings.Contains(err.Error(), "timeout") { + t.Fatal("engine failure lost") + } + } +} + +func TestSearchSettingsValidateAndResolve(t *testing.T) { + svc := New(newFakeRepo(), nil) + resolver := NewWebSearchResolver(svc) + per, total := resolver.SearchTimeouts(t.Context()) + if per != 20*time.Second || total != 45*time.Second || resolver.FallbackProvider(t.Context()) != "" { + t.Fatal("unsafe defaults") + } + for _, tc := range []struct { + key, value string + valid bool + }{ + {model.KeyWebSearchFallbackProvider, "brave", true}, + {model.KeyWebSearchFallbackProvider, "unknown", false}, + {model.KeySearxngEngines, "360search,sogou", true}, + {model.KeySearxngEngines, "bing&other=1", false}, + {model.KeyWebSearchRequestTimeout, "0", false}, + {model.KeyWebSearchRequestTimeout, "30", true}, + {model.KeyWebSearchTotalTimeout, "9999", false}, + {model.KeyWebSearchTotalTimeout, "60", true}, + } { + err := svc.Set(t.Context(), model.CategoryWebSearch, tc.key, tc.value, false) + if (err == nil) != tc.valid { + t.Fatalf("%s=%s err=%v", tc.key, tc.value, err) + } + } + per, total = resolver.SearchTimeouts(t.Context()) + if per != 30*time.Second || total != 60*time.Second || resolver.FallbackProvider(t.Context()) != "brave" { + t.Fatal("saved policy not used") + } +} + +func TestSearchConfigurationSaveRejectsFailedBackupWithoutPartialWrites(t *testing.T) { + svc := New(newFakeRepo(), nil) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"results":[{"title":"Kubernetes","url":"https://kubernetes.io"}]}`) + })) + defer srv.Close() + p := NewWebSearchProbe(NewWebSearchResolver(svc)) + in := WebSearchProbeInput{Provider: "searxng", SearxngURL: srv.URL, FallbackProvider: "brave", SearxngEngines: "360search"} + if _, _, err := p.ValidateAndSaveConfiguration(t.Context(), in); err == nil { + t.Fatal("accepted missing backup key") + } + if rows, err := svc.List(t.Context(), model.CategoryWebSearch); err != nil || len(rows) != 0 { + t.Fatal("failed draft partially saved") + } + in.FallbackProvider = "" + if _, _, err := p.ValidateAndSaveConfiguration(t.Context(), in); err != nil { + t.Fatal(err) + } + if rows, err := svc.List(t.Context(), model.CategoryWebSearch); err != nil || len(rows) != 8 { + t.Fatal("configuration not saved atomically") + } +} diff --git a/internal/manager/model/setting/model.go b/internal/manager/model/setting/model.go index 70ccd3ed9..cae2894a4 100644 --- a/internal/manager/model/setting/model.go +++ b/internal/manager/model/setting/model.go @@ -73,6 +73,8 @@ const ( // tools. Only "true" enables write tools; unset / any other value keeps the // agent read-only (only Class=="read" tools exposed to the LLM). KeyAgentWriteEnabled = "write_enabled" + // KeyAgentRuntimeVersion selects the runtime for new sessions only. + KeyAgentRuntimeVersion = "runtime_version" // KeyAgentLLMTimeoutSeconds bounds an assistant LLM request and report // generation in seconds. The typed setting accessor owns its default and // range validation so callers cannot accidentally accept an unbounded value. @@ -246,10 +248,14 @@ const ( // Empty / unset provider falls through to "searxng" (the zero-config // baseline). Tavily / Brave are opt-in via UI. const ( - KeyWebSearchProvider = "provider" // "searxng" | "tavily" | "brave" - KeySearxngURL = "searxng_url" // e.g. http://searxng:8080 (default) - KeyTavilyAPIKey = "tavily_api_key" // sensitive — Tavily Search API key - KeyBraveAPIKey = "brave_api_key" // sensitive — Brave Search API key + KeyWebSearchFallbackProvider = "fallback_provider" + KeySearxngEngines = "searxng_engines" + KeyWebSearchRequestTimeout = "request_timeout_seconds" + KeyWebSearchTotalTimeout = "total_timeout_seconds" + KeyWebSearchProvider = "provider" // "searxng" | "tavily" | "brave" + KeySearxngURL = "searxng_url" // e.g. http://searxng:8080 (default) + KeyTavilyAPIKey = "tavily_api_key" // sensitive — Tavily Search API key + KeyBraveAPIKey = "brave_api_key" // sensitive — Brave Search API key ) // WebSearch provider names, exported as constants so callers don't pass diff --git a/internal/manager/server/integration/http.go b/internal/manager/server/integration/http.go index 5e38fa2af..c8b18c153 100644 --- a/internal/manager/server/integration/http.go +++ b/internal/manager/server/integration/http.go @@ -124,6 +124,7 @@ func (h *Handler) Register(r chi.Router) { r.Post("/v1/integrations/loki/test", h.testLoki) r.Post("/v1/integrations/tempo/test", h.testTempo) r.Post("/v1/integrations/websearch/test", h.testWebSearch) + r.Post("/v1/integrations/websearch/validate-and-save", h.saveWebSearch) r.Post("/v1/integrations/llm/test", h.testLLMConfiguration) r.Post("/v1/integrations/llm/models", h.fetchLLMModels) r.Post("/v1/integrations/llm/validate-and-save", h.validateAndSaveLLMConfiguration) @@ -580,3 +581,32 @@ func writeErr(w http.ResponseWriter, err error) { } writeJSON(w, status, errorBody{Error: err.Error(), Code: slug}) } + +// @Summary Validate and atomically save search providers (admin only) +// @Router /api/v1/integrations/websearch/validate-and-save [post] +// @Accept json +// @Produce json +// @Param body body bizsetting.WebSearchProbeInput true "Complete search configuration" +// @Success 200 {object} map[string]string +func (h *Handler) saveWebSearch(w http.ResponseWriter, r *http.Request) { + if !h.requireAdmin(w, r) { + return + } + saver, ok := h.webSearch.(interface { + ValidateAndSaveConfiguration(context.Context, bizsetting.WebSearchProbeInput) (string, string, error) + }) + if !ok { + writeJSON(w, http.StatusServiceUnavailable, errorBody{Error: "search settings not wired", Code: "websearch-disabled"}) + return + } + in, ok := decodeDraft[bizsetting.WebSearchProbeInput](w, r) + if !ok { + return + } + provider, sample, err := saver.ValidateAndSaveConfiguration(r.Context(), in) + if err != nil { + writeErr(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "saved", "provider": provider, "sample": sample}) +} diff --git a/internal/manager/server/integration/http_test.go b/internal/manager/server/integration/http_test.go index b690125cb..aa0973381 100644 --- a/internal/manager/server/integration/http_test.go +++ b/internal/manager/server/integration/http_test.go @@ -529,3 +529,29 @@ func TestLLMConfigurationValidateAndSaveUsesOneDraftAndInvalidatesRouter(t *test t.Fatalf("response = %s", rec.Body.String()) } } + +type searchConfigurationSaver struct{ calls int } + +func (s *searchConfigurationSaver) ProbeConfiguration(context.Context, bizsetting.WebSearchProbeInput) (string, string, error) { + return "searxng", "sample", nil +} +func (s *searchConfigurationSaver) ValidateAndSaveConfiguration(_ context.Context, in bizsetting.WebSearchProbeInput) (string, string, error) { + s.calls++ + return in.Provider, "sample", nil +} +func TestSearchConfigurationSaveRequiresAdmin(t *testing.T) { + saver := &searchConfigurationSaver{} + h := NewHandler(nil, nil, nil, nil, saver) + for _, role := range []string{"viewer", "admin"} { + req := httptest.NewRequest(http.MethodPost, "/v1/integrations/websearch/validate-and-save", strings.NewReader(`{"provider":"searxng","fallback_provider":"brave"}`)) + req = req.WithContext(tenantctx.With(t.Context(), tenantctx.Tenant{UserID: 7, Role: role})) + rec := httptest.NewRecorder() + newRouter(h).ServeHTTP(rec, req) + if role == "viewer" && (rec.Code != http.StatusForbidden || saver.calls != 0) { + t.Fatal("non-admin saved settings") + } + if role == "admin" && (rec.Code != http.StatusOK || saver.calls != 1) { + t.Fatalf("admin save failed: %d %s", rec.Code, rec.Body.String()) + } + } +} diff --git a/internal/manager/service/aiops/approval_adapters.go b/internal/manager/service/aiops/approval_adapters.go index a89115de0..ed931650e 100644 --- a/internal/manager/service/aiops/approval_adapters.go +++ b/internal/manager/service/aiops/approval_adapters.go @@ -11,6 +11,7 @@ import ( aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" managerbizapproval "github.com/ongridio/ongrid/internal/manager/biz/approval" managerbizmcp "github.com/ongridio/ongrid/internal/manager/biz/mcp" managerbizsecret "github.com/ongridio/ongrid/internal/manager/biz/secret" @@ -21,9 +22,11 @@ import ( // TYPE inject rule into env vars at approve time. Credential (singular) is a // legacy field kept so an in-flight pre-upgrade approval still resolves. type cloudBashPayload struct { - Command string `json:"command"` - Credentials []string `json:"credentials,omitempty"` - Credential string `json:"credential,omitempty"` // legacy single + SkillAssets []basetool.SkillAssetRef `json:"skill_assets,omitempty"` + SkillCredentials []string `json:"skill_credentials,omitempty"` + Command string `json:"command"` + Credentials []string `json:"credentials,omitempty"` + Credential string `json:"credential,omitempty"` // legacy single // SessionID is the chat session that proposed the command (HLD-019). The // executor maps it to a persistent per-session working directory so files // a tool writes in one command survive to the next, instead of running in @@ -140,7 +143,7 @@ func (s CloudBashProposer) ProposeAndAwait(ctx context.Context, command string, Kind: "cloud_bash", Title: title, Summary: strings.Join(credentials, ", "), // plain names; card shows them - Payload: cloudBashPayload{Command: command, Credentials: credentials, SessionID: sessionID}, + Payload: cloudBashPayload{SkillCredentials: basetool.BoundCredentialsFromContext(ctx), SkillAssets: basetool.SkillAssetsFromContext(ctx), Command: command, Credentials: credentials, SessionID: sessionID}, Source: "agent", SessionID: sessionID, ProposedBy: userID, @@ -287,6 +290,7 @@ func (s InstallSkillProposer) ProposeInstall(ctx context.Context, url, sourceTyp // P2). Server/Tool/Arguments drive the executor; Command is a human-readable // one-liner the inline approval card shows (reuses the cloud_bash card). type mcpCallPayload struct { + Revision string `json:"revision,omitempty"` Server string `json:"server"` Tool string `json:"tool"` Arguments map[string]any `json:"arguments,omitempty"` @@ -302,9 +306,20 @@ func (s MCPCaller) CallMCPTool(ctx context.Context, server, tool string, args ma // MCPProposer queues an MCP call into the human approval inbox (default, // untrusted path) — same propose-confirm model as cloud_bash. -type MCPProposer struct{ Usecase *managerbizapproval.Usecase } +type MCPProposer struct { + Usecase *managerbizapproval.Usecase + MCP *managerbizmcp.Usecase +} func (s MCPProposer) ProposeMCPCallAndAwait(ctx context.Context, server, tool string, args map[string]any, sessionID, toolCallID string, userID uint64) (string, error) { + var revision string + if s.MCP != nil { + var err error + revision, err = s.MCP.Revision(ctx, server) + if err != nil { + return "", err + } + } argsJSON, err := json.Marshal(args) if err != nil { return "", fmt.Errorf("marshal MCP approval arguments: %w", err) @@ -317,7 +332,7 @@ func (s MCPProposer) ProposeMCPCallAndAwait(ctx context.Context, server, tool st RuntimeVersion: string(aiopschatruntime.VersionFromContext(ctx)), Kind: "mcp_call", Title: cmd, - Payload: mcpCallPayload{Server: server, Tool: tool, Arguments: args, Command: cmd}, + Payload: mcpCallPayload{Revision: revision, Server: server, Tool: tool, Arguments: args, Command: cmd}, Source: "agent", SessionID: sessionID, ProposedBy: userID, diff --git a/internal/manager/service/aiops/approval_executors.go b/internal/manager/service/aiops/approval_executors.go index ee15c1327..0d20a0bea 100644 --- a/internal/manager/service/aiops/approval_executors.go +++ b/internal/manager/service/aiops/approval_executors.go @@ -6,11 +6,13 @@ import ( "fmt" "log/slog" "os" + "path/filepath" "strings" "time" "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" managerbizapproval "github.com/ongridio/ongrid/internal/manager/biz/approval" managerbizdevice "github.com/ongridio/ongrid/internal/manager/biz/device" managerbizedge "github.com/ongridio/ongrid/internal/manager/biz/edge" @@ -22,12 +24,18 @@ import ( "github.com/ongridio/ongrid/internal/pkg/workspace" ) +// SkillAssetLeaser validates runtime-owned package snapshots at execution time. +type SkillAssetLeaser interface { + AcquireSkillAssets(context.Context, []basetool.SkillAssetRef) (func(), error) +} + // ApprovalExecutors supplies the existing execution paths for approved actions. type ApprovalExecutors struct { Secrets *managerbizsecret.Usecase Runner runner.Runner Workspace *workspace.Manager SkillsRoot string + SkillAssets SkillAssetLeaser Frontier *managersvcfb.Client Edges *managerbizedge.Usecase Devices *managerbizdevice.Usecase @@ -59,6 +67,35 @@ func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { if err := json.Unmarshal([]byte(payloadJSON), &p); err != nil { return "", err } + if len(p.SkillAssets) > 0 { + if e.SkillAssets == nil { + return "", fmt.Errorf("skill snapshot runtime unavailable; submit a new request") + } + release, err := e.SkillAssets.AcquireSkillAssets(ctx, p.SkillAssets) + if err != nil { + return "", err + } + defer release() + } + if len(p.SkillCredentials) > 0 && len(p.SkillAssets) > 0 { + if e.Marketplace == nil { + return "", fmt.Errorf("skill credential bindings unavailable") + } + skillNames := make([]string, 0, len(p.SkillAssets)) + for _, ref := range p.SkillAssets { + skillNames = append(skillNames, ref.Name) + } + current := e.Marketplace.BoundCredentialNamesForSkills(ctx, skillNames) + allowed := make(map[string]bool, len(current)) + for _, name := range current { + allowed[name] = true + } + for _, name := range p.SkillCredentials { + if !allowed[name] { + return "", fmt.Errorf("skill credential binding was revoked; submit a new request") + } + } + } names := append([]string(nil), p.Credentials...) if p.Credential != "" { // legacy single-credential approvals names = append(names, p.Credential) @@ -87,6 +124,22 @@ func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { // bin dir go on PATH. PIP_BREAK_SYSTEM_PACKAGES sidesteps PEP 668 so a // non-root --user install isn't refused. env["PATH"] = cloudBashToolsDir + "/bin:" + skillBinPATH(e.SkillsRoot) + if len(p.SkillAssets) > 0 { + var bins []string + for _, ref := range p.SkillAssets { + if ref.Directory != "" { + bins = append(bins, filepath.Join(ref.Directory, "bin")) + if ref.PackageDirectory != "" && ref.PackageDirectory != ref.Directory { + bins = append(bins, filepath.Join(ref.PackageDirectory, "bin")) + } + } + } + // Pinned packages precede legacy installed bins. Explicit absolute paths + // from load_skill also point into the same immutable snapshot. + if len(bins) > 0 { + env["PATH"] = strings.Join(bins, ":") + ":" + env["PATH"] + } + } env["PYTHONUSERBASE"] = cloudBashToolsDir env["PIP_USER"] = "1" env["PIP_BREAK_SYSTEM_PACKAGES"] = "1" @@ -138,7 +191,10 @@ func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { if err := json.Unmarshal([]byte(payloadJSON), &p); err != nil { return "", err } - out, err := e.MCP.CallTool(ctx, p.Server, p.Tool, p.Arguments) + if contract.VersionFromContext(ctx) == contract.V3 && p.Revision == "" { + return "", fmt.Errorf("MCP approval has no registration revision; submit a new request") + } + out, err := e.MCP.CallToolRevision(ctx, p.Server, p.Tool, p.Arguments, p.Revision) if err != nil { return "", err } diff --git a/internal/manager/service/aiops/mcp_refresh.go b/internal/manager/service/aiops/mcp_refresh.go index 952cb2f38..bd40e1da0 100644 --- a/internal/manager/service/aiops/mcp_refresh.go +++ b/internal/manager/service/aiops/mcp_refresh.go @@ -63,20 +63,12 @@ func (r *MCPToolRefresher) LoadEnabled(ctx context.Context, mcpUC *managerbizmcp if servers, err := mcpUC.ListEnabled(ctx); err == nil { for _, srv := range servers { connCtx, cancel := context.WithTimeout(ctx, 15*time.Second) - cli, berr := mcpUC.BuildClient(connCtx, srv) - if berr == nil { - berr = cli.Initialize(connCtx) - } - var mtools []mcpclient.Tool - if berr == nil { - mtools, berr = cli.ListTools(connCtx) - } + mtools, berr := mcpUC.TestConnection(connCtx, srv.ID) cancel() if berr != nil { r.log.Warn("mcp: connect failed, skipping server", slog.String("server", srv.Name), slog.Any("err", berr)) continue } - r.Refresh(ctx, srv.Name, srv, mtools) r.log.Info("mcp: server connected", slog.String("server", srv.Name), slog.Int("tools", len(mtools)), slog.Bool("trusted", srv.Trusted)) } } else { diff --git a/internal/manager/service/aiops/mcp_watch.go b/internal/manager/service/aiops/mcp_watch.go new file mode 100644 index 000000000..f4097819a --- /dev/null +++ b/internal/manager/service/aiops/mcp_watch.go @@ -0,0 +1,130 @@ +package aiops + +import ( + "context" + "errors" + "log/slog" + "sync" + "time" + + managerbizmcp "github.com/ongridio/ongrid/internal/manager/biz/mcp" +) + +// RunRefresh watches registered servers until shutdown. Each server has one +// worker; periodic discovery also covers servers without notification support. +// Registration CRUD still revokes the execution catalog synchronously. +func (r *MCPToolRefresher) RunRefresh(ctx context.Context, uc *managerbizmcp.Usecase, interval time.Duration) { + if interval <= 0 { + return + } + type watcher struct { + revision string + cancel context.CancelFunc + } + watchers := map[uint64]watcher{} + var wg sync.WaitGroup + defer func() { + for _, w := range watchers { + w.cancel() + } + wg.Wait() + }() + reconcile := func() { + servers, err := uc.ListEnabled(ctx) + if err != nil { + r.log.Warn("list MCP refresh targets", slog.Any("err", err)) + return + } + present := map[uint64]bool{} + for _, server := range servers { + present[server.ID] = true + revision, err := uc.Revision(ctx, server.Name) + if err != nil { + if w, ok := watchers[server.ID]; ok { + w.cancel() + delete(watchers, server.ID) + } + continue + } + if w, ok := watchers[server.ID]; ok { + if w.revision == revision { + continue + } + w.cancel() + } + child, cancel := context.WithCancel(ctx) + watchers[server.ID] = watcher{revision: revision, cancel: cancel} + id := server.ID + wg.Add(1) + go func() { + defer wg.Done() + defer func() { + if recovered := recover(); recovered != nil { + r.log.Error("MCP refresh worker panic", slog.Uint64("server_id", id), slog.Any("panic", recovered)) + } + }() + r.watchServer(child, uc, id, interval) + }() + } + for id, w := range watchers { + if !present[id] { + w.cancel() + delete(watchers, id) + } + } + } + reconcile() + timer := time.NewTicker(interval) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + reconcile() + } + } +} + +func (r *MCPToolRefresher) watchServer(ctx context.Context, uc *managerbizmcp.Usecase, id uint64, interval time.Duration) { + refresh := func() { + probe, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + if _, err := uc.RefreshConnection(probe, id); err != nil && !errors.Is(err, context.Canceled) { + r.log.Warn("MCP catalog refresh failed", slog.Uint64("server_id", id), slog.Any("err", err)) + } + } + for ctx.Err() == nil { + cycle := time.NewTimer(interval) + refresh() + server, err := uc.Get(ctx, id) + if err == nil && server.Enabled { + watch, cancel := context.WithTimeout(ctx, interval) + cli, err := uc.BuildClient(watch, server) + if err == nil { + err = cli.Initialize(watch) + } + if err == nil { + // Coalesce notification bursts: only one sequential discovery can be + // active in this worker. Further events observe the resulting catalog. + last := time.Time{} + err = cli.WatchToolsChanged(watch, func() { + if time.Since(last) >= time.Second { + last = time.Now() + refresh() + } + }) + } + cancel() + if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + r.log.Debug("MCP notification watch ended; polling remains enabled", slog.Uint64("server_id", id), slog.Any("err", err)) + } + } + select { + case <-ctx.Done(): + cycle.Stop() + return + case <-cycle.C: + } + } +} diff --git a/internal/manager/service/aiops/runtime_router.go b/internal/manager/service/aiops/runtime_router.go index 174ac2e10..64c51702f 100644 --- a/internal/manager/service/aiops/runtime_router.go +++ b/internal/manager/service/aiops/runtime_router.go @@ -31,15 +31,24 @@ type RuntimeSessionReader interface { } // RuntimeRouter owns execution dispatch, never tool or persona policy. -// Its registry and default are immutable; historical-session classification -// is still required before exposing a live default-version setting. +// Its engine registry is immutable. New-session policy can follow a persisted +// setting, while historical unversioned sessions retain the initial default. type RuntimeRouter struct { - sessions RuntimeSessionReader - engines map[contract.Version]RuntimeEngine - defaultVersion contract.Version + sessions RuntimeSessionReader + engines map[contract.Version]RuntimeEngine + defaultVersion contract.Version + defaultProvider func(context.Context) (contract.Version, error) } -func NewRuntimeRouter(sessions RuntimeSessionReader, defaultVersion contract.Version, engines map[contract.Version]RuntimeEngine) (*RuntimeRouter, error) { +type RuntimeRouterOption func(*RuntimeRouter) + +// WithRuntimeDefaultProvider configures new-session policy at construction time. +// An empty returned version follows the release default; errors fail closed. +func WithRuntimeDefaultProvider(provider func(context.Context) (contract.Version, error)) RuntimeRouterOption { + return func(r *RuntimeRouter) { r.defaultProvider = provider } +} + +func NewRuntimeRouter(sessions RuntimeSessionReader, defaultVersion contract.Version, engines map[contract.Version]RuntimeEngine, options ...RuntimeRouterOption) (*RuntimeRouter, error) { copy := make(map[contract.Version]RuntimeEngine, len(engines)) for version, engine := range engines { if version != contract.V1 && version != contract.V2 && version != contract.V3 { @@ -53,7 +62,11 @@ func NewRuntimeRouter(sessions RuntimeSessionReader, defaultVersion contract.Ver if copy[defaultVersion] == nil { return nil, fmt.Errorf("%w: default runtime %q unavailable", errs.ErrNotWiredYet, defaultVersion) } - return &RuntimeRouter{sessions: sessions, engines: copy, defaultVersion: defaultVersion}, nil + router := &RuntimeRouter{sessions: sessions, engines: copy, defaultVersion: defaultVersion} + for _, option := range options { + option(router) + } + return router, nil } func (r *RuntimeRouter) resolve(ctx context.Context) (context.Context, RuntimeEngine, contract.Version, error) { @@ -61,6 +74,13 @@ func (r *RuntimeRouter) resolve(ctx context.Context) (context.Context, RuntimeEn return ctx, nil, "", errs.ErrNotWiredYet } version := contract.VersionFromContext(ctx) + if version == "" && r.defaultProvider != nil { + var err error + version, err = r.defaultProvider(ctx) + if err != nil { + return ctx, nil, "", fmt.Errorf("resolve runtime default: %w", err) + } + } if version == "" { version = r.defaultVersion } @@ -77,8 +97,8 @@ func (r *RuntimeRouter) SessionVersion(ctx context.Context) (contract.Version, e return version, err } -// sessionContext honors durable ownership. Empty historical values retain -// the current single-version behavior until a provenance migration is ready. +// sessionContext honors durable ownership. Unclassified historical sessions +// remain on the initial runtime, independent of new-session settings. func (r *RuntimeRouter) sessionContext(ctx context.Context, id string) (context.Context, error) { if r == nil { return ctx, errs.ErrNotWiredYet @@ -93,10 +113,10 @@ func (r *RuntimeRouter) sessionContext(ctx context.Context, id string) (context. if session == nil { return ctx, errs.ErrNotFound } - if session.RuntimeVersion == "" { - return ctx, nil - } version := contract.Version(session.RuntimeVersion) + if version == "" { + version = r.defaultVersion + } if requested := contract.VersionFromContext(ctx); requested != "" && requested != version { return ctx, fmt.Errorf("%w: session runtime mismatch", errs.ErrInvalid) } diff --git a/internal/manager/service/aiops/runtime_router_test.go b/internal/manager/service/aiops/runtime_router_test.go index 919ee51af..9e53f37b0 100644 --- a/internal/manager/service/aiops/runtime_router_test.go +++ b/internal/manager/service/aiops/runtime_router_test.go @@ -290,3 +290,46 @@ func TestCreateSession_PinsValidatedRuntimeDefault(t *testing.T) { t.Fatalf("unavailable version=%v", err) } } + +func TestRuntimeDefaultSettingDoesNotMoveExistingSessions(t *testing.T) { + repo := &memSessions{sessions: map[string]*model.Session{ + "legacy": {ID: "legacy"}, "pinned": {ID: "pinned", RuntimeVersion: "v3"}, + }} + providerCalls := 0 + selected := contract.V2 + providerErr := error(nil) + engine := &routerEngine{handle: func(ctx context.Context, _ *contract.Request) (*contract.Reply, error) { + if contract.VersionFromContext(ctx) != contract.V3 { + t.Fatal("existing session moved") + } + return &contract.Reply{}, nil + }} + router, err := NewRuntimeRouter(repo, contract.V3, map[contract.Version]RuntimeEngine{contract.V3: engine}, + WithRuntimeDefaultProvider(func(context.Context) (contract.Version, error) { providerCalls++; return selected, providerErr })) + if err != nil { + t.Fatal(err) + } + if _, err := router.SessionVersion(t.Context()); !errors.Is(err, errs.ErrNotWiredYet) { + t.Fatalf("unavailable selection: %v", err) + } + providerErr = errors.New("settings unavailable") + if _, err := router.SessionVersion(t.Context()); !errors.Is(err, providerErr) { + t.Fatalf("lost settings error: %v", err) + } + before := providerCalls + for _, id := range []string{"legacy", "pinned"} { + if _, err := router.Handle(t.Context(), &contract.Request{SessionID: id}); err != nil { + t.Fatal(err) + } + } + if providerCalls != before { + t.Fatal("existing sessions consulted new-session policy") + } + providerErr = nil + for _, value := range []contract.Version{"", contract.V3} { + selected = value + if got, err := router.SessionVersion(t.Context()); err != nil || got != contract.V3 { + t.Fatalf("selection %q: %q %v", value, got, err) + } + } +} diff --git a/internal/pkg/mcpclient/client.go b/internal/pkg/mcpclient/client.go index 261268f73..90698c167 100644 --- a/internal/pkg/mcpclient/client.go +++ b/internal/pkg/mcpclient/client.go @@ -22,15 +22,16 @@ import ( ) // ProtocolVersion is the MCP revision ongrid advertises at initialize. -const ProtocolVersion = "2024-11-05" +const ProtocolVersion = "2025-03-26" // Client is an MCP client bound to one server endpoint. type Client struct { - endpoint string - headers map[string]string // static auth headers (credential-injected) - http *http.Client - sessionID string // captured from initialize, echoed afterwards - nextID int + endpoint string + headers map[string]string // static auth headers (credential-injected) + http *http.Client + sessionID string // captured from initialize, echoed afterwards + nextID int + listChanged bool } // NewHTTP builds a Streamable-HTTP MCP client. headers carries any @@ -92,27 +93,59 @@ func (c *Client) Initialize(ctx context.Context) error { "capabilities": map[string]any{}, "clientInfo": map[string]any{"name": "ongrid", "version": "1"}, } - if _, err := c.call(ctx, "initialize", params); err != nil { + raw, err := c.call(ctx, "initialize", params) + if err != nil { return err } + var result struct { + Capabilities struct { + Tools struct { + ListChanged bool `json:"listChanged"` + } `json:"tools"` + } `json:"capabilities"` + } + if err := json.Unmarshal(raw, &result); err != nil { + return fmt.Errorf("decode MCP initialize: %w", err) + } + c.listChanged = result.Capabilities.Tools.ListChanged // initialized is a notification (no id, no response expected). - _ = c.notify(ctx, "notifications/initialized", map[string]any{}) - return nil + return c.notify(ctx, "notifications/initialized", map[string]any{}) } // ListTools returns the server's advertised tools. func (c *Client) ListTools(ctx context.Context) ([]Tool, error) { - raw, err := c.call(ctx, "tools/list", map[string]any{}) - if err != nil { - return nil, err - } - var out struct { - Tools []Tool `json:"tools"` - } - if err := json.Unmarshal(raw, &out); err != nil { - return nil, fmt.Errorf("mcp: decode tools/list: %w", err) + var tools []Tool + cursor := "" + seen := map[string]bool{} + for { + params := map[string]any{} + if cursor != "" { + params["cursor"] = cursor + } + raw, err := c.call(ctx, "tools/list", params) + if err != nil { + return nil, err + } + var page struct { + Tools []Tool `json:"tools"` + Next string `json:"nextCursor"` + } + if err := json.Unmarshal(raw, &page); err != nil { + return nil, fmt.Errorf("mcp: decode tools/list: %w", err) + } + tools = append(tools, page.Tools...) + if len(tools) > 10000 { + return nil, fmt.Errorf("mcp: tool catalog exceeds 10000 entries") + } + if page.Next == "" { + return tools, nil + } + if seen[page.Next] || len(seen) >= 100 { + return nil, fmt.Errorf("mcp: invalid or excessive tools/list pagination") + } + seen[page.Next] = true + cursor = page.Next } - return out.Tools, nil } // CallTool invokes a tool by name with JSON arguments. diff --git a/internal/pkg/mcpclient/notifications.go b/internal/pkg/mcpclient/notifications.go new file mode 100644 index 000000000..37e1ecc68 --- /dev/null +++ b/internal/pkg/mcpclient/notifications.go @@ -0,0 +1,93 @@ +package mcpclient + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +// WatchToolsChanged listens to the optional server-initiated SSE stream after +// Initialize. The caller owns reconnection and polling fallback. It must not +// concurrently call Initialize on this client. +func (c *Client) WatchToolsChanged(ctx context.Context, changed func()) error { + if !c.listChanged { + return nil + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.endpoint, nil) + if err != nil { + return err + } + for k, v := range c.headers { + req.Header.Set(k, v) + } + req.Header.Set("Accept", "text/event-stream") + if c.sessionID != "" { + req.Header.Set("Mcp-Session-Id", c.sessionID) + } + client := *c.http + client.Timeout = 0 // watch lifetime belongs to ctx, not the RPC timeout. + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("MCP notification stream: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusMethodNotAllowed { + return nil + } + if resp.StatusCode != http.StatusOK || !strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") { + return fmt.Errorf("MCP notification stream HTTP %d", resp.StatusCode) + } + scanner := bufio.NewScanner(resp.Body) + scanner.Buffer(make([]byte, 4096), 1<<20) + var data strings.Builder + flush := func() error { + raw := strings.TrimSpace(data.String()) + data.Reset() + if raw == "" { + return nil + } + var messages []json.RawMessage + if strings.HasPrefix(raw, "[") { + if err := json.Unmarshal([]byte(raw), &messages); err != nil { + return err + } + } else { + messages = []json.RawMessage{json.RawMessage(raw)} + } + for _, msg := range messages { + var notification struct { + Method string `json:"method"` + } + if err := json.Unmarshal(msg, ¬ification); err != nil { + return err + } + if notification.Method == "notifications/tools/list_changed" && changed != nil { + changed() + } + } + return nil + } + for scanner.Scan() { + line := scanner.Text() + if line == "" { + if err := flush(); err != nil { + return fmt.Errorf("decode MCP notification: %w", err) + } + continue + } + if strings.HasPrefix(line, "data:") { + if data.Len() > 1<<20 { + return fmt.Errorf("MCP notification exceeds 1 MiB") + } + data.WriteString(strings.TrimSpace(strings.TrimPrefix(line, "data:"))) + data.WriteByte('\n') + } + } + if err := scanner.Err(); err != nil { + return err + } + return flush() +} diff --git a/internal/pkg/mcpclient/notifications_test.go b/internal/pkg/mcpclient/notifications_test.go new file mode 100644 index 000000000..059871eb9 --- /dev/null +++ b/internal/pkg/mcpclient/notifications_test.go @@ -0,0 +1,85 @@ +package mcpclient + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestListPaginationAndToolChangeNotification(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + if r.Header.Get("Mcp-Session-Id") != "session" { + t.Error("watch lost session") + } + w.Header().Set("Content-Type", "text/event-stream") + if _, err := fmt.Fprint(w, "data: {\"jsonrpc\":\"2.0\",\"method\":\"notifications/tools/list_changed\"}\n\n"); err != nil { + t.Error(err) + } + return + } + var req struct { + ID int `json:"id"` + Method string `json:"method"` + Params struct { + Cursor string `json:"cursor"` + } `json:"params"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + switch req.Method { + case "initialize": + w.Header().Set("Mcp-Session-Id", "session") + writeJSONRPC(w, req.ID, map[string]any{"protocolVersion": ProtocolVersion, "capabilities": map[string]any{"tools": map[string]any{"listChanged": true}}}) + case "notifications/initialized": + w.WriteHeader(http.StatusAccepted) + case "tools/list": + if req.Params.Cursor == "" { + writeJSONRPC(w, req.ID, map[string]any{"tools": []Tool{{Name: "first"}}, "nextCursor": "page2"}) + } else { + writeJSONRPC(w, req.ID, map[string]any{"tools": []Tool{{Name: "second"}}}) + } + } + })) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + client := NewHTTP(server.URL, nil, 0) + if err := client.Initialize(ctx); err != nil { + t.Fatal(err) + } + tools, err := client.ListTools(ctx) + if err != nil || len(tools) != 2 || tools[1].Name != "second" { + t.Fatalf("tools=%v err=%v", tools, err) + } + changes := 0 + if err := client.WatchToolsChanged(ctx, func() { changes++ }); err != nil { + t.Fatal(err) + } + if changes != 1 { + t.Fatalf("changes=%d", changes) + } +} + +func TestNotificationWatchCancellation(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + <-r.Context().Done() + })) + defer server.Close() + client := NewHTTP(server.URL, nil, 0) + client.listChanged = true + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := client.WatchToolsChanged(ctx, nil); err == nil { + t.Fatal("watch ignored cancellation") + } +} diff --git a/internal/skill/builtin/web_search.go b/internal/skill/builtin/web_search.go index 59d4f0cef..9cbc9d8ab 100644 --- a/internal/skill/builtin/web_search.go +++ b/internal/skill/builtin/web_search.go @@ -11,7 +11,6 @@ import ( "net/url" "strings" "sync" - "time" "github.com/ongridio/ongrid/internal/skill" ) @@ -156,6 +155,7 @@ type webSearchSkill struct { braveEndpoint string cfgResolver WebSearchConfigResolver httpClient *http.Client + health map[string]searchProviderHealth } // Metadata returns the framework-visible spec. @@ -163,7 +163,7 @@ func (s *webSearchSkill) Metadata() skill.Metadata { return skill.Metadata{ Key: "web_search", Name: "联网搜索", - Description: "Search the public web and return top results with title + url + snippet. Default provider is SearXNG (self-hosted, zero-key); operators may switch to Tavily or Brave Search via Settings → Integrations. When the provider supports it (Tavily) an auto-generated `answer` field is also returned.", + Description: "Search the public web and return top results with title + url + snippet. Check status and engine_failures: failed is not no_results; partial results remain usable. Cite returned URLs, prefer official sources, and use context to disambiguate short acronyms. Do not claim no public information exists after a failed search. Default provider is SearXNG (self-hosted, zero-key); operators may switch to Tavily or Brave Search via Settings → Integrations. When the provider supports it (Tavily) an auto-generated `answer` field is also returned.", Class: skill.ClassSafe, Scope: skill.ScopeManager, Category: "web", @@ -190,7 +190,7 @@ func (s *webSearchSkill) Metadata() skill.Metadata { Desc: "强制使用哪个 provider；留空 = 走系统设置选择", }}, }, - ResultPreview: "{provider, results:[{title,url,snippet,published_date}], answer?, skipped_reason?}", + ResultPreview: "{provider,status:success|partial|no_results|failed,results:[{title,url,snippet,published_date}],engine_failures?,attempts?,skipped_reason?}", } } @@ -216,10 +216,13 @@ type WebSearchResult struct { // providers that support it (Tavily). `skipped_reason` is set when the // skill cannot run — empty results, no error. type webSearchResponse struct { - Provider string `json:"provider"` - Results []WebSearchResult `json:"results"` - Answer string `json:"answer,omitempty"` - SkippedReason string `json:"skipped_reason,omitempty"` + Provider string `json:"provider"` + Results []WebSearchResult `json:"results"` + Answer string `json:"answer,omitempty"` + SkippedReason string `json:"skipped_reason,omitempty"` + Status string `json:"status"` + EngineFailures [][]string `json:"engine_failures,omitempty"` + Attempts []searchAttempt `json:"attempts,omitempty"` } // Execute dispatches to the chosen provider and packages the result. @@ -240,7 +243,7 @@ func (s *webSearchSkill) Execute(ctx context.Context, params json.RawMessage) (j braveURL = "https://api.search.brave.com/res/v1/web/search" } if client == nil { - client = &http.Client{Timeout: 60 * time.Second} + client = s.defaultSearchHTTPClient() } var p webSearchParams @@ -270,16 +273,7 @@ func (s *webSearchSkill) Execute(ctx context.Context, params json.RawMessage) (j provider = providerSearxng } - switch provider { - case providerTavily: - return s.searchTavily(ctx, client, tavilyURL, resolver, p) - case providerBrave: - return s.searchBrave(ctx, client, braveURL, resolver, p) - case providerSearxng: - return s.searchSearxng(ctx, client, resolver, p) - default: - return nil, fmt.Errorf("web_search: unknown provider %q", provider) - } + return s.searchWithPolicy(ctx, client, resolver, provider, tavilyURL, braveURL, p) } // ---- SearXNG sub-fn ------------------------------------------------- @@ -313,6 +307,11 @@ func (s *webSearchSkill) searchSearxng( q.Set("format", "json") q.Set("safesearch", "1") q.Set("pageno", "1") + if policy, ok := resolver.(webSearchPolicyResolver); ok { + if engines := policy.SearxngEngines(ctx); engines != "" { + q.Set("engines", engines) + } + } full := base + "/search?" + q.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, full, nil) @@ -333,19 +332,25 @@ func (s *webSearchSkill) searchSearxng( Results: []WebSearchResult{}, SkippedReason: fmt.Sprintf( "SearXNG 不可达 (%s) — 检查 docker-compose 是否启动 searxng 服务，或在设置 → 集成 → 联网搜索 改为 Tavily / Brave。原始错误: %v", - base, err, + "configured endpoint", safeSearchError(err), ), }) } defer resp.Body.Close() - respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024)) + respBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024+1)) + if readErr != nil { + return nil, fmt.Errorf("web_search: read response: %w", readErr) + } + if len(respBody) > 4*1024*1024 { + return nil, errors.New("web_search: response exceeds 4 MiB") + } if resp.StatusCode < 200 || resp.StatusCode >= 300 { return json.Marshal(webSearchResponse{ Provider: providerSearxng, Results: []WebSearchResult{}, SkippedReason: fmt.Sprintf( - "SearXNG 返回 %d: %s", - resp.StatusCode, truncate(string(respBody), 200), + "SearXNG 返回 HTTP %d", + resp.StatusCode, ), }) } @@ -355,8 +360,9 @@ func (s *webSearchSkill) searchSearxng( return nil, fmt.Errorf("web_search: decode searxng response: %w", err) } out := webSearchResponse{ - Provider: providerSearxng, - Results: make([]WebSearchResult, 0, len(sr.Results)), + Provider: providerSearxng, + Results: make([]WebSearchResult, 0, len(sr.Results)), + EngineFailures: sr.UnresponsiveEngines, } limit := p.MaxResults for i, r := range sr.Results { @@ -375,10 +381,11 @@ func (s *webSearchSkill) searchSearxng( // searxngResponse is the subset of SearXNG's JSON output we consume. // The full upstream shape carries a lot more (suggestions, infoboxes, -// engine timings); we only need results[]. +// engine timings); preserve engine failures as well as results. type searxngResponse struct { - Query string `json:"query"` - Results []struct { + Query string `json:"query"` + UnresponsiveEngines [][]string `json:"unresponsive_engines"` + Results []struct { Title string `json:"title"` URL string `json:"url"` Content string `json:"content"` @@ -426,12 +433,18 @@ func (s *webSearchSkill) searchTavily( resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("web_search: call Tavily: %w", err) + return nil, fmt.Errorf("web_search: Tavily transport: %w", safeSearchError(err)) } defer resp.Body.Close() - respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024)) + respBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024+1)) + if readErr != nil { + return nil, fmt.Errorf("web_search: read response: %w", readErr) + } + if len(respBody) > 4*1024*1024 { + return nil, errors.New("web_search: response exceeds 4 MiB") + } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("web_search: Tavily returned %d: %s", resp.StatusCode, truncate(string(respBody), 200)) + return nil, fmt.Errorf("web_search: Tavily returned HTTP %d", resp.StatusCode) } var tr tavilyResponse @@ -515,12 +528,18 @@ func (s *webSearchSkill) searchBrave( resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("web_search: call Brave: %w", err) + return nil, fmt.Errorf("web_search: Brave transport: %w", safeSearchError(err)) } defer resp.Body.Close() - respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024)) + respBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024+1)) + if readErr != nil { + return nil, fmt.Errorf("web_search: read response: %w", readErr) + } + if len(respBody) > 4*1024*1024 { + return nil, errors.New("web_search: response exceeds 4 MiB") + } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("web_search: Brave returned %d: %s", resp.StatusCode, truncate(string(respBody), 200)) + return nil, fmt.Errorf("web_search: Brave returned HTTP %d", resp.StatusCode) } var br braveResponse diff --git a/internal/skill/builtin/web_search_policy.go b/internal/skill/builtin/web_search_policy.go new file mode 100644 index 000000000..415aaab3d --- /dev/null +++ b/internal/skill/builtin/web_search_policy.go @@ -0,0 +1,231 @@ +package builtin + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +// Optional interface keeps existing integrations compatible. A configured key +// alone never authorizes fallback: the administrator must select the provider. +type webSearchPolicyResolver interface { + FallbackProvider(context.Context) string + SearxngEngines(context.Context) string + SearchTimeouts(context.Context) (time.Duration, time.Duration) +} + +type searchAttempt struct { + Provider string `json:"provider"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + EngineFailures [][]string `json:"engine_failures,omitempty"` +} + +type searchProviderHealth struct { + fingerprint [32]byte + failures int + retryAt time.Time +} + +func safeSearchError(err error) error { + var u *url.Error + if errors.As(err, &u) { + return u.Err + } + return err +} + +func (s *webSearchSkill) searchWithPolicy(ctx context.Context, client *http.Client, resolver WebSearchConfigResolver, primary, tavilyURL, braveURL string, p webSearchParams) (json.RawMessage, error) { + providers := []string{primary} + perAttempt, total := 20*time.Second, 45*time.Second + engines := "" + if policy, ok := resolver.(webSearchPolicyResolver); ok { + fallback := policy.FallbackProvider(ctx) + // An explicit override to a different provider stays pinned. Explicitly + // naming the administrator-selected primary still uses its backup. + if (p.Provider == "" || primary == resolver.Provider(ctx)) && fallback != "" && fallback != primary { + providers = append(providers, fallback) + } + engines = policy.SearxngEngines(ctx) + perAttempt, total = policy.SearchTimeouts(ctx) + } + if perAttempt <= 0 || total <= 0 { + return nil, errors.New("web_search: invalid timeout policy") + } + for _, provider := range providers { + if provider != providerSearxng && provider != providerTavily && provider != providerBrave { + return nil, fmt.Errorf("web_search: unknown provider %q", provider) + } + } + ctx, cancel := context.WithTimeout(ctx, total) + defer cancel() + attempts := make([]searchAttempt, 0, len(providers)) + var last webSearchResponse + var lastErr error + var empty *webSearchResponse + for _, provider := range providers { + if err := ctx.Err(); err != nil { + return nil, err + } + config := provider + "|" + engines + "|" + tavilyURL + "|" + braveURL + if resolver != nil { + config += "|" + resolver.SearxngURL(ctx) + "|" + resolver.TavilyAPIKey(ctx) + "|" + resolver.BraveAPIKey(ctx) + } + fingerprint := sha256.Sum256([]byte(config)) + if !s.allowSearch(provider, fingerprint, time.Now()) { + attempts = append(attempts, searchAttempt{Provider: provider, Status: "failed", Reason: "cooldown"}) + last = webSearchResponse{Provider: provider, Status: "failed", Results: []WebSearchResult{}, SkippedReason: "Search provider temporarily unavailable; retry after cooldown."} + continue + } + cctx, stop := context.WithTimeout(ctx, perAttempt) + var raw json.RawMessage + var err error + switch provider { + case providerSearxng: + raw, err = s.searchSearxng(cctx, client, resolver, p) + case providerTavily: + raw, err = s.searchTavily(cctx, client, tavilyURL, resolver, p) + case providerBrave: + raw, err = s.searchBrave(cctx, client, braveURL, resolver, p) + } + stop() + if ctx.Err() != nil { + return nil, ctx.Err() + } + last = webSearchResponse{Provider: provider, Results: []WebSearchResult{}} + if err == nil { + err = json.Unmarshal(raw, &last) + } + lastErr = err + reason := "" + if err != nil { + last.Status = "failed" + reason = "upstream_error" + } else { + last.Results = uniqueSearchResults(last.Results, p.MaxResults) + switch { + case len(last.Results) > 0: + last.Status = "success" + if len(last.EngineFailures) > 0 { + last.Status = "partial" + } + case last.SkippedReason != "" || len(last.EngineFailures) > 0: + last.Status = "failed" + reason = "upstream_unavailable" + if last.SkippedReason == "" { + last.SkippedReason = "Search engines failed; empty results do not mean no matching information exists." + } + default: + last.Status = "no_results" + } + } + s.recordSearch(provider, fingerprint, last.Status == "failed", time.Now()) + attempts = append(attempts, searchAttempt{Provider: provider, Status: last.Status, Reason: reason, EngineFailures: last.EngineFailures}) + if last.Status == "success" || last.Status == "partial" { + if len(attempts) > 1 { + last.Status = "partial" + } + last.Attempts = attempts + return json.Marshal(last) + } + if last.Status == "no_results" { + cp := last + empty = &cp + } + } + // A successful empty response is distinct from a total outage, even if the + // optional cross-check failed. Attempts retain that limitation for the model. + if empty != nil { + last = *empty + lastErr = nil + } + last.Attempts = attempts + if lastErr != nil { + return nil, fmt.Errorf("web_search: configured providers failed: %w", lastErr) + } + return json.Marshal(last) +} + +func (s *webSearchSkill) allowSearch(provider string, fingerprint [32]byte, now time.Time) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.health == nil { + s.health = make(map[string]searchProviderHealth) + } + h := s.health[provider] + if h.fingerprint != fingerprint { + s.health[provider] = searchProviderHealth{fingerprint: fingerprint} + return true + } + if !h.retryAt.IsZero() && now.Before(h.retryAt) { + return false + } + // After cooldown admit one probe; concurrent callers use the backup. + if h.failures >= 3 { + h.retryAt = now.Add(time.Minute) + s.health[provider] = h + } + return true +} + +func (s *webSearchSkill) recordSearch(provider string, fingerprint [32]byte, failed bool, now time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + h := s.health[provider] + if h.fingerprint != fingerprint { + return + } + if failed { + h.failures++ + if h.failures >= 3 { + h.retryAt = now.Add(time.Minute) + } + } else { + h.failures = 0 + h.retryAt = time.Time{} + } + s.health[provider] = h +} + +func uniqueSearchResults(in []WebSearchResult, limit int) []WebSearchResult { + out := make([]WebSearchResult, 0, len(in)) + seen := make(map[string]bool) + for _, result := range in { + u, err := url.Parse(result.URL) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" || strings.TrimSpace(result.Title) == "" { + continue + } + u.Fragment = "" + key := u.String() + if seen[key] { + continue + } + seen[key] = true + out = append(out, result) + if len(out) >= limit { + break + } + } + return out +} + +// Shared per-skill transport separates connection establishment from the +// configurable request and total budgets, and reuses idle connections. +func (s *webSearchSkill) defaultSearchHTTPClient() *http.Client { + s.mu.Lock() + defer s.mu.Unlock() + if s.httpClient == nil { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DialContext = (&net.Dialer{Timeout: 5 * time.Second, KeepAlive: 30 * time.Second}).DialContext + transport.TLSHandshakeTimeout = 5 * time.Second + s.httpClient = &http.Client{Transport: transport} + } + return s.httpClient +} diff --git a/internal/skill/builtin/web_search_policy_test.go b/internal/skill/builtin/web_search_policy_test.go new file mode 100644 index 000000000..e45a648d8 --- /dev/null +++ b/internal/skill/builtin/web_search_policy_test.go @@ -0,0 +1,159 @@ +package builtin + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +type searchPolicyStub struct { + stubConfigResolver + fallback, engines string + per, total time.Duration +} + +func (p searchPolicyStub) FallbackProvider(context.Context) string { return p.fallback } +func (p searchPolicyStub) SearxngEngines(context.Context) string { return p.engines } +func (p searchPolicyStub) SearchTimeouts(context.Context) (time.Duration, time.Duration) { + if p.per == 0 { + return time.Second, 3 * time.Second + } + return p.per, p.total +} + +func TestSearchEngineFailureClassification(t *testing.T) { + for _, tc := range []struct { + name, body, status string + count int + }{ + {"all failed", `{"results":[],"unresponsive_engines":[["bing","timeout"],["baidu","CAPTCHA"]]}`, "failed", 0}, + {"empty", `{"results":[]}`, "no_results", 0}, + {"partial", `{"results":[{"title":"official","url":"https://example.com/a"}],"unresponsive_engines":[["bing","timeout"]]}`, "partial", 1}, + {"deduplicate", `{"results":[{"title":"a","url":"https://example.com/a"},{"title":"a","url":"https://example.com/a#x"},{"title":"unsafe","url":"javascript:alert(1)"}]}`, "success", 1}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("engines") != "360search,sogou" { + t.Error("engine selection lost") + } + fmt.Fprint(w, tc.body) + })) + defer srv.Close() + s := &webSearchSkill{cfgResolver: searchPolicyStub{stubConfigResolver: stubConfigResolver{searxngURL: srv.URL}, engines: "360search,sogou"}} + raw, err := s.Execute(t.Context(), json.RawMessage(`{"query":"test"}`)) + if err != nil { + t.Fatal(err) + } + var out webSearchResponse + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatal(err) + } + if out.Status != tc.status || len(out.Results) != tc.count { + t.Fatalf("%s", raw) + } + if tc.status == "failed" && (len(out.EngineFailures) != 2 || out.SkippedReason == "") { + t.Fatalf("failure hidden: %s", raw) + } + }) + } +} + +func TestSearchFallbackIsExplicitAndBounded(t *testing.T) { + for _, enable := range []bool{false, true} { + t.Run(fmt.Sprint(enable), func(t *testing.T) { + var primaryCalls, backupCalls atomic.Int32 + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + primaryCalls.Add(1) + fmt.Fprint(w, `{"results":[],"unresponsive_engines":[["bing","timeout"]]}`) + })) + defer primary.Close() + backup := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + backupCalls.Add(1) + fmt.Fprint(w, `{"web":{"results":[{"title":"official","url":"https://example.com/answer"}]}}`) + })) + defer backup.Close() + policy := searchPolicyStub{stubConfigResolver: stubConfigResolver{searxngURL: primary.URL, braveKey: "configured-key"}} + if enable { + policy.fallback = "brave" + } + s := &webSearchSkill{cfgResolver: policy, braveEndpoint: backup.URL} + raw, err := s.Execute(t.Context(), json.RawMessage(`{"query":"test","provider":"searxng"}`)) + if err != nil { + t.Fatal(err) + } + var out webSearchResponse + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatal(err) + } + if primaryCalls.Load() != 1 { + t.Fatal("primary retry") + } + if enable { + if backupCalls.Load() != 1 || out.Provider != "brave" || out.Status != "partial" || len(out.Attempts) != 2 { + t.Fatalf("fallback: %s", raw) + } + } else if backupCalls.Load() != 0 || out.Status != "failed" { + t.Fatal("unapproved fallback") + } + }) + } +} + +func TestSearchAttemptTimeoutUsesBackupAndCancellationStops(t *testing.T) { + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() })) + defer primary.Close() + var calls atomic.Int32 + backup := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + fmt.Fprint(w, `{"web":{"results":[{"title":"ok","url":"https://example.com"}]}}`) + })) + defer backup.Close() + s := &webSearchSkill{braveEndpoint: backup.URL, cfgResolver: searchPolicyStub{stubConfigResolver: stubConfigResolver{searxngURL: primary.URL, braveKey: "key"}, fallback: "brave", per: 20 * time.Millisecond, total: time.Second}} + raw, err := s.Execute(t.Context(), json.RawMessage(`{"query":"test"}`)) + if err != nil || !strings.Contains(string(raw), `"provider":"brave"`) { + t.Fatalf("timeout fallback: %s %v", raw, err) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := s.Execute(ctx, json.RawMessage(`{"query":"test"}`)); err != context.Canceled { + t.Fatalf("cancel: %v", err) + } + if calls.Load() != 1 { + t.Fatal("called backup after cancellation") + } +} + +func TestSearchCircuitCooldownRecoveryAndConfigurationChange(t *testing.T) { + s := &webSearchSkill{} + fingerprint := [32]byte{1} + now := time.Now() + for i := 0; i < 3; i++ { + if !s.allowSearch("brave", fingerprint, now) { + t.Fatal("closed too soon") + } + s.recordSearch("brave", fingerprint, true, now) + } + if s.allowSearch("brave", fingerprint, now) { + t.Fatal("failed provider not suspended") + } + later := now.Add(61 * time.Second) + if !s.allowSearch("brave", fingerprint, later) || s.allowSearch("brave", fingerprint, later) { + t.Fatal("must admit one recovery probe") + } + s.recordSearch("brave", fingerprint, false, later) + if !s.allowSearch("brave", fingerprint, later) { + t.Fatal("recovery not accepted") + } + for i := 0; i < 3; i++ { + s.recordSearch("brave", fingerprint, true, later) + } + if !s.allowSearch("brave", [32]byte{2}, later) { + t.Fatal("new credentials inherited failure state") + } +} diff --git a/web/e2e/agent-runtime-settings.spec.ts b/web/e2e/agent-runtime-settings.spec.ts new file mode 100644 index 000000000..4957aec2e --- /dev/null +++ b/web/e2e/agent-runtime-settings.spec.ts @@ -0,0 +1,49 @@ +import { expect, test } from '@playwright/test'; + +test.use({ storageState: { cookies: [], origins: [] } }); + +for (const theme of ['light', 'dark']) { + for (const width of [390, 768, 1280, 1920]) { + test(`runtime selector ${theme} ${width}`, async ({ page }) => { + const english = width === 768 || width === 1920; + let selection = 'release'; + await page.setViewportSize({ width, height: 1000 }); + await page.addInitScript(({ theme, english }) => { + localStorage.setItem('ongrid-theme-preference', theme); + localStorage.setItem('ongrid-locale', english ? 'en-US' : 'zh-CN'); + localStorage.setItem('ongrid.auth', JSON.stringify({ state: { + token: 'e2e-token', role: 'admin', email: 'e2e@example.com', refreshToken: null, + }, version: 0 })); + }, { theme, english }); + await page.route('**/api/v1/**', async (route) => { + const path = new URL(route.request().url()).pathname; + if (path === '/api/v1/system-settings/agent/runtime_version') { + selection = route.request().postDataJSON().value; + await route.fulfill({ json: {} }); + } else if (path === '/api/v1/system-settings') { + await route.fulfill({ json: { items: [{ category: 'agent', key: 'runtime_version', value: selection }], total: 1 } }); + } else { + await route.fulfill({ json: { items: [], total: 0 } }); + } + }); + await page.goto('/settings/agent'); + const selector = page.getByRole('combobox', { name: english ? 'Runtime version' : '运行时版本', exact: true }); + await expect(selector).toBeEnabled(); + await selector.scrollIntoViewIfNeeded(); + await selector.click(); + await expect(page.getByRole('option', { name: /v2/ })).toBeDisabled(); + await expect(page.getByRole('option', { name: /v1/ })).toBeDisabled(); + await page.screenshot({ path: `/tmp/ongrid-runtime-${theme}-${width}.png`, fullPage: true }); + await page.getByRole('option', { name: english ? 'v3 · Pinned version' : 'v3 · 固定版本', exact: true }).click(); + await page.getByRole('button', { name: english ? 'Save runtime' : '保存版本', exact: true }).click(); + await expect(page.getByRole('status')).toContainText(english ? 'Saved.' : '已保存'); + await page.reload(); + await expect(selector).toContainText(english ? 'v3 · Pinned version' : 'v3 · 固定版本'); + const box = await selector.boundingBox(); + expect(box).not.toBeNull(); + expect(box!.x).toBeGreaterThanOrEqual(0); + expect(box!.x + box!.width).toBeLessThanOrEqual(width); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + }); + } +} diff --git a/web/e2e/web-search-settings.spec.ts b/web/e2e/web-search-settings.spec.ts new file mode 100644 index 000000000..a474b14e5 --- /dev/null +++ b/web/e2e/web-search-settings.spec.ts @@ -0,0 +1,67 @@ +import { expect, test } from '@playwright/test'; + +test.use({ storageState: { cookies: [], origins: [] } }); +for (const theme of ['light', 'dark']) { + for (const width of [390, 768, 1280, 1920]) { + test(`search settings ${theme} ${width}`, async ({ page }) => { + const en = width === 768 || width === 1920; + let saved: Record = {}; + let probeFails = false; + await page.setViewportSize({ width, height: 1000 }); + await page.addInitScript(({ theme, en }) => { + localStorage.setItem('ongrid-theme-preference', theme); + localStorage.setItem('ongrid-locale', en ? 'en-US' : 'zh-CN'); + localStorage.setItem('ongrid.auth', JSON.stringify({ state: { token: 'test-token', role: 'admin', email: 'e2e@example.com', refreshToken: null }, version: 0 })); + }, { theme, en }); + await page.route('**/api/v1/**', async (route) => { + const url = new URL(route.request().url()); + if (url.pathname === '/api/v1/system-settings') { + const items = url.searchParams.get('category') === 'websearch' + ? Object.entries(saved).map(([key, value]) => ({ category: 'websearch', key, value })) : []; + return route.fulfill({ json: { items, total: items.length } }); + } + if (url.pathname.endsWith('/reveal')) return route.fulfill({ json: { value: 'test-only-key' } }); + if (url.pathname === '/api/v1/integrations/websearch/test') { + const body = route.request().postDataJSON(); + expect(body.fallback_provider).toBe('brave'); + expect(body.searxng_engines).toBe('360search,sogou'); + return route.fulfill(probeFails ? { status: 502, json: { error: 'CAPTCHA: engine unavailable' } } : { json: { status: 'ok', provider: 'searxng', sample: 'Kubernetes' } }); + } + if (url.pathname === '/api/v1/integrations/websearch/validate-and-save') { + saved = route.request().postDataJSON(); + return route.fulfill({ json: { status: 'saved', provider: 'searxng', sample: 'Kubernetes' } }); + } + await route.fulfill({ json: { items: [], total: 0 } }); + }); + await page.goto('/settings/integrations'); + const card = page.locator('section').filter({ has: page.getByRole('heading', { name: en ? 'Web search (web_search skill)' : '联网搜索（web_search skill）', exact: true }) }); + const fallback = card.getByRole('combobox', { name: en ? 'Backup search provider' : '备用搜索来源' }); + await fallback.click(); + await page.getByRole('option', { name: 'Brave', exact: true }).click(); + await expect(card.getByLabel('Brave API Key', { exact: false })).toBeVisible(); + await card.getByLabel('Brave API Key', { exact: false }).fill('test-only-key'); + await card.getByLabel(en ? 'SearXNG engines' : 'SearXNG 引擎', { exact: false }).fill('360search,sogou'); + const probe = card.getByRole('button', { name: en ? 'Test connection' : '测试连接', exact: true }); + const save = card.getByRole('button', { name: en ? 'Save' : '保存', exact: true }); + await expect(save).toBeDisabled(); + await probe.click(); + await expect(save).toBeEnabled(); + await save.click(); + await expect.poll(() => saved.fallback_provider).toBe('brave'); + await expect.poll(() => saved.request_timeout_seconds).toBe('20'); + await expect(card.getByRole('button', { name: en ? 'Saved' : '已保存', exact: true })).toBeVisible(); + await fallback.scrollIntoViewIfNeeded(); + await page.screenshot({ path: `/tmp/ongrid-search-${theme}-${width}-top.png` }); + await card.getByLabel('Brave API Key', { exact: false }).scrollIntoViewIfNeeded(); + await page.screenshot({ path: `/tmp/ongrid-search-${theme}-${width}-bottom.png` }); + const box = await card.boundingBox(); + expect(box!.x).toBeGreaterThanOrEqual(0); + expect(box!.x + box!.width).toBeLessThanOrEqual(width); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + probeFails = true; + await probe.click(); + await expect(card.getByText('✗ CAPTCHA: engine unavailable')).toBeVisible(); + await expect(card.getByRole('button', { name: en ? 'Saved' : '已保存', exact: true })).toBeDisabled(); + }); + } +} diff --git a/web/src/api/settings.ts b/web/src/api/settings.ts index f373465e4..c8311653b 100644 --- a/web/src/api/settings.ts +++ b/web/src/api/settings.ts @@ -142,6 +142,10 @@ export type WebSearchProbeResult = { }; export type WebSearchConfigurationProbeInput = { + fallback_provider?: string; + searxng_engines?: string; + request_timeout_seconds?: string; + total_timeout_seconds?: string; provider: string; searxng_url: string; tavily_api_key: string; @@ -203,3 +207,7 @@ export function saveLLMConfiguration( ): Promise { return request('POST', '/integrations/llm/validate-and-save', input); } + +export function saveWebSearchConfiguration(input: WebSearchConfigurationProbeInput): Promise { + return request('POST', '/integrations/websearch/validate-and-save', input); +} diff --git a/web/src/pages/SettingsLayout.tsx b/web/src/pages/SettingsLayout.tsx index caeb81942..6af095221 100644 --- a/web/src/pages/SettingsLayout.tsx +++ b/web/src/pages/SettingsLayout.tsx @@ -50,7 +50,7 @@ const RAIL_ITEMS: RailItem[] = [ // the previous /communications and /bots paths live in App.tsx. { to: 'notifications', icon: Bell, labelZh: '通知', labelEn: 'Notifications', hintZh: '飞书 / 钉钉 / 企业微信 / Slack / Telegram / Webhook（告警推送）', hintEn: 'Slack / Telegram / Larksuite / DingTalk / WeCom / Webhook (alert delivery)' }, { to: 'channels', icon: MessagesSquare, labelZh: 'IM', labelEn: 'Channels', hintZh: '飞书 / 钉钉 / Telegram / Slack 双向多轮', hintEn: 'Slack / Telegram / Larksuite / DingTalk — two-way multi-turn' }, - { to: 'agent', icon: Bot, labelZh: '助理', labelEn: 'Agent', hintZh: '输出语言、写操作与超时', hintEn: 'Output language, writes, and timeouts' }, + { to: 'agent', icon: Bot, labelZh: '助理', labelEn: 'Agent', hintZh: '运行时版本、语言与执行权限', hintEn: 'Runtime version, language, and permissions' }, { to: 'preferences', icon: Gauge, labelZh: '偏好', labelEn: 'Preferences', hintZh: '默认时间窗 / 自动刷新', hintEn: 'Default time window / auto-refresh' }, { to: 'about', icon: Info, labelZh: '关于', labelEn: 'About', hintZh: '版本 / GitHub / 许可证', hintEn: 'Version / GitHub / license' }, ]; diff --git a/web/src/pages/settings/Agent.test.tsx b/web/src/pages/settings/Agent.test.tsx index f8e7b1e4a..1f36029f2 100644 --- a/web/src/pages/settings/Agent.test.tsx +++ b/web/src/pages/settings/Agent.test.tsx @@ -127,3 +127,40 @@ describe('SettingsAgent output language', () => { await waitFor(() => expect(saved).toEqual({ value: 'zh', sensitive: false })); }); }); + +describe('SettingsAgent runtime selection', () => { + it('persists a pinned runtime and can return to release defaults', async () => { + const saved: unknown[] = []; + server.use( + http.get('/api/v1/system-settings', () => HttpResponse.json({ items: [], total: 0 })), + http.put('/api/v1/system-settings/agent/runtime_version', async ({ request }) => { + saved.push(await request.json()); + return HttpResponse.json({}); + }), + ); + const user = userEvent.setup(); + render(); + await waitFor(() => expect(screen.getByRole('combobox', { name: 'Runtime version' })).toBeEnabled()); + await selectOption(screen.getByRole('combobox', { name: 'Runtime version' }), 'v3 · Pinned version'); + await user.click(screen.getByRole('button', { name: 'Save runtime' })); + await screen.findByText('Saved. Applies to new conversations.'); + expect(saved).toEqual([{ value: 'v3', sensitive: false }]); + await selectOption(screen.getByRole('combobox', { name: 'Runtime version' }), 'Follow release default (currently v3)'); + await user.click(screen.getByRole('button', { name: 'Save runtime' })); + await waitFor(() => expect(saved).toEqual([{ value: 'v3', sensitive: false }, { value: 'release', sensitive: false }])); + }); + + it('retains the selection and shows an error when saving fails', async () => { + server.use( + http.get('/api/v1/system-settings', () => HttpResponse.json({ items: [{ key: 'runtime_version', value: 'v3' }], total: 1 })), + http.put('/api/v1/system-settings/agent/runtime_version', () => HttpResponse.json({ error: 'Runtime save failed' }, { status: 500 })), + ); + const user = userEvent.setup(); + render(); + await waitFor(() => expect(screen.getByRole('combobox', { name: 'Runtime version' })).toHaveTextContent('v3 · Pinned version')); + await selectOption(screen.getByRole('combobox', { name: 'Runtime version' }), 'Follow release default (currently v3)'); + await user.click(screen.getByRole('button', { name: 'Save runtime' })); + await screen.findByText('Runtime save failed'); + expect(screen.getByRole('button', { name: 'Save runtime' })).toBeEnabled(); + }); +}); diff --git a/web/src/pages/settings/Agent.tsx b/web/src/pages/settings/Agent.tsx index 1a5799cd7..c332dae10 100644 --- a/web/src/pages/settings/Agent.tsx +++ b/web/src/pages/settings/Agent.tsx @@ -19,6 +19,7 @@ const CATEGORY = 'agent'; const KEY = 'write_enabled'; const LLM_TIMEOUT_KEY = 'llm_timeout_seconds'; const OUTPUT_LOCALE_KEY = 'output_locale'; +const RUNTIME_VERSION_KEY = 'runtime_version'; const DEFAULT_LLM_TIMEOUT_SECONDS = 120; const MIN_LLM_TIMEOUT_SECONDS = 30; const MAX_LLM_TIMEOUT_SECONDS = 900; @@ -28,6 +29,11 @@ export default function SettingsAgent() { const [writeEnabled, setWriteEnabled] = useState(false); const [llmTimeoutSeconds, setLLMTimeoutSeconds] = useState(String(DEFAULT_LLM_TIMEOUT_SECONDS)); const [outputLocale, setOutputLocale] = useState<'' | 'zh' | 'en'>(''); + const [runtimeVersion, setRuntimeVersion] = useState('release'); + const [savedRuntimeVersion, setSavedRuntimeVersion] = useState('release'); + const [savingRuntime, setSavingRuntime] = useState(false); + const [runtimeSaved, setRuntimeSaved] = useState(false); + const [loadFailed, setLoadFailed] = useState(false); const [loading, setLoading] = useState(true); const [saving, setSaving] = useState(false); const [savingTimeout, setSavingTimeout] = useState(false); @@ -39,6 +45,10 @@ export default function SettingsAgent() { const res = await listSettings(CATEGORY); const row = res.items.find((i) => i.key === KEY); const timeoutRow = res.items.find((i) => i.key === LLM_TIMEOUT_KEY); + const runtimeRow = res.items.find((i) => i.key === RUNTIME_VERSION_KEY); + setRuntimeVersion(runtimeRow?.value ?? 'release'); + setSavedRuntimeVersion(runtimeRow?.value ?? 'release'); + setLoadFailed(false); const localeRow = res.items.find((i) => i.key === OUTPUT_LOCALE_KEY); // Missing row → server default is DISABLED (fail-safe). setWriteEnabled(row ? row.value === 'true' : false); @@ -47,6 +57,7 @@ export default function SettingsAgent() { setOutputLocale(localeRow?.value === 'zh' || localeRow?.value === 'en' ? localeRow.value : ''); setErr(null); } catch (e) { + setLoadFailed(true); setErr(e instanceof Error ? e.message : String(e)); } finally { setLoading(false); @@ -110,6 +121,21 @@ export default function SettingsAgent() { } }, [outputLocale]); + const onSaveRuntime = async () => { + setSavingRuntime(true); + setRuntimeSaved(false); + setErr(null); + try { + await setSetting(CATEGORY, RUNTIME_VERSION_KEY, runtimeVersion, false); + setSavedRuntimeVersion(runtimeVersion); + setRuntimeSaved(true); + } catch (e) { + setErr(e instanceof Error ? e.message : String(e)); + } finally { + setSavingRuntime(false); + } + }; + return (
{err &&
{err}
} @@ -161,6 +187,50 @@ export default function SettingsAgent() {

+ +
+ +

{tr('Agent 运行时版本', 'Agent runtime version')}

+
+

+ {tr('选择所有用户新会话使用的 Agent 运行时。已有会话及其子任务继续使用原来的版本。', + 'Choose the Agent runtime for new conversations for all users. Existing conversations and their workers retain their original runtime.')} +

+ + +
+ + + {runtimeSaved ? tr('已保存，对新会话生效', 'Saved. Applies to new conversations.') : tr('无需重启', 'No restart required')} + +
+

+ {tr('当前构建仅接入 v3。v1、v2 源码已隔离，完成执行适配前不可选择。跟随 Release 默认会在升级后使用该发行版的默认运行时。', + 'This build runs v3 only. v1 and v2 source is isolated, but execution adapters are not available yet. Following the release default uses the default runtime shipped with each upgrade.')} +

+ +
diff --git a/web/src/pages/settings/Integrations.tsx b/web/src/pages/settings/Integrations.tsx index e9c5597ae..252d2f9fd 100644 --- a/web/src/pages/settings/Integrations.tsx +++ b/web/src/pages/settings/Integrations.tsx @@ -1,4 +1,4 @@ -import { Label, Input, Radio } from '@/components/ui'; +import { Label, Input, Radio, Select } from '@/components/ui'; import { Tabs, TabsList, TabsTrigger, TabsContent } from '@/components/ui/Tabs'; import { Checkbox } from '@/components/ui/Checkbox'; import { useCallback, useEffect, useRef, useState, type ReactNode } from 'react'; @@ -39,6 +39,7 @@ import { testLokiConnection, testTempoConnection, testWebSearchConnection, + saveWebSearchConfiguration, type SystemSetting, type GrafanaSyncResult, } from '@/api/settings'; @@ -1948,6 +1949,10 @@ function PluginCountLine({ label, count }: { label: string; count: number | null // a 1-result probe to the unsaved draft before the save button is enabled. type WebSearchForm = { + fallback_provider: string; + searxng_engines: string; + request_timeout_seconds: string; + total_timeout_seconds: string; provider: string; // "searxng" | "tavily" | "brave" searxng_url: string; tavily_api_key: string; @@ -1955,6 +1960,7 @@ type WebSearchForm = { }; const WEBSEARCH_KEYS: (keyof WebSearchForm)[] = [ + 'fallback_provider', 'searxng_engines', 'request_timeout_seconds', 'total_timeout_seconds', 'provider', 'searxng_url', 'tavily_api_key', @@ -1965,6 +1971,10 @@ const WEBSEARCH_SENSITIVE: Set = new Set([ 'brave_api_key', ]); const emptyWebSearchForm: WebSearchForm = { + fallback_provider: '', + searxng_engines: '', + request_timeout_seconds: '20', + total_timeout_seconds: '45', provider: 'searxng', searxng_url: 'http://searxng:8080', tavily_api_key: '', @@ -2036,6 +2046,7 @@ function WebSearchCard() { const dirty = WEBSEARCH_KEYS.some((k) => draft[k] !== server[k]); const update = (k: keyof WebSearchForm, v: string) => { + if (saving) return; probeRequest.current += 1; setProbe({ kind: 'idle' }); setSavedOk(false); @@ -2047,10 +2058,7 @@ function WebSearchCard() { setSaving(true); setErr(null); try { - for (const k of WEBSEARCH_KEYS) { - if (draft[k] === server[k]) continue; - await setSetting('websearch', k, draft[k], WEBSEARCH_SENSITIVE.has(k)); - } + await saveWebSearchConfiguration(draft); await refresh(); setSavedOk(true); } catch (e) { @@ -2066,6 +2074,10 @@ function WebSearchCard() { try { const r = await testWebSearchConnection(draft); if (requestID !== probeRequest.current) return; + if (!r.sample) { + setProbe({ kind: 'error', msg: tr('测试未返回有效搜索结果，不能确认来源可用。', 'The probe returned no usable results; availability is unconfirmed.') }); + return; + } setProbe({ kind: 'ok', provider: r.provider, sample: r.sample }); } catch (e) { if (requestID !== probeRequest.current) return; @@ -2113,15 +2125,29 @@ function WebSearchCard() {
) : (
+
+ + +

{tr('主来源失败或返回空结果时，最多向所选备用来源查询一次。查询内容会发送给该来源；API 服务需要先配置密钥。测试会分别验证主备来源。', 'If the primary fails or returns no results, query the selected backup at most once. Queries are sent to this provider; API providers require a configured key. Testing verifies both providers.')}

+ update('request_timeout_seconds', v)} /> + update('total_timeout_seconds', v)} /> + update('searxng_engines', v)} /> +
update('provider', 'searxng')} description={tr( - 'docker-compose 里跟 Loki/Tempo/Prom 一起跑的元搜索聚合器，把查询发到 Bing/DuckDuckGo/Brave/百度/搜狗 等，再合并结果。无 API key、无额度限制。', - 'Meta-search aggregator that runs alongside Loki/Tempo/Prom in docker-compose; fans a query out to Bing/DuckDuckGo/Brave/Baidu/Sogou and merges the results. No API key, no quota.', + '自托管的元搜索服务，无需 API key。引擎可用性受网络、验证码和上游网页变化影响，请通过实际搜索验证。', + 'Self-hosted meta-search without API keys. Engine availability depends on network access, CAPTCHA and upstream changes. Test actual search results before using it.', )} > update('provider', 'tavily')} description={ <> @@ -2173,6 +2200,7 @@ function WebSearchCard() { label="Brave Search" badge={tr('需 API key · 2000 次/月免费', 'API key required · 2000 calls/month free')} checked={draft.provider === 'brave'} + expanded={draft.fallback_provider === 'brave'} onSelect={() => update('provider', 'brave')} description={ <> @@ -2208,7 +2236,7 @@ function WebSearchCard() {
- {checked &&
{children}
} + {(checked || expanded) &&
{children}
}
); } @@ -2304,7 +2334,7 @@ function WebSearchProbeLine({ {tr('· 示例结果：', '· Sample result: ')}{probe.sample} ) : ( - tr(' · 上游返回 0 结果（key 工作正常，但探针 query 没匹配到）', ' · Upstream returned 0 results (key works, but probe query found no matches)') + tr(' · 未验证有效结果', ' · Usable results not verified') )}

); From d9f24e47e039136cb8f665984d6ead6b1e80dba2 Mon Sep 17 00:00:00 2001 From: singchia Date: Sat, 17 Oct 2026 19:51:47 +0800 Subject: [PATCH 24/24] feat(aiops): wire dual runtimes and validate dynamic capability lifecycles Wire v2 routing, runtime settings and version-owned approvals; mark v1 unsupported. Fix MCP discovery wakeups, preserve built-in agents during reload, and reconcile interrupted workers on restart. Add dynamic capability and dual-runtime E2E coverage with test deployment evidence. --- api/manager/setting/v1/setting.proto | 2 +- cmd/ongrid/bootstrap_extensions.go | 88 ++++-- cmd/ongrid/bootstrap_imbridge.go | 3 + cmd/ongrid/bootstrap_runtime.go | 41 ++- cmd/ongrid/runtime_builder.go | 214 +++++++------ cmd/ongrid/runtime_builder_v2.go | 233 ++++++++++++++ docs/rfc/RFC-009-agent-runtime-versions.md | 53 +++- docs/test/e2e-catalog.md | 69 +++- docs/test/runtime-v2v3-deploy-20261017-5.md | 38 +++ docs/test/v3-test-deploy-20261017-4.md | 20 ++ internal/manager/biz/aiops/runtime/README.md | 12 +- .../manager/biz/aiops/runtime/v1/README.md | 4 +- .../manager/biz/aiops/runtime/v2/README.md | 14 +- .../runtime/v2/chatruntime/agent_registry.go | 41 ++- .../v2/chatruntime/agent_registry_test.go | 36 +++ .../aiops/runtime/v2/chatruntime/runtime.go | 230 +++----------- .../v2/chatruntime/runtime_worker_test.go | 18 +- .../runtime/v2/chatruntime/tool_lifecycle.go | 168 ++++++++++ .../v2/chatruntime/tool_lifecycle_test.go | 229 ++++++++++++++ .../aiops/runtime/v2/chatruntime/worker.go | 149 ++------- .../v2/chatruntime/worker_context_test.go | 48 +++ .../runtime/v3/chatruntime/agent_registry.go | 41 ++- .../v3/chatruntime/agent_registry_test.go | 36 +++ internal/manager/biz/mcp/lifecycle_test.go | 65 +++- internal/manager/biz/mcp/usecase.go | 44 ++- internal/manager/biz/setting/agent.go | 4 +- internal/manager/biz/setting/service_test.go | 6 +- .../data/aiops/store/session_recovery.go | 59 ++++ .../data/aiops/store/session_recovery_test.go | 131 ++++++++ .../service/aiops/approval_executors.go | 3 +- internal/manager/service/aiops/mcp_watch.go | 2 + .../service/aiops/runtime_v2_adapter.go | 42 +++ .../service/aiops/runtime_v2_approval_test.go | 131 ++++++++ tests/e2e/assistant_chat_test.go | 9 +- tests/e2e/dynamic_capabilities_test.go | 297 ++++++++++++++++++ tests/e2e/dynamic_skills_test.go | 173 ++++++++++ tests/e2e/mcp_test.go | 6 +- tests/e2e/runtime_versions_test.go | 242 ++++++++++++++ tests/e2e/testenv/fakes.go | 25 +- web/e2e/README.md | 2 + web/e2e/agent-runtime-settings.spec.ts | 8 +- web/e2e/live-navigation.spec.ts | 30 +- web/src/pages/settings/Agent.test.tsx | 8 +- web/src/pages/settings/Agent.tsx | 10 +- 44 files changed, 2587 insertions(+), 497 deletions(-) create mode 100644 cmd/ongrid/runtime_builder_v2.go create mode 100644 docs/test/runtime-v2v3-deploy-20261017-5.md create mode 100644 docs/test/v3-test-deploy-20261017-4.md create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/tool_lifecycle.go create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/tool_lifecycle_test.go create mode 100644 internal/manager/biz/aiops/runtime/v2/chatruntime/worker_context_test.go create mode 100644 internal/manager/data/aiops/store/session_recovery.go create mode 100644 internal/manager/data/aiops/store/session_recovery_test.go create mode 100644 internal/manager/service/aiops/runtime_v2_adapter.go create mode 100644 internal/manager/service/aiops/runtime_v2_approval_test.go create mode 100644 tests/e2e/dynamic_capabilities_test.go create mode 100644 tests/e2e/dynamic_skills_test.go create mode 100644 tests/e2e/runtime_versions_test.go diff --git a/api/manager/setting/v1/setting.proto b/api/manager/setting/v1/setting.proto index f13cb5c50..141a38dfd 100644 --- a/api/manager/setting/v1/setting.proto +++ b/api/manager/setting/v1/setting.proto @@ -6,7 +6,7 @@ option go_package = "github.com/ongridio/ongrid/api/gen/manager/setting/v1;setti // SettingService 管理管理员可编辑的运行时配置。 // agent.runtime_version accepts "release" (follow this release default) or -// "v3" (pin new sessions to v3). v1/v2 are not wired and are rejected. +// "v3" or "v2" (pin new sessions). v1 is no longer supported and is rejected. // Existing sessions retain their runtime; legacy unversioned sessions stay v3. // Read/write via the existing system-settings category/key API. // Legacy platform.auto_apm_enabled values are retained but no longer affect discovery. diff --git a/cmd/ongrid/bootstrap_extensions.go b/cmd/ongrid/bootstrap_extensions.go index a22dcfb34..501cc4417 100644 --- a/cmd/ongrid/bootstrap_extensions.go +++ b/cmd/ongrid/bootstrap_extensions.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "log/slog" "os" "path/filepath" @@ -20,6 +21,7 @@ import ( managermarketplacedata "github.com/ongridio/ongrid/internal/manager/data/marketplace/store" managermcpdata "github.com/ongridio/ongrid/internal/manager/data/mcp/store" pagesdata "github.com/ongridio/ongrid/internal/manager/data/pages" + managermodelmcp "github.com/ongridio/ongrid/internal/manager/model/mcp" managerserverapproval "github.com/ongridio/ongrid/internal/manager/server/approval" managerserverk8s "github.com/ongridio/ongrid/internal/manager/server/k8s" managerservermarketplace "github.com/ongridio/ongrid/internal/manager/server/marketplace" @@ -33,6 +35,7 @@ import ( managersvck8s "github.com/ongridio/ongrid/internal/manager/service/k8s" managersvcskill "github.com/ongridio/ongrid/internal/manager/service/skill" "github.com/ongridio/ongrid/internal/pkg/config" + "github.com/ongridio/ongrid/internal/pkg/mcpclient" "github.com/ongridio/ongrid/internal/pkg/notify" "github.com/ongridio/ongrid/internal/pkg/runner" "github.com/ongridio/ongrid/internal/pkg/secretbox" @@ -64,22 +67,11 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. // marketplace wiring. Install / List / Uninstall on // /v1/marketplace/*. The usecase reloads the chatruntime registries // after every mutation so newly installed skills appear in the next - // chat without a restart. When the graph kernel didn't build (no - // LLM provider configured) the registries are nil, the marketplace - // still works for List/Install but the hot-reload is a no-op until - // the next chatruntime construction picks the disk state up. - var mpSkillReg managerbizmarketplace.SkillRegistry - var skillAssetLeaser managersvcaiops.SkillAssetLeaser - var mpAgentReg managerbizmarketplace.AgentRegistry - if rt := runtime.chatRT; rt != nil { - if sk := rt.SkillRegistry(); sk != nil { - mpSkillReg = sk - skillAssetLeaser = sk - } - if ag := rt.AgentRegistry(); ag != nil { - mpAgentReg = ag - } - } + // chat without a restart. Registries exist independently of runtime assembly. + // Reload both version-local registries, even when LLM assembly is unavailable. + var mpSkillReg managerbizmarketplace.SkillRegistry = registryReloaders{runtime.bootstrapSkillReg, runtime.bootstrapV2SkillReg} + var mpAgentReg managerbizmarketplace.AgentRegistry = registryReloaders{runtime.bootstrapAgentReg, runtime.bootstrapV2AgentReg} + var skillAssetLeaser managersvcaiops.SkillAssetLeaser = runtime.bootstrapSkillReg mpRepo := managermarketplacedata.NewRepo(storage.db) mpDevMode := strings.EqualFold(os.Getenv("ONGRID_MARKETPLACE_DEVMODE"), "true") || os.Getenv("ONGRID_MARKETPLACE_DEVMODE") == "" @@ -142,6 +134,7 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. runtime.humanApprovalBroker.SetUsecase(approvalUC) approvalUC.RegisterExecutor(managersvcaiops.GenericToolApprovalKind, runtime.humanApprovalBroker.Execute) managersvcaiops.RegisterRuntimeApprovalExecutor(approvalUC, managersvcaiops.GenericToolApprovalKind, contract.V3, runtime.humanApprovalBroker.Execute) + managersvcaiops.RegisterRuntimeApprovalExecutor(approvalUC, managersvcaiops.GenericToolApprovalKind, contract.V2, runtime.humanApprovalBroker.Execute) approvalHandler := managerserverapproval.NewHandler(approvalUC) // HLD-017 cloud_bash producer: register the execute-on-approve executor // (resolve the bound credential → inject into the Runner sandbox → run) @@ -204,7 +197,17 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. // live bag here, exactly like the AgentTool trio above. The coordinator // (coordinatorToolNames) and specialist-ops (persona Tools list) filters // both whitelist cloud_bash, so this single append reaches both. + var runtimes []extensionRuntime if runtime.chatRT != nil { + runtime.chatRT.SetCredentialBinder(mpUC) + runtimes = append(runtimes, runtime.chatRT) + } + if runtime.chatV2 != nil { + runtime.chatV2.SetCredentialBinder(mpUC) + runtimes = append(runtimes, runtime.chatV2) + } + var refreshers []*managersvcaiops.MCPToolRefresher + for _, rt := range runtimes { cbDeps := aiopstoolsdec.Deps{ // HLD-021: cloud_bash now BLOCKS in-tool until the human approves // (synchronous propose-confirm), so the per-call timeout must @@ -230,7 +233,7 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. Registerer: reg, HumanApproval: runtime.humanApprovalBroker, } - if err := runtime.chatRT.AppendToolBag([]aiopstoolsbase.BaseTool{ + if err := rt.AppendToolBag([]aiopstoolsbase.BaseTool{ aiopstoolsdec.Wrap(aiopstools.NewBashToolWithProposer(tunnel.fbClient, devices.edgeUC, devices.deviceUC, managersvcaiops.HostBashProposer{Usecase: approvalUC}, log), cbDeps), aiopstoolsdec.Wrap(aiopstools.NewCloudBashTool(managersvcaiops.CloudBashProposer{Usecase: approvalUC}, log), cbDeps), aiopstoolsdec.Wrap(aiopstools.NewInstallSkillTool(managersvcaiops.InstallSkillProposer{Usecase: approvalUC}, log), cbDeps), @@ -240,16 +243,12 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. }); err != nil { log.Error("append runtime extension tools failed", slog.Any("err", err)) } - log.Info("host_bash + cloud_bash + install_skill + serve_page + messaging tools bolted onto chat runtime bag", slog.Int("tool_count", runtime.chatRT.ToolCount())) - // HLD-017: wire the active-skill → bound-credentials resolver so - // cloud_bash auto-injects the credentials an active skill was bound - // to at install time (design-time binding, no run-time choice). - runtime.chatRT.SetCredentialBinder(mpUC) + log.Info("host_bash + cloud_bash + install_skill + serve_page + messaging tools bolted onto chat runtime bag", slog.Int("tool_count", rt.ToolCount())) // Admin write-action gate: consult the agent/write_enabled system // setting live on every chat request. When an admin turns it off the // agent goes read-only (all non-read tools stripped from the LLM's - // toolbag). Default (unset) resolves to enabled, preserving behaviour. - runtime.chatRT.SetAgentWriteEnabledProvider(func(ctx context.Context) bool { + // toolbag). Missing settings fail closed. + rt.SetAgentWriteEnabledProvider(func(ctx context.Context) bool { return settings.settingSvc.AgentWriteEnabled(ctx) }) // HLD-018 P2: connect each enabled MCP server, pull its tools, and @@ -268,19 +267,24 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. // Keep the graph-facing decorators and ToolSearch's unredacted catalog // in sync. Registration changes call this with one server's fresh test // result, so an admin save never reconnects every MCP endpoint. - mcpTools := managersvcaiops.NewMCPToolRefresher(runtime.chatRT, mcpDeps, mcpCaller, mcpProposer, log) - mcpUC.SetToolChangeHook(mcpTools.Refresh) - - // Chat path: load enabled servers at boot; later changes use the same - // per-server refresh hook. The FLOW path uses LIVE managersvcflow.MCPSource. - mcpTools.LoadEnabled(rootCtx, mcpUC) + mcpTools := managersvcaiops.NewMCPToolRefresher(rt, mcpDeps, mcpCaller, mcpProposer, log) + refreshers = append(refreshers, mcpTools) + } + mcpUC.SetToolChangeHook(func(ctx context.Context, name string, srv *managermodelmcp.Server, discovered []mcpclient.Tool) { + for _, refresher := range refreshers { + refresher.Refresh(ctx, name, srv, discovered) + } + }) + if len(refreshers) > 0 { + // Discovery has one supervisor; every change fans out to both catalogs. + refreshers[0].LoadEnabled(rootCtx, mcpUC) go func() { defer func() { if recovered := recover(); recovered != nil { log.Error("MCP refresh supervisor panic", slog.Any("panic", recovered)) } }() - mcpTools.RunRefresh(rootCtx, mcpUC, time.Minute) + refreshers[0].RunRefresh(rootCtx, mcpUC, time.Minute) }() } @@ -319,3 +323,25 @@ func bootstrapExtensions(rootCtx context.Context, cfg *config.Config, log *slog. pageStore: pageStore, } } + +// extensionRuntime exposes only version-independent startup hooks. +type extensionRuntime interface { + AppendToolBag([]aiopstoolsbase.BaseTool) error + ReplaceToolSource(string, []aiopstoolsbase.BaseTool, []aiopstoolsbase.BaseTool) error + ToolCount() int + SetAgentWriteEnabledProvider(func(context.Context) bool) +} +type registryReloader interface{ Reload(string, ...string) error } +type registryReloaders []registryReloader + +func (rs registryReloaders) Reload(root string, extras ...string) error { + var errs []error + for _, r := range rs { + if r != nil { + if err := r.Reload(root, extras...); err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} diff --git a/cmd/ongrid/bootstrap_imbridge.go b/cmd/ongrid/bootstrap_imbridge.go index 79e4c786b..11532793a 100644 --- a/cmd/ongrid/bootstrap_imbridge.go +++ b/cmd/ongrid/bootstrap_imbridge.go @@ -99,6 +99,9 @@ func bootstrapIMBridge(rootCtx context.Context, log *slog.Logger, storage *stora if rt := runtime.chatRT; rt != nil { rt.SetMentionResolver(managersvcaiops.ChatMentionResolver{Inner: mentionSearcher}) } + if rt := runtime.chatV2; rt != nil { + rt.SetMentionResolver(managersvcaiops.ChatMentionResolver{Inner: mentionSearcher}) + } return &imbridgeModule{ imbridgeHandler: imbridgeHandler, diff --git a/cmd/ongrid/bootstrap_runtime.go b/cmd/ongrid/bootstrap_runtime.go index 9c1a0550a..37bb46adf 100644 --- a/cmd/ongrid/bootstrap_runtime.go +++ b/cmd/ongrid/bootstrap_runtime.go @@ -5,7 +5,9 @@ import ( "log/slog" "time" + einocallbacks "github.com/cloudwego/eino/callbacks" "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" + v2 "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/chatruntime" aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v3/chatruntime" aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" @@ -21,12 +23,22 @@ type runtimeModule struct { bootstrapAgentReg *aiopschatruntime.AgentRegistry aiopsRuntime *managersvcaiops.RuntimeRouter chatRT *aiopschatruntime.Runtime + chatV2 *v2.Runtime + bootstrapV2SkillReg *v2.SkillRegistry + bootstrapV2AgentReg *v2.AgentRegistry humanApprovalBroker *managersvcaiops.HumanApprovalBroker aiopsSvc *managersvcaiops.Service aiopsHandler *managerserveraiops.Handler } func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Logger, reg *prometheus.Registry, tracing *tracingModule, models *modelsModule, devices *devicesModule, telemetry *telemetryModule, tunnel *tunnelModule, aiops *aiopsModule, settings *settingsModule) *runtimeModule { + // 在创建任何新 Worker 前收尾上一进程的内部会话，避免重启后永久显示运行中。 + if n, err := aiops.aiopsRepo.ReconcileInterruptedWorkers(rootCtx, time.Now().UTC()); err != nil { + log.Error("interrupted worker recovery failed", slog.Any("err", err)) + } else if n > 0 { + log.Info("interrupted workers closed on startup", slog.Int64("sessions", n)) + } + // AgentRegistry + SkillRegistry are loaded UNCONDITIONALLY from // ./agents + configured skill roots + the marketplace root. These are // YAML+markdown metadata, so the /v1/agents @@ -39,6 +51,8 @@ func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Log // building its own; the chat coordinator + worker dispatch references // the same in-memory instances we hand to aiopsHandler below. bootstrapSkillReg, bootstrapAgentReg := loadBootstrapRegistries(log) + v2Skills, v2Agents := loadV2BootstrapRegistries(log) + var chatV2 *v2.Runtime var ( aiopsRuntime *managersvcaiops.RuntimeRouter @@ -52,7 +66,19 @@ func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Log if rterr != nil { log.Error("aiops runtime unavailable", slog.Any("err", rterr)) } else { - router, err := managersvcaiops.NewRuntimeRouter(aiops.aiopsRepo, contract.V3, map[contract.Version]managersvcaiops.RuntimeEngine{contract.V3: rt}, + var v2Trace einocallbacks.Handler + if tracing.agentTraceCallback != nil { + v2Trace = tracing.agentTraceCallback + } + v2rt, err := buildV2AIOpsRuntime(rootCtx, cfg, models.llmClient, models.llmRouter, aiops.toolsReg, aiops.aiopsRepo, aiops.mutatingProposalRepo, tunnel.fbClient, devices.edgeUC, devices.deviceUC, reg, log, v2Skills, v2Agents, models.llmSettingsResolver, humanApprovalBroker, v2Trace) + engines := map[contract.Version]managersvcaiops.RuntimeEngine{contract.V3: rt} + if err != nil { + log.Error("v2 runtime unavailable", slog.Any("err", err)) + } else { + chatV2 = v2rt + engines[contract.V2] = managersvcaiops.V2Runtime{Runtime: v2rt} + } + router, err := managersvcaiops.NewRuntimeRouter(aiops.aiopsRepo, contract.V3, engines, managersvcaiops.WithRuntimeDefaultProvider(func(ctx context.Context) (contract.Version, error) { version, err := settings.settingSvc.AgentRuntimeVersion(ctx) return contract.Version(version), err @@ -101,6 +127,16 @@ func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Log if err := rt.AppendToolBag(wrappedCoord); err != nil { log.Error("append coordinator tools failed", slog.Any("err", err)) } + if chatV2 != nil { + v2Coord := []aiopstoolsbase.BaseTool{ + aiopstoolsdec.Wrap(aiopstools.NewAgentTool(managersvcaiops.WorkerSpawner{Runtime: aiopsRuntime}, managersvcaiops.V2AgentRegistry{Inner: chatV2.AgentRegistry()}, log), coordDepsDispatch), + aiopstoolsdec.Wrap(aiopstools.NewSendMessageTool(managersvcaiops.WorkerSpawner{Runtime: aiopsRuntime}, log), coordDepsFast), + aiopstoolsdec.Wrap(aiopstools.NewTaskStopTool(managersvcaiops.WorkerSpawner{Runtime: aiopsRuntime}, log), coordDepsFast), + } + if err := chatV2.AppendToolBag(v2Coord); err != nil { + log.Error("append v2 coordinator tools failed", slog.Any("err", err)) + } + } log.Info("aiops runtime wired", slog.Int("tool_count", rt.ToolCount()), slog.Any("tool_names", rt.ToolNames(rootCtx)), @@ -121,6 +157,9 @@ func bootstrapRuntime(rootCtx context.Context, cfg *config.Config, log *slog.Log bootstrapAgentReg: bootstrapAgentReg, aiopsRuntime: aiopsRuntime, chatRT: chatRT, + chatV2: chatV2, + bootstrapV2SkillReg: v2Skills, + bootstrapV2AgentReg: v2Agents, humanApprovalBroker: humanApprovalBroker, aiopsSvc: aiopsSvc, aiopsHandler: aiopsHandler, diff --git a/cmd/ongrid/runtime_builder.go b/cmd/ongrid/runtime_builder.go index b63a915c2..1055d8ddb 100644 --- a/cmd/ongrid/runtime_builder.go +++ b/cmd/ongrid/runtime_builder.go @@ -91,108 +91,9 @@ func buildAIOpsRuntime( humanApproval aiopstoolsdec.HumanApprovalProposer, traceCallback *aiopsgraphcb.TraceHandler, ) (*aiopschatruntime.Runtime, error) { - // 1. RoutingChatModel — one inner per provider that exists. We - // layer managersvcaiops.ProviderClient around the existing - // llmRouter so each inner ChatModel routes its Chat() call - // to the correct sub-Client. Models stamp their default model - // name from cfg so a per-call model.WithModel still wins. - innerModels := map[string]einomodel.ChatModel{} - addInner := func(provider, defaultModel string) { - ic := &managersvcaiops.ProviderClient{Inner: llmClient, Provider: provider} - m, err := llm.NewClientChatModel(llm.ClientChatModelConfig{ - Client: ic, - Model: defaultModel, - }) - if err != nil { - log.Warn("chatruntime: build inner ChatModel", - slog.String("provider", provider), slog.Any("err", err)) - return - } - innerModels[provider] = m - } - // Build inners from the RESOLVED provider set (env + Settings-UI/DB), - // the same source the SPA model picker uses. Previously this gated on - // boot-time env keys only — so a provider configured via the UI (e.g. - // anthropic, with its key in the DB and an empty env var) showed in the - // picker but had no inner ChatModel, and picking it failed with - // "unknown provider". The per-call key is resolved by the - // resolver-backed llmClient, so registering the inner is all that's - // needed. defProv comes from the resolved default (DB default_provider). - defProv := cfg.LLM.Default - if resolver != nil { - if provCfgs, resolvedDefault, rerr := resolver.ResolveProviders(ctx); rerr == nil { - for _, pc := range provCfgs { - addInner(pc.ID, pc.Model) - } - if id, _ := llm.PickProviderDefault(provCfgs, resolvedDefault); id != "" { - defProv = id - } - } else { - log.Warn("chatruntime: resolve providers for inner models", slog.Any("err", rerr)) - } - } - // Safety net: if the resolver gave nothing (error / no rows), fall back - // to the boot-time env-keyed providers so the kernel still wires. - if len(innerModels) == 0 { - if cfg.OpenAI.APIKey != "" { - addInner(llm.ProviderOpenAI, firstNonEmpty(cfg.OpenAI.Model, "gpt-5.4")) - } - if cfg.LLM.Anthropic.APIKey != "" { - addInner(llm.ProviderAnthropic, firstNonEmpty(cfg.LLM.Anthropic.Model, "claude-sonnet-4-6")) - } - if cfg.LLM.Zhipu.APIKey != "" { - addInner(llm.ProviderZhipu, firstNonEmpty(cfg.LLM.Zhipu.Model, "glm-4.7")) - } - if cfg.LLM.Gemini.APIKey != "" { - addInner(llm.ProviderGemini, firstNonEmpty(cfg.LLM.Gemini.Model, "gemini-2.5-pro")) - } - } - // Pre-register an inner for every known provider id (incl. the generic - // "custom" endpoint) even if unconfigured at boot, so a provider whose key - // is added via the UI AFTER boot routes immediately — no restart. Only the - // inner's existence is boot-time; the per-call key/baseURL is resolved - // dynamically by llmClient. Unconfigured providers never reach the picker - // (the /v1/aiops/models catalog gates on ResolveProviders), so they're - // never selected; a stray call to one fails cleanly at key resolution. - for _, id := range knownLLMProviderIDs() { - if _, ok := innerModels[id]; !ok { - addInner(id, "") // model supplied per-call (picker / DefaultResolver) - } - } - if len(innerModels) == 0 { - return nil, fmt.Errorf("chatruntime: no LLM provider configured") - } - if defProv == "" { - defProv = llm.ProviderOpenAI - } - if _, ok := innerModels[defProv]; !ok { - // Default provider not configured — pick the first configured - // provider alphabetically so the result is deterministic across - // restarts (Go map iteration order is randomized). - keys := make([]string, 0, len(innerModels)) - for k := range innerModels { - keys = append(keys, k) - } - sort.Strings(keys) - defProv = keys[0] - } - // DefaultResolver lets calls that omit a provider (the RCA investigator - // worker, query_translate) track the LIVE configured default — the model - // the home-page picker writes to default_provider / _default_model - // — instead of the boot-time defProv. The chat picker pins a provider - // per-message and is unaffected. Resolved per-call (cheap: a settings read, - // and only on default-routed calls, which are low-frequency). - var defaultResolver func(context.Context) (string, string) - if resolver != nil { - defaultResolver = resolver.DefaultModel - } - chatModel, err := llm.NewRoutingChatModel(llm.RoutingChatModelConfig{ - Inner: innerModels, - DefaultProvider: defProv, - DefaultResolver: defaultResolver, - }) + chatModel, err := buildRuntimeChatModel(ctx, cfg, llmClient, resolver, log) if err != nil { - return nil, fmt.Errorf("chatruntime: NewRoutingChatModel: %w", err) + return nil, err } // 2. Tool bag — Registry.BuildBaseTools, @@ -242,7 +143,7 @@ func buildAIOpsRuntime( // An empty Tools whitelist inherits the complete tool bag; permission // decorators and ToolSearch govern execution and progressive disclosure. // Specialists improve depth but are never a prerequisite for closure. - agentReg.Add(&aiopschatruntime.Agent{ + agentReg.AddBuiltin(&aiopschatruntime.Agent{ Name: "default", Description: "默认助理", WhenToUse: "首页发起的会话默认绑定它；适合任何运维 / 排查 / 知识库查询场景。", @@ -332,3 +233,112 @@ func buildAIOpsRuntime( ) return rt, nil } + +// buildRuntimeChatModel shares provider configuration, not version-local execution state. +func buildRuntimeChatModel(ctx context.Context, cfg *config.Config, llmClient llm.Client, resolver *managerbizsetting.LLMSettingsResolver, log *slog.Logger) (einomodel.ToolCallingChatModel, error) { + // 1. RoutingChatModel — one inner per provider that exists. We + // layer managersvcaiops.ProviderClient around the existing + // llmRouter so each inner ChatModel routes its Chat() call + // to the correct sub-Client. Models stamp their default model + // name from cfg so a per-call model.WithModel still wins. + innerModels := map[string]einomodel.ChatModel{} + addInner := func(provider, defaultModel string) { + ic := &managersvcaiops.ProviderClient{Inner: llmClient, Provider: provider} + m, err := llm.NewClientChatModel(llm.ClientChatModelConfig{ + Client: ic, + Model: defaultModel, + }) + if err != nil { + log.Warn("chatruntime: build inner ChatModel", + slog.String("provider", provider), slog.Any("err", err)) + return + } + innerModels[provider] = m + } + // Build inners from the RESOLVED provider set (env + Settings-UI/DB), + // the same source the SPA model picker uses. Previously this gated on + // boot-time env keys only — so a provider configured via the UI (e.g. + // anthropic, with its key in the DB and an empty env var) showed in the + // picker but had no inner ChatModel, and picking it failed with + // "unknown provider". The per-call key is resolved by the + // resolver-backed llmClient, so registering the inner is all that's + // needed. defProv comes from the resolved default (DB default_provider). + defProv := cfg.LLM.Default + if resolver != nil { + if provCfgs, resolvedDefault, rerr := resolver.ResolveProviders(ctx); rerr == nil { + for _, pc := range provCfgs { + addInner(pc.ID, pc.Model) + } + if id, _ := llm.PickProviderDefault(provCfgs, resolvedDefault); id != "" { + defProv = id + } + } else { + log.Warn("chatruntime: resolve providers for inner models", slog.Any("err", rerr)) + } + } + // Safety net: if the resolver gave nothing (error / no rows), fall back + // to the boot-time env-keyed providers so the kernel still wires. + if len(innerModels) == 0 { + if cfg.OpenAI.APIKey != "" { + addInner(llm.ProviderOpenAI, firstNonEmpty(cfg.OpenAI.Model, "gpt-5.4")) + } + if cfg.LLM.Anthropic.APIKey != "" { + addInner(llm.ProviderAnthropic, firstNonEmpty(cfg.LLM.Anthropic.Model, "claude-sonnet-4-6")) + } + if cfg.LLM.Zhipu.APIKey != "" { + addInner(llm.ProviderZhipu, firstNonEmpty(cfg.LLM.Zhipu.Model, "glm-4.7")) + } + if cfg.LLM.Gemini.APIKey != "" { + addInner(llm.ProviderGemini, firstNonEmpty(cfg.LLM.Gemini.Model, "gemini-2.5-pro")) + } + } + // Pre-register an inner for every known provider id (incl. the generic + // "custom" endpoint) even if unconfigured at boot, so a provider whose key + // is added via the UI AFTER boot routes immediately — no restart. Only the + // inner's existence is boot-time; the per-call key/baseURL is resolved + // dynamically by llmClient. Unconfigured providers never reach the picker + // (the /v1/aiops/models catalog gates on ResolveProviders), so they're + // never selected; a stray call to one fails cleanly at key resolution. + for _, id := range knownLLMProviderIDs() { + if _, ok := innerModels[id]; !ok { + addInner(id, "") // model supplied per-call (picker / DefaultResolver) + } + } + if len(innerModels) == 0 { + return nil, fmt.Errorf("chatruntime: no LLM provider configured") + } + if defProv == "" { + defProv = llm.ProviderOpenAI + } + if _, ok := innerModels[defProv]; !ok { + // Default provider not configured — pick the first configured + // provider alphabetically so the result is deterministic across + // restarts (Go map iteration order is randomized). + keys := make([]string, 0, len(innerModels)) + for k := range innerModels { + keys = append(keys, k) + } + sort.Strings(keys) + defProv = keys[0] + } + // DefaultResolver lets calls that omit a provider (the RCA investigator + // worker, query_translate) track the LIVE configured default — the model + // the home-page picker writes to default_provider / _default_model + // — instead of the boot-time defProv. The chat picker pins a provider + // per-message and is unaffected. Resolved per-call (cheap: a settings read, + // and only on default-routed calls, which are low-frequency). + var defaultResolver func(context.Context) (string, string) + if resolver != nil { + defaultResolver = resolver.DefaultModel + } + chatModel, err := llm.NewRoutingChatModel(llm.RoutingChatModelConfig{ + Inner: innerModels, + DefaultProvider: defProv, + DefaultResolver: defaultResolver, + }) + if err != nil { + return nil, fmt.Errorf("chatruntime: NewRoutingChatModel: %w", err) + } + + return chatModel, nil +} diff --git a/cmd/ongrid/runtime_builder_v2.go b/cmd/ongrid/runtime_builder_v2.go new file mode 100644 index 000000000..c08cd0ca0 --- /dev/null +++ b/cmd/ongrid/runtime_builder_v2.go @@ -0,0 +1,233 @@ +package main + +import ( + "context" + "log/slog" + "os" + "time" + + einocallbacks "github.com/cloudwego/eino/callbacks" + managerbizaiops "github.com/ongridio/ongrid/internal/manager/biz/aiops" + aiopschatruntime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/chatruntime" + aiopsgraph "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph" + aiopsgraphcb "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph/callbacks" + aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" + aiopstoolsbase "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + aiopstoolsdec "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/decorators" + managerbizdevice "github.com/ongridio/ongrid/internal/manager/biz/device" + managerbizedge "github.com/ongridio/ongrid/internal/manager/biz/edge" + managerbizsetting "github.com/ongridio/ongrid/internal/manager/biz/setting" + managersvcaiops "github.com/ongridio/ongrid/internal/manager/service/aiops" + managersvcfb "github.com/ongridio/ongrid/internal/manager/service/frontierbound" + "github.com/ongridio/ongrid/internal/pkg/config" + "github.com/ongridio/ongrid/internal/pkg/llm" + "github.com/prometheus/client_golang/prometheus" +) + +// loadV2BootstrapRegistries walks ./agents + configured skills + the marketplace +// skill root and returns populated registries. Called once at boot +// independently of runtime availability, so /v1/agents has data to render even +// when the chat runtime can't build (no LLM provider, build failure). +// +// Env knobs mirror the values buildV2AIOpsRuntime used to read inline: +// +// ONGRID_BUILTIN_AGENTS_ROOT default ./agents +// ONGRID_BUILTIN_SKILLS_ROOT optional extra skills directory (no default) +// ONGRID_SKILLS_ROOT default /var/lib/ongrid/skills (marketplace mutable root) +func loadV2BootstrapRegistries(log *slog.Logger) (*aiopschatruntime.SkillRegistry, *aiopschatruntime.AgentRegistry) { + builtinSkillsRoot := os.Getenv("ONGRID_BUILTIN_SKILLS_ROOT") + builtinAgentsRoot := firstNonEmpty(os.Getenv("ONGRID_BUILTIN_AGENTS_ROOT"), "./agents") + marketplaceSkillsRoot := firstNonEmpty(os.Getenv("ONGRID_SKILLS_ROOT"), "/var/lib/ongrid/skills") + skillReg := aiopschatruntime.NewSkillRegistry() + agentReg := aiopschatruntime.NewAgentRegistry() + loadRes, loadErr := aiopschatruntime.LoadAll(aiopschatruntime.LoadAllConfig{ + SkillsRoot: builtinSkillsRoot, + AgentsRoot: builtinAgentsRoot, + ExtraSkillsRoots: []string{marketplaceSkillsRoot}, + }) + if loadErr != nil { + log.Warn("chatruntime: load all", slog.Any("err", loadErr)) + return skillReg, agentReg + } + skillReg.AddAll(loadRes.Skills) + agentReg.AddAll(loadRes.Agents) + skillReg.AddWarnings(loadRes.Warnings) + log.Info("chatruntime: loaded skills + agents", + slog.Int("skills", len(loadRes.Skills)), + slog.Int("agents", len(loadRes.Agents)), + slog.Int("warnings", len(loadRes.Warnings))) + for _, w := range skillReg.Warnings() { + log.Warn("chatruntime: skill warning", + slog.String("path", w.Path), slog.String("code", w.Code), slog.String("reason", w.Reason)) + } + for _, w := range agentReg.Warnings() { + log.Warn("chatruntime: agent warning", + slog.String("path", w.Path), slog.String("code", w.Code), slog.String("reason", w.Reason)) + } + return skillReg, agentReg +} + +// Heavy on parameters because every dep flows through this site +// exactly once — the alternative (build the runtime inline in main) +// would balloon main() further and make wiring harder to read. +func buildV2AIOpsRuntime( + ctx context.Context, + cfg *config.Config, + llmClient llm.Client, + llmRouter *llm.MultiClient, + toolsReg *aiopstools.Registry, + sessions managerbizaiops.SessionRepo, + mutatingProposals managersvcaiops.MutatingProposalRepo, + fbClient *managersvcfb.Client, + edgeUC *managerbizedge.Usecase, + deviceUC *managerbizdevice.Usecase, + reg prometheus.Registerer, + log *slog.Logger, + skillReg *aiopschatruntime.SkillRegistry, + agentReg *aiopschatruntime.AgentRegistry, + resolver *managerbizsetting.LLMSettingsResolver, + humanApproval aiopstoolsdec.HumanApprovalProposer, + traceCallback einocallbacks.Handler, +) (*aiopschatruntime.Runtime, error) { + chatModel, err := buildRuntimeChatModel(ctx, cfg, llmClient, resolver, log) + if err != nil { + return nil, err + } + + // 2. Tool bag — Registry.BuildBaseTools, + // then wrap the whole thing through the standard decorator + // chain so audit / timeout / rate-limit / metric apply + // uniformly. chat_tool_calls writes still happen via the + // graph persistence callback, while mutating proposal decisions + // are written by ReviewGate through chat_mutating_proposals. + // + // BuildBaseTools now returns a *tools.ToolBag + // (deferred-loading wrapper). When the bag size is below the + // deferral threshold (default 30, override via + // ONGRID_TOOLBAG_DEFERRAL_THRESHOLD) the LLM still sees full + // schemas for everything; once the marketplace pushes us past + // threshold the specialty tier auto-redacts and the LLM + // fetches schemas via the always-loaded ToolSearch tool. + bag := toolsReg.BuildBaseTools() + baseTools := bag.SchemasForLLM() + reviewSpawner := &managersvcaiops.ReviewSpawner{} + reviewSink := managersvcaiops.NewMutatingProposalSink(mutatingProposals) + deps := aiopstoolsdec.Deps{ + Limiter: aiopstoolsdec.NewTokenBucketLimiter(0), + Registerer: reg, + ReviewSpawner: reviewSpawner, + ReviewSink: reviewSink, + HumanApproval: humanApproval, + } + wrapped := make([]aiopstoolsbase.BaseTool, 0, len(baseTools)) + for _, t := range baseTools { + toolDeps := deps + if info, err := t.Info(context.Background()); err == nil && info != nil && info.Name == aiopstools.ToolNameCapturePCAP { + // A short capture includes capture duration plus parser work. Keep + // this operation-specific ceiling; other tools own their execution + // budgets without a blanket runtime deadline. + toolDeps.Timeout = 45 * time.Second + } + wrapped = append(wrapped, aiopstoolsdec.Wrap(t, toolDeps)) + } + + // 3. Skill / Agent registries — pre-loaded by loadV2BootstrapRegistries + // in main() so /v1/agents populates even when the chat runtime + // can't build (no LLM provider configured yet, etc). We just take + // the references handed in here and continue with the chat-wiring + // work below. + + // Register the virtual "default" persona — the user-facing root agent. + // An empty Tools whitelist inherits the complete tool bag; permission + // decorators and ToolSearch govern execution and progressive disclosure. + // Specialists improve depth but are never a prerequisite for closure. + agentReg.AddBuiltin(&aiopschatruntime.Agent{ + Name: "default", + Description: "默认助理", + WhenToUse: "首页发起的会话默认绑定它；适合任何运维 / 排查 / 知识库查询场景。", + Capabilities: []aiopschatruntime.AgentCapability{ + { + ID: "operate_and_investigate", + Description: "Own the user conversation, resolve facts, execute permitted tools, and integrate expert evidence.", + MaxToolCalls: managersvcaiops.DefaultCoordinatorMaxTurns, + }, + }, + Tools: nil, + // Keep the coordinator ceiling aligned with the global ReAct + // budget. Loop prevention belongs in graph-level guards + // (identical-call memo, per-tool execution caps, AgentTool + // dedupe), not in a lower persona turn limit that can clip + // legitimate long investigations. + MaxTurns: managersvcaiops.DefaultCoordinatorMaxTurns, + Source: "builtin", + }) + + // 4. Callback deps. Persistence/Audit/Metrics use the same + // SessionRepo + Registerer threaded everywhere. Budget gate is + // wired when LLM.DailyTokenLimit > 0 — single global UTC-day cap + // enforced against llm.InMemoryBudget (sufficient for the + // private-MVP single-tenant scope). + cbDeps := aiopsgraphcb.Deps{ + Persistence: aiopsgraphcb.PersistenceDeps{ + Repo: sessions, + Logger: log.With(slog.String("comp", "chatruntime-persist")), + Registerer: reg, + }, + Audit: aiopsgraphcb.AuditDeps{ + Logger: log.With(slog.String("comp", "chatruntime-audit")), + }, + Metrics: aiopsgraphcb.MetricsDeps{ + Registerer: reg, + }, + } + if traceCallback != nil { + cbDeps.Tracing = traceCallback + } + if cfg.LLM.DailyTokenLimit > 0 { + cbDeps.BudgetChecker = llm.NewInMemoryBudget(cfg.LLM.DailyTokenLimit) + log.Info("aiops: daily token budget enabled", + slog.Int("daily_limit", cfg.LLM.DailyTokenLimit), + ) + } + + // 5. Stitch the runtime. + // ctx + llmRouter are reserved for future runtime hooks (e.g. + // per-call provider catalog refresh). Reference them so unused- + // param lints stay quiet across edits. + _ = ctx + _ = llmRouter + rt, err := aiopschatruntime.NewRuntime(aiopschatruntime.Config{ + SkillRegistry: skillReg, + AgentRegistry: agentReg, + Sessions: sessions, + ChatModel: chatModel, + ToolBag: wrapped, + MentionResolver: nil, // wired below if we have a searcher + BasePrompt: managersvcaiops.BasePrompt(), + HistoryLimit: 50, + GraphCfg: aiopsgraph.Config{ + Model: cfg.OpenAI.Model, + Temperature: 0.1, + MaxIterations: 30, + }, + CallbackDeps: cbDeps, + Logger: log.With(slog.String("comp", "chatruntime")), + }) + if err != nil { + return nil, err + } + reviewSpawner.SetRuntime(managersvcaiops.V2Runtime{Runtime: rt}) + // — hand the unredacted *ToolBag to the runtime so + // future introspection paths can query the full tool universe even + // when the LLM-facing slice is the deferred / redacted view. + // ToolSearch already holds its own bag handle (registered inside + // BuildBaseTools); SetToolBag here is for runtime-level callers. + rt.SetToolBag(bag) + log.Info("aiops toolbag", + slog.Bool("deferring", bag.IsDeferring()), + slog.Int("threshold", bag.Threshold()), + slog.Int("total_tools", len(bag.AllTools())), + slog.Int("deferred_tools", len(bag.DeferredTools())), + ) + return rt, nil +} diff --git a/docs/rfc/RFC-009-agent-runtime-versions.md b/docs/rfc/RFC-009-agent-runtime-versions.md index 1b4b6c9a9..8b6428725 100644 --- a/docs/rfc/RFC-009-agent-runtime-versions.md +++ b/docs/rfc/RFC-009-agent-runtime-versions.md @@ -1,13 +1,13 @@ # RFC-009：Agent Runtime v1/v2/v3 隔离与显式切换 -- 状态：提议；用户已确认仅管理员设置入口、release 默认最新稳定 Runtime，源码隔离、新会话版本持久化已实现，版本切换尚未实现 +- 状态：v2/v3 切换已部署测试环境；v1 不再支持，保留源码归档；未推送、未合并 - 日期：2026-10-16 - 关联：PR #503、RFC-008、ADR-035；PRD-009 - 基线：正式发布标签 v0.18.2（`3b41874cae792329fd61cfe0450e57335efa3f1d`）的 Graph；不把已退役的 legacy for-loop 当作 v2 ## 背景与目标 -用户需要在新版 Agent 效果不佳时切回熟悉的行为，同时隔离实现。Agent Runtime 版本独立于 Ongrid 产品版本和模型版本：v2 是已发布 Graph 的行为基线，v3 是当前统一目录、权限和工具生命周期改进。三个版本源码已隔离，当前业务入口仅装配 v3；版本切换尚未交付。 +用户需要在新版 Agent 效果不佳时切回熟悉的行为，同时隔离实现。Agent Runtime 版本独立于 Ongrid 产品版本和模型版本：v2 是已发布 Graph 的行为基线，v3 是当前统一目录、权限和工具生命周期改进。三个版本源码已隔离；当前本地实现同时装配 v2/v3，release 默认 v3。下文按日期保留阶段记录，最新切换行为见末节。 ## 版本与隔离边界 @@ -32,7 +32,7 @@ v1/v2/v3 不互相 import；入口只依赖契约。代码位置仍需结合实 - 仅在设置 → AI / Agent 设置提供管理员版本切换；每次 release 的默认值为该发布包含的最新稳定 Runtime。建议保留管理员显式指定的版本，不在升级时覆盖回退选择。 - 聊天页不提供版本选择器；新会话采用管理员设置或 release 默认值。建议已有会话保持绑定版本，默认不复制工具历史或待审批动作。 -- `chat_sessions.runtime_version` 持久化选择。明确来自既有发布、且无版本标记的会话归 v2；PR 试验环境中已生成的 v3 会话需显式标记，不能统一回填为 v2。 +- `chat_sessions.runtime_version` 持久化选择。历史空值不推断、不批量回填；保持此前实际执行默认 v3，避免切换设置时迁移既有会话。确需恢复旧发布会话归属时，必须另行验证来源后迁移。 - coordinator、同步/后台 worker、报告、告警和工作流 Agent 节点均绑定版本。子任务继承父版本；无父会话的任务在创建时解析一次默认版本。 - IM 既有会话保持版本，新会话使用默认；将来如加用户偏好也只影响新会话。 - v3 不可用时明确失败，不静默路由到 v2。停用版本仅阻止新建，已有任务/审批不跨版本迁移。 @@ -142,3 +142,50 @@ Worker 查询、继续、停止按实例实际归属分派，覆盖内部 review - MCP 按 [`tools/list` 分页与变更通知](https://modelcontextprotocol.io/specification/2025-03-26/server/tools)发现全部工具；支持 [`Streamable HTTP GET SSE`](https://modelcontextprotocol.io/specification/2025-03-26/basic/transports) 的服务可主动通知刷新，每分钟发现作为无通知服务及断线后的兜底。每个服务一条顺序刷新链路，关闭时取消并等待退出。 回滚：测试部署保留旧 Manager/前端镜像、Edge 二进制、Compose 配置和数据库备份。新增 Skill 加载模式只用于明确配置的 Skill；旧 runtime 不识别 `on_demand`，回滚前应还原这类新配置。临时资源副本不是持久化发行版目录，不承诺跨进程续跑。测试部署与现场验证结果单独记录，不以本地测试通过代替现场验收。 + +### v2 可用回退与 v1 退役（2026-10-17，本地实现） + +「设置 → 助理」第二张卡片现在支持 `release`、`v3`、`v2`；v1 文案为「不再支持」，后端继续拒绝。v1 源码和来源清单保留。release 仍默认 v3，用户主动选择 v2 后只有新会话及无父会话的新任务采用 v2。 + +v2 的 Graph/Worker 真实独立执行，Service 仅适配协议与 Worker 快照。两套 Skill/Agent registry、工具袋和 reviewer 分别装配；共享 LLM 配置、业务服务、底层工具、审批存储及可复用指标。MCP 只有一个发现监督器，结果分别发布到两套目录；安装/卸载同时重新加载两套 registry。每个版本的工具撤销独立生效，旧 schema 引用不能调用新端点。 + +v2 回补会话/父任务版本校验、Worker 版本持久化、后台身份与权限快照、当前联网开关、工具目录一致性及撤销保护。审批来源固定为 v2 或 v3；cloud_bash、host_bash、安装 Skill、MCP 和通用工具分别注册两个版本的执行键，默认切换不会改写待审批行为。v1 不注册执行器。 + +v2 保留旧 Skill 自动加载策略，未移植 v3 `load_skill`、按需目录或资源副本。旧解析器对 `on_demand` 会按未知模式自动激活，不具备 v3 按需语义；切回 v2 时须预期上下文占用和资源一致性策略不同。来源 blob 清单继续描述原始提取，兼容回补列在 v2 README 中。 + +历史空版本继续归 v3，不随管理员新会话设置变化；本次不做数据回填或跨版本续跑。版本切换无需重启，不等于旧镜像回滚：回退到只支持 v3 的构建会使已创建的 v2 会话不可执行。恢复 release 设置只影响新会话，已经存在的 v2 会话和审批仍需保留 v2 引擎/执行器，或另行制定迁移方案。 + +验证范围：共享协议/路由、v2 工具撤销与并发、联网开关、后台权限与任务版本、v2 审批 SSE 恢复；真实 Manager/MySQL + 模型/MCP 替身验证默认切换、双版本动态工具与待审批来源；Chrome 深浅主题、四种宽度及中英文设置交互。测试结果见本轮验证记录，不将替身测试视为真实模型效果验收。 + +本轮结果：Linux `make test-agent-runtime`（race）通过；原生 cmd、setting、三版本 runtime、service/aiops race 回归通过；全量 E2E 31 个顶层测试 / 95 项通过（191.8 秒）。设置页 6 项单测、8 组 Chrome E2E、前端 build、修改文件 ESLint 通过，深浅主题及中英文截图已实看。Proto 已通过 `make proto`（本机 protoc 回退）和 Buf lint，未产生需提交的生成代码差异。 + +架构检查在仅包含受版本管理源码和本轮新增源码的干净导出上通过；直接扫描开发目录会被被忽略的 `api/gen` 内部导入和 `web/node_modules/flatted` Go 文件触发既有规则错误。全仓前端 lint 仍有原有 9 errors / 43 warnings，本轮未扩大修复范围。未重新执行全仓 `go test ./...`，不将上述专项回归表述为全仓检查通过。未推送、未部署、未合并。 + +2026-10-17 用户授权部署后，测试环境已更新至 `.5`；默认仍为 v3，v2/v3 真实模型只读查询通过。详见 [部署与回滚记录](../test/runtime-v2v3-deploy-20261017-5.md)。 + +### MCP 就绪与热加载收尾（2026-10-17，测试环境 .6/.7） + +纠正现场归因：创建 MCP 一直只保存注册，需探测后才能发布工具；顺序立即测试 v2、稍后测试 v3 会把发现延迟误判为版本能力差异。手动验证后两版均可调用。另一次超时是模型改用已有审批工具后等待人工批准，并非 v2 引擎卡死，测试提案已拒绝。 + +- `.6`：注册 CRUD 唤醒共享发现监督器，保留一分钟轮询兜底；后台探测避让正在进行的手动验证，避免自动发现使手动请求返回 409。现场无需调用 `/test`，约 0.11 秒就绪，两版实际调用通过。 +- `.7`：显式 `AddBuiltin` 注册保留角色，Load/Reload 保留代码内置配置并屏蔽磁盘同名覆盖；插件角色仍随卸载移除。两版均有配置、能力卡、同名覆盖与卸载回归，v1 不变。 +- 历史空版本仍按 v3 解释，不进行无法确认来源的数据回填。v2 的 on_demand 自动激活差异和 Skill 内存资源副本不跨进程恢复仍是明确边界，不宣称本轮解决。 +- `.7` 全量后端 E2E（race）31 个顶层 / 95 项通过，约 124.4 秒；v2/v3 registry 与启动装配 race 通过。Chrome 真实页面 21 通过 / 1 跳过 / 0 失败，跳过项为测试环境没有已接入 Kubernetes 集群。设备页 profiles 顶层行已由页面设计移除，不能据此判定采集缺失。 +- 现场 v2/v3 SSE 断开后执行继续、结果持久化并可继续下一轮；v3 在途 MCP 主动停止后下一轮可用。验证的是重新读取持久化历史，不是 SSE 事件 ID 重放。 +- 在 v2 MCP 提案等待审批时将 Manager 从 `.6` 更新为 `.7`：审批版本仍为 v2，批准后只读测试端点执行一次，结果写入审批记录，原会话可继续。此结果不代表 Skill 临时资源或全部后台 Worker 能跨重启恢复。 + +部署与回滚：`/opt/ongrid/test-deploy-20261017-runtime-hardening-7/` 保存前一镜像与 Compose 配置；`python3 .../ongrid-deploy7.py rollback` 恢复 `.6` Manager。前端仍为 `.5`，未修改数据库 schema。现场测试生成的会话、MCP 注册和 Skill 包均清理，默认运行时恢复；审批审计按现有机制保留。未推送、未合并。 + + +### Worker 重启收尾与剩余验收（2026-10-17，测试环境 .8） + +按用户要求，Kubernetes 以代码验收为主：Clusters/Kubernetes 前端 51 项测试通过，后端 biz/service/server/store 四层 `-race` 测试通过；不将无真实集群环境下的页面跳过算作现场通过。 + +真实环境发现并修复：工作流运行中的 Agent Worker 遇到 Manager 重启，工作流已被标为失败，但内部 Worker 会话未关闭。启动装配在创建新 runtime/Worker 前，分批、事务化关闭旧进程未结束的内部 Worker，收尾 pending 工具并写入一条中断消息。操作幂等，保留原 runtime_version，不改用户会话及独立审批记录，不重放外部工具。此机制沿用单 Manager 部署假设；多 Manager 并行运行需要先增加持久化执行归属/租约，不能直接复用全局启动收尾。 + +- 单测覆盖 v2/v3、用户与非 Worker 会话隔离、启动边界后新会话排除、已关闭会话、工具状态、重复启动幂等与审计失败时事务回滚；store 与启动模块 `-race`、`go vet` 均通过。 +- `.8` 现场 6 个场景通过：v2/v3 委派 Worker 取消后子会话关闭且父会话可继续；v2 on_demand Skill 正文兼容加载；历史空版本会话在新会话默认切为 v2 时仍实际执行 v3 的 load_skill；v2/v3 工作流 Agent 节点运行中重启后，工作流失败、Worker 关闭、pending 工具清零、中断记录恰好一条、外部调用重放次数为零。 +- 当前聊天 AgentTool 未开放 background=true；后台验收走真实工作流 Agent 节点。这里保证中断后状态收尾，不承诺跨进程自动续跑。 +- 测试会话、工作流、动态 MCP、插件及临时文件已清理，默认设置恢复 release/v3。 + +Manager 镜像 `ongrid:v0.18.2-v3-test.20261017.8`，前端仍为 `.5`，无 schema 变更。现场证据与脚本位于 `/opt/ongrid/test-deploy-20261017-worker-recovery-8/`，结果为 `worker-environment.json`。回滚命令 `python3 /opt/ongrid/test-deploy-20261017-worker-recovery-8/ongrid-deploy8.py rollback` 恢复 `.7` Manager；已记录的中断状态保留，不恢复执行。未推送、未合并。 diff --git a/docs/test/e2e-catalog.md b/docs/test/e2e-catalog.md index 00c86534c..c23fbb855 100644 --- a/docs/test/e2e-catalog.md +++ b/docs/test/e2e-catalog.md @@ -162,7 +162,7 @@ |---|---|---|---|---|---| | L1 | 加载内置 skill registry | `GET /v1/skills` | 列出 ~18 内置 | P1 | ✅ `tests/e2e/skills_registry_test.go` | | L2 | skill execute(read-only) | `POST /skills/{key}/execute` | edge 跑 + 返结果 | P1 | | -| L3 | marketplace install | `POST /marketplace/install` | pack 下载、registry 行 | P2 | | +| L3 | marketplace 本地安装 / 热加载 / 卸载 | `POST /marketplace/install` + 聊天 `load_skill` | 已有会话立即可见；按需正文与版本持久化；卸载撤销旧回合加载 | P2 | ✅ `tests/e2e/dynamic_skills_test.go`（本地包；远程下载未覆盖） | ## 域 M — Webshell @@ -225,3 +225,70 @@ > - 写操作通过浏览器侧 API stub 验证交互和请求体,不写入本地环境数据 详见 `tests/e2e/README.md` 和 `web/e2e/README.md`。 + + +## v3 动态能力回归 + +全部经真实 Manager 的公开 HTTP API、真实临时 MySQL 和脚本化 LLM/MCP 服务执行；不重启 Manager、不直接修改运行时注册表。测试注册聊天、MCP、安装包的清理；文件使用临时目录。 + +| 场景 | 关键断言 | 测试 | +|---|---|---| +| 启动后注册 MCP / 分页发现 | 同一个已有会话通过 ToolSearch 取得第二页工具 schema；真实工具调用及结果进入模型上下文 | `TestDynamicMCP_HotRegistrationLifecycle` | +| 发现临时失败 | API 报错，最后验证的工具仍可发现和执行 | 同上 | +| 工具列表替换 | 新工具可用；旧工具 schema 消失，强制输出旧调用也不能到达 MCP endpoint | 同上 | +| 禁用 / 重新启用 | 禁用立即撤销目录与执行，重新启用恢复有效工具 | 同上 | +| 替换端点 / 删除 | 旧工具不能执行；新端点验证后可执行；删除后再次撤销 | 同上 | +| 后台首次发现 | 不调用手动连接测试，按生产一分钟周期发现新注册服务 | `TestDynamicMCP_BackgroundDiscoveryNotificationAndPolling` | +| MCP 列表变更通知 | 服务端 SSE 通知后 12 秒内更新；新工具可调用，旧工具撤销 | 同上 | +| 无通知的轮询兜底 | 不发送通知，85 秒内发现新目录并更新执行能力 | 同上 | +| Skill 安装 / 按需加载 | 已有会话目录热更新，正文不提前注入；load_skill 返回版本、revision 和资源目录，工具消息持久化 | `TestDynamicSkills_InstallLoadAndUninstall` | +| 回合内卸载 Skill | 模型请求阻塞期间卸载，旧回合 load_skill 返回失败观察，正文不得进入模型；后续目录不含该 Skill | 同上 | + +运行： + +```bash +go test -race -tags=e2e -count=1 -timeout=8m ./tests/e2e -run TestDynamic +``` + +后台测试保留真实一分钟刷新周期，通常需要约两分钟。MCP 执行场景使用 trusted 注册的只读工具；外部服务是测试替身，不代表真实供应商互操作性验收；审批期间配置/凭据变化、Skill 执行中资源引用与重启恢复边界仍由已有单元/集成测试覆盖，不将其计入这些 E2E 的覆盖范围。 + +本轮验证（2026-10-17）：上述动态能力专项通过；`go test -race -tags=e2e -count=1 -timeout=12m -json ./tests/e2e/...` 全量通过，29 个顶层测试、含子测试共 93 项，零失败 / 跳过，耗时约 180.6 秒。 + + +## v2/v3 版本切换回归 + +| 场景 | 关键断言 | 测试 | +|---|---|---| +| release → v2 → release | 新会话选择生效，既有 v2/v3 会话继续各自运行 | `TestRuntimeVersions_SwitchAndDynamicMCP` | +| 双版本动态 MCP | 同一注册更新两套目录；真实结果进入模型；禁用后两边均不能执行 | 同上 | +| 待审批期间切换默认 | v2 审批仍标记 v2，批准后实际 MCP 调用一次，结果回到 v2；子任务仍可委派 | 同上 | +| Skill / persona 热加载 | 安装后两版 Skill 正文与专家 prompt 实际进入模型，卸载后不能再调用；v3 原生 load_skill 不泄露到 v2 | `TestRuntimeVersions_SkillAndPersonaReload` | +| 设置 UI | v2/v3 保存与恢复 release；v1「不再支持」禁用；深浅主题、四宽度、中英文 | `web/e2e/agent-runtime-settings.spec.ts`、`web/src/pages/settings/Agent.test.tsx` | + +后端专项：`go test -race -tags=e2e -count=1 -timeout=8m ./tests/e2e -run TestRuntimeVersions`。使用真实 Manager/MySQL、脚本模型/MCP；不操作测试部署环境。新建会话、MCP 注册、安装包与临时文件均清理。 + +本轮双版本验证（2026-10-17）：全量 `go test -race -tags=e2e -count=1 -timeout=12m -json ./tests/e2e/...` 通过，31 个顶层测试、含子测试共 95 项，零失败 / 跳过，耗时约 191.8 秒。 + +### 2026-10-17 收尾复验（Manager .7） + +新增注册无需 `/test`，要求 10 秒内自动发现，并分别用已有 v2/v3 会话调用；保留一分钟通知/轮询兜底验证。两版 AgentRegistry 增加内置角色热加载保留、同名插件隔离、卸载删除外部角色测试。全量后端 E2E（race）95 项通过，约 124.4 秒。 + +真实 Chrome 页面回归：21 通过、1 跳过、0 失败。旧 Kubernetes 路由跳转到统一集群页；设备与拓扑用 `tab` 角色定位；profiles 不属于设备页顶层插件。Kubernetes 资源页从真实 API 选择集群，环境无集群时明确跳过，不伪造通过。 + +现场 SSE 断线后继续执行并重读历史、v3 主动停止后继续对话、v2 MCP 待审批跨 Manager 更新后执行均通过。未将这些结果外推为所有 Worker/Skill 内存资源的跨进程续跑；详见 RFC-009 最新阶段。 + + +### 2026-10-17 Worker 与版本边界现场复验（Manager .8） + +真实模型、Manager/MySQL 和临时只读 MCP：6 个场景通过，零失败。 + +| 场景 | 关键断言 | +|---|---| +| v2/v3 委派取消（2 项） | 子 Worker 会话关闭，父会话后续 query_devices 成功 | +| v2 on_demand Skill 兼容 | 正文中的私有证据标记可用，无 load_skill 调用 | +| 历史空版本 | 新会话默认改为 v2，既有空版本会话仍执行 v3 load_skill | +| v2/v3 工作流 Worker 重启（2 项） | 工作流失败、Worker 关闭、pending 工具清零、唯一中断记录、零外部重放 | + +脚本与 JSON 证据：测试主机 `/opt/ongrid/test-deploy-20261017-worker-recovery-8/ongrid-worker-environment8.py`、`worker-environment.json`。脚本会重启 Manager，仅用于授权测试环境；临时会话、工作流、注册和安装包全部清理，默认设置恢复。`.7` 首次现场复现的未关闭 Worker 问题已在 `.8` 修复；存储层回归见 `session_recovery_test.go`，覆盖范围、幂等和事务失败回滚，race 通过。 + +Kubernetes 按用户要求完成代码验收：前端 51 项，Go 四层 race 通过；真实集群验收仍未执行。聊天 AgentTool 没有开放 background=true，本次后台场景来自工作流 Agent 节点；不把中断收尾表述为自动续跑。 diff --git a/docs/test/runtime-v2v3-deploy-20261017-5.md b/docs/test/runtime-v2v3-deploy-20261017-5.md new file mode 100644 index 000000000..9164e18b6 --- /dev/null +++ b/docs/test/runtime-v2v3-deploy-20261017-5.md @@ -0,0 +1,38 @@ +# v2/v3 测试环境部署：2026-10-17 .5 + +- 目标：`https://101.34.63.91/`，测试环境；用户明确要求部署。 +- 来源：`refactor/unified-agent-runtime`，基于 `45ec9ad4` 加本地未提交的 v2/v3 切换改动。没有推送或合并。 +- Manager：`ongrid:v0.18.2-v3-test.20261017.5`，镜像 ID `sha256:c56803c1a3db39c77e6031db33556b27084b12248de803795feb867f16fec0b9`。 +- 前端：`ongrid-web:v0.18.2-v3-test.20261017.5`，镜像 ID `sha256:ac7d3cc2fb7c03759d18c58fc171014d3899543e104ba49046a8bb6079ca78c3`。 +- Edge 和其他基础服务未更新；本次没有新增 schema。 +- Manager 使用 Linux builder 的 `make build-ongrid`；测试机通过 `make docker-ongrid` 和 `make docker-build-web` 将产物叠加到原镜像。 +- 二进制 SHA256：`b208040ce5e7d6f5bdc1d42240faf548710874a21d3d2ba401f389c5e14761b8`；本地与远端一致。 +- 部署目录：`/opt/ongrid/test-deploy-20261017-runtime-v2v3-5/`，含原 Compose、镜像记录、构建日志、部署/回滚及验证脚本。 + +## 验证 + +- 部署前：95 项隔离 E2E、runtime race、设置页单测和 Chrome 交互验证已通过，详见 RFC-009。 +- 部署后：`/`、`/healthz`、`/readyz` 均 200；Manager/前端 running，restart count 均 0，近 10 分钟未发现 panic。 +- 原运行时设置不存在，即跟随 release 默认 v3。验证时分别短暂选择 v2/v3 创建会话，立即恢复原设置，再执行真实模型调用。 +- 两版均实际执行 `query_devices` 成功，返回两台在线设备；耗时分别约 1.49 秒和 2.12 秒。 +- 两个临时会话已删除，原设置已恢复；当前默认仍为 v3。 +- 线上 index 和包含「v1 · 不再支持」的 JS 资源逐字节匹配新构建。页面支持选择 v2、v3，v1 禁用。 +- 初次前端资源校验遇到 macOS 打包附属文件；已清理附属文件并重建前端镜像，等待 Nginx 就绪后重新验证通过。 +- 本轮现场冒烟没有重跑动态 MCP/Skill 生命周期；这些由前述隔离 E2E 覆盖。 + +## 回滚 + +恢复之前 Manager `.4` 和前端 `.2`： + +```bash +python3 /opt/ongrid/test-deploy-20261017-runtime-v2v3-5/ongrid-deploy5.py rollback +``` + +旧镜像没有 v2 执行入口。后续用户若已创建需保留的 v2 会话或待审批动作，不能直接将此镜像回滚当作运行时切换；应优先将新会话默认恢复 `release`，保留当前 v2 引擎处理既有任务，再决定迁移或停用。此次验证创建的 v2 会话已清理，不回滚数据库数据。 + +## 后续测试环境版本 + +2026-10-17 后续 Manager 已更新为 `.7`，前端继续使用 `.5`。`.6` 修复 MCP 注册发现唤醒与手动验证竞争，`.7` 修复两版默认助理热加载保留。最终现场安装/卸载 Skill 前后 `/agents/default` 响应一致；MCP 自动发现约 0.11 秒，双版本调用通过。部署、回滚及生命周期验收详见 RFC-009「MCP 就绪与热加载收尾」；当前回滚入口在 `/opt/ongrid/test-deploy-20261017-runtime-hardening-7/ongrid-deploy7.py`，恢复 `.6`，不要误用本文件早期的 `.5` 回滚路径。 + + +2026-10-17 再次更新 Manager 至 `.8`：修复重启后孤立 Worker 未关闭。镜像 ID `sha256:97cc8a61b2ce2e1f9b032456c32210bf39222b72fc338d33e2465cd1b26258d5`，二进制 SHA256 `e3ce9dda30635336d4a6cef559bf94af2edd0af68b2b3d5ff4bb71d8174ff77a`。双版本取消、重启收尾、历史空版本和 v2 Skill 兼容共 6 个现场场景通过，临时数据清理完成。当前回滚入口为 `/opt/ongrid/test-deploy-20261017-worker-recovery-8/ongrid-deploy8.py rollback`，恢复 `.7` Manager；保留已完成的状态收尾，不恢复被中断的执行。前端继续 `.5`，无 schema 变更、未推送或合并。 diff --git a/docs/test/v3-test-deploy-20261017-4.md b/docs/test/v3-test-deploy-20261017-4.md new file mode 100644 index 000000000..6250aa949 --- /dev/null +++ b/docs/test/v3-test-deploy-20261017-4.md @@ -0,0 +1,20 @@ +# v3 测试环境更新：2026-10-17 .4 + +- 目标：`https://101.34.63.91/`，仅测试环境。 +- 源码：需求分支 `refactor/unified-agent-runtime`，生产代码提交 `45ec9ad4`；本轮本地 E2E 与覆盖文档尚未提交、未推送。 +- 变更：同步 main #524/#525，移除协调控制工具残留的默认 15 秒限制；保留显式 AgentTool 预算及 v1/v2 源码快照。 +- Manager 镜像：`ongrid:v0.18.2-v3-test.20261017.4`；镜像 ID `sha256:908f4dd740c41d28de1c0ea9cca0bc2412f7ebf0b10f6bbdbb73fffb776d9b84`。 +- 前端及 Edge 保持 `.2`，它们没有本轮生产代码变化。 +- 构建使用 Linux builder 的 `make build-ongrid`，远端使用 `make docker-ongrid`；仅重建 Manager 服务。 +- Compose 原配置、镜像构建日志、部署脚本与结果：`/opt/ongrid/test-deploy-20261017-v3-45ec9ad4/`。健康检查失败自动恢复 `.3`。 +- 手动回滚：在测试机执行 `python3 /opt/ongrid/test-deploy-20261017-v3-45ec9ad4/ongrid-v3-deploy4.py rollback`。本轮没有新增 schema，不回滚数据库数据。 + +## 验证与边界 + +- 部署前 Linux runtime race（含 Flow）通过；新增动态注册/MCP/Skill E2E 与全量 E2E race 通过（93 项）。 +- 部署后 `/healthz`、`/readyz` 通过。 +- 真实模型创建新聊天记录为 v3；`query_devices` 返回两台在线设备，`ToolSearch` 和 `web_search` 实际调用成功；临时聊天均在 finally 中删除。 +- 搜索返回的目标质量仍受 SearXNG 上游限制：本次没有直接命中 Kubernetes DNS 排查官方子页面；模型明确说明缺失，第二次查询报告 partial 与 sogou CAPTCHA。不能把工具执行成功等同于搜索目标命中。 +- 现有 MCP `approval_demo` 为 enabled/ok；当前没有安装 Skill 包。因此本次部署冒烟不声称现场再次覆盖 Skill 安装/卸载或真实 MCP 工具执行，动态生命周期由隔离 E2E 覆盖。 +- 预检曾误用应用不支持的 `version` 子命令，附加进程初始化后退出；版本最终按镜像及部署记录确认，未将该命令当作版本查询接口。 +- 本轮没有部署生产环境、推送代码或合并 PR。 diff --git a/internal/manager/biz/aiops/runtime/README.md b/internal/manager/biz/aiops/runtime/README.md index 67aff0462..9b196ce6f 100644 --- a/internal/manager/biz/aiops/runtime/README.md +++ b/internal/manager/biz/aiops/runtime/README.md @@ -2,12 +2,14 @@ | 目录 | 基线 | 当前接线 | | --- | --- | --- | -| [v1](v1/README.md) | GitHub 最早正式 release `v0.7.158` 的 Eino Graph | 编译、测试；未启用 | -| [v2](v2/README.md) | `v0.18.2` 的 Eino Graph | 编译、测试；未启用 | -| [v3](v3/README.md) | PR #503 当前统一运行时 | 所有现有调用方使用 | +| [v1](v1/README.md) | GitHub 最早正式 release `v0.7.158` 的 Eino Graph | 源码归档；不再支持 | +| [v2](v2/README.md) | `v0.18.2` 的 Eino Graph | 可选回退版本；独立装配 | +| [v3](v3/README.md) | PR #503 当前统一运行时 | 当前 release 默认 | 编号是本次引入的运行时版本，不是 release 当年的命名。legacy 手写循环是并存的历史旁支，不属于这里的 v1。 -每个版本独立持有 `chatruntime`（上下文、prompt、Worker、目录策略）和 `graph`（Eino 编排、工具适配、回调）。禁止版本间 import，`isolation_test.go` 在标准 Go 测试中检查该边界，并验证三版本指标注册共存。业务执行能力与基础设施继续共享；尚未声称三版本已可切换。 +每个版本独立持有 `chatruntime`（上下文、prompt、Worker、目录策略）和 `graph`（Eino 编排、工具适配、回调）。禁止版本间 import，`isolation_test.go` 在标准 Go 测试中检查该边界，并验证三版本指标注册共存。业务执行能力与基础设施继续共享；仅 v2/v3 可切换，v1 不再支持。 -验证：Linux 环境执行 `make test-agent-runtime`。不新增 schema/API/设置，不改变当前运行时选择；需要撤销本次隔离时回退该源码迁移提交即可。执行入口已通过统一契约和路由，当前只注册 v3。会话持久化版本、旧版本装配及管理设置的后续方案见 [RFC-009](../../../../../docs/rfc/RFC-009-agent-runtime-versions.md)。 +验证：Linux 环境执行 `make test-agent-runtime`，真实 Manager + MySQL 的 `TestRuntimeVersions_*` 覆盖切换、动态 MCP、审批、Skill 与 persona 热加载。管理员在「设置 → 助理」第二张卡片选择；新会话采用所选版本，既有会话/子任务/待审批动作不迁移。历史空版本固定使用 v3，不随设置改变。 + +回到 `release` 只影响新会话，不能据此卸载仍承载既有会话的 v2 引擎。旧镜像回滚限制和实现记录见 [RFC-009](../../../../../docs/rfc/RFC-009-agent-runtime-versions.md)。 diff --git a/internal/manager/biz/aiops/runtime/v1/README.md b/internal/manager/biz/aiops/runtime/v1/README.md index 6decec827..b573644ae 100644 --- a/internal/manager/biz/aiops/runtime/v1/README.md +++ b/internal/manager/biz/aiops/runtime/v1/README.md @@ -6,4 +6,6 @@ 适配：两个包的内部 import 改为本版本路径。v1 的两个仓储测试替身补充当前接口要求的 `UpdateSessionModel`，返回未接线错误，生产逻辑未修改。 -当前与主模块共享 Eino/LLM 依赖、领域模型、业务工具和存储接口，因此不是旧 release 的完整可执行镜像。此目录参与编译和 race 测试，但尚未接入生产路由或管理员切换。接入前必须验证工具装配、权限、审批及后台任务生命周期，不能用历史策略绕过当前安全约束。 +**不再支持执行或切换。** 源码保留归档并参与编译、race 测试，便于历史追溯；不注册运行时和审批执行器。 + +当前与主模块共享 Eino/LLM 依赖、领域模型、业务工具和存储接口，因此不是旧 release 的完整可执行镜像。 diff --git a/internal/manager/biz/aiops/runtime/v2/README.md b/internal/manager/biz/aiops/runtime/v2/README.md index 28edf2dae..d95530be0 100644 --- a/internal/manager/biz/aiops/runtime/v2/README.md +++ b/internal/manager/biz/aiops/runtime/v2/README.md @@ -4,6 +4,16 @@ 提取该 release 的 `biz/aiops/chatruntime/` 与 `biz/aiops/graph/`，包括测试、testdata、prompt、Worker 和回调；不提取 legacy `agent/`。原文件清单与 Git blob 记录在 [source.json](source.json)。 -适配：两个包的内部 import 改为本版本路径。无其他生产逻辑修改。 +v2 已接入管理员运行时选择，执行本目录的 Eino Graph、上下文策略和 Worker；不转发到 v3。`cmd/ongrid/runtime_builder_v2.go` 独立装配注册器、工具目录、装饰器和回调。来源清单记录的是提取时的原始 Git blob，后续兼容修改不改写来源。 -当前与主模块共享 Eino/LLM 依赖、领域模型、业务工具和存储接口，因此不是旧 release 的完整可执行镜像。此目录参与编译和 race 测试，但尚未接入生产路由或管理员切换。接入前必须验证工具装配、权限、审批及后台任务生命周期，不能用历史策略绕过当前安全约束。 +为接入当前系统而回补的兼容与安全能力： + +- 请求、回复、事件使用共享 `runtime/contract`；Worker 保持版本内私有锁和取消句柄，通过 Service 适配快照。 +- 校验会话/请求/父任务版本，子任务持久化 v2；后台任务保留身份、版本及读写/联网权限，不继承请求取消和 callback。 +- 工具搜索与执行目录同步更新；撤销/替换后拒绝旧执行句柄；联网开关在实际执行入口检查。 +- 采用当前共享审批、业务工具和操作超时。审批按来源版本执行；MCP 批准前校验配置指纹。 +- marketplace 热加载与 MCP 发现同时更新两套独立目录；不共享可变工具袋。 + +Skill 继续采用 v2 的自动加载策略（`always`/`keyword`）；没有 v3 的 `load_skill` 和资源快照机制。原解析器对未知模式按自动激活处理，因此 `on_demand` 在 v2 不具有按需语义。切换会改变这一行为，但不会把 v2 选项接到 v3。 + +当前仍共享 Eino/LLM 依赖、领域模型、底层业务工具和存储接口，因此不是旧 release 的完整镜像或逐字节行为复现。v2 仅接受必要兼容、安全修复；新增编排行为继续在 v3 开发。 diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry.go index c6ca9c8d9..78948b7a6 100644 --- a/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry.go +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry.go @@ -17,6 +17,7 @@ import ( type AgentRegistry struct { mu sync.RWMutex agents []*Agent + builtins []*Agent warnings []LoadWarning } @@ -42,7 +43,7 @@ func (r *AgentRegistry) Load(agentsRoot string) error { return fmt.Errorf("chatruntime: agent registry load: %w", err) } r.mu.Lock() - r.agents = append([]*Agent(nil), res.Agents...) + r.agents = r.withBuiltinsLocked(res.Agents) r.warnings = append([]LoadWarning(nil), res.Warnings...) r.mu.Unlock() return nil @@ -69,7 +70,7 @@ func (r *AgentRegistry) Reload(agentsRoot string, extras ...string) error { newAgents := append([]*Agent(nil), res.Agents...) newWarnings := append([]LoadWarning(nil), res.Warnings...) r.mu.Lock() - r.agents = newAgents + r.agents = r.withBuiltinsLocked(newAgents) r.warnings = newWarnings r.mu.Unlock() return nil @@ -144,6 +145,42 @@ func (r *AgentRegistry) Add(ag *Agent) { r.mu.Unlock() } +// AddBuiltin 注册代码内置角色，磁盘热加载不能删除或覆盖这些保留名称。 +func (r *AgentRegistry) AddBuiltin(ag *Agent) { + if ag == nil || ag.Name == "" { + return + } + r.mu.Lock() + defer r.mu.Unlock() + replaced := false + for i, old := range r.builtins { + if old.Name == ag.Name { + r.builtins[i] = ag + replaced = true + break + } + } + if !replaced { + r.builtins = append(r.builtins, ag) + } + r.agents = r.withBuiltinsLocked(r.agents) +} + +// 调用方持有写锁。只保留显式注册的内置角色，已卸载插件不能残留。 +func (r *AgentRegistry) withBuiltinsLocked(loaded []*Agent) []*Agent { + out := append([]*Agent(nil), r.builtins...) + reserved := make(map[string]bool, len(r.builtins)) + for _, ag := range r.builtins { + reserved[ag.Name] = true + } + for _, ag := range loaded { + if ag != nil && !reserved[ag.Name] { + out = append(out, ag) + } + } + return out +} + // AddAll inserts a batch of agents. Nil entries are skipped. Used by // LoadAll to merge plugin-container output into the registry. func (r *AgentRegistry) AddAll(agents []*Agent) { diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry_test.go index bbf811b7d..4a0d8e836 100644 --- a/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry_test.go +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/agent_registry_test.go @@ -1,6 +1,7 @@ package chatruntime import ( + "os" "path/filepath" "testing" ) @@ -62,3 +63,38 @@ func TestAgentRegistryCapabilityCardsCarryOwner(t *testing.T) { t.Errorf("registry card mutated through snapshot: %q", got) } } + +func TestAgentRegistryReloadPreservesBuiltinAndRemovesUninstalledPersona(t *testing.T) { + r := NewAgentRegistry() + builtin := &Agent{Name: "default", Source: "builtin", MaxTurns: 30, Capabilities: []AgentCapability{{ID: "operate"}}} + r.AddBuiltin(builtin) + root := t.TempDir() + body := "---\nname: default\ndescription: shadow\nwhen_to_use: shadow\n---\nshadow" + if err := os.WriteFile(filepath.Join(root, "default.md"), []byte(body), 0600); err != nil { + t.Fatal(err) + } + if err := r.Reload(root, fixtureAgentRoot("multi")); err != nil { + t.Fatal(err) + } + if got, _ := r.ByName("default"); got != builtin { + t.Fatal("plugin replaced builtin persona") + } + if _, ok := r.ByName("reviewer"); !ok { + t.Fatal("plugin persona was not loaded") + } + if err := r.Reload(t.TempDir()); err != nil { + t.Fatal(err) + } + if got, _ := r.ByName("default"); got != builtin || got.MaxTurns != 30 { + t.Fatal("reload lost builtin configuration") + } + if len(r.All()) != 1 || len(r.CapabilityCards()) != 1 { + t.Fatal("uninstall left stale personas or removed builtin capabilities") + } + if err := r.Load(t.TempDir()); err != nil { + t.Fatal(err) + } + if got, _ := r.ByName("default"); got != builtin { + t.Fatal("load lost builtin persona") + } +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime.go index b9e894370..4f3b1dce8 100644 --- a/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime.go +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime.go @@ -53,6 +53,7 @@ import ( "go.opentelemetry.io/otel/attribute" biz "github.com/ongridio/ongrid/internal/manager/biz/aiops" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph" "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph/callbacks" "github.com/ongridio/ongrid/internal/manager/biz/aiops/toolreplay" @@ -63,123 +64,25 @@ import ( "github.com/ongridio/ongrid/internal/pkg/tracing" ) -// Mention mirrors the legacy agent.Mention shape so chatruntime can -// be called without dragging the manager/biz/aiops/agent package as -// a dependency. The service-layer adapter does the type translation. -type Mention struct { - Type string - ID string - Label string -} - -// MentionResolver hydrates a list of @-mentions into one bullet line -// each. Mirrors agent.MentionResolver. nil = no inlining. -type MentionResolver interface { - Resolve(ctx context.Context, mentions []Mention) []string -} - -// EventType enumerates streaming events emitted by the runtime. The -// values are byte-equal to the legacy agent.EventType strings so the -// SSE adapter at the service layer can route both kernels through -// the same wire format. New event names introduced in PR-6 -// (assistant_start / assistant_delta / assistant_end) are NOT -// surfaced through this enum yet — the cutover keeps the legacy -// frame names so the SPA round-trips without changes. -type EventType string +// Public protocol aliases preserve existing callers while ownership moves to contract. +type Mention = contract.Mention +type MentionResolver = contract.MentionResolver +type EventType = contract.EventType +type AssistantEvent = contract.AssistantEvent +type ToolEvent = contract.ToolEvent +type Event = contract.Event +type Emit = contract.Emit +type Request = contract.Request +type Reply = contract.Reply const ( - // EventAssistant fires after every assistant turn is persisted. - EventAssistant EventType = "assistant" - // EventToolStart fires after a tool_call row is persisted as - // pending, before the tool actually runs. - EventToolStart EventType = "tool_start" - // EventToolEnd fires once the tool finishes (success/error/timeout). - EventToolEnd EventType = "tool_end" - // EventDone fires once at terminal success with the final Reply. - EventDone EventType = "done" - // EventError fires on terminal failure. - EventError EventType = "error" + EventAssistant = contract.EventAssistant + EventToolStart = contract.EventToolStart + EventToolEnd = contract.EventToolEnd + EventDone = contract.EventDone + EventError = contract.EventError ) -// AssistantEvent describes one assistant turn's persisted row. Same -// shape as legacy agent.AssistantEvent. -type AssistantEvent struct { - Iteration int - MessageID string - Content string - CreatedAt time.Time - PendingToolCalls int -} - -// ToolEvent describes one tool_call lifecycle event. Same shape as -// legacy agent.ToolEvent. -type ToolEvent struct { - ToolCallID string - Name string - DeviceID *uint64 - Status string - StartedAt time.Time - EndedAt *time.Time - DurationMs int64 - Error string - ArgsJSON string - ResultJSON string -} - -// Event is the streaming envelope. Mirrors legacy agent.Event so the -// service-layer adapter can copy fields one-to-one without a switch. -// -// Notification carries the task_notification payload — set when -// Type == EventTaskNotification and a background worker reaches a -// terminal state. The SPA / SSE adapter renders it as a separate frame -// type next to assistant / tool / done. -type Event struct { - Type EventType - Assistant *AssistantEvent - Tool *ToolEvent - Done *Reply - Error string - Notification *TaskNotification - // Approval carries the approval_pending payload — set when - // Type == EventApprovalPending and a blocking tool (cloud_bash) has - // queued a proposal and is awaiting the human decision (HLD-021). - Approval *ApprovalPending -} - -// Emit is the streaming callback. nil-safe. -type Emit func(Event) - -// Request bundles every per-call input the runtime needs. -type Request struct { - SessionID string - UserID uint64 - // Role is the caller's system role (admin | user | viewer). - // gates the toolbag against this — viewer drops Class!=read so the - // LLM cannot reach mutating tools no matter what the persona allows. - Role string - UserText string - Attachments []aiopsmodel.Attachment - Mentions []Mention - Provider string - Model string - WebSearchEnabled bool - // Locale is the UI language ("en-US"/"zh-CN") the reply should use; - // threaded into graph.Input so the assembler adds a language directive. - Locale string - // Emit is the streaming sink. nil = no streaming (blocking call). - Emit Emit -} - -// Reply is the terminal result returned by Handle. Mirrors the -// legacy agent.Reply shape so the service layer can fan it back out -// through the same DTO without translation losses. -type Reply struct { - Message *aiopsmodel.Message - Usage llm.Usage - Iterations int - ToolCalls []*aiopsmodel.ToolCall -} - // Config tunes runtime behaviour. ToolBag is the wrapped (decorated) // tool list; the runtime does NOT wrap tools itself — that's the // caller's job at construction time so the decorator deps (audit @@ -271,6 +174,7 @@ type ToolBagProvider interface { type mutableToolBagProvider interface { ToolBagProvider ReplaceToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) + UpsertTools([]basetool.BaseTool) } // CredentialBinder resolves the vault credential NAMES bound to whichever @@ -304,7 +208,9 @@ type Runtime struct { // when the caller wires the runtime without calling SetToolBag — // not all code paths need it (legacy callers only feed // cfg.ToolBag through the graph). - bag ToolBagProvider + bag ToolBagProvider + toolVersions map[string]uint64 + nextToolVersion uint64 } // NewRuntime builds a Runtime. Returns an error when required deps @@ -323,7 +229,13 @@ func NewRuntime(cfg Config) (*Runtime, error) { if log == nil { log = slog.Default() } - return &Runtime{cfg: cfg, log: log}, nil + initial := cfg.ToolBag + cfg.ToolBag = nil + rt := &Runtime{cfg: cfg, log: log, toolVersions: map[string]uint64{}} + if err := rt.AppendToolBag(initial); err != nil { + return nil, fmt.Errorf("initialize tool catalog: %w", err) + } + return rt, nil } // SetMentionResolver wires the @-mention hydrator post-construction. @@ -372,79 +284,26 @@ func (rt *Runtime) SetToolBag(bag ToolBagProvider) { if rt == nil { return } + rt.toolMu.Lock() + defer rt.toolMu.Unlock() rt.bag = bag -} - -// AppendToolBag appends extra BaseTools to the coordinator tool bag. -// — used by main.go to bolt the coordinator-only AgentTool -// / SendMessageTool / TaskStopTool onto the Runtime AFTER construction -// (the chicken-and-egg: those tools need the Runtime as their spawner, -// the Runtime needs the tool bag at NewRuntime time). Workers do NOT -// observe these added tools — filterToolsForAgent strips them -// unconditionally via coordinatorOnlyTools (see worker.go). -// -// This is intentionally distinct from a generic "AddTool" method: it -// signals "these are coordinator-only" so a future reader of main.go -// understands why three more tools appear after NewRuntime returned. -func (rt *Runtime) AppendToolBag(tools []basetool.BaseTool) { - if rt == nil || len(tools) == 0 { + if bag == nil { return } - rt.toolMu.Lock() - defer rt.toolMu.Unlock() - index := map[string]int{} - for i, t := range rt.cfg.ToolBag { - if t == nil { - continue - } + full := map[string]*basetool.ToolInfo{} + for _, t := range bag.AllTools() { if info, err := t.Info(context.Background()); err == nil && info != nil { - index[info.Name] = i + full[info.Name] = info } } - for _, t := range tools { - if t == nil { - continue - } - info, err := t.Info(context.Background()) - if err == nil && info != nil { - if i, ok := index[info.Name]; ok { - rt.cfg.ToolBag[i] = t - continue + for i, t := range rt.cfg.ToolBag { + if current, ok := t.(*runtimeTool); ok { + if info := full[current.name]; info != nil { + copy := *current + copy.fullInfo = cloneToolInfo(info) + rt.cfg.ToolBag[i] = © } - index[info.Name] = len(rt.cfg.ToolBag) } - rt.cfg.ToolBag = append(rt.cfg.ToolBag, t) - } -} - -// ReplaceToolsByNamePrefix replaces tools in the graph-facing bag whose wire -// names share prefix. MCP uses one prefix per server, preventing a refresh of -// one registration from removing tools offered by other servers. -func (rt *Runtime) ReplaceToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) { - if rt == nil || strings.TrimSpace(prefix) == "" { - return - } - rt.toolMu.Lock() - defer rt.toolMu.Unlock() - next := make([]basetool.BaseTool, 0, len(rt.cfg.ToolBag)+len(replacements)) - for _, tool := range rt.cfg.ToolBag { - if toolNameHasPrefix(tool, prefix) { - continue - } - next = append(next, tool) - } - rt.cfg.ToolBag = append(next, replacements...) -} - -// ReplaceCatalogToolsByNamePrefix updates the unredacted catalog used by -// ToolSearch. It is separate from ReplaceToolsByNamePrefix because the graph -// sees decorated tools while ToolSearch must retain raw schemas. -func (rt *Runtime) ReplaceCatalogToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) { - if rt == nil || strings.TrimSpace(prefix) == "" { - return - } - if bag, ok := rt.bag.(mutableToolBagProvider); ok { - bag.ReplaceToolsByNamePrefix(prefix, replacements) } } @@ -537,6 +396,8 @@ func (rt *Runtime) Handle(ctx context.Context, req *Request) (result *Reply, ret // when a background worker terminates. / ctx = withEmit(ctx, req.Emit) + ctx = basetool.WithWebSearchAllowed(ctx, req.WebSearchEnabled) + // 1. Ownership check. sess, err := rt.cfg.Sessions.GetSession(ctx, req.SessionID) if err != nil { @@ -547,6 +408,13 @@ func (rt *Runtime) Handle(ctx context.Context, req *Request) (result *Reply, ret // ErrForbidden) so non-owners can't fingerprint sessions. return nil, errs.ErrNotFound } + if version := contract.VersionFromContext(ctx); version != "" && version != contract.V2 { + return nil, fmt.Errorf("%w: request runtime mismatch", errs.ErrInvalid) + } + if sess.RuntimeVersion != "" && sess.RuntimeVersion != string(contract.V2) { + return nil, fmt.Errorf("%w: session runtime mismatch", errs.ErrInvalid) + } + ctx = contract.WithVersion(ctx, contract.V2) ctx, span := startAgentSpan(ctx, sess, "default") tracing.RecordAgentContent(span, "input.value", req.UserText) var runErr error @@ -639,7 +507,7 @@ func (rt *Runtime) Handle(ctx context.Context, req *Request) (result *Reply, ret // the coordinator path now honors it too. Stale agent ids fall back // to the global default with a log line — never crash a session. basePrompt := rt.cfg.BasePrompt - runtimeToolBag := rt.toolBagSnapshot() + runtimeToolBag := basetool.FilterWebSearch(ctx, rt.toolBagSnapshot()) sessionToolBag := runtimeToolBag agentReminderForPersona := "" // Coordinator iff session is pinned to "default" or has no diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_worker_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_worker_test.go index b36b64ad9..20fda346f 100644 --- a/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_worker_test.go +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/runtime_worker_test.go @@ -55,11 +55,14 @@ func TestSpawnWorker_Sync(t *testing.T) { t.Errorf("session_id = %q, want worker-* prefix", w.SessionID) } - // chat_sessions audit row — assert all three columns are set. + // chat_sessions audit row — assert version and ownership are preserved. row, err := store.GetSession(context.Background(), w.SessionID) if err != nil || row == nil { t.Fatalf("worker chat_sessions row missing: %v", err) } + if row.RuntimeVersion != "v2" { + t.Fatalf("worker runtime = %q, want v2", row.RuntimeVersion) + } if row.AgentID == nil || *row.AgentID != "incident-investigator" { t.Errorf("row.AgentID = %v, want incident-investigator", row.AgentID) } @@ -631,3 +634,16 @@ func newSeededRuntimeStore(t *testing.T, userID uint64) (*memSessions, string) { store := newMemSessions(&model.Session{ID: parentID, UserID: userID, Title: "parent", RootSessionID: &rootID, OwnerAgentID: &owner}) return store, parentID } + +func TestSpawnWorker_RejectsParentFromAnotherRuntime(t *testing.T) { + store := newMemSessions(&model.Session{ID: "parent", UserID: 7, RuntimeVersion: "v3"}) + rt := newRuntimeWithStore(t, "incident-investigator", newScriptedChatModel(&schema.Message{Role: schema.Assistant, Content: "must not run"}), nil, store) + _, err := rt.SpawnWorker(context.Background(), SpawnRequest{AgentName: "incident-investigator", Prompt: "work", ParentSession: "parent"}) + if err == nil || !strings.Contains(err.Error(), "runtime mismatch") { + t.Fatalf("cross-runtime spawn = %v", err) + } + children, listErr := store.ListByParent(context.Background(), "parent") + if listErr != nil || len(children) != 0 { + t.Fatalf("rejected spawn persisted children: %+v %v", children, listErr) + } +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/tool_lifecycle.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/tool_lifecycle.go new file mode 100644 index 000000000..6fbfb240a --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/tool_lifecycle.go @@ -0,0 +1,168 @@ +package chatruntime + +import ( + "context" + "fmt" + "strings" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// runtimeTool 保留本轮 schema；撤销或替换后，旧引用不得开始新的调用。 +// 已经进入执行的调用继续沿用原超时/取消机制，不在持锁期间执行外部 IO。 +type runtimeTool struct { + inner basetool.BaseTool + runtime *Runtime + name string + version uint64 + info *basetool.ToolInfo + fullInfo *basetool.ToolInfo +} + +func cloneToolInfo(info *basetool.ToolInfo) *basetool.ToolInfo { + copy := *info + copy.Parameters = append([]byte(nil), info.Parameters...) + return © +} + +func (rt *Runtime) versionedToolLocked(inner basetool.BaseTool, info, full *basetool.ToolInfo) basetool.BaseTool { + rt.nextToolVersion++ + rt.toolVersions[info.Name] = rt.nextToolVersion + return &runtimeTool{inner: inner, runtime: rt, name: info.Name, version: rt.nextToolVersion, info: cloneToolInfo(info), fullInfo: cloneToolInfo(full)} +} +func (t *runtimeTool) Info(context.Context) (*basetool.ToolInfo, error) { + return cloneToolInfo(t.info), nil +} + +// FullInfo 保留此执行版本的完整 schema，避免旧回合读到新版本参数。 +func (t *runtimeTool) FullInfo(context.Context) (*basetool.ToolInfo, error) { + return cloneToolInfo(t.fullInfo), nil +} + +// inspectToolBatch 在修改任何视图前校验整批元数据，错误不破坏现有工具。 +type toolBatch struct { + infos map[string]*basetool.ToolInfo + names []string +} + +func inspectToolBatch(tools []basetool.BaseTool, prefix string) (*toolBatch, error) { + infos := make(map[string]*basetool.ToolInfo, len(tools)) + names := make([]string, 0, len(tools)) + for _, t := range tools { + if t == nil { + return nil, fmt.Errorf("nil tool in registration") + } + info, err := t.Info(context.Background()) + if err != nil { + return nil, fmt.Errorf("read tool metadata: %w", err) + } + if info == nil || strings.TrimSpace(info.Name) == "" { + return nil, fmt.Errorf("tool name required") + } + if !strings.HasPrefix(info.Name, prefix) { + return nil, fmt.Errorf("tool %q is outside source %q", info.Name, prefix) + } + if infos[info.Name] != nil { + return nil, fmt.Errorf("duplicate tool %q", info.Name) + } + infos[info.Name] = cloneToolInfo(info) + names = append(names, info.Name) + } + return &toolBatch{infos: infos, names: names}, nil +} + +// AppendToolBag 同步注册执行与搜索视图；同名工具替换，整批重复名称被拒绝。 +func (rt *Runtime) AppendToolBag(tools []basetool.BaseTool) error { + if rt == nil { + return fmt.Errorf("runtime unavailable") + } + infos, err := inspectToolBatch(tools, "") + if err != nil { + return err + } + rt.toolMu.Lock() + defer rt.toolMu.Unlock() + if rt.toolVersions == nil { + rt.toolVersions = map[string]uint64{} + } + index := map[string]int{} + for i, t := range rt.cfg.ToolBag { + index[t.(*runtimeTool).name] = i + } + for j, t := range tools { + info := infos.infos[infos.names[j]] + wrapped := rt.versionedToolLocked(t, info, info) + if i, exists := index[info.Name]; exists { + rt.cfg.ToolBag[i] = wrapped + } else { + rt.cfg.ToolBag = append(rt.cfg.ToolBag, wrapped) + } + } + if bag, ok := rt.bag.(mutableToolBagProvider); ok { + bag.UpsertTools(tools) + } + return nil +} + +func (rt *Runtime) ReplaceToolsByNamePrefix(prefix string, replacements []basetool.BaseTool) error { + return rt.ReplaceToolSource(prefix, replacements, replacements) +} + +// ReplaceToolSource 校验来源边界与两份视图，成功后一起替换并撤销旧版本。 +func (rt *Runtime) ReplaceToolSource(prefix string, raw, decorated []basetool.BaseTool) error { + if rt == nil || strings.TrimSpace(prefix) == "" { + return fmt.Errorf("runtime and tool source required") + } + full, err := inspectToolBatch(raw, prefix) + if err != nil { + return err + } + infos, err := inspectToolBatch(decorated, prefix) + if err != nil { + return err + } + if len(full.infos) != len(infos.infos) { + return fmt.Errorf("tool source %q catalog/executor mismatch", prefix) + } + for name := range infos.infos { + if full.infos[name] == nil { + return fmt.Errorf("tool %q has no catalog entry", name) + } + } + rt.toolMu.Lock() + defer rt.toolMu.Unlock() + if rt.toolVersions == nil { + rt.toolVersions = map[string]uint64{} + } + next := make([]basetool.BaseTool, 0, len(rt.cfg.ToolBag)+len(decorated)) + for _, t := range rt.cfg.ToolBag { + current := t.(*runtimeTool) + if strings.HasPrefix(current.name, prefix) { + delete(rt.toolVersions, current.name) + } else { + next = append(next, t) + } + } + for j, t := range decorated { + name := infos.names[j] + next = append(next, rt.versionedToolLocked(t, infos.infos[name], full.infos[name])) + } + if bag, ok := rt.bag.(mutableToolBagProvider); ok { + bag.ReplaceToolsByNamePrefix(prefix, raw) + } + rt.cfg.ToolBag = next + return nil +} + +func (t *runtimeTool) InvokableRun(ctx context.Context, args string, opts ...basetool.InvokeOption) (string, error) { + t.runtime.toolMu.RLock() + current := t.runtime.toolVersions[t.name] + t.runtime.toolMu.RUnlock() + if current != t.version { + return "", fmt.Errorf("tool %q was removed or replaced; start a new turn", t.name) + } + if t.name == basetool.ToolWebSearch && !basetool.WebSearchAllowed(ctx) { + return "", fmt.Errorf("web_search disabled for this request") + } + return t.inner.InvokableRun(ctx, args, opts...) +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/tool_lifecycle_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/tool_lifecycle_test.go new file mode 100644 index 000000000..538523356 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/tool_lifecycle_test.go @@ -0,0 +1,229 @@ +package chatruntime + +import ( + "context" + "encoding/json" + "fmt" + aiopstools "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cloudwego/eino/schema" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + model "github.com/ongridio/ongrid/internal/manager/model/aiops" +) + +func TestRuntimeRemovedOrReplacedToolRejectsStaleReference(t *testing.T) { + rt, err := NewRuntime(Config{Sessions: newMemSessions(nil), ChatModel: newScriptedChatModel(), ToolBag: []basetool.BaseTool{&fakeTool{name: "mcp__test__search", schema: `{"type":"object"}`}}}) + if err != nil { + t.Fatal(err) + } + old := rt.toolBagSnapshot()[0] + if err := rt.ReplaceToolSource("mcp__test__", nil, nil); err != nil { + t.Fatal(err) + } + if _, err := old.InvokableRun(context.Background(), `{}`); err == nil { + t.Fatal("removed tool executed") + } + fresh := &fakeTool{name: "mcp__test__search", schema: `{"type":"object"}`} + if err := rt.ReplaceToolSource("mcp__test__", []basetool.BaseTool{fresh}, []basetool.BaseTool{fresh}); err != nil { + t.Fatal(err) + } + if _, err := old.InvokableRun(context.Background(), `{}`); err == nil { + t.Fatal("old generation became executable again") + } + if _, err := rt.toolBagSnapshot()[0].InvokableRun(context.Background(), `{}`); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeConcurrentToolRefreshAndSnapshot(t *testing.T) { + rt, err := NewRuntime(Config{Sessions: newMemSessions(nil), ChatModel: newScriptedChatModel()}) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + tool := &fakeTool{name: "mcp__test__search", schema: `{"type":"object"}`} + if err := rt.ReplaceToolSource("mcp__test__", []basetool.BaseTool{tool}, []basetool.BaseTool{tool}); err != nil { + t.Error(err) + return + } + for _, entry := range rt.toolBagSnapshot() { + // 并发替换允许明确拒绝旧引用，不能数据竞争或执行已撤销版本。 + _, err := entry.InvokableRun(context.Background(), `{}`) + if err != nil { + continue + } + } + } + }() + } + wg.Wait() +} + +type countingSearchTool struct{ calls atomic.Int32 } + +func (t *countingSearchTool) Info(context.Context) (*basetool.ToolInfo, error) { + return &basetool.ToolInfo{Name: basetool.ToolWebSearch, Description: "search", Class: "read", Parameters: []byte(`{"type":"object","properties":{"query":{"type":"string"}}}`)}, nil +} +func (t *countingSearchTool) InvokableRun(context.Context, string, ...basetool.InvokeOption) (string, error) { + t.calls.Add(1) + return `{"result":"found"}`, nil +} + +func TestRuntimeWebSearchToggleControlsActualDispatch(t *testing.T) { + for _, enabled := range []bool{false, true} { + t.Run(map[bool]string{false: "disabled", true: "enabled"}[enabled], func(t *testing.T) { + tool := &countingSearchTool{} + modelStub := newScriptedChatModel(&schema.Message{Role: schema.Assistant, ToolCalls: []schema.ToolCall{{ID: "tc-search", Type: "function", Function: schema.FunctionCall{Name: "web_search", Arguments: `{"query":"test"}`}}}}, &schema.Message{Role: schema.Assistant, Content: "done"}) + rt, err := NewRuntime(Config{Sessions: newMemSessions(&model.Session{ID: "s1", UserID: 7}), ChatModel: modelStub, ToolBag: []basetool.BaseTool{tool}}) + if err != nil { + t.Fatal(err) + } + if _, err := rt.Handle(context.Background(), &Request{SessionID: "s1", UserID: 7, Role: "admin", UserText: "search", WebSearchEnabled: enabled}); err != nil { + t.Fatal(err) + } + want := int32(0) + if enabled { + want = 1 + } + if tool.calls.Load() != want { + t.Fatalf("calls=%d want=%d", tool.calls.Load(), want) + } + // 即使持有旧 schema 或直接调用工具引用，也不能越过执行边界。 + if _, err := rt.toolBagSnapshot()[0].InvokableRun(context.Background(), `{}`); err == nil { + t.Fatal("missing execution gate") + } + }) + } +} + +func TestWorkerInheritsWebSearchGate(t *testing.T) { + for _, background := range []bool{false, true} { + for _, enabled := range []bool{false, true} { + t.Run(fmt.Sprintf("background=%t/enabled=%t", background, enabled), func(t *testing.T) { + tool := &countingSearchTool{} + scripted := newScriptedChatModel(&schema.Message{Role: schema.Assistant, ToolCalls: []schema.ToolCall{{ID: "worker-search", Type: "function", Function: schema.FunctionCall{Name: "web_search", Arguments: `{"query":"test"}`}}}}, &schema.Message{Role: schema.Assistant, Content: "done"}) + store, parent := newSeededRuntimeStore(t, 7) + rt := newRuntimeWithStore(t, "network", scripted, []basetool.BaseTool{tool}, store) + ctx := basetool.WithWebSearchAllowed(context.Background(), enabled) + done := make(chan struct{}, 1) + _, err := rt.SpawnWorker(ctx, SpawnRequest{AgentName: "network", ParentSession: parent, Prompt: "search", Background: background, ParentEmit: func(ev Event) { + if ev.Type == EventTaskNotification { + done <- struct{}{} + } + }}) + if err != nil { + t.Fatal(err) + } + if background { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("worker did not finish") + } + } + want := int32(0) + if enabled { + want = 1 + } + if tool.calls.Load() != want { + t.Fatalf("calls=%d want=%d", tool.calls.Load(), want) + } + }) + } + } +} + +func TestToolSearchKeepsTurnSchemaAcrossRefresh(t *testing.T) { + old := &fakeTool{name: "mcp__test__search", schema: `{"type":"object","properties":{"old_param":{"type":"string"}}}`} + bag := aiopstools.NewToolBag([]basetool.BaseTool{old}, 0) + rt, err := NewRuntime(Config{Sessions: newMemSessions(nil), ChatModel: newScriptedChatModel(), ToolBag: bag.SchemasForLLM()}) + if err != nil { + t.Fatal(err) + } + rt.SetToolBag(bag) + before := rt.toolBagSnapshot() + fresh := &fakeTool{name: old.name, schema: `{"type":"object","properties":{"new_param":{"type":"string"}}}`} + if err := rt.ReplaceToolSource("mcp__test__", []basetool.BaseTool{fresh}, []basetool.BaseTool{fresh}); err != nil { + t.Fatal(err) + } + search := aiopstools.NewToolSearchTool(bag, nil) + for _, tc := range []struct { + view []basetool.BaseTool + want string + }{{before, "old_param"}, {rt.toolBagSnapshot(), "new_param"}} { + result, err := search.InvokableRun(basetool.WithFilteredTools(context.Background(), tc.view), `{"query":"select:mcp__test__search"}`) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(result, tc.want) { + t.Fatalf("want %s, got %s", tc.want, result) + } + } + if _, err := before[0].InvokableRun(context.Background(), `{}`); err == nil { + t.Fatal("old executable survived refresh") + } + // An empty child view must clear the inherited parent view. + ctx := basetool.WithFilteredTools(context.Background(), rt.toolBagSnapshot()) + result, err := search.InvokableRun(basetool.WithFilteredTools(ctx, nil), `{"query":"select:mcp__test__search"}`) + if err != nil { + t.Fatal(err) + } + var response struct { + Tools []json.RawMessage `json:"tools"` + } + if err := json.Unmarshal([]byte(result), &response); err != nil { + t.Fatal(err) + } + if len(response.Tools) != 0 { + t.Fatalf("empty view leaked parent schemas: %s", result) + } +} + +func TestInvalidToolSourceUpdatePreservesExistingViews(t *testing.T) { + old := &fakeTool{name: "mcp__test__old", schema: `{"type":"object"}`} + other := &fakeTool{name: "mcp__other__tool", schema: `{"type":"object"}`} + for _, tc := range []struct { + name string + raw, executed []basetool.BaseTool + }{ + {"duplicate", []basetool.BaseTool{old, old}, []basetool.BaseTool{old, old}}, + {"outside source", []basetool.BaseTool{other}, []basetool.BaseTool{other}}, + {"missing execution", []basetool.BaseTool{old}, nil}, + {"missing schema", nil, []basetool.BaseTool{old}}, + {"nil entry", []basetool.BaseTool{nil}, []basetool.BaseTool{nil}}, + } { + t.Run(tc.name, func(t *testing.T) { + bag := aiopstools.NewToolBag([]basetool.BaseTool{old}, 30) + rt, err := NewRuntime(Config{Sessions: newMemSessions(nil), ChatModel: newScriptedChatModel(), ToolBag: bag.AllTools()}) + if err != nil { + t.Fatal(err) + } + rt.SetToolBag(bag) + ref := rt.toolBagSnapshot()[0] + if err := rt.ReplaceToolSource("mcp__test__", tc.raw, tc.executed); err == nil { + t.Fatal("invalid update accepted") + } + if _, err := ref.InvokableRun(context.Background(), `{}`); err != nil { + t.Fatalf("existing executable revoked: %v", err) + } + all := bag.AllTools() + info, err := all[0].Info(context.Background()) + if err != nil || len(all) != 1 || info.Name != old.name { + t.Fatal("catalog changed after rejected update") + } + }) + } + if _, err := NewRuntime(Config{Sessions: newMemSessions(nil), ChatModel: newScriptedChatModel(), ToolBag: []basetool.BaseTool{old, old}}); err == nil { + t.Fatal("startup accepted duplicate tools") + } +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/worker.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/worker.go index 5c7f527ad..95a3674da 100644 --- a/internal/manager/biz/aiops/runtime/v2/chatruntime/worker.go +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/worker.go @@ -42,6 +42,8 @@ import ( "encoding/json" "errors" "fmt" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" + "github.com/ongridio/ongrid/internal/pkg/tenantctx" "strings" "sync" "time" @@ -62,7 +64,7 @@ import ( // WorkerStatus enumerates the worker state machine. See package header // for the transition diagram. -type WorkerStatus string +type WorkerStatus = contract.WorkerStatus const ( // WorkerStatusPending — accepted but not yet started. Observable @@ -121,121 +123,15 @@ type Worker struct { mu sync.Mutex } -// SpawnRequest describes a worker to spawn. See AgentTool.InvokableRun -// for the LLM-facing argument shape that produces this struct. -type SpawnRequest struct { - // AgentName is the persona name (frontmatter `name`). Must be - // resolvable through the Runtime's AgentRegistry. - AgentName string - // Prompt is the initial user message for the worker. - Prompt string - // Background = true: fire-and-forget; SpawnWorker returns immediately - // with status=running. Background = false: SpawnWorker blocks until - // the worker reaches a terminal state. - Background bool - // ParentSession is the coordinator's chat session id. - ParentSession string - // ParentEmit is the SSE emitter the coordinator runtime threads - // through. Used to deliver the task_notification frame when the - // worker (background=true) finishes. nil = no notification (sync - // callers don't need it). - ParentEmit Emit - // SessionKind overrides the persisted chat_sessions.kind for the - // worker session. Empty = "work". The investigator usecase - // sets "investigation" so auto-spawned RCA transcripts stay out of - // the /chat list. - SessionKind string - // OwnerUserID overrides the persisted chat_sessions.user_id when - // the worker has no ParentSession to inherit from. The investigator - // usecase passes 0 to mark these rows as "system-owned" — they - // don't belong to any operator and the internal audience prevents - // them from appearing in the primary chat list. - OwnerUserID uint64 - // Locale is the UI language ("en", "zh-CN", ...) the worker's reply - // should use. Threaded from the parent coordinator's request so a - // sub-agent answers in the same language the user is typing in - // (otherwise GLM defaults to zh on English questions). Empty = no - // directive, back-compat with investigator/auto-spawned workers. - Locale string - // Provider + Model are the coordinator's resolved LLM choice. Threaded - // into runWorker's g.Invoke as chatModelOpts so the sub-agent runs - // against the same LLM the user picked for the coordinator — without - // this, runWorker falls through to the RoutingChatModel default and - // installs without an OpenAI key see specialists fail with - // `provider "openai" not configured`. Empty = no override; the worker - // uses whatever the routing default is. - Provider string - Model string -} - -// TaskNotification is the payload of a task_notification streaming -// frame. Mirrors the schema laid out in — coordinator's -// SSE listener fans this back through the legacy SSE envelope so the -// SPA only needs to learn one new event type. -type TaskNotification struct { - TaskID string `json:"task_id"` - Status WorkerStatus `json:"status"` - Summary string `json:"summary"` - Result string `json:"result,omitempty"` - Err string `json:"error,omitempty"` - Usage map[string]any `json:"usage,omitempty"` -} - -// EventTaskNotification is the new streaming event type the runtime -// emits when a background worker reaches a terminal state. The SPA -// listens for this in addition to the existing assistant / tool / -// done frames. -const EventTaskNotification EventType = "task_notification" - -// EventApprovalPending fires when a synchronous-blocking tool (HLD-021, -// e.g. cloud_bash) has queued a human-approval proposal and is about to -// block waiting for the decision. The frame surfaces the inline approve/ -// reject card LIVE — the tool no longer returns a pending_approval result -// blob (it blocks, then returns the real executor result), so the card -// must be driven by this frame instead. ToolCallID ties the card to the -// tool call's existing streaming card so the UI shows a single card. -const EventApprovalPending EventType = "approval_pending" - -// ApprovalPending is the EventApprovalPending payload. Emitted by the -// proposer shim (cmd/main.go) via EmitFromContext while a blocking tool -// awaits the human decision. -type ApprovalPending struct { - ApprovalID string - ToolCallID string - Kind string - ToolName string - Command string - Credentials []string -} +type SpawnRequest = contract.SpawnRequest +type TaskNotification = contract.TaskNotification +type ApprovalPending = contract.ApprovalPending -// emitCtxKey is the unexported context key that threads the active -// per-request Emit through the graph layer down into AgentTool's -// InvokableRun via the WorkerSpawner shim. The shim reads it through -// EmitFromContext to populate SpawnRequest.ParentEmit so a background -// worker can fire its task_notification through the SSE channel that -// owns the user's chat session. -type emitCtxKeyT struct{} +const EventTaskNotification = contract.EventTaskNotification +const EventApprovalPending = contract.EventApprovalPending -var emitCtxKey = emitCtxKeyT{} - -// withEmit returns ctx augmented with emit. Internal — Handle threads -// the request emitter in before invoking the graph. -func withEmit(ctx context.Context, emit Emit) context.Context { - if emit == nil { - return ctx - } - return context.WithValue(ctx, emitCtxKey, emit) -} - -// EmitFromContext retrieves the active streaming Emit from ctx, if any. -// Returns nil when no emitter was attached (e.g. blocking call without -// SSE). The wiring shim at the cmd/main.go boundary uses this to thread -// task_notification frames through the same SSE channel that owns the -// coordinator's chat session. -func EmitFromContext(ctx context.Context) Emit { - v, _ := ctx.Value(emitCtxKey).(Emit) - return v -} +func withEmit(ctx context.Context, emit Emit) context.Context { return contract.WithEmit(ctx, emit) } +func EmitFromContext(ctx context.Context) Emit { return contract.EmitFromContext(ctx) } // Locale ctx propagation lives in the tools package (next to AgentTool, // the consumer). Runtime calls tools.WithLocale before g.Invoke; the @@ -253,6 +149,10 @@ func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, if rt == nil { return nil, errors.New("chatruntime: nil runtime") } + if version := contract.VersionFromContext(ctx); version != "" && version != contract.V2 { + return nil, fmt.Errorf("chatruntime: worker runtime mismatch") + } + ctx = contract.WithVersion(ctx, contract.V2) if rt.cfg.ChatModel == nil { return nil, errors.New("chatruntime: ChatModel is required") } @@ -281,6 +181,9 @@ func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, if req.ParentSession != "" && rt.cfg.Sessions != nil { parentSess, err := rt.cfg.Sessions.GetSession(ctx, req.ParentSession) if err == nil && parentSess != nil { + if parentSess.RuntimeVersion != "" && parentSess.RuntimeVersion != string(contract.V2) { + return nil, fmt.Errorf("chatruntime: parent runtime mismatch") + } if parentSess.ParentSessionID != nil && *parentSess.ParentSessionID != "" { return nil, fmt.Errorf("chatruntime: delegation depth exceeded for parent session %q", req.ParentSession) } @@ -323,6 +226,7 @@ func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, } row := &aiopsmodel.Session{ ID: sessID, + RuntimeVersion: string(contract.V2), APMSource: sourceScope, UserID: effectiveOwner, Title: fmt.Sprintf("Worker: %s", agentDef.Name), @@ -348,7 +252,7 @@ func (rt *Runtime) SpawnWorker(ctx context.Context, req SpawnRequest) (*Worker, parent := ctx if req.Background { // Keep trace ancestry without inheriting cancellation or callback state. - parent = trace.ContextWithSpanContext(context.Background(), trace.SpanContextFromContext(ctx)) + parent = detachedWorkerContext(ctx) } workerCtx, cancel := context.WithCancel(parent) @@ -602,7 +506,7 @@ func (rt *Runtime) runWorker(ctx context.Context, agentDef *Agent, sessID, userT // Workers are not an authorization bypass. The coordinator stamps the // resolved gate on context; absent wiring fails closed so a direct spawn // cannot expose mutating tools unexpectedly. - workerTools := filterToolsForAgentRole(rt.toolBagSnapshot(), agentDef, false, !basetool.AgentWriteAllowedFromContext(ctx)) + workerTools := filterToolsForAgentRole(basetool.FilterWebSearch(ctx, rt.toolBagSnapshot()), agentDef, false, !basetool.AgentWriteAllowedFromContext(ctx)) // Thread the persona-filtered tool view onto ctx so ToolSearch // (which runs inside the worker graph) only returns tools the @@ -1078,3 +982,16 @@ func workerChatModelOpts(ctx context.Context) []model.Option { } return opts } + +// detachedWorkerContext preserves authorization and tracing without retaining +// request cancellation, callback handlers, or mutable request slots. +func detachedWorkerContext(ctx context.Context) context.Context { + parent := trace.ContextWithSpanContext(context.Background(), trace.SpanContextFromContext(ctx)) + parent = contract.WithVersion(parent, contract.VersionFromContext(ctx)) + parent = basetool.WithWebSearchAllowed(parent, basetool.WebSearchAllowed(ctx)) + parent = basetool.WithAgentWriteAllowed(parent, basetool.AgentWriteAllowedFromContext(ctx)) + if tenant, ok := tenantctx.From(ctx); ok { + parent = tenantctx.With(parent, tenant) + } + return parent +} diff --git a/internal/manager/biz/aiops/runtime/v2/chatruntime/worker_context_test.go b/internal/manager/biz/aiops/runtime/v2/chatruntime/worker_context_test.go new file mode 100644 index 000000000..969c56099 --- /dev/null +++ b/internal/manager/biz/aiops/runtime/v2/chatruntime/worker_context_test.go @@ -0,0 +1,48 @@ +package chatruntime + +import ( + "context" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + "github.com/ongridio/ongrid/internal/pkg/tenantctx" + "go.opentelemetry.io/otel/trace" + "testing" +) + +func TestDetachedWorkerContextPreservesAuthorizationWithoutRequestState(t *testing.T) { + type requestKey struct{} + for _, allowed := range []bool{false, true} { + ctx, cancel := context.WithCancel(context.Background()) + ctx = context.WithValue(ctx, requestKey{}, "callback state") + ctx = contract.WithVersion(ctx, contract.V2) + ctx = contract.WithEmit(ctx, func(contract.Event) {}) + span := trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{2}, TraceFlags: trace.FlagsSampled}) + ctx = trace.ContextWithSpanContext(ctx, span) + ctx = basetool.WithWebSearchAllowed(ctx, allowed) + ctx = basetool.WithAgentWriteAllowed(ctx, allowed) + ctx = tenantctx.WithSlot(ctx) + identity := tenantctx.Tenant{UserID: 42, Role: "admin", IsSuperuser: true} + tenantctx.SetOnSlot(ctx, identity) + worker := detachedWorkerContext(ctx) + cancel() + tenantctx.SetOnSlot(ctx, tenantctx.Tenant{UserID: 99}) + if worker.Err() != nil || worker.Done() != nil { + t.Fatal("worker inherited request cancellation") + } + if contract.VersionFromContext(worker) != contract.V2 || contract.EmitFromContext(worker) != nil { + t.Fatal("worker version lost or request emitter retained") + } + if worker.Value(requestKey{}) != nil { + t.Fatal("worker retained request-only state") + } + if basetool.WebSearchAllowed(worker) != allowed || basetool.AgentWriteAllowedFromContext(worker) != allowed { + t.Fatal("worker capability gates changed") + } + if got, ok := tenantctx.From(worker); !ok || got != identity { + t.Fatalf("identity snapshot=%+v, present=%t", got, ok) + } + if got := trace.SpanContextFromContext(worker); !got.Equal(span) { + t.Fatalf("trace ancestry=%v, want %v", got, span) + } + } +} diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry.go index c6ca9c8d9..78948b7a6 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry.go @@ -17,6 +17,7 @@ import ( type AgentRegistry struct { mu sync.RWMutex agents []*Agent + builtins []*Agent warnings []LoadWarning } @@ -42,7 +43,7 @@ func (r *AgentRegistry) Load(agentsRoot string) error { return fmt.Errorf("chatruntime: agent registry load: %w", err) } r.mu.Lock() - r.agents = append([]*Agent(nil), res.Agents...) + r.agents = r.withBuiltinsLocked(res.Agents) r.warnings = append([]LoadWarning(nil), res.Warnings...) r.mu.Unlock() return nil @@ -69,7 +70,7 @@ func (r *AgentRegistry) Reload(agentsRoot string, extras ...string) error { newAgents := append([]*Agent(nil), res.Agents...) newWarnings := append([]LoadWarning(nil), res.Warnings...) r.mu.Lock() - r.agents = newAgents + r.agents = r.withBuiltinsLocked(newAgents) r.warnings = newWarnings r.mu.Unlock() return nil @@ -144,6 +145,42 @@ func (r *AgentRegistry) Add(ag *Agent) { r.mu.Unlock() } +// AddBuiltin 注册代码内置角色，磁盘热加载不能删除或覆盖这些保留名称。 +func (r *AgentRegistry) AddBuiltin(ag *Agent) { + if ag == nil || ag.Name == "" { + return + } + r.mu.Lock() + defer r.mu.Unlock() + replaced := false + for i, old := range r.builtins { + if old.Name == ag.Name { + r.builtins[i] = ag + replaced = true + break + } + } + if !replaced { + r.builtins = append(r.builtins, ag) + } + r.agents = r.withBuiltinsLocked(r.agents) +} + +// 调用方持有写锁。只保留显式注册的内置角色，已卸载插件不能残留。 +func (r *AgentRegistry) withBuiltinsLocked(loaded []*Agent) []*Agent { + out := append([]*Agent(nil), r.builtins...) + reserved := make(map[string]bool, len(r.builtins)) + for _, ag := range r.builtins { + reserved[ag.Name] = true + } + for _, ag := range loaded { + if ag != nil && !reserved[ag.Name] { + out = append(out, ag) + } + } + return out +} + // AddAll inserts a batch of agents. Nil entries are skipped. Used by // LoadAll to merge plugin-container output into the registry. func (r *AgentRegistry) AddAll(agents []*Agent) { diff --git a/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry_test.go b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry_test.go index bbf811b7d..4a0d8e836 100644 --- a/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry_test.go +++ b/internal/manager/biz/aiops/runtime/v3/chatruntime/agent_registry_test.go @@ -1,6 +1,7 @@ package chatruntime import ( + "os" "path/filepath" "testing" ) @@ -62,3 +63,38 @@ func TestAgentRegistryCapabilityCardsCarryOwner(t *testing.T) { t.Errorf("registry card mutated through snapshot: %q", got) } } + +func TestAgentRegistryReloadPreservesBuiltinAndRemovesUninstalledPersona(t *testing.T) { + r := NewAgentRegistry() + builtin := &Agent{Name: "default", Source: "builtin", MaxTurns: 30, Capabilities: []AgentCapability{{ID: "operate"}}} + r.AddBuiltin(builtin) + root := t.TempDir() + body := "---\nname: default\ndescription: shadow\nwhen_to_use: shadow\n---\nshadow" + if err := os.WriteFile(filepath.Join(root, "default.md"), []byte(body), 0600); err != nil { + t.Fatal(err) + } + if err := r.Reload(root, fixtureAgentRoot("multi")); err != nil { + t.Fatal(err) + } + if got, _ := r.ByName("default"); got != builtin { + t.Fatal("plugin replaced builtin persona") + } + if _, ok := r.ByName("reviewer"); !ok { + t.Fatal("plugin persona was not loaded") + } + if err := r.Reload(t.TempDir()); err != nil { + t.Fatal(err) + } + if got, _ := r.ByName("default"); got != builtin || got.MaxTurns != 30 { + t.Fatal("reload lost builtin configuration") + } + if len(r.All()) != 1 || len(r.CapabilityCards()) != 1 { + t.Fatal("uninstall left stale personas or removed builtin capabilities") + } + if err := r.Load(t.TempDir()); err != nil { + t.Fatal(err) + } + if got, _ := r.ByName("default"); got != builtin { + t.Fatal("load lost builtin persona") + } +} diff --git a/internal/manager/biz/mcp/lifecycle_test.go b/internal/manager/biz/mcp/lifecycle_test.go index f9d6ea128..49411c838 100644 --- a/internal/manager/biz/mcp/lifecycle_test.go +++ b/internal/manager/biz/mcp/lifecycle_test.go @@ -15,7 +15,7 @@ import ( ) func TestDiscoveryCannotRestoreDisabledOrDeletedServer(t *testing.T) { - for _, action := range []string{"disable", "delete", "replace", "refresh"} { + for _, action := range []string{"disable", "delete", "replace", "refresh", "background"} { t.Run(action, func(t *testing.T) { entered, release := make(chan struct{}), make(chan struct{}) first := true // httptest requests are sequential until the first tools/list blocks. @@ -72,6 +72,8 @@ func TestDiscoveryCannotRestoreDisabledOrDeletedServer(t *testing.T) { err = uc.Delete(ctx, row.ID) case "refresh": _, err = uc.TestConnection(ctx, row.ID) + case "background": + _, err = uc.RefreshConnection(ctx, row.ID) default: patch := *row if action == "disable" { @@ -85,11 +87,19 @@ func TestDiscoveryCannotRestoreDisabledOrDeletedServer(t *testing.T) { if err != nil { t.Fatal(err) } - if err := <-done; !errors.Is(err, errs.ErrConflict) { - t.Fatalf("stale discovery: %v", err) + probeErr := <-done + if action == "background" { + if probeErr != nil { + t.Fatalf("background refresh superseded manual verification: %v", probeErr) + } + } else if !errors.Is(probeErr, errs.ErrConflict) { + t.Fatalf("stale discovery: %v", probeErr) + } + if len(uc.manualProbes) != 0 { + t.Fatal("manual discovery ownership leaked") } want := 0 - if action == "refresh" { + if action == "refresh" || action == "background" { want = 1 } if published != want { @@ -145,3 +155,50 @@ func TestApprovalRevisionRejectsChangedEndpointAndCredential(t *testing.T) { }) } } + +func TestRegistrationChangesWakeDiscoveryWithoutConnecting(t *testing.T) { + ctx := t.Context() + repo := newFakeRepo() + uc := NewUsecase(repo, nil, nil) + // 无可连接的端点：注册保存不能被远端网络阻塞。 + server, err := uc.Create(ctx, &model.Server{Name: "wake", Transport: TransportHTTP, Endpoint: "http://127.0.0.1:1/mcp", Enabled: true}) + if err != nil { + t.Fatal(err) + } + assertWake := func() { + t.Helper() + select { + case <-uc.RegistrationChanges(): + default: + t.Fatal("registration change did not wake discovery") + } + } + assertWake() + for _, enabled := range []bool{false, true} { + if err := uc.Update(ctx, server.ID, &model.Server{Name: "wake", Transport: TransportHTTP, Endpoint: "http://127.0.0.1:2/mcp", Enabled: enabled}); err != nil { + t.Fatal(err) + } + assertWake() + } + // 消费者暂停时连续写入仍能完成，唤醒合并后由 supervisor 读取最新状态。 + for i := 0; i < 3; i++ { + if err := uc.Update(ctx, server.ID, &model.Server{Name: "wake", Transport: TransportHTTP, Endpoint: "http://127.0.0.1:2/mcp", Enabled: true}); err != nil { + t.Fatal(err) + } + } + if err := uc.Delete(ctx, server.ID); err != nil { + t.Fatal(err) + } + assertWake() + if _, err := uc.Get(ctx, server.ID); !errors.Is(err, errs.ErrNotFound) { + t.Fatalf("latest registration state: %v", err) + } + if err := uc.Delete(ctx, server.ID); !errors.Is(err, errs.ErrNotFound) { + t.Fatalf("missing delete: %v", err) + } + select { + case <-uc.RegistrationChanges(): + t.Fatal("failed mutation woke discovery") + default: + } +} diff --git a/internal/manager/biz/mcp/usecase.go b/internal/manager/biz/mcp/usecase.go index 5fdf1f1ff..4c336eb61 100644 --- a/internal/manager/biz/mcp/usecase.go +++ b/internal/manager/biz/mcp/usecase.go @@ -50,15 +50,17 @@ type ToolChangeHook func(ctx context.Context, serverName string, server *model.S // Usecase is the MCP-server-registration facade. type Usecase struct { - repo Repo - secrets SecretResolver - log *slog.Logger - onToolsChange ToolChangeHook + repo Repo + secrets SecretResolver + log *slog.Logger + onToolsChange ToolChangeHook + registrationChanges chan struct{} // commitGate serializes registration changes and discovery publication. // Remote MCP requests never hold it; waiting respects request cancellation. commitGate chan struct{} probeSequence uint64 latestProbe map[uint64]uint64 + manualProbes map[uint64]int } // SetToolChangeHook wires the optional runtime catalog updater. It is set @@ -80,7 +82,18 @@ func NewUsecase(repo Repo, secrets SecretResolver, log *slog.Logger) *Usecase { if log == nil { log = slog.Default() } - return &Usecase{repo: repo, secrets: secrets, log: log, commitGate: make(chan struct{}, 1), latestProbe: make(map[uint64]uint64)} + return &Usecase{repo: repo, secrets: secrets, log: log, registrationChanges: make(chan struct{}, 1), commitGate: make(chan struct{}, 1), latestProbe: make(map[uint64]uint64), manualProbes: make(map[uint64]int)} +} + +// RegistrationChanges 唤醒唯一的发现 supervisor。合并连续变更，由消费者重新读取 +// 最新注册表；不在 HTTP 请求中等待远端连接，也不为每次保存启动 goroutine。 +func (u *Usecase) RegistrationChanges() <-chan struct{} { return u.registrationChanges } + +func (u *Usecase) notifyRegistrationChanged() { + select { + case u.registrationChanges <- struct{}{}: + default: + } } func (u *Usecase) enterCommit(ctx context.Context) error { @@ -105,6 +118,7 @@ func (u *Usecase) Create(ctx context.Context, s *model.Server) (*model.Server, e if err := u.repo.Create(ctx, s); err != nil { return nil, err } + u.notifyRegistrationChanged() return s, nil } @@ -127,6 +141,7 @@ func (u *Usecase) Update(ctx context.Context, id uint64, patch *model.Server) er if err := u.repo.Update(ctx, id, patch); err != nil { return err } + defer u.notifyRegistrationChanged() // Invalidate discoveries started before this change, including disable/re-enable. delete(u.latestProbe, id) updated, err := u.repo.Get(ctx, id) @@ -168,6 +183,7 @@ func (u *Usecase) Delete(ctx context.Context, id uint64) error { if err := u.repo.Delete(ctx, id); err != nil { return err } + defer u.notifyRegistrationChanged() delete(u.latestProbe, id) u.notifyToolsChanged(ctx, server.Name, nil, nil) return nil @@ -330,6 +346,24 @@ func (u *Usecase) refreshConnection(ctx context.Context, id uint64, forcePublish if err := u.enterCommit(ctx); err != nil { return nil, err } + // 后台刷新不能废弃用户正在执行的连接验证。手动验证负责发布结果， + // 后台稍后仍会轮询；重复手动请求继续沿用最新请求优先的语义。 + if !forcePublish && u.manualProbes[id] > 0 { + u.leaveCommit() + return nil, nil + } + if forcePublish { + u.manualProbes[id]++ + defer func() { + // 这是内存锁的资源清理，不执行 IO；请求取消后也必须归还计数。 + u.commitGate <- struct{}{} + defer u.leaveCommit() + u.manualProbes[id]-- + if u.manualProbes[id] == 0 { + delete(u.manualProbes, id) + } + }() + } s, err := u.repo.Get(ctx, id) if err != nil { u.leaveCommit() diff --git a/internal/manager/biz/setting/agent.go b/internal/manager/biz/setting/agent.go index 4e29d80d9..0d498f48f 100644 --- a/internal/manager/biz/setting/agent.go +++ b/internal/manager/biz/setting/agent.go @@ -94,8 +94,8 @@ func (s *Service) AgentRuntimeVersion(ctx context.Context) (string, error) { func validateAgentSetting(key, value string) error { switch key { case model.KeyAgentRuntimeVersion: - if value != "release" && value != "v3" { - return fmt.Errorf("%w: runtime_version must be release or v3; v1/v2 are not available", errs.ErrInvalid) + if value != "release" && value != "v3" && value != "v2" { + return fmt.Errorf("%w: runtime_version must be release, v3 or v2; v1 is no longer supported", errs.ErrInvalid) } return nil case model.KeyAgentOutputLocale: diff --git a/internal/manager/biz/setting/service_test.go b/internal/manager/biz/setting/service_test.go index cbd6c0e38..c2cd0654a 100644 --- a/internal/manager/biz/setting/service_test.go +++ b/internal/manager/biz/setting/service_test.go @@ -382,7 +382,7 @@ func TestAgentRuntimeVersionSelection(t *testing.T) { if got, err := svc.AgentRuntimeVersion(ctx); err != nil || got != "" { t.Fatalf("default: %q %v", got, err) } - for _, value := range []string{"v3", "release", "v3"} { + for _, value := range []string{"v3", "v2", "release", "v3"} { if err := svc.Set(ctx, model.CategoryAgent, model.KeyAgentRuntimeVersion, value, false); err != nil { t.Fatal(err) } @@ -394,7 +394,7 @@ func TestAgentRuntimeVersionSelection(t *testing.T) { t.Fatalf("saved: %q %v", got, err) } } - for _, value := range []string{"v1", "v2", "v99", "", "V3"} { + for _, value := range []string{"v1", "v99", "", "V3"} { if err := svc.Set(ctx, model.CategoryAgent, model.KeyAgentRuntimeVersion, value, false); !errors.Is(err, errs.ErrInvalid) { t.Fatalf("accepted %q: %v", value, err) } @@ -409,7 +409,7 @@ func TestAgentRuntimeVersionSelection(t *testing.T) { t.Fatalf("deleted: %q %v", got, err) } // Unsupported persisted values cannot silently fall back to v3. - if _, err := repo.Set(ctx, model.CategoryAgent, model.KeyAgentRuntimeVersion, "v2", false); err != nil { + if _, err := repo.Set(ctx, model.CategoryAgent, model.KeyAgentRuntimeVersion, "v1", false); err != nil { t.Fatal(err) } if _, err := svc.AgentRuntimeVersion(ctx); !errors.Is(err, errs.ErrInvalid) { diff --git a/internal/manager/data/aiops/store/session_recovery.go b/internal/manager/data/aiops/store/session_recovery.go new file mode 100644 index 000000000..4e2ab9899 --- /dev/null +++ b/internal/manager/data/aiops/store/session_recovery.go @@ -0,0 +1,59 @@ +package store + +import ( + "context" + "fmt" + "time" + + model "github.com/ongridio/ongrid/internal/manager/model/aiops" + "gorm.io/gorm" +) + +// ReconcileInterruptedWorkers 仅在单 Manager 启动、装配运行时之前调用。 +// 旧进程的内存 Worker 无法继续执行：关闭其内部会话、收尾工具状态并保留 +// 中断证据。不重放工具，不修改用户会话、runtime 版本或独立审批记录。 +func (r *SessionRepo) ReconcileInterruptedWorkers(ctx context.Context, before time.Time) (int64, error) { + reason := "Worker interrupted by manager restart; execution was not resumed." + var total int64 + for { + var count int64 + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var ids []string + if err := tx.Model(&model.Session{}). + Where("audience = ? AND initiator = ?", model.SessionAudienceInternal, model.SessionInitiatorAgent). + Where("id LIKE ? AND agent_id IS NOT NULL", "worker-%"). + Where("closed_at IS NULL AND created_at < ?", before). + Order("id ASC").Limit(256).Pluck("id", &ids).Error; err != nil { + return fmt.Errorf("list interrupted workers: %w", err) + } + if len(ids) == 0 { + return nil + } + now := time.Now().UTC() + messages := tx.Model(&model.Message{}).Select("id").Where("session_id IN ?", ids) + if err := tx.Model(&model.ToolCall{}).Where("status = ? AND message_id IN (?)", model.StatusPending, messages). + Updates(map[string]any{"status": model.StatusError, "error": reason, "ended_at": now}).Error; err != nil { + return fmt.Errorf("finalize interrupted worker tools: %w", err) + } + for _, id := range ids { + message := &model.Message{SessionID: id, Role: model.RoleSystem, Content: &reason, CreatedAt: now} + if err := tx.Create(message).Error; err != nil { + return fmt.Errorf("record worker interruption: %w", err) + } + } + res := tx.Model(&model.Session{}).Where("id IN ? AND closed_at IS NULL", ids).Update("closed_at", now) + if res.Error != nil { + return fmt.Errorf("close interrupted worker sessions: %w", res.Error) + } + count = res.RowsAffected + return nil + }) + if err != nil { + return total, fmt.Errorf("reconcile interrupted workers: %w", err) + } + total += count + if count == 0 { + return total, nil + } + } +} diff --git a/internal/manager/data/aiops/store/session_recovery_test.go b/internal/manager/data/aiops/store/session_recovery_test.go new file mode 100644 index 000000000..c4b117da8 --- /dev/null +++ b/internal/manager/data/aiops/store/session_recovery_test.go @@ -0,0 +1,131 @@ +package store + +import ( + "strings" + "testing" + "time" + + model "github.com/ongridio/ongrid/internal/manager/model/aiops" +) + +func TestInterruptedWorkersRecoveryScopeAndIdempotence(t *testing.T) { + repo := newTestRepo(t) + ctx := t.Context() + cutoff := time.Now().UTC() + old := cutoff.Add(-time.Hour) + for _, tc := range []struct { + id, version, audience, initiator string + future, closed bool + }{ + {"worker-v2", "v2", "internal", "agent", false, false}, + {"worker-v3", "v3", "internal", "agent", false, false}, + {"worker-user", "v3", "user", "user", false, false}, + {"ordinary-internal", "v3", "internal", "agent", false, false}, + {"worker-new", "v3", "internal", "agent", true, false}, + {"worker-done", "v2", "internal", "agent", false, true}, + } { + created := old + if tc.future { + created = cutoff.Add(time.Hour) + } + row := &model.Session{ID: tc.id, UserID: 42, RuntimeVersion: tc.version, Audience: tc.audience, Initiator: tc.initiator, AgentID: strp("test"), Kind: "flow", CreatedAt: created} + if tc.closed { + row.ClosedAt = &old + } + if err := repo.CreateSession(ctx, row); err != nil { + t.Fatal(err) + } + m := &model.Message{SessionID: row.ID, Role: model.RoleAssistant, CreatedAt: created} + if err := repo.AppendMessage(ctx, m); err != nil { + t.Fatal(err) + } + for _, status := range []string{model.StatusPending, model.StatusSuccess} { + call := &model.ToolCall{MessageID: m.ID, ToolName: status, ArgumentsJSON: "{}", Status: status, StartedAt: created, CreatedAt: created} + if err := repo.CreateToolCall(ctx, call); err != nil { + t.Fatal(err) + } + } + } + n, err := repo.ReconcileInterruptedWorkers(ctx, cutoff) + if err != nil || n != 2 { + t.Fatalf("recovered=%d err=%v", n, err) + } + for _, id := range []string{"worker-v2", "worker-v3", "worker-user", "ordinary-internal", "worker-new", "worker-done"} { + row, err := repo.GetSession(ctx, id) + if err != nil { + t.Fatal(err) + } + recovered := id == "worker-v2" || id == "worker-v3" + if (row.ClosedAt != nil) != (recovered || id == "worker-done") { + t.Fatalf("unexpected closed state: %+v", row) + } + messages, err := repo.ListMessages(ctx, id, 0) + if err != nil { + t.Fatal(err) + } + audits := 0 + for _, m := range messages { + if m.Role == model.RoleSystem && m.Content != nil && strings.Contains(*m.Content, "interrupted by manager restart") { + audits++ + } + for _, call := range m.ToolCalls { + want := call.ToolName + if recovered && want == model.StatusPending { + want = model.StatusError + } + if call.Status != want { + t.Fatalf("%s tool status=%s want=%s", id, call.Status, want) + } + } + } + if (audits == 1) != recovered { + t.Fatalf("%s interruption audit count=%d", id, audits) + } + if id == "worker-v2" && row.RuntimeVersion != "v2" { + t.Fatal("recovery changed runtime ownership") + } + } + n, err = repo.ReconcileInterruptedWorkers(ctx, cutoff) + if err != nil || n != 0 { + t.Fatalf("second recovery=%d err=%v", n, err) + } +} + +func TestInterruptedWorkerRecoveryRollsBackOnAuditFailure(t *testing.T) { + repo := newTestRepo(t) + ctx := t.Context() + old := time.Now().UTC().Add(-time.Hour) + if err := repo.CreateSession(ctx, &model.Session{ID: "worker-rollback", UserID: 42, Audience: "internal", Initiator: "agent", AgentID: strp("test"), CreatedAt: old}); err != nil { + t.Fatal(err) + } + m := &model.Message{SessionID: "worker-rollback", Role: model.RoleAssistant, CreatedAt: old} + if err := repo.AppendMessage(ctx, m); err != nil { + t.Fatal(err) + } + call := &model.ToolCall{MessageID: m.ID, ToolName: "read", ArgumentsJSON: "{}", Status: model.StatusPending, StartedAt: old, CreatedAt: old} + if err := repo.CreateToolCall(ctx, call); err != nil { + t.Fatal(err) + } + // 审计落库失败时，会话和工具更新必须一起回滚。 + if err := repo.db.Exec("CREATE TRIGGER reject_recovery_audit BEFORE INSERT ON chat_messages BEGIN SELECT RAISE(ABORT, 'audit unavailable'); END").Error; err != nil { + t.Fatal(err) + } + n, err := repo.ReconcileInterruptedWorkers(ctx, time.Now().UTC()) + if err == nil || n != 0 { + t.Fatalf("recovery=%d err=%v", n, err) + } + row, err := repo.GetSession(ctx, "worker-rollback") + if err != nil { + t.Fatal(err) + } + if row.ClosedAt != nil { + t.Fatal("recovery committed without audit") + } + messages, err := repo.ListMessages(ctx, "worker-rollback", 0) + if err != nil { + t.Fatal(err) + } + if len(messages) != 1 || len(messages[0].ToolCalls) != 1 || messages[0].ToolCalls[0].Status != model.StatusPending { + t.Fatalf("tool update escaped rollback: %+v", messages) + } +} diff --git a/internal/manager/service/aiops/approval_executors.go b/internal/manager/service/aiops/approval_executors.go index 0d20a0bea..c143c214b 100644 --- a/internal/manager/service/aiops/approval_executors.go +++ b/internal/manager/service/aiops/approval_executors.go @@ -61,6 +61,7 @@ func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { // Preserve the existing unversioned route for historical/non-runtime proposals. approvalUC.RegisterExecutor(kind, execute) RegisterRuntimeApprovalExecutor(approvalUC, kind, contract.V3, execute) + RegisterRuntimeApprovalExecutor(approvalUC, kind, contract.V2, execute) } register("cloud_bash", func(ctx context.Context, payloadJSON string) (string, error) { var p cloudBashPayload @@ -191,7 +192,7 @@ func (e ApprovalExecutors) Register(approvalUC *managerbizapproval.Usecase) { if err := json.Unmarshal([]byte(payloadJSON), &p); err != nil { return "", err } - if contract.VersionFromContext(ctx) == contract.V3 && p.Revision == "" { + if (contract.VersionFromContext(ctx) == contract.V3 || contract.VersionFromContext(ctx) == contract.V2) && p.Revision == "" { return "", fmt.Errorf("MCP approval has no registration revision; submit a new request") } out, err := e.MCP.CallToolRevision(ctx, p.Server, p.Tool, p.Arguments, p.Revision) diff --git a/internal/manager/service/aiops/mcp_watch.go b/internal/manager/service/aiops/mcp_watch.go index f4097819a..094cdc121 100644 --- a/internal/manager/service/aiops/mcp_watch.go +++ b/internal/manager/service/aiops/mcp_watch.go @@ -80,6 +80,8 @@ func (r *MCPToolRefresher) RunRefresh(ctx context.Context, uc *managerbizmcp.Use select { case <-ctx.Done(): return + case <-uc.RegistrationChanges(): + reconcile() case <-timer.C: reconcile() } diff --git a/internal/manager/service/aiops/runtime_v2_adapter.go b/internal/manager/service/aiops/runtime_v2_adapter.go new file mode 100644 index 000000000..c5a9e8bc4 --- /dev/null +++ b/internal/manager/service/aiops/runtime_v2_adapter.go @@ -0,0 +1,42 @@ +package aiops + +import ( + "context" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/contract" + v2 "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/chatruntime" +) + +// V2Runtime adapts immutable worker snapshots; execution remains in the v2 Graph. +type V2Runtime struct{ *v2.Runtime } + +var _ RuntimeEngine = V2Runtime{} + +func (r V2Runtime) SpawnWorker(ctx context.Context, req contract.SpawnRequest) (*contract.Worker, error) { + w, err := r.Runtime.SpawnWorker(ctx, req) + if err != nil { + return nil, err + } + return v2WorkerSnapshot(w), nil +} +func (r V2Runtime) GetWorker(id string) (*contract.Worker, bool) { + w, ok := r.Runtime.GetWorker(id) + return v2WorkerSnapshot(w), ok +} +func v2WorkerSnapshot(w *v2.Worker) *contract.Worker { + if w == nil { + return nil + } + return &contract.Worker{ID: w.ID, AgentName: w.AgentName, SessionID: w.SessionID, + ParentSessionID: w.ParentSessionID, Prompt: w.Prompt, Status: w.Status, Background: w.Background, + StartedAt: w.StartedAt, EndedAt: w.EndedAt, Result: w.Result, Err: w.Err} +} + +type V2AgentRegistry struct{ Inner *v2.AgentRegistry } + +func (r V2AgentRegistry) HasAgent(name string) bool { + if r.Inner == nil { + return false + } + _, ok := r.Inner.ByName(name) + return ok +} diff --git a/internal/manager/service/aiops/runtime_v2_approval_test.go b/internal/manager/service/aiops/runtime_v2_approval_test.go new file mode 100644 index 000000000..86fedd76d --- /dev/null +++ b/internal/manager/service/aiops/runtime_v2_approval_test.go @@ -0,0 +1,131 @@ +package aiops + +import ( + "context" + "path/filepath" + "sync" + "testing" + "time" + + einomodel "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + "github.com/glebarez/sqlite" + runtime "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/chatruntime" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/runtime/v2/graph" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools" + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" + "github.com/ongridio/ongrid/internal/manager/biz/approval" + chatstore "github.com/ongridio/ongrid/internal/manager/data/aiops/store" + approvalstore "github.com/ongridio/ongrid/internal/manager/data/approval/store" + chatmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type v2MCPReviewModel struct { + mu sync.Mutex + turns int + sawResult bool +} + +func (m *v2MCPReviewModel) Generate(_ context.Context, input []*schema.Message, _ ...einomodel.Option) (*schema.Message, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.turns++ + if m.turns == 1 { + return &schema.Message{Role: schema.Assistant, ToolCalls: []schema.ToolCall{{ID: "call-445", Type: "function", Function: schema.FunctionCall{Name: tools.MCPToolName("test", "cluster.list"), Arguments: `{}`}}}}, nil + } + for _, msg := range input { + if msg.Role == schema.Tool && msg.Content == `{"clusters":["test"]}` { + m.sawResult = true + } + } + return &schema.Message{Role: schema.Assistant, Content: "Cluster result received"}, nil +} +func (m *v2MCPReviewModel) Stream(ctx context.Context, input []*schema.Message, opts ...einomodel.Option) (*schema.StreamReader[*schema.Message], error) { + msg, err := m.Generate(ctx, input, opts...) + if err != nil { + return nil, err + } + return schema.StreamReaderFromArray([]*schema.Message{msg}), nil +} +func (m *v2MCPReviewModel) WithTools([]*schema.ToolInfo) (einomodel.ToolCallingChatModel, error) { + return m, nil +} + +func TestV2MCPApprovalRuntimeResumesWithResult(t *testing.T) { + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "runtime.db")), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatal(err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + sqlDB.SetMaxOpenConns(1) + t.Cleanup(func() { + if err := sqlDB.Close(); err != nil { + t.Error(err) + } + }) + if err := approvalstore.Migrate(db); err != nil { + t.Fatal(err) + } + if err := chatstore.Migrate(db); err != nil { + t.Fatal(err) + } + uc := approval.NewUsecase(approvalstore.NewRepo(db), nil) + uc.RegisterRuntimeExecutor("mcp_call", "v2", func(context.Context, string) (string, error) { return `{"clusters":["test"]}`, nil }) + repo := chatstore.NewSessionRepoWithAttachmentRoot(db, t.TempDir()) + sess := &chatmodel.Session{ID: "session-445", UserID: 7} + if err := repo.CreateSession(context.Background(), sess); err != nil { + t.Fatal(err) + } + model := &v2MCPReviewModel{} + rt, err := runtime.NewRuntime(runtime.Config{Sessions: repo, ChatModel: model, GraphCfg: graph.Config{MaxIterations: 5}, ToolBag: []basetool.BaseTool{tools.NewMCPTool("test", "cluster.list", "List clusters", nil, false, nil, MCPProposer{Usecase: uc}, nil)}}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + events := make(chan *runtime.ApprovalPending, 2) + done := make(chan error, 1) + go func() { + _, err := rt.Handle(ctx, &runtime.Request{SessionID: sess.ID, UserID: 7, UserText: "List clusters", Emit: func(ev runtime.Event) { + if ev.Type == runtime.EventApprovalPending { + events <- ev.Approval + } + }}) + done <- err + }() + select { + case event := <-events: + if event.ToolCallID != "call-445" || event.Kind != "mcp_call" || event.ToolName != tools.MCPToolName("test", "cluster.list") { + t.Fatalf("invalid live card: %+v", event) + } + row, err := uc.Get(ctx, event.ApprovalID) + if err != nil || row.SessionID != sess.ID { + t.Fatalf("approval cannot recover by session: %+v %v", row, err) + } + if _, err := uc.Approve(ctx, 7, event.ApprovalID); err != nil { + t.Fatal(err) + } + case err := <-done: + t.Fatalf("chat ended without card: %v", err) + case <-ctx.Done(): + t.Fatal("no live approval event") + } + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal("chat did not resume after approval") + } + model.mu.Lock() + defer model.mu.Unlock() + if !model.sawResult || model.turns != 2 { + t.Fatalf("model did not receive MCP result: turns=%d sawResult=%v", model.turns, model.sawResult) + } +} diff --git a/tests/e2e/assistant_chat_test.go b/tests/e2e/assistant_chat_test.go index b11be4c78..4c051cd00 100644 --- a/tests/e2e/assistant_chat_test.go +++ b/tests/e2e/assistant_chat_test.go @@ -1069,13 +1069,18 @@ func TestAssistantChat_RemainingScenarioCoverage(t *testing.T) { } func createChatSession(t *testing.T, env *testenv.Env, token, title string) string { + t.Helper() + return createVersionedChatSession(t, env, token, title, "v3") +} + +func createVersionedChatSession(t *testing.T, env *testenv.Env, token, title, version string) string { t.Helper() status, body, err := env.DoJSON("POST", "/api/v1/chat/sessions", map[string]any{"title": title}, token) if err != nil || status != http.StatusCreated { t.Fatalf("create chat session: status=%d body=%v err=%v", status, body, err) } - if body["runtime_version"] != "v3" { - t.Fatalf("new session runtime = %v, want v3", body["runtime_version"]) + if body["runtime_version"] != version { + t.Fatalf("new session runtime = %v, want %s", body["runtime_version"], version) } id, _ := body["id"].(string) if id == "" { diff --git a/tests/e2e/dynamic_capabilities_test.go b/tests/e2e/dynamic_capabilities_test.go new file mode 100644 index 000000000..5b75f6525 --- /dev/null +++ b/tests/e2e/dynamic_capabilities_test.go @@ -0,0 +1,297 @@ +//go:build e2e + +package e2e + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/ongridio/ongrid/tests/e2e/testenv" +) + +// dynamicMCP is controlled through locks/channels, but all Manager interactions +// go through public HTTP APIs. Each list page contains one tool. +type dynamicMCP struct { + mu sync.Mutex + names []string + calls []string + fail bool + notify chan struct{} + watching chan struct{} + once sync.Once + server *httptest.Server +} + +func newDynamicMCP(t *testing.T, names ...string) *dynamicMCP { + t.Helper() + m := &dynamicMCP{names: names, notify: make(chan struct{}, 1), watching: make(chan struct{})} + m.server = httptest.NewServer(http.HandlerFunc(m.serve)) + t.Cleanup(m.server.Close) + return m +} +func (m *dynamicMCP) set(names ...string) { + m.mu.Lock() + defer m.mu.Unlock() + m.names = append([]string(nil), names...) +} +func (m *dynamicMCP) count() int { m.mu.Lock(); defer m.mu.Unlock(); return len(m.calls) } +func (m *dynamicMCP) serve(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + m.once.Do(func() { close(m.watching) }) + for { + select { + case <-r.Context().Done(): + return + case <-m.notify: + if _, err := fmt.Fprint(w, "data: {\"jsonrpc\":\"2.0\",\"method\":\"notifications/tools/list_changed\"}\n\n"); err != nil { + return + } + w.(http.Flusher).Flush() + } + } + } + var req struct { + ID int `json:"id"` + Method string `json:"method"` + Params struct { + Cursor string `json:"cursor"` + Name string `json:"name"` + } `json:"params"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), 400) + return + } + m.mu.Lock() + defer m.mu.Unlock() + switch req.Method { + case "initialize": + mcpWriteRPC(w, req.ID, map[string]any{"protocolVersion": "2025-03-26", "capabilities": map[string]any{"tools": map[string]bool{"listChanged": true}}}) + case "notifications/initialized": + w.WriteHeader(202) + case "tools/list": + if m.fail { + http.Error(w, "temporary discovery failure", 503) + return + } + page := 0 + if req.Params.Cursor != "" { + if _, err := fmt.Sscanf(req.Params.Cursor, "page-%d", &page); err != nil { + http.Error(w, "bad cursor", 400) + return + } + } + list := []map[string]any{} + if page < len(m.names) { + list = append(list, map[string]any{"name": m.names[page], "description": "dynamic evidence " + m.names[page], "inputSchema": map[string]any{"type": "object", "properties": map[string]any{}}}) + } + result := map[string]any{"tools": list} + if page+1 < len(m.names) { + result["nextCursor"] = fmt.Sprintf("page-%d", page+1) + } + mcpWriteRPC(w, req.ID, result) + case "tools/call": + m.calls = append(m.calls, req.Params.Name) + mcpWriteRPC(w, req.ID, map[string]any{"content": []map[string]string{{"type": "text", "text": "dynamic-evidence:" + req.Params.Name}}}) + default: + http.Error(w, "unsupported method", 400) + } +} + +func dynamicJSON(t *testing.T, env *testenv.Env, token, method, path string, input any) map[string]any { + t.Helper() + status, body, err := env.DoJSON(method, path, input, token) + if err != nil || status < 200 || status >= 300 { + t.Fatalf("%s %s: status=%d err=%v body=%v", method, path, status, err, body) + } + return body +} +func registerDynamicMCP(t *testing.T, env *testenv.Env, token, name string, m *dynamicMCP) string { + t.Helper() + dynamicJSON(t, env, token, "POST", "/api/v1/mcp/servers", map[string]any{"name": name, "transport": "http", "endpoint": m.server.URL, "enabled": true, "trusted": true}) + list := dynamicJSON(t, env, token, "GET", "/api/v1/mcp/servers", nil) + items, _ := list["items"].([]any) + id := mcpFindID(items, name) + if id == 0 { + t.Fatal("new MCP registration missing") + } + path := fmt.Sprintf("/api/v1/mcp/servers/%d", id) + t.Cleanup(func() { + status, _, err := env.DoJSON("DELETE", path, nil, token) + if err != nil || (status != 200 && status != 204 && status != 404) { + t.Errorf("cleanup MCP: %d %v", status, err) + } + }) + return path +} +func dynamicToolReply(name, args string) testenv.LLMReply { + return testenv.LLMReply{ToolCalls: []testenv.LLMToolCall{{ID: "dynamic-call", Name: name, Arguments: args}}} +} +func dynamicSearch(t *testing.T, env *testenv.Env, token, session, name string, want bool) { + t.Helper() + env.FakeLLM().SetScript(dynamicToolReply("ToolSearch", fmt.Sprintf(`{"query":"select:%s"}`, name)), testenv.LLMReply{Content: "catalog inspected"}) + dynamicJSON(t, env, token, "POST", chatMessagesPath(session), map[string]string{"content": "查询可用工具目录"}) + reqs := env.FakeLLM().Requests() + if len(reqs) != 2 { + t.Fatalf("search model requests=%d want 2", len(reqs)) + } + if got := containsString(reqs[1].ToolNames, name); got != want { + t.Fatalf("discovered schema %s=%v want %v; schemas=%v", name, got, want, reqs[1].ToolNames) + } +} +func dynamicExecute(t *testing.T, env *testenv.Env, token, session, name string, m *dynamicMCP) { + t.Helper() + before := m.count() + env.FakeLLM().SetScript(dynamicToolReply("ToolSearch", fmt.Sprintf(`{"query":"select:%s"}`, name)), dynamicToolReply(name, `{}`), testenv.LLMReply{Content: "evidence received"}) + body := dynamicJSON(t, env, token, "POST", chatMessagesPath(session), map[string]string{"content": "调用工具获取证据"}) + if !containsToolCall(body["tool_calls"], name) || m.count() != before+1 { + t.Fatalf("dynamic execution missing: calls=%d body=%v", m.count()-before, body) + } + reqs := env.FakeLLM().Requests() + if len(reqs) != 3 { + t.Fatalf("execution requests=%d want 3", len(reqs)) + } + found := false + for _, msg := range reqs[2].Messages { + if msg.Role == "tool" && strings.Contains(msg.Content, "dynamic-evidence:"+name[strings.LastIndex(name, "__")+2:]) { + found = true + } + } + if !found { + t.Fatal("actual MCP evidence did not reach the model") + } +} +func dynamicRevoked(t *testing.T, env *testenv.Env, token, session, name string, m *dynamicMCP) { + t.Helper() + dynamicSearch(t, env, token, session, name, false) + before := m.count() + env.FakeLLM().SetScript(dynamicToolReply(name, `{}`), testenv.LLMReply{Content: "stale call rejected"}) + // Even a model that emits a stale tool call must not reach the endpoint. + _, _, err := env.DoJSON("POST", chatMessagesPath(session), map[string]string{"content": "尝试旧工具"}, token) + if err != nil { + t.Fatal(err) + } + if m.count() != before { + t.Fatal("revoked tool reached MCP endpoint") + } +} + +func TestDynamicMCP_HotRegistrationLifecycle(t *testing.T) { + first := newDynamicMCP(t, "get_first", "get_second") + replacement := newDynamicMCP(t, "get_replacement") + env := testenv.Start(t, testenv.WithEnv("ONGRID_TOOLBAG_DEFERRAL_THRESHOLD", "1")) + token := env.LoginAdmin().AccessToken + session := createChatSession(t, env, token, "dynamic MCP lifecycle") + const name = "e2e_dynamic" + oldTool := "mcp__" + name + "__get_second" + dynamicSearch(t, env, token, session, oldTool, false) + path := registerDynamicMCP(t, env, token, name, first) + t.Run("register and discover paginated tools in existing chat", func(t *testing.T) { + body := dynamicJSON(t, env, token, "POST", path+"/test", nil) + tools, _ := body["tools"].([]any) + if len(tools) != 2 || !mcpHasTool(tools, "get_second") { + t.Fatalf("pagination incomplete: %v", body) + } + dynamicExecute(t, env, token, session, oldTool, first) + }) + t.Run("temporary discovery failure retains executable catalog", func(t *testing.T) { + first.mu.Lock() + first.fail = true + first.mu.Unlock() + status, _, err := env.DoJSON("POST", path+"/test", nil, token) + first.mu.Lock() + first.fail = false + first.mu.Unlock() + if err != nil || status < 400 { + t.Fatalf("expected discovery failure: %d %v", status, err) + } + dynamicExecute(t, env, token, session, oldTool, first) + }) + t.Run("schema refresh removes stale tools", func(t *testing.T) { + first.set("get_new_tool") + dynamicJSON(t, env, token, "POST", path+"/test", nil) + dynamicRevoked(t, env, token, session, oldTool, first) + dynamicExecute(t, env, token, session, "mcp__"+name+"__get_new_tool", first) + }) + patch := func(t *testing.T, endpoint string, enabled bool) { + t.Helper() + dynamicJSON(t, env, token, "PUT", path, map[string]any{"name": name, "transport": "http", "endpoint": endpoint, "enabled": enabled, "trusted": true}) + } + t.Run("disable and reenable without restart", func(t *testing.T) { + patch(t, first.server.URL, false) + dynamicRevoked(t, env, token, session, "mcp__"+name+"__get_new_tool", first) + patch(t, first.server.URL, true) + dynamicExecute(t, env, token, session, "mcp__"+name+"__get_new_tool", first) + }) + t.Run("replace endpoint then delete", func(t *testing.T) { + patch(t, replacement.server.URL, true) + dynamicRevoked(t, env, token, session, "mcp__"+name+"__get_new_tool", first) + dynamicJSON(t, env, token, "POST", path+"/test", nil) + dynamicExecute(t, env, token, session, "mcp__"+name+"__get_replacement", replacement) + dynamicJSON(t, env, token, "DELETE", path, nil) + dynamicRevoked(t, env, token, session, "mcp__"+name+"__get_replacement", replacement) + }) +} + +func TestDynamicMCP_BackgroundDiscoveryNotificationAndPolling(t *testing.T) { + m := newDynamicMCP(t, "get_before_notification") + env := testenv.Start(t, testenv.WithEnv("ONGRID_TOOLBAG_DEFERRAL_THRESHOLD", "1")) + token := env.LoginAdmin().AccessToken + session := createChatSession(t, env, token, "MCP notification refresh") + dynamicJSON(t, env, token, "PUT", "/api/v1/system-settings/agent/runtime_version", map[string]any{"value": "v2", "sensitive": false}) + v2 := createVersionedChatSession(t, env, token, "v2 automatic MCP discovery", "v2") + registerDynamicMCP(t, env, token, "e2e_notify", m) + // 不调用 /test；注册必须唤醒发现流程，而不是等待一分钟轮询。 + select { + case <-m.watching: + case <-time.After(10 * time.Second): + t.Fatal("registration did not promptly wake MCP discovery") + } + t.Log("background supervisor discovered registration and opened SSE") + for _, sid := range []string{session, v2} { + dynamicExecute(t, env, token, sid, "mcp__e2e_notify__get_before_notification", m) + } + m.set("get_after_notification") + m.notify <- struct{}{} + waitDynamicCatalog(t, env, token, "get_after_notification", 12*time.Second) + t.Log("notification published replacement catalog") + + dynamicRevoked(t, env, token, session, "mcp__e2e_notify__get_before_notification", m) + dynamicExecute(t, env, token, session, "mcp__e2e_notify__get_after_notification", m) + // No notification this time: only the production polling fallback can + // discover this change. Keep its actual one-minute cadence in E2E. + m.set("get_after_poll") + waitDynamicCatalog(t, env, token, "get_after_poll", 85*time.Second) + t.Log("polling discovered change without a notification") + dynamicRevoked(t, env, token, session, "mcp__e2e_notify__get_after_notification", m) + dynamicExecute(t, env, token, session, "mcp__e2e_notify__get_after_poll", m) +} + +// waitDynamicCatalog polls the public cache to observe background publication. +func waitDynamicCatalog(t *testing.T, env *testenv.Env, token, marker string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for { + body := dynamicJSON(t, env, token, "GET", "/api/v1/mcp/servers", nil) + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), marker) { + return + } + if time.Now().After(deadline) { + t.Fatalf("background catalog did not publish %s within %s", marker, timeout) + } + time.Sleep(100 * time.Millisecond) + } +} diff --git a/tests/e2e/dynamic_skills_test.go b/tests/e2e/dynamic_skills_test.go new file mode 100644 index 000000000..b1ee98c3c --- /dev/null +++ b/tests/e2e/dynamic_skills_test.go @@ -0,0 +1,173 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ongridio/ongrid/tests/e2e/testenv" +) + +func TestDynamicSkills_InstallLoadAndUninstall(t *testing.T) { + env := testenv.Start(t, testenv.WithEnv("ONGRID_SKILLS_ROOT", t.TempDir())) + token := env.LoginAdmin().AccessToken + session := createChatSession(t, env, token, "skill hot registration") + const packID = "e2e-dynamic-skill" + const skillName = "e2e_on_demand" + const marker = "PRIVATE_SKILL_BODY_7e93" + source := t.TempDir() + files := map[string]string{ + ".claude-plugin/plugin.json": `{"id":"` + packID + `","name":"` + packID + `","version":"1.0.0","description":"dynamic E2E fixture"}`, + "skills/e2e_on_demand/SKILL.md": "---\nname: " + skillName + "\ndescription: E2E on-demand instructions\nversion: 1.0.0\nactivation:\n mode: on_demand\n---\n" + marker + "\nUse the bundled resource.\n", + "bin/resource.txt": "shared-plugin-asset", + } + for name, content := range files { + path := filepath.Join(source, name) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + assertCatalog := func(want bool) { + t.Helper() + env.FakeLLM().SetScript(testenv.LLMReply{Content: "catalog checked"}) + dynamicJSON(t, env, token, "POST", chatMessagesPath(session), map[string]string{"content": "请说明当前可用能力"}) + reqs := env.FakeLLM().Requests() + if len(reqs) != 1 { + t.Fatalf("requests=%d", len(reqs)) + } + system := "" + for _, msg := range reqs[0].Messages { + if msg.Role == "system" { + system += msg.Content + } + } + if strings.Contains(system, skillName) != want { + t.Fatalf("skill catalog membership=%v want %v", strings.Contains(system, skillName), want) + } + if strings.Contains(system, marker) { + t.Fatal("on-demand skill body was eagerly injected") + } + } + assertCatalog(false) + dynamicJSON(t, env, token, "POST", "/api/v1/marketplace/install", map[string]string{"type": "local", "path": source}) + installed := true + t.Cleanup(func() { + if installed { + dynamicJSON(t, env, token, "DELETE", "/api/v1/marketplace/installed/"+packID, nil) + } + }) + assertCatalog(true) + + t.Run("load body and persist versioned tool result", func(t *testing.T) { + env.FakeLLM().SetScript(dynamicToolReply("load_skill", `{"name":"`+skillName+`"}`), testenv.LLMReply{Content: "skill loaded"}) + body := dynamicJSON(t, env, token, "POST", chatMessagesPath(session), map[string]string{"content": "加载按需操作说明"}) + if !containsToolCall(body["tool_calls"], "load_skill") { + t.Fatalf("load_skill not executed: %v", body) + } + reqs := env.FakeLLM().Requests() + if len(reqs) != 2 { + t.Fatalf("requests=%d", len(reqs)) + } + result := "" + for _, msg := range reqs[1].Messages { + if msg.Role == "tool" && strings.Contains(msg.Content, marker) { + result = msg.Content + } + } + if result == "" { + t.Fatal("loaded skill instructions never reached model") + } + var loaded struct{ Name, Version, Revision, Directory, Body string } + if err := json.Unmarshal([]byte(result), &loaded); err != nil { + t.Fatalf("skill result: %v; %s", err, result) + } + if loaded.Name != skillName || loaded.Version != "1.0.0" || loaded.Revision == "" || loaded.Directory == "" { + t.Fatalf("missing versioned resource identity: %+v", loaded) + } + history := dynamicJSON(t, env, token, "GET", chatMessagesPath(session), nil) + items, _ := history["items"].([]any) + persisted := false + for _, raw := range items { + row, _ := raw.(map[string]any) + content, _ := row["content"].(string) + if row["role"] == "tool" && strings.Contains(content, loaded.Revision) && strings.Contains(content, marker) { + persisted = true + } + } + if !persisted { + t.Fatal("skill body and revision were not persisted as tool evidence") + } + }) + + t.Run("uninstall revokes an already constructed turn catalog", func(t *testing.T) { + // A fresh session avoids confusing historical tool evidence with a new load. + blockedSession := createChatSession(t, env, token, "skill removed during inference") + env.FakeLLM().SetScript(dynamicToolReply("load_skill", `{"name":"`+skillName+`"}`), testenv.LLMReply{Content: "removed skill rejected"}) + block := env.FakeLLM().BlockNext() + defer block.Release() + type response struct { + status int + body map[string]any + err error + } + done := make(chan response, 1) + go func() { + status, body, err := env.DoJSON("POST", chatMessagesPath(blockedSession), map[string]string{"content": "加载按需操作说明"}, token) + done <- response{status, body, err} + }() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := block.WaitStarted(ctx); err != nil { + t.Fatal(err) + } + dynamicJSON(t, env, token, "DELETE", "/api/v1/marketplace/installed/"+packID, nil) + installed = false + block.Release() + select { + case res := <-done: + if res.err != nil { + t.Fatal(res.err) + } + if res.status != 200 || !containsToolCall(res.body["tool_calls"], "load_skill") { + t.Fatalf("load attempt missing: status=%d body=%v", res.status, res.body) + } + // Graph reports recoverable tool failures as JSON observations so + // the model can explain them; inspect that result, not SSE status. + rejected := false + for _, req := range env.FakeLLM().Requests() { + for _, msg := range req.Messages { + if msg.Role != "tool" { + continue + } + var result struct{ Status, Error string } + if json.Unmarshal([]byte(msg.Content), &result) == nil && result.Status == "failed" && strings.Contains(result.Error, "was removed") { + rejected = true + } + } + } + if !rejected { + t.Fatal("removed skill did not produce a failed observation") + } + + case <-ctx.Done(): + t.Fatal("chat did not finish after removing skill") + } + for _, req := range env.FakeLLM().Requests() { + for _, msg := range req.Messages { + if strings.Contains(msg.Content, marker) { + t.Fatal("removed skill body leaked into model context") + } + } + } + }) + assertCatalog(false) +} diff --git a/tests/e2e/mcp_test.go b/tests/e2e/mcp_test.go index 59ef8007c..b08c81b1e 100644 --- a/tests/e2e/mcp_test.go +++ b/tests/e2e/mcp_test.go @@ -9,9 +9,9 @@ // credential injection: header_template_json {{field}} is filled from the // bound vault credential and actually reaches the MCP server's HTTP headers. // -// The Agent-calls-an-MCP-tool path (P2) is boot-time tool registration and is -// validated on the live deploy instead; here we pin the deterministic API + -// the protocol round-trip + the credential→header injection. +// Runtime hot registration, discovery/execution consistency and background +// refresh are covered in dynamic_capabilities_test.go. This file protects +// CRUD, protocol round-trips and credential-to-header injection. package e2e import ( diff --git a/tests/e2e/runtime_versions_test.go b/tests/e2e/runtime_versions_test.go new file mode 100644 index 000000000..0f6864bea --- /dev/null +++ b/tests/e2e/runtime_versions_test.go @@ -0,0 +1,242 @@ +//go:build e2e + +package e2e + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ongridio/ongrid/tests/e2e/testenv" +) + +func TestRuntimeVersions_SwitchAndDynamicMCP(t *testing.T) { + m := newDynamicMCP(t, "get_evidence") + env := testenv.Start(t, testenv.WithEnv("ONGRID_TOOLBAG_DEFERRAL_THRESHOLD", "1")) + token := env.LoginAdmin().AccessToken + setVersion := func(version string) { + dynamicJSON(t, env, token, "PUT", "/api/v1/system-settings/agent/runtime_version", map[string]any{"value": version, "sensitive": false}) + } + v3 := createChatSession(t, env, token, "release remains v3") + setVersion("v2") + v2 := createVersionedChatSession(t, env, token, "v2 fallback", "v2") + // Selecting a fallback changes only new sessions. Both original sessions remain usable. + setVersion("release") + createChatSession(t, env, token, "return to release") + for _, session := range []struct{ id, version string }{{v2, "v2"}, {v3, "v3"}} { + list := dynamicJSON(t, env, token, "GET", "/api/v1/chat/sessions", nil) + var row map[string]any + for _, item := range list["items"].([]any) { + candidate := item.(map[string]any) + if candidate["id"] == session.id { + row = candidate + break + } + } + if row["runtime_version"] != session.version { + t.Fatalf("session changed version: %v", row) + } + env.FakeLLM().SetScript(testenv.LLMReply{Content: "versioned response"}) + reply := dynamicJSON(t, env, token, "POST", chatMessagesPath(session.id), map[string]string{"content": "你好"}) + if nestedString(reply, "assistant_message", "content") != "versioned response" { + t.Fatalf("versioned reply: %v", reply) + } + } + path := registerDynamicMCP(t, env, token, "versioned", m) + dynamicJSON(t, env, token, "POST", path+"/test", nil) + const tool = "mcp__versioned__get_evidence" + for _, session := range []string{v2, v3} { + dynamicExecute(t, env, token, session, tool, m) + } + dynamicJSON(t, env, token, "PUT", path, map[string]any{"name": "versioned", "transport": "http", "endpoint": m.server.URL, "enabled": false, "trusted": true}) + for _, session := range []string{v2, v3} { + dynamicRevoked(t, env, token, session, tool, m) + } + dynamicJSON(t, env, token, "PUT", path, map[string]any{"name": "versioned", "transport": "http", "endpoint": m.server.URL, "enabled": true, "trusted": false}) + dynamicJSON(t, env, token, "POST", path+"/test", nil) + dynamicJSON(t, env, token, "PUT", "/api/v1/system-settings/agent/write_enabled", map[string]any{"value": "true", "sensitive": false}) + // A pending v2 approval must execute through v2 after the default becomes v3. + env.FakeLLM().SetScript(dynamicToolReply("ToolSearch", `{"query":"select:mcp__versioned__get_evidence"}`), dynamicToolReply(tool, `{}`), testenv.LLMReply{Content: "approved evidence received"}) + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, "POST", env.BaseURL()+chatMessagesPath(v2)+"/stream", bytes.NewBufferString(`{"content":"调用 MCP 工具获取证据"}`)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("stream: %d", resp.StatusCode) + } + before := m.count() + scanner := bufio.NewScanner(resp.Body) + var event string + var approved, done bool + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "event: ") { + event = strings.TrimPrefix(line, "event: ") + continue + } + if !strings.HasPrefix(line, "data: ") { + continue + } + if event == "approval_pending" { + var proposal struct { + ApprovalID string `json:"approval_id"` + } + if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &proposal); err != nil { + t.Fatal(err) + } + row := dynamicJSON(t, env, token, "GET", "/api/v1/approvals/"+proposal.ApprovalID, nil) + if row["runtime_version"] != "v2" { + t.Fatalf("approval lost v2 ownership: %v", row) + } + setVersion("v3") + dynamicJSON(t, env, token, "POST", "/api/v1/approvals/"+proposal.ApprovalID+"/approve", map[string]any{}) + approved = true + } + if event == "done" { + done = true + } + } + if err := scanner.Err(); err != nil { + t.Fatal(err) + } + if !approved || !done || m.count() != before+1 { + t.Fatalf("approval=%t done=%t actual calls=%d", approved, done, m.count()-before) + } + reqs := env.FakeLLM().Requests() + found := false + for _, request := range reqs { + for _, msg := range request.Messages { + if msg.Role == "tool" && strings.Contains(msg.Content, "dynamic-evidence:get_evidence") { + found = true + } + } + } + if !found { + t.Fatal("approved result did not reach v2 model") + } + env.FakeLLM().SetScript( + dynamicToolReply("AgentTool", `{"description":"证据复核","subagent_type":"specialist-sre","prompt":"只读复核已有证据"}`), + testenv.LLMReply{Content: "v2 worker evidence"}, testenv.LLMReply{Content: "v2 synthesized worker evidence"}) + reply := dynamicJSON(t, env, token, "POST", chatMessagesPath(v2), map[string]string{"content": "请 SRE 专家复核证据"}) + if nestedString(reply, "assistant_message", "content") != "v2 synthesized worker evidence" || !containsToolCall(reply["tool_calls"], "AgentTool") { + t.Fatalf("v2 delegation failed: %v", reply) + } + workerSeen := false + for _, req := range env.FakeLLM().Requests() { + for _, msg := range req.Messages { + if msg.Role == "tool" && strings.Contains(msg.Content, "v2 worker evidence") { + workerSeen = true + } + } + } + if !workerSeen { + t.Fatal("v2 worker did not return actual evidence") + } +} + +// Registry fanout must preserve the native skill strategy in each runtime and +// make newly installed personas callable without restarting either engine. +func TestRuntimeVersions_SkillAndPersonaReload(t *testing.T) { + env := testenv.Start(t, testenv.WithEnv("ONGRID_SKILLS_ROOT", t.TempDir())) + token := env.LoginAdmin().AccessToken + v3 := createChatSession(t, env, token, "v3 dynamic registry") + dynamicJSON(t, env, token, "PUT", "/api/v1/system-settings/agent/runtime_version", map[string]any{"value": "v2", "sensitive": false}) + v2 := createVersionedChatSession(t, env, token, "v2 dynamic registry", "v2") + dynamicJSON(t, env, token, "PUT", "/api/v1/system-settings/agent/write_enabled", map[string]any{"value": "true", "sensitive": false}) + source := t.TempDir() + const pack = "e2e-versioned-registry" + const marker = "VERSIONED_SKILL_BODY_37d92" + files := map[string]string{ + ".claude-plugin/plugin.json": `{"id":"e2e-versioned-registry","name":"e2e-versioned-registry","version":"1.0.0"}`, + "skills/versioned/SKILL.md": "---\nname: versioned-skill\ndescription: test instructions\nactivation:\n mode: keyword\n keywords: [versioned-probe]\n---\n" + marker, + "agents/versioned-reviewer.md": "---\nname: versioned-reviewer\ndescription: Evidence reviewer\nwhen_to_use: Review evidence\ntools: [query_devices]\nmax_turns: 4\n---\nVERSIONED_PERSONA_BODY_37d92\n", + } + for name, content := range files { + path := filepath.Join(source, name) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + dynamicJSON(t, env, token, "POST", "/api/v1/marketplace/install", map[string]string{"type": "local", "path": source}) + installed := true + t.Cleanup(func() { + if installed { + dynamicJSON(t, env, token, "DELETE", "/api/v1/marketplace/installed/"+pack, nil) + } + }) + assertSkill := func(session string, want bool) { + t.Helper() + env.FakeLLM().SetScript(testenv.LLMReply{Content: "skill probe"}) + dynamicJSON(t, env, token, "POST", chatMessagesPath(session), map[string]string{"content": "versioned-probe"}) + found := false + for _, req := range env.FakeLLM().Requests() { + for _, msg := range req.Messages { + if msg.Role == "system" && strings.Contains(msg.Content, marker) { + found = true + } + } + } + if found != want { + t.Fatalf("dynamic skill body=%t want=%t", found, want) + } + } + for _, session := range []string{v2, v3} { + assertSkill(session, true) + // A registered skill exposes v3 native loading only in v3; v2 keeps its own strategy. + if got := containsString(env.FakeLLM().Requests()[0].ToolNames, "load_skill"); got != (session == v3) { + t.Fatalf("native skill loader present=%t for session %s", got, session) + } + env.FakeLLM().SetScript(dynamicToolReply("AgentTool", `{"description":"review","subagent_type":"versioned-reviewer","prompt":"review the evidence"}`), testenv.LLMReply{Content: "registered worker evidence"}, testenv.LLMReply{Content: "integrated evidence"}) + reply := dynamicJSON(t, env, token, "POST", chatMessagesPath(session), map[string]string{"content": "请专家复核证据"}) + if nestedString(reply, "assistant_message", "content") != "integrated evidence" { + for _, req := range env.FakeLLM().Requests() { + for _, msg := range req.Messages { + if msg.Role == "tool" { + t.Logf("worker tool result: %s", msg.Content) + } + } + } + t.Fatalf("dynamic worker reply: %v", reply) + } + found := false + for _, req := range env.FakeLLM().Requests() { + for _, msg := range req.Messages { + if msg.Role == "system" && strings.Contains(msg.Content, "VERSIONED_PERSONA_BODY_37d92") { + found = true + } + } + } + if !found { + t.Fatal("new persona did not execute") + } + } + dynamicJSON(t, env, token, "DELETE", "/api/v1/marketplace/installed/"+pack, nil) + installed = false + for _, session := range []string{v2, v3} { + assertSkill(session, false) + env.FakeLLM().SetScript(dynamicToolReply("AgentTool", `{"description":"review","subagent_type":"versioned-reviewer","prompt":"must not execute"}`), testenv.LLMReply{Content: "removed persona rejected"}) + dynamicJSON(t, env, token, "POST", chatMessagesPath(session), map[string]string{"content": "尝试旧专家"}) + if len(env.FakeLLM().Requests()) != 2 { + t.Fatal("removed persona spawned a worker") + } + } +} diff --git a/tests/e2e/testenv/fakes.go b/tests/e2e/testenv/fakes.go index e0db9f60a..d8ffa5d28 100644 --- a/tests/e2e/testenv/fakes.go +++ b/tests/e2e/testenv/fakes.go @@ -53,10 +53,18 @@ type LLMReply struct { ToolCalls []LLMToolCall } +// LLMMessage captures model input so lifecycle tests assert actual tool evidence, +// rather than trusting the fake model's scripted final answer. +type LLMMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + // LLMRequest is the observable subset of an OpenAI completion request used // by E2E assertions. ToolNames establishes what schemas the agent exposed to // the model on that turn without binding tests to provider wire details. type LLMRequest struct { + Messages []LLMMessage Model string ToolNames []string } @@ -161,13 +169,17 @@ func (f *FakeLLM) Requests() []LLMRequest { defer f.mu.Unlock() out := make([]LLMRequest, len(f.requests)) for i, req := range f.requests { - out[i] = LLMRequest{Model: req.Model, ToolNames: append([]string(nil), req.ToolNames...)} + out[i] = LLMRequest{Model: req.Model, ToolNames: append([]string(nil), req.ToolNames...), Messages: append([]LLMMessage(nil), req.Messages...)} } return out } func (f *FakeLLM) openaiChat(w http.ResponseWriter, r *http.Request) { var req struct { + Messages []struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + } `json:"messages"` Model string `json:"model"` Tools []struct { Function struct { @@ -188,7 +200,16 @@ func (f *FakeLLM) openaiChat(w http.ResponseWriter, r *http.Request) { f.mu.Lock() f.calls++ f.gotModels = append(f.gotModels, req.Model) - f.requests = append(f.requests, LLMRequest{Model: req.Model, ToolNames: toolNames}) + messages := make([]LLMMessage, 0, len(req.Messages)) + for _, message := range req.Messages { + content := string(message.Content) + var plain string + if json.Unmarshal(message.Content, &plain) == nil { + content = plain + } + messages = append(messages, LLMMessage{Role: message.Role, Content: content}) + } + f.requests = append(f.requests, LLMRequest{Model: req.Model, ToolNames: toolNames, Messages: messages}) reply := LLMReply{Content: f.reply} if len(f.script) > 0 { reply = f.script[0] diff --git a/web/e2e/README.md b/web/e2e/README.md index 3ba6502a0..55c13821f 100644 --- a/web/e2e/README.md +++ b/web/e2e/README.md @@ -14,3 +14,5 @@ The test runner reads `ONGRID_ADMIN_EMAIL` and `ONGRID_ADMIN_PASSWORD` from the Read-only navigation and resource views run against the live local environment. Deterministic create/test workflows intercept their API calls in the browser, so they validate UI behavior and request payloads without changing local data or calling external Slack services. Reports, screenshots, traces, videos, and the temporary login state are written to `output/playwright/`. + +Live navigation follows the `/kubernetes` redirect to `/clusters` and uses tab roles for device/topology panels. The Kubernetes detail test requires an enrolled cluster from the real `/api/v1/k8s/clusters` API; an empty list produces an explicit skip. Report that skip separately from passing tests. The device plugin page intentionally excludes top-level `profiles` and `autoapm` rows. diff --git a/web/e2e/agent-runtime-settings.spec.ts b/web/e2e/agent-runtime-settings.spec.ts index 4957aec2e..d5305fbe2 100644 --- a/web/e2e/agent-runtime-settings.spec.ts +++ b/web/e2e/agent-runtime-settings.spec.ts @@ -31,14 +31,14 @@ for (const theme of ['light', 'dark']) { await expect(selector).toBeEnabled(); await selector.scrollIntoViewIfNeeded(); await selector.click(); - await expect(page.getByRole('option', { name: /v2/ })).toBeDisabled(); - await expect(page.getByRole('option', { name: /v1/ })).toBeDisabled(); + await expect(page.getByRole('option', { name: /v2/ })).toBeEnabled(); + await expect(page.getByRole('option', { name: english ? 'v1 · No longer supported' : 'v1 · 不再支持' })).toBeDisabled(); await page.screenshot({ path: `/tmp/ongrid-runtime-${theme}-${width}.png`, fullPage: true }); - await page.getByRole('option', { name: english ? 'v3 · Pinned version' : 'v3 · 固定版本', exact: true }).click(); + await page.getByRole('option', { name: english ? 'v2 · Pinned version' : 'v2 · 固定版本', exact: true }).click(); await page.getByRole('button', { name: english ? 'Save runtime' : '保存版本', exact: true }).click(); await expect(page.getByRole('status')).toContainText(english ? 'Saved.' : '已保存'); await page.reload(); - await expect(selector).toContainText(english ? 'v3 · Pinned version' : 'v3 · 固定版本'); + await expect(selector).toContainText(english ? 'v2 · Pinned version' : 'v2 · 固定版本'); const box = await selector.boundingBox(); expect(box).not.toBeNull(); expect(box!.x).toBeGreaterThanOrEqual(0); diff --git a/web/e2e/live-navigation.spec.ts b/web/e2e/live-navigation.spec.ts index 21ac7c698..dcf10189e 100644 --- a/web/e2e/live-navigation.spec.ts +++ b/web/e2e/live-navigation.spec.ts @@ -21,7 +21,7 @@ test.describe('live frontend interaction', () => { const routes: Array<[string, RegExp]> = [ ['/dashboard', /仪表盘|Dashboard/], ['/devices', /全部设备|All devices/], - ['/kubernetes', /Kubernetes 集群|Kubernetes clusters/], + ['/kubernetes', /^集群$|^Clusters$/], ['/topology', /拓扑|Topology/], ['/monitor', /监控|Monitor/], ['/logs', /日志|Logs/], @@ -52,32 +52,36 @@ test.describe('live frontend interaction', () => { await expect(rows.first()).toBeVisible(); await rows.first().click(); await expect(page).toHaveURL(/\/devices\/\d+/); - await expect(page.getByRole('button', { name: /指标|Metrics/ })).toBeVisible(); - await page.getByRole('button', { name: /插件|Plugins/ }).click(); - for (const plugin of ['metrics', 'logs', 'traces', 'profiles']) { + await expect(page.getByRole('tab', { name: /指标|Metrics/ })).toBeVisible(); + await page.getByRole('tab', { name: /插件|Plugins/ }).click(); + for (const plugin of ['metrics', 'logs', 'traces']) { await expect(page.getByText(plugin, { exact: true }).first()).toBeVisible(); } - await expect(page.getByText(/subprocess otelcol-contrib/)).toBeVisible(); + // profiles/autoapm 已移到 APM 入口，不再是设备插件页的顶层行。 + await expect(page.getByText(/subprocess otelcol-contrib/).first()).toBeVisible(); }); test('Kubernetes cluster detail switches every resource view', async ({ page }) => { - await page.goto('/kubernetes'); - const details = page.getByRole('link', { name: /详情|Details/ }); - await expect(details.first()).toBeVisible(); - await details.first().click(); - await expect(page).toHaveURL(/\/kubernetes\/\d+/); + await page.goto('/clusters'); + const token = await page.evaluate(() => JSON.parse(localStorage.getItem('ongrid.auth')!).state.token); + const response = await page.request.get('/api/v1/k8s/clusters', { headers: { Authorization: `Bearer ${token}` } }); + expect(response.ok()).toBe(true); + const { items } = await response.json(); + expect(Array.isArray(items)).toBe(true); + test.skip(items.length === 0, '测试环境没有已接入的 Kubernetes 集群，资源页需在具备该前置条件的环境验收'); + await page.goto(`/kubernetes/${items[0].id}`); for (const tab of ['Nodes', 'Workloads', 'Pods', 'Events', 'Namespaces', 'Actions']) { - await page.getByRole('button', { name: new RegExp(`^${tab}\\s+\\d+`) }).click(); + await page.getByRole('tab', { name: new RegExp(`^${tab}\\s+\\d+`) }).click(); await expect(page).toHaveURL(new RegExp(`tab=${tab.toLowerCase()}`)); - await expect(page.getByRole('button', { name: new RegExp(`^${tab}\\s+\\d+`) })).toHaveClass(/bg-zinc-100/); + await expect(page.getByRole('tab', { name: new RegExp(`^${tab}\\s+\\d+`) })).toHaveAttribute('aria-selected', 'true'); } }); test('topology view switches graph, nodes, and type management', async ({ page }) => { await page.goto('/topology'); for (const tab of [/图谱|Graph/, /节点 \+ 关系|Nodes \+ relations/, /类型管理|Types/]) { - await page.getByRole('button', { name: tab }).click(); + await page.getByRole('tab', { name: tab }).click(); } await expect(page.getByText(/节点类型|Node types/).first()).toBeVisible(); }); diff --git a/web/src/pages/settings/Agent.test.tsx b/web/src/pages/settings/Agent.test.tsx index 1f36029f2..9d840a5ac 100644 --- a/web/src/pages/settings/Agent.test.tsx +++ b/web/src/pages/settings/Agent.test.tsx @@ -129,7 +129,7 @@ describe('SettingsAgent output language', () => { }); describe('SettingsAgent runtime selection', () => { - it('persists a pinned runtime and can return to release defaults', async () => { + it.each(['v2', 'v3'])('persists %s and can return to release defaults', async (version) => { const saved: unknown[] = []; server.use( http.get('/api/v1/system-settings', () => HttpResponse.json({ items: [], total: 0 })), @@ -141,13 +141,13 @@ describe('SettingsAgent runtime selection', () => { const user = userEvent.setup(); render(); await waitFor(() => expect(screen.getByRole('combobox', { name: 'Runtime version' })).toBeEnabled()); - await selectOption(screen.getByRole('combobox', { name: 'Runtime version' }), 'v3 · Pinned version'); + await selectOption(screen.getByRole('combobox', { name: 'Runtime version' }), `${version} · Pinned version`); await user.click(screen.getByRole('button', { name: 'Save runtime' })); await screen.findByText('Saved. Applies to new conversations.'); - expect(saved).toEqual([{ value: 'v3', sensitive: false }]); + expect(saved).toEqual([{ value: version, sensitive: false }]); await selectOption(screen.getByRole('combobox', { name: 'Runtime version' }), 'Follow release default (currently v3)'); await user.click(screen.getByRole('button', { name: 'Save runtime' })); - await waitFor(() => expect(saved).toEqual([{ value: 'v3', sensitive: false }, { value: 'release', sensitive: false }])); + await waitFor(() => expect(saved).toEqual([{ value: version, sensitive: false }, { value: 'release', sensitive: false }])); }); it('retains the selection and shows an error when saving fails', async () => { diff --git a/web/src/pages/settings/Agent.tsx b/web/src/pages/settings/Agent.tsx index c332dae10..231a2ae02 100644 --- a/web/src/pages/settings/Agent.tsx +++ b/web/src/pages/settings/Agent.tsx @@ -208,15 +208,15 @@ export default function SettingsAgent() { > - - + + {!['release', 'v1', 'v2', 'v3'].includes(runtimeVersion) && ( )}

- {tr('当前构建仅接入 v3。v1、v2 源码已隔离，完成执行适配前不可选择。跟随 Release 默认会在升级后使用该发行版的默认运行时。', - 'This build runs v3 only. v1 and v2 source is isolated, but execution adapters are not available yet. Following the release default uses the default runtime shipped with each upgrade.')} + {tr('可选择 v3 或切回 v2，新会话生效。v1 不再支持，源码保留归档。跟随 Release 默认会在升级后使用该发行版的默认运行时。', + 'Choose v3 or switch back to v2 for new conversations. v1 is no longer supported; its source remains archived. Following the release default uses the default runtime shipped with each upgrade.')}