Desktop: Phase 3 (step 2) broker serves /api/procedures

desktop/broker.js

@@ -14,12 +14,15 @@ const crypto = require('crypto')
 const store = require('./store')
 const { token } = require('./resolve')
 
-// GET endpoints the broker answers itself, from the local store, in FastAPI's exact response shape
-// (parity-tested against the live FastAPI). settings/procedures/pending follow in later increments:
-// settings reads an environment-detected terminal, so it isn't a pure static read yet.
+// GET endpoints the broker answers itself, from the local store, in FastAPI's response shape
+// (parity-tested against the live FastAPI). Cost/autonomy numbers ride as JSON numbers, so a whole
+// dollar serializes as `1` here vs `1.0` in Python -- semantically identical after JSON.parse (what
+// the SPA does), so the parity check compares parsed values, not bytes. settings/pending follow
+// later: settings reads an environment-detected terminal, so it isn't a pure static read yet.
 const SERVED = {
   '/api/plate': (sopDir) => ({ plate: store.plate(sopDir) }),
   '/api/queue': (sopDir) => ({ queue: store.queue(sopDir) }),
+  '/api/procedures': (sopDir) => ({ procedures: store.procedures(sopDir) }),
 }
 
 // Constant-time token compare (the broker now gates the reads it serves, mirroring FastAPI's check).

desktop/store.js

@@ -13,6 +13,13 @@ const { DatabaseSync } = require('node:sqlite')
 const path = require('path')
 const fs = require('fs')
 
+// Strict UTF-8 read: THROWS on invalid bytes (Node's 'utf8' silently substitutes U+FFFD), so a
+// corrupt SOP/queue file is skipped by the caller's try/catch, matching Python read_text(encoding=utf-8).
+const utf8Strict = new TextDecoder('utf-8', { fatal: true })
+function readTextStrict(p) {
+  return utf8Strict.decode(fs.readFileSync(p))
+}
+
 function dbPath(sopDir) { return path.join(sopDir, 'state.db') }
 
 function withDb(sopDir, fn) {
@@ -38,14 +45,113 @@ function queue(sopDir) {
   const out = []
   for (const f of files) {
     let text
-    try { text = fs.readFileSync(path.join(qdir, f), 'utf8') } catch (_) { continue }
+    try { text = readTextStrict(path.join(qdir, f)) } catch (_) { continue }  // skip unreadable/non-utf8
     const m = parseFrontmatter(text)
     if (String(m.status || '').trim() !== 'queued') continue
     out.push({ file: f, sop: m.sop || f.replace(/\.md$/, ''), project: m.project ? path.basename(m.project) : '' })
   }
   return out
 }
 
+// SOP files for the Procedures view, skipping runtime dirs / index / template / dotfiles (mirrors
+// smbos_lib.iter_sops: rglob *.md, sorted, with the same skip set).
+const SKIP_NAMES = new Set(['INDEX.md', '_template.md', 'DIGEST.md'])
+const SKIP_DIRS = new Set(['pending', 'payloads', 'triggers', 'queue', 'work', 'active-runs', 'active-sessions', 'archive'])
+
+function collectMd(dir, acc) {
+  let entries
+  try { entries = fs.readdirSync(dir, { withFileTypes: true }) } catch (_) { return acc }
+  for (const e of entries) {
+    const full = path.join(dir, e.name)
+    if (e.isDirectory()) {
+      if (!SKIP_DIRS.has(e.name)) collectMd(full, acc)  // prune runtime/archive dirs (don't descend)
+    } else if (e.name.endsWith('.md')) {
+      acc.push(full)
+    }
+  }
+  return acc
+}
+
+function iterSops(sopDir) {
+  return collectMd(sopDir, [])
+    // sort by path COMPONENTS, mirroring Python's sorted(Path.rglob) tuple compare -- a full-string
+    // sort orders '/' (0x2F) vs '-' (0x2D) differently, so nested-vs-prefix-sharing paths would diverge.
+    .sort((a, b) => {
+      const pa = path.relative(sopDir, a).split(path.sep)
+      const pb = path.relative(sopDir, b).split(path.sep)
+      for (let i = 0; i < Math.min(pa.length, pb.length); i++) {
+        if (pa[i] < pb[i]) return -1
+        if (pa[i] > pb[i]) return 1
+      }
+      return pa.length - pb.length
+    })
+    // SKIP_DIRS are already pruned above; here just drop index/template/dotfiles by name (a dot-DIR is
+    // kept, matching Python iter_sops, which only skips dotfiles by filename).
+    .filter((full) => {
+      const name = path.basename(full)
+      return !SKIP_NAMES.has(name) && !name.startsWith('.')
+    })
+}
+
+// Derive a procedure's autonomy level from its frontmatter (mirrors smbos_lib.autonomy_level_from_meta).
+const AUTONOMY_LEVELS = new Set(['with_me', 'prepare_ask', 'on_its_own'])
+function autonomyFromMeta(meta) {
+  const val = String(meta.autonomy || '').trim().toLowerCase()
+  if (AUTONOMY_LEVELS.has(val)) return val
+  if (['true', 'yes', '1'].includes(String(meta.interactive_only || '').trim().toLowerCase())) return 'with_me'
+  const status = String(meta.status || 'draft').trim().toLowerCase()
+  return (status === 'active' || status === 'trusted') ? 'on_its_own' : 'prepare_ask'
+}
+
+// Per-SOP cost estimate (MEDIAN of prior 'ok' runs) from runs.jsonl (mirrors dashboard_app._cost_estimates).
+function costEstimates(sopDir) {
+  let lines = []
+  try { lines = fs.readFileSync(path.join(sopDir, 'runs.jsonl'), 'utf8').split('\n') } catch (_) { /* none yet */ }
+  const bySop = {}
+  for (const line of lines) {
+    let r
+    try { r = JSON.parse(line) } catch (_) { continue }
+    const cost = r.cost_usd
+    if (typeof cost !== 'number' || !Number.isFinite(cost) || cost < 0) continue  // typeof excludes bool
+    if (r.result === 'ok') {
+      const s = String(r.sop || '')
+      ;(bySop[s] = bySop[s] || []).push(cost)
+    }
+  }
+  const estimates = {}
+  for (const sop of Object.keys(bySop)) {
+    const costs = bySop[sop].sort((a, b) => a - b)  // NUMERIC sort (JS default is lexicographic)
+    const n = costs.length
+    const med = n % 2 ? costs[(n - 1) / 2] : (costs[n / 2 - 1] + costs[n / 2]) / 2
+    // round-half-up vs Python's round() half-to-even can differ by $0.0001 only on an exact-half
+    // median (rare; sub-cent; the UI shows cents) -- accepted tolerance, not chased to banker's rounding.
+    estimates[sop] = { estimate: Math.round(med * 1e4) / 1e4, n }
+  }
+  return estimates
+}
+
+// The Procedures view: each SOP with the facts the UI needs to pick its action. Mirrors
+// dashboard_app._procedures (id/title/draft/interactive/needs_inputs/cost/autonomy), sorted by title.
+function procedures(sopDir) {
+  const ests = costEstimates(sopDir)
+  const out = []
+  for (const p of iterSops(sopDir)) {
+    let m
+    try { m = parseFrontmatter(readTextStrict(p)) } catch (_) { continue }  // skip unreadable/non-utf8 SOP
+    const sid = m.id || path.basename(p, '.md')
+    out.push({
+      id: sid,
+      title: m.title || sid,
+      draft: !['active', 'trusted'].includes(String(m.status || '').trim().toLowerCase()),
+      interactive: ['true', 'yes', '1'].includes(String(m.interactive_only || '').trim().toLowerCase()),
+      needs_inputs: Boolean(m.run_inputs),
+      cost: Object.prototype.hasOwnProperty.call(ests, sid) ? ests[sid] : null,  // {estimate,n} or null
+      autonomy: autonomyFromMeta(m),
+    })
+  }
+  return out.sort((a, b) => (a.title.toLowerCase() < b.title.toLowerCase() ? -1 : a.title.toLowerCase() > b.title.toLowerCase() ? 1 : 0))
+}
+
 // Minimal frontmatter parser for the simple `key: value` lines we read (matches smbos_lib for these).
 function parseFrontmatter(text) {
   const m = /^---\r?\n([\s\S]*?)\r?\n---/.exec(text)
@@ -61,4 +167,4 @@ function parseFrontmatter(text) {
   return out
 }
 
-module.exports = { plate, queue }
+module.exports = { plate, queue, procedures }

desktop/store.test.js

@@ -46,3 +46,45 @@ test('queue: #-comment frontmatter lines are skipped (parity with smbos_lib.pars
   assert.deepEqual(store.queue(d), [{ file: 'a.md', sop: 'weekly', project: '' }])
 })
 
+test('procedures: derives autonomy, computes the cost median, skips INDEX, sorts by title', () => {
+  const d = tmpSop(); fs.mkdirSync(path.join(d, 'ops'))
+  const sop = (sid, title, status, extra = '') => fs.writeFileSync(path.join(d, 'ops', sid + '.md'),
+    `---\nid: ${sid}\ntitle: ${title}\nstatus: ${status}\n${extra}---\n# ${title}\n`)
+  sop('b-weekly', 'Weekly', 'active', 'autonomy: on_its_own\n')
+  sop('a-draft', 'Draft proc', 'draft')
+  sop('c-inter', 'Inbox', 'active', 'interactive_only: true\n')
+  fs.writeFileSync(path.join(d, 'INDEX.md'), 'skip me')  // iter_sops skips INDEX.md
+  fs.writeFileSync(path.join(d, 'runs.jsonl'), [
+    { sop: 'b-weekly', result: 'ok', cost_usd: 0.1 },
+    { sop: 'b-weekly', result: 'ok', cost_usd: 0.3 },
+    { sop: 'b-weekly', result: 'error', cost_usd: 9 },  // non-ok -> excluded from the median
+  ].map((r) => JSON.stringify(r)).join('\n'))
+  const procs = store.procedures(d)
+  assert.deepEqual(procs.map((p) => p.title), ['Draft proc', 'Inbox', 'Weekly'])  // sorted by title.lower()
+  const weekly = procs.find((p) => p.id === 'b-weekly')
+  assert.equal(weekly.autonomy, 'on_its_own'); assert.equal(weekly.draft, false)
+  assert.deepEqual(weekly.cost, { estimate: 0.2, n: 2 })          // median of 0.1, 0.3
+  const draft = procs.find((p) => p.id === 'a-draft')
+  assert.equal(draft.autonomy, 'prepare_ask'); assert.equal(draft.cost, null)  // draft, no runs
+  const inter = procs.find((p) => p.id === 'c-inter')
+  assert.equal(inter.interactive, true); assert.equal(inter.autonomy, 'with_me')  // interactive_only -> with_me
+})
+
+test('procedures: same-title tie-break follows Python path-component order (nested before prefix-sibling)', () => {
+  const d = tmpSop(); fs.mkdirSync(path.join(d, 'clients'))
+  fs.writeFileSync(path.join(d, 'clients', 'nested.md'), '---\nid: nested\ntitle: Same\nstatus: active\n---\n')
+  fs.writeFileSync(path.join(d, 'clients-flat.md'), '---\nid: flat\ntitle: Same\nstatus: active\n---\n')
+  // Python sorted(rglob) compares components: 'clients' < 'clients-flat.md', so the nested file is
+  // first; a full-string sort would put 'clients-flat.md' first ('-' < '/'). Stable title sort keeps it.
+  assert.deepEqual(store.procedures(d).map((p) => p.id), ['nested', 'flat'])
+})
+
+test('procedures/queue skip a non-UTF-8 file (parity: Python read_text raises -> skipped)', () => {
+  const d = tmpSop(); fs.mkdirSync(path.join(d, 'ops')); fs.mkdirSync(path.join(d, 'queue'))
+  fs.writeFileSync(path.join(d, 'ops', 'good.md'), '---\nid: good\ntitle: Good\nstatus: active\n---\n')
+  fs.writeFileSync(path.join(d, 'ops', 'bad.md'), Buffer.from([0x2d, 0x2d, 0x2d, 0x0a, 0xff, 0xfe, 0x0a, 0x2d, 0x2d, 0x2d]))
+  fs.writeFileSync(path.join(d, 'queue', 'q.md'), Buffer.from([0xff, 0xfe]))  // invalid UTF-8
+  assert.deepEqual(store.procedures(d).map((p) => p.id), ['good'])  // bad.md skipped, not garbled-in
+  assert.deepEqual(store.queue(d), [])                              // bad queue file skipped
+})
+