fix: harden MCP against DB corruption crashes and orphaned workers

.env.1password

@@ -0,0 +1,4 @@
+COHERE_API_KEY=op://Dev/COHERE_API_KEY/credential
+KIMI_API_KEY=op://Dev/KIMI_API_KEY/credential
+OLLAMA_API_KEY=op://Dev/OLLAMA_API_KEY/credential
+VOYAGE_API_KEY=op://Dev/VOYAGE_API_KEY/credential

.github/workflows/ci.yml

@@ -11,7 +11,14 @@ env:
   CARGO_INCREMENTAL: 0
 
 jobs:
+  optimize_ci:
+    uses: shilin23061991/common-actions/.github/workflows/optimize-ci.yml@main
+    secrets:
+      graphite_token: ${{ secrets.GRAPHITE_TOKEN }}
+
   fmt:
+    needs: optimize_ci
+    if: needs.optimize_ci.outputs.skip == 'false'
     name: Format
     runs-on: ubuntu-latest
     steps:
@@ -22,6 +29,8 @@ jobs:
       - run: cargo fmt --all -- --check
 
   check:
+    needs: optimize_ci
+    if: needs.optimize_ci.outputs.skip == 'false'
     name: Check (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     strategy:

.github/workflows/pages.yml

@@ -25,7 +25,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  optimize_ci:
+    uses: shilin23061991/common-actions/.github/workflows/optimize-ci.yml@main
+    secrets:
+      graphite_token: ${{ secrets.GRAPHITE_TOKEN }}
+
   build:
+    needs: optimize_ci
+    if: needs.optimize_ci.outputs.skip == 'false'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

crates/infigraph-core/src/embed/mod.rs

@@ -6,6 +6,9 @@ use anyhow::{Context, Result};
 
 use crate::model::Symbol;
 
+mod voyage; // local-dev add-on: remote Voyage AI embedder
+pub use voyage::VoyageEmbedder;
+
 struct CachedEmbeddings {
     path: PathBuf,
     modified: std::time::SystemTime,
@@ -54,8 +57,19 @@ pub fn invalidate_embeddings_cache() {
 /// Embedding engine trait. Implementations can use ONNX, API calls, etc.
 pub trait EmbedProvider: Send + Sync {
     fn dimension(&self) -> usize;
+    /// Stable identity of the provider+model producing vectors. Persisted next
+    /// to `embeddings.bin` so switching providers/models invalidates old
+    /// vectors even when the dimension is unchanged.
+    fn identity(&self) -> String;
     fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>>;
 
+    /// Whether `embed_batch` makes network calls to a remote API. Remote
+    /// providers are batched sequentially during index builds so parallel
+    /// chunks don't hammer the API into rate-limit failures.
+    fn is_remote(&self) -> bool {
+        false
+    }
+
     fn embed(&self, text: &str) -> Result<Vec<f32>> {
         let mut results = self.embed_batch(&[text])?;
         results
@@ -175,6 +189,10 @@ impl EmbedProvider for TrigramEmbedder {
         self.dim
     }
 
+    fn identity(&self) -> String {
+        format!("trigram:{}", self.dim)
+    }
+
     fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
         Ok(texts.iter().map(|t| trigram_embed(t, self.dim)).collect())
     }
@@ -298,6 +316,10 @@ impl EmbedProvider for Model2VecEmbedder {
         256 // potion-base-8M outputs 256-dim
     }
 
+    fn identity(&self) -> String {
+        format!("model2vec:potion-base-8M:{}", self.dimension())
+    }
+
     fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
         let owned: Vec<String> = texts.iter().map(|s| s.to_string()).collect();
         Ok(self.model.encode(&owned))
@@ -309,6 +331,9 @@ static DOC_EMBEDDER: OnceLock<Arc<dyn EmbedProvider>> = OnceLock::new();
 
 /// Factory: select Model2Vec if available, otherwise fall back to TrigramEmbedder.
 pub fn init_embedder() -> Arc<dyn EmbedProvider> {
+    if let Some(v) = VoyageEmbedder::from_env() {
+        return Arc::new(v); // local-dev hook: VOYAGE_API_KEY takes precedence
+    }
     match Model2VecEmbedder::new() {
         Ok(m) => Arc::new(m),
         Err(e) => {
@@ -330,6 +355,9 @@ pub fn doc_embedder() -> Arc<dyn EmbedProvider> {
 
 /// Create the best available embedder: Model2Vec if possible, fallback to trigram.
 pub fn best_embedder() -> Box<dyn EmbedProvider> {
+    if let Some(v) = VoyageEmbedder::from_env() {
+        return Box::new(v); // local-dev hook: VOYAGE_API_KEY takes precedence
+    }
     match Model2VecEmbedder::new() {
         Ok(m) => Box::new(m),
         Err(e) => {
@@ -339,6 +367,30 @@ pub fn best_embedder() -> Box<dyn EmbedProvider> {
     }
 }
 
+/// Sidecar recording which embedder (provider:model:dim) produced `embeddings.bin`.
+fn embedder_meta_path(infigraph_dir: &Path) -> PathBuf {
+    infigraph_dir.join("embeddings.provider")
+}
+
+/// Whether the recorded embedder identity differs from `identity`.
+///
+/// A missing or unreadable record is treated as stale: existing vectors of
+/// unknown provenance may live in an incompatible embedding space, so the
+/// fail-safe is a one-time re-embed. Callers only consult this when
+/// embeddings already exist, so fresh indexes are unaffected.
+pub fn embedder_identity_stale(infigraph_dir: &Path, identity: &str) -> bool {
+    match std::fs::read_to_string(embedder_meta_path(infigraph_dir)) {
+        Ok(prev) => prev.trim() != identity,
+        Err(_) => true,
+    }
+}
+
+/// Record the embedder identity that produced the current `embeddings.bin`.
+pub fn write_embedder_identity(infigraph_dir: &Path, identity: &str) -> Result<()> {
+    std::fs::write(embedder_meta_path(infigraph_dir), identity)
+        .context("write embeddings provider sidecar")
+}
+
 /// Count the number of embeddings in the binary file at `root/.infigraph/embeddings.bin`.
 pub fn embedding_count(root: &Path) -> usize {
     let path = root.join(".infigraph").join("embeddings.bin");
@@ -425,12 +477,22 @@ pub fn update_embeddings(
         return Ok(0);
     }
 
-    let emb_path = root.join(".infigraph").join("embeddings.bin");
+    let embedder: Arc<Box<dyn EmbedProvider>> = Arc::new(best_embedder());
+    let identity = embedder.identity();
+    let infigraph_dir = root.join(".infigraph");
+    let emb_path = infigraph_dir.join("embeddings.bin");
     let mut existing: std::collections::HashMap<String, Vec<f32>> = load_embeddings(&emb_path)
         .unwrap_or_default()
         .into_iter()
         .collect();
 
+    // Provider/model changed since the last build: old vectors live in an
+    // incompatible embedding space (even at the same dimension) — discard them
+    // and re-embed everything.
+    if !existing.is_empty() && embedder_identity_stale(&infigraph_dir, &identity) {
+        existing.clear();
+    }
+
     let changed_set: std::collections::HashSet<&str> = changed_files.iter().copied().collect();
 
     let to_embed: Vec<(String, String)> = rows
@@ -453,21 +515,38 @@ pub fn update_embeddings(
         .collect();
 
     if !to_embed.is_empty() {
-        let embedder: Arc<Box<dyn EmbedProvider>> = Arc::new(best_embedder());
         const BATCH: usize = 256;
-        let results: Vec<Vec<(String, Vec<f32>)>> = to_embed
-            .par_chunks(BATCH)
-            .map(|chunk| {
-                let emb = Arc::clone(&embedder);
-                let texts: Vec<&str> = chunk.iter().map(|(_, t)| t.as_str()).collect();
-                let vecs = emb.embed_batch(&texts).unwrap_or_default();
-                chunk
-                    .iter()
-                    .enumerate()
-                    .filter_map(|(i, (id, _))| vecs.get(i).map(|v| (id.clone(), v.clone())))
-                    .collect()
-            })
-            .collect();
+        // Propagate batch failures instead of silently dropping vectors: a
+        // partial index must never be persisted (nor its identity recorded)
+        // as a successful build.
+        let embed_chunk = |chunk: &[(String, String)]| -> Result<Vec<(String, Vec<f32>)>> {
+            let texts: Vec<&str> = chunk.iter().map(|(_, t)| t.as_str()).collect();
+            let vecs = embedder.embed_batch(&texts)?;
+            anyhow::ensure!(
+                vecs.len() == chunk.len(),
+                "embedder returned {} vectors for {} texts",
+                vecs.len(),
+                chunk.len()
+            );
+            Ok(chunk
+                .iter()
+                .zip(vecs)
+                .map(|((id, _), v)| (id.clone(), v))
+                .collect())
+        };
+        // Remote APIs get sequential batches (rate limits); local models get
+        // the full rayon pool.
+        let results: Vec<Vec<(String, Vec<f32>)>> = if embedder.is_remote() {
+            to_embed
+                .chunks(BATCH)
+                .map(embed_chunk)
+                .collect::<Result<Vec<_>>>()?
+        } else {
+            to_embed
+                .par_chunks(BATCH)
+                .map(embed_chunk)
+                .collect::<Result<Vec<_>>>()?
+        };
         for batch in results {
             for (id, v) in batch {
                 existing.insert(id, v);
@@ -481,6 +560,7 @@ pub fn update_embeddings(
     let symbol_embeddings: Vec<(String, Vec<f32>)> = existing.into_iter().collect();
     let count = symbol_embeddings.len();
     save_embeddings(&emb_path, &symbol_embeddings)?;
+    write_embedder_identity(&infigraph_dir, &identity)?;
 
     // Below 200K symbols, brute-force rayon dot-product is faster than HNSW.
     // Build/rebuild HNSW only when above threshold OR when an existing index