diff --git a/.github/workflows/fork-pre-pr.yml b/.github/workflows/fork-pre-pr.yml new file mode 100644 index 00000000..8458637e --- /dev/null +++ b/.github/workflows/fork-pre-pr.yml @@ -0,0 +1,35 @@ +name: Fork Pre-PR Check + +# 在 fork 仓库推送 feat/* 分支时自动运行， +# 提 PR 到上游前提前发现 clippy / 编译问题。 +on: + push: + branches: + - "feat/**" + - "fix/**" + - "chore/**" + +env: + CARGO_TERM_COLOR: always + SQLX_OFFLINE: "true" + # 无需真实 DB / Embedding key，仅做静态检查 + DATABASE_URL: mysql://root:111@localhost:6001/memoria_test + +jobs: + check-and-clippy: + name: Check & Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@1.85.0 + with: + components: clippy + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: memoria + shared-key: fork-pre-pr + + - name: Check & Clippy + run: cd memoria && cargo check && cargo clippy -- -D warnings diff --git a/README.md b/README.md index c8e983a6..e68467a6 100644 --- a/README.md +++ b/README.md @@ -378,7 +378,7 @@ If you're an AI agent helping a user set up Memoria: 3. **Run `memoria init -i`** in the user's project directory 4. **Tell user to restart** their AI tool, then **verify** with `memory_retrieve("test")` -> **Self-Hosted only:** Configure embedding BEFORE first MCP server start — dimension is locked into schema. +> **Self-Hosted only:** Embedding dimension defaults to auto-infer (`EMBEDDING_DIM=0`): Memoria probes the embedding service on startup. Set `EMBEDDING_DIM` explicitly if the service may be unavailable at boot. The dimension is locked into the database schema on first run. --- diff --git a/memoria/crates/memoria-cli/src/main.rs b/memoria/crates/memoria-cli/src/main.rs index 6c567b8b..c443c6ba 100644 --- a/memoria/crates/memoria-cli/src/main.rs +++ b/memoria/crates/memoria-cli/src/main.rs @@ -532,6 +532,9 @@ async fn cmd_serve(db_url: Option, port: u16, master_key: String) -> Res validate_embedding_config(&cfg)?; bootstrap_runtime_topology(&mut cfg).await?; + if cfg.embedding_dim == 0 && cfg.has_embedding() { + cfg.embedding_dim = probe_embedding_dim(&cfg).await?; + } let redacted_db_url = redact_url(&cfg.db_url); let redacted_shared_db_url = redact_url(&cfg.shared_db_url); @@ -541,7 +544,9 @@ async fn cmd_serve(db_url: Option, port: u16, master_key: String) -> Res multi_db = cfg.multi_db, port = port, instance_id = %cfg.instance_id, + embedding_dim = cfg.embedding_dim, has_llm = cfg.has_llm(), + has_embedding = cfg.has_embedding(), embedding_provider = %cfg.embedding_provider, governance_plugin_binding = %cfg.governance_plugin_binding, "Starting Memoria API server" @@ -571,6 +576,7 @@ async fn cmd_serve(db_url: Option, port: u16, master_key: String) -> Res "shared_db_merged_pool", ); store.migrate_shared().await?; + store.check_embedding_dim_compat().await?; store.set_db_router(router.clone()); let git = Arc::new(GitForDataService::new( shared_pool, @@ -582,6 +588,7 @@ async fn cmd_serve(db_url: Option, port: u16, master_key: String) -> Res SqlMemoryStore::connect(&cfg.db_url, cfg.embedding_dim, cfg.instance_id.clone()) .await?; store.migrate().await?; + store.check_embedding_dim_compat().await?; let pool = connect_git_pool(&cfg.db_url, false).await?; let git_db_name = parse_db_name(&cfg.db_url).unwrap_or_else(|| cfg.db_name.clone()); let git = Arc::new(GitForDataService::new(pool, git_db_name)); @@ -738,11 +745,17 @@ async fn cmd_mcp( let redacted_db_url = redact_url(&cfg.db_url); let redacted_shared_db_url = redact_url(&cfg.shared_db_url); + // Auto-infer embedding dimension when EMBEDDING_DIM=0 (or unset). + if cfg.embedding_dim == 0 && cfg.has_embedding() { + cfg.embedding_dim = probe_embedding_dim(&cfg).await?; + } + tracing::info!( db_url = %redacted_db_url, shared_db_url = %redacted_shared_db_url, multi_db = cfg.multi_db, embedding_provider = %cfg.embedding_provider, + embedding_dim = cfg.embedding_dim, has_llm = cfg.has_llm(), governance_plugin_binding = %cfg.governance_plugin_binding, user = %cfg.user, @@ -773,6 +786,7 @@ async fn cmd_mcp( "shared_db_merged_pool", ); store.migrate_shared().await?; + store.check_embedding_dim_compat().await?; store.set_db_router(router.clone()); let git = Arc::new(GitForDataService::new( shared_pool, @@ -784,6 +798,7 @@ async fn cmd_mcp( SqlMemoryStore::connect(&cfg.db_url, cfg.embedding_dim, cfg.instance_id.clone()) .await?; store.migrate().await?; + store.check_embedding_dim_compat().await?; let pool = connect_git_pool(&cfg.db_url, false).await?; let git_db_name = parse_db_name(&cfg.db_url).unwrap_or_else(|| cfg.db_name.clone()); let git = Arc::new(GitForDataService::new(pool, git_db_name)); @@ -1256,6 +1271,56 @@ fn cmd_plugin_dev_keygen(dir: &Path) -> Result<()> { // ── Shared helpers ──────────────────────────────────────────────────────────── +/// Probe the configured embedding service to determine the vector dimension. +/// +/// Called when `EMBEDDING_DIM=0` (the default). Makes a single embedding +/// request with a short probe string and returns `vec.len()` as the +/// actual dimension, which is then used to create or validate the database +/// schema. +/// +/// # Errors +/// Returns an error if the embedding service is unreachable or returns an +/// empty vector, with a suggestion to set `EMBEDDING_DIM` explicitly. +async fn probe_embedding_dim(cfg: &memoria_service::Config) -> Result { + use memoria_core::interfaces::EmbeddingProvider; + use memoria_embedding::HttpEmbedder; + + // Build a temporary embedder with dim=0 (dim is not used by embed()). + let embedder = HttpEmbedder::new( + &cfg.embedding_base_url, + &cfg.embedding_api_key, + &cfg.embedding_model, + 0, + ); + + tracing::info!( + model = %cfg.embedding_model, + base_url = %cfg.embedding_base_url, + "EMBEDDING_DIM=0: probing embedding service to auto-infer dimension" + ); + + let vec = embedder + .embed("dimension probe") + .await + .map_err(|e| anyhow::anyhow!( + "EMBEDDING_DIM=0 but the embedding probe failed: {e}. \ + Set EMBEDDING_DIM explicitly (e.g. EMBEDDING_DIM=768 for \ + nomic-embed-text, EMBEDDING_DIM=1024 for BAAI/bge-m3) or \ + check that your embedding service is reachable." + ))?; + + if vec.is_empty() { + return Err(anyhow::anyhow!( + "EMBEDDING_DIM=0: embedding service returned an empty vector. \ + Set EMBEDDING_DIM explicitly." + )); + } + + let dim = vec.len(); + tracing::info!(embedding_dim = dim, "Auto-inferred embedding dimension"); + Ok(dim) +} + fn build_embedder( cfg: &memoria_service::Config, ) -> Option> { diff --git a/memoria/crates/memoria-service/src/config.rs b/memoria/crates/memoria-service/src/config.rs index 7a3c7fcc..2024f3e2 100644 --- a/memoria/crates/memoria-service/src/config.rs +++ b/memoria/crates/memoria-service/src/config.rs @@ -95,7 +95,7 @@ impl Config { let embedding_dim = std::env::var("EMBEDDING_DIM") .ok() .and_then(|s| s.parse().ok()) - .unwrap_or(1024usize); + .unwrap_or(0usize); // 0 = auto-infer from embedding service at startup let llm_api_key = std::env::var("LLM_API_KEY").ok().filter(|s| !s.is_empty()); diff --git a/memoria/crates/memoria-storage/src/store.rs b/memoria/crates/memoria-storage/src/store.rs index c805d0bc..09159403 100644 --- a/memoria/crates/memoria-storage/src/store.rs +++ b/memoria/crates/memoria-storage/src/store.rs @@ -2050,6 +2050,52 @@ impl SqlMemoryStore { Ok(()) } + /// Check that the configured embedding dimension matches the dimension + /// already stored in the database schema. + /// + /// If `mem_memories` already exists with a different dimension, returning + /// an error here is far better than silently failing on the first INSERT. + /// Called after `migrate()` so the table is guaranteed to exist. + /// + /// # Errors + /// Returns [`MemoriaError::Internal`] when a mismatch is detected, + /// with a human-readable message explaining how to resolve it. + pub async fn check_embedding_dim_compat(&self) -> Result<(), MemoriaError> { + // Query the actual column type stored in the schema, e.g. "vecf32(768)" + let col_type: Option = sqlx::query_scalar( + "SELECT column_type \ + FROM information_schema.columns \ + WHERE table_schema = DATABASE() \ + AND table_name = 'mem_memories' \ + AND column_name = 'embedding'", + ) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + + if let Some(ct) = col_type { + // Parse "vecf32(768)" → 768 + if let Ok(schema_dim) = ct + .trim_start_matches("vecf32(") + .trim_end_matches(')') + .parse::() + { + if schema_dim != self.embedding_dim { + return Err(MemoriaError::Internal(format!( + "Embedding dimension mismatch: the database schema has \ + {}d vectors but Memoria is configured for {}d. \ + To fix: either set EMBEDDING_DIM={} to match the \ + existing schema, or drop the database (data loss) and \ + restart to rebuild with the new dimension.", + schema_dim, self.embedding_dim, schema_dim + ))); + } + } + } + + Ok(()) + } + // ── Audit log ───────────────────────────────────────────────────────────── /// Create a safety snapshot before destructive operations. Best-effort. diff --git a/skills/deployment/SKILL.md b/skills/deployment/SKILL.md index d89d28cb..ba2c6167 100644 --- a/skills/deployment/SKILL.md +++ b/skills/deployment/SKILL.md @@ -40,7 +40,7 @@ Services: API on `:8100`, MatrixOne on `:6001`. Verify: `curl http://localhost:8 | `MEMORIA_EMBEDDING_API_KEY` | — | Required if provider is `openai` (single-backend) | | `MEMORIA_EMBEDDING_BASE_URL` | — | Custom endpoint, OpenAI-compatible (single-backend) | | `MEMORIA_EMBEDDING_ENDPOINTS` | — | JSON array for multi-backend round-robin. When set, supersedes `BASE_URL`/`API_KEY`. Format: `[{"url":"https://api1.example.com/v1","api_key":"sk-1"},{"url":"https://api2.example.com/v1","api_key":"sk-2"}]`. All endpoints must serve the same model. Requests rotate round-robin; failed/rate-limited endpoints are skipped automatically. | -| `MEMORIA_EMBEDDING_DIM` | `0` (auto) | Embedding dimension | +| `MEMORIA_EMBEDDING_DIM` | `0` (auto) | Embedding dimension. `0` = auto-infer: Memoria probes the embedding service on startup and uses the returned vector length. Set explicitly (e.g. `768`, `1024`) to skip the probe or when the embedding service may be unavailable at boot time. | ### Distributed