diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31066c202..9d711ff43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push: - branches: [ "main", "release/**" ] + branches: [ "main", "release/**", "codex/deploy" ] pull_request: branches: [ "main" ] @@ -14,8 +14,8 @@ env: RUST_BACKTRACE: 1 RUST_LIB_BACKTRACE: 1 -# 9 parallel jobs. See doc/working/test.md § "Current CI Test Design" -# for the assignment rule and how to add new test tasks. +# 10 parallel jobs. See doc/working/test.md § "Current CI Test Design" +# for the component assignment rule and how to add new test tasks. jobs: Lint: runs-on: ubuntu-latest @@ -83,7 +83,16 @@ jobs: with: name: runtime-cpp-${{ github.run_attempt }} path: | - .crowdb-runtime/ + .crowdb-runtime/**/log/** + .crowdb-runtime/**/*.log + .crowdb-runtime/**/*.log.* + .crowdb-runtime/**/*.out + .crowdb-runtime/**/*.toml + .crowdb-runtime/**/*.json + .crowdb-runtime/**/*.png + .crowdb-runtime/**/*.md + .crowdb-runtime/**/*.html + .crowdb-runtime/**/trace.zip include-hidden-files: true if-no-files-found: ignore retention-days: 7 @@ -108,8 +117,8 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Clean env run: pixi run clean-env - - name: Run unit tests - run: pixi run test-unit + - name: Run core component tests + run: pixi run test-core - name: Capture runner state on failure if: failure() run: | @@ -127,12 +136,21 @@ jobs: with: name: runtime-unit-${{ github.run_attempt }} path: | - .crowdb-runtime/ + .crowdb-runtime/**/log/** + .crowdb-runtime/**/*.log + .crowdb-runtime/**/*.log.* + .crowdb-runtime/**/*.out + .crowdb-runtime/**/*.toml + .crowdb-runtime/**/*.json + .crowdb-runtime/**/*.png + .crowdb-runtime/**/*.md + .crowdb-runtime/**/*.html + .crowdb-runtime/**/trace.zip include-hidden-files: true if-no-files-found: ignore retention-days: 7 - ServerTests: + StorageTests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -154,8 +172,8 @@ jobs: run: pixi run clean-env - name: Build C++ libs (crowdb-diskio subprocess) run: pixi run build-cpp - - name: Run server tests - run: pixi run test-server + - name: Run storage component tests + run: pixi run test-storage - name: Capture runner state on failure if: failure() run: | @@ -171,9 +189,73 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: runtime-server-${{ github.run_attempt }} + name: runtime-storage-${{ github.run_attempt }} path: | - .crowdb-runtime/ + .crowdb-runtime/**/log/** + .crowdb-runtime/**/*.log + .crowdb-runtime/**/*.log.* + .crowdb-runtime/**/*.out + .crowdb-runtime/**/*.toml + .crowdb-runtime/**/*.json + .crowdb-runtime/**/*.png + .crowdb-runtime/**/*.md + .crowdb-runtime/**/*.html + .crowdb-runtime/**/trace.zip + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + - name: Clean subprocesses + if: always() + run: pixi run clean-env + + AccessTests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache + sudo apt-get clean + df -h / + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: latest + - uses: Swatinem/rust-cache@v2 + - name: Clean env + run: pixi run clean-env + - name: Run access component tests + run: pixi run test-access + - name: Capture runner state on failure + if: failure() + run: | + echo "=== Listening ports ===" + ss -tlnp 2>/dev/null || true + echo "=== Process list (crowdb) ===" + ps aux | grep -E 'crowdb|cargo' | grep -v grep || true + echo "=== Disk space ===" + df -h + echo "=== CROWDB runtime files ===" + find .crowdb-runtime -maxdepth 4 -type f -print 2>/dev/null || true + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: runtime-access-${{ github.run_attempt }} + path: | + .crowdb-runtime/**/log/** + .crowdb-runtime/**/*.log + .crowdb-runtime/**/*.log.* + .crowdb-runtime/**/*.out + .crowdb-runtime/**/*.toml + .crowdb-runtime/**/*.json + .crowdb-runtime/**/*.png + .crowdb-runtime/**/*.md + .crowdb-runtime/**/*.html + .crowdb-runtime/**/trace.zip include-hidden-files: true if-no-files-found: ignore retention-days: 7 @@ -220,7 +302,16 @@ jobs: with: name: runtime-s3-e2e-${{ github.run_attempt }} path: | - .crowdb-runtime/ + .crowdb-runtime/**/log/** + .crowdb-runtime/**/*.log + .crowdb-runtime/**/*.log.* + .crowdb-runtime/**/*.out + .crowdb-runtime/**/*.toml + .crowdb-runtime/**/*.json + .crowdb-runtime/**/*.png + .crowdb-runtime/**/*.md + .crowdb-runtime/**/*.html + .crowdb-runtime/**/trace.zip include-hidden-files: true if-no-files-found: ignore retention-days: 7 @@ -251,7 +342,16 @@ jobs: with: name: runtime-iceberge2e-${{ github.run_attempt }} path: | - .crowdb-runtime/ + .crowdb-runtime/**/log/** + .crowdb-runtime/**/*.log + .crowdb-runtime/**/*.log.* + .crowdb-runtime/**/*.out + .crowdb-runtime/**/*.toml + .crowdb-runtime/**/*.json + .crowdb-runtime/**/*.png + .crowdb-runtime/**/*.md + .crowdb-runtime/**/*.html + .crowdb-runtime/**/trace.zip target/iceberg-rck/open-api/build/reports/tests/ target/iceberg-rck/open-api/build/test-results/ include-hidden-files: true @@ -284,7 +384,16 @@ jobs: with: name: runtime-icebergsdk-${{ github.run_attempt }} path: | - .crowdb-runtime/ + .crowdb-runtime/**/log/** + .crowdb-runtime/**/*.log + .crowdb-runtime/**/*.log.* + .crowdb-runtime/**/*.out + .crowdb-runtime/**/*.toml + .crowdb-runtime/**/*.json + .crowdb-runtime/**/*.png + .crowdb-runtime/**/*.md + .crowdb-runtime/**/*.html + .crowdb-runtime/**/trace.zip ${{ runner.temp }}/iceberg-rck/open-api/build/reports/tests/ ${{ runner.temp }}/iceberg-rck/open-api/build/test-results/ include-hidden-files: true @@ -335,7 +444,17 @@ jobs: with: name: runtime-console-${{ github.run_attempt }} path: | - .crowdb-runtime/ + .crowdb-runtime/**/log/** + .crowdb-runtime/**/*.log + .crowdb-runtime/**/*.log.* + .crowdb-runtime/**/*.out + .crowdb-runtime/**/*.toml + .crowdb-runtime/**/*.json + .crowdb-runtime/**/*.png + .crowdb-runtime/**/*.md + .crowdb-runtime/**/*.html + .crowdb-runtime/**/trace.zip + app/crowdb-web/ui/e2e/test-results/ include-hidden-files: true if-no-files-found: ignore retention-days: 7 @@ -387,7 +506,16 @@ jobs: with: name: runtime-ui-${{ github.run_attempt }} path: | - .crowdb-runtime/ + .crowdb-runtime/**/log/** + .crowdb-runtime/**/*.log + .crowdb-runtime/**/*.log.* + .crowdb-runtime/**/*.out + .crowdb-runtime/**/*.toml + .crowdb-runtime/**/*.json + .crowdb-runtime/**/*.png + .crowdb-runtime/**/*.md + .crowdb-runtime/**/*.html + .crowdb-runtime/**/trace.zip include-hidden-files: true if-no-files-found: ignore retention-days: 7 @@ -399,6 +527,7 @@ jobs: path: | app/crowdb-web/ui/playwright-report app/crowdb-web/ui/test-results + app/crowdb-web/ui/e2e/test-results retention-days: 7 - name: Clean subprocesses if: always() diff --git a/.github/workflows/iceberg-rust-sdk.yml b/.github/workflows/iceberg-rust-sdk.yml index 9e0c265eb..ce6cb35df 100644 --- a/.github/workflows/iceberg-rust-sdk.yml +++ b/.github/workflows/iceberg-rust-sdk.yml @@ -34,7 +34,17 @@ jobs: uses: actions/upload-artifact@v4 with: name: runtime-iceberg-rust-sdk-${{ github.run_attempt }} - path: .crowdb-runtime/ + path: | + .crowdb-runtime/**/log/** + .crowdb-runtime/**/*.log + .crowdb-runtime/**/*.log.* + .crowdb-runtime/**/*.out + .crowdb-runtime/**/*.toml + .crowdb-runtime/**/*.json + .crowdb-runtime/**/*.png + .crowdb-runtime/**/*.md + .crowdb-runtime/**/*.html + .crowdb-runtime/**/trace.zip if-no-files-found: ignore retention-days: 7 - name: Clean subprocesses diff --git a/CHANGELOG.md b/CHANGELOG.md index 3238dc878..9a94a77dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ # Changelog -CROWDB is preparing `0.2.2`. This is a development version, not a +CROWDB is developing `0.3.0`. This is a development version, not a production release or a compatibility promise. CROWDB does not yet maintain compatibility for persisted data, WAL, metadata, @@ -21,17 +21,33 @@ policy. ## [Unreleased] -### 0.2.2 preparation +The following changes are being prepared for `0.3.0`. -- Build service binaries before stream and console acceptance tests in CI. -- Wait for stopped KV children before deleting console workspaces during reset. +### Added -### 0.2.1 preparation +- Dataset access support with namespace, manifest, cursor, retention, transport, + and publication primitives. +- A Python dataset client surface for iterating dataset records. -- S3 and Iceberg own separate chunk types, storage policies, and write pools +### Changed + +- S3 and Iceberg use separate chunk types, storage policies, and write pools behind one access-server process. -- Explicit single-node test and three-node production protection profiles, - with strip-level mirror and EC I/O and repairable degraded EC placement. +- Single-node and three-node profiles now make storage protection explicit, + including strip-level mirror and EC I/O with repairable degraded placement. + +### Internal + +- Local and CI tests are organized around component tasks rather than package- + level Pixi tasks. + +## [0.2.2] - 2026-10-06 + +### Fixed + +- CI builds service binaries before stream and console acceptance tests. +- Console workspace reset waits for stopped KV children before deleting runtime + state. ## [0.1.0] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6ee384641..de5c99a6a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Thank you for contributing to CROWDB. ## Development status -CROWDB is under active development at version `0.2.2`. It has not reached +CROWDB is under active development at version `0.3.0`. It has not reached alpha, is not recommended for production, and must be tested with disposable data. Compatibility is not yet maintained for persisted data, WAL, metadata, or other on-disk formats. A change may deliberately replace an unreleased format diff --git a/Cargo.lock b/Cargo.lock index c49cfceec..183744f3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -607,9 +607,25 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crowdb-access-dataset" +version = "0.3.0" +dependencies = [ + "async-trait", + "bincode", + "crowdb-chunk-kv-client", + "crowdb-protocol", + "md-5", + "reqwest", + "serde", + "thiserror 2.0.18", + "tokio", + "uuid", +] + [[package]] name = "crowdb-access-iceberg" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "async-trait", @@ -645,7 +661,7 @@ dependencies = [ [[package]] name = "crowdb-access-multipart" -version = "0.2.2" +version = "0.3.0" dependencies = [ "crowdb-protocol", "md-5", @@ -655,7 +671,7 @@ dependencies = [ [[package]] name = "crowdb-access-s3" -version = "0.2.2" +version = "0.3.0" dependencies = [ "aes-gcm", "arc-swap", @@ -693,7 +709,7 @@ dependencies = [ [[package]] name = "crowdb-access-server" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "async-trait", @@ -701,6 +717,7 @@ dependencies = [ "bincode", "chrono", "crc", + "crowdb-access-dataset", "crowdb-access-iceberg", "crowdb-access-s3", "crowdb-access-server", @@ -739,7 +756,7 @@ dependencies = [ [[package]] name = "crowdb-chunk-client" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "async-trait", @@ -763,7 +780,7 @@ dependencies = [ [[package]] name = "crowdb-chunk-kv" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "async-trait", @@ -774,6 +791,7 @@ dependencies = [ "crowdb-tree-ffi", "futures", "serde", + "serde_bytes", "sha2", "thiserror 2.0.18", "tokio", @@ -782,7 +800,7 @@ dependencies = [ [[package]] name = "crowdb-chunk-kv-client" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "async-trait", @@ -801,7 +819,7 @@ dependencies = [ [[package]] name = "crowdb-chunk-kv-server" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "async-trait", @@ -810,11 +828,13 @@ dependencies = [ "clap", "crowdb-chunk-client", "crowdb-chunk-kv", + "crowdb-chunk-kv-server", "crowdb-chunk-stream", "crowdb-common", "crowdb-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", + "crowdb-test-harness", "crowdb-tree-ffi", "serde", "serde_json", @@ -827,7 +847,7 @@ dependencies = [ [[package]] name = "crowdb-chunk-stream" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "async-trait", @@ -849,7 +869,7 @@ dependencies = [ [[package]] name = "crowdb-chunkdb" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "axum", @@ -886,7 +906,7 @@ dependencies = [ [[package]] name = "crowdb-chunkdb-client" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "bytes", @@ -901,7 +921,7 @@ dependencies = [ [[package]] name = "crowdb-cli" -version = "0.2.2" +version = "0.3.0" dependencies = [ "axum", "chrono", @@ -932,7 +952,7 @@ dependencies = [ [[package]] name = "crowdb-common" -version = "0.2.2" +version = "0.3.0" dependencies = [ "crowdb-test-harness", "flate2", @@ -951,7 +971,7 @@ dependencies = [ [[package]] name = "crowdb-console-shared" -version = "0.2.2" +version = "0.3.0" dependencies = [ "async-trait", "axum", @@ -976,7 +996,7 @@ dependencies = [ [[package]] name = "crowdb-diskdb" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "axum", @@ -1012,7 +1032,7 @@ dependencies = [ [[package]] name = "crowdb-diskdb-client" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "bytes", @@ -1031,7 +1051,7 @@ dependencies = [ [[package]] name = "crowdb-diskio-client" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "bytes", @@ -1052,7 +1072,7 @@ dependencies = [ [[package]] name = "crowdb-kv" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "bytes", @@ -1085,7 +1105,7 @@ dependencies = [ [[package]] name = "crowdb-kv-client" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "axum", @@ -1109,7 +1129,7 @@ dependencies = [ [[package]] name = "crowdb-kv-server" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "axum", @@ -1134,7 +1154,7 @@ dependencies = [ [[package]] name = "crowdb-monitor" -version = "0.2.2" +version = "0.3.0" dependencies = [ "clap", "crowdb-diskio-client", @@ -1157,7 +1177,7 @@ dependencies = [ [[package]] name = "crowdb-protocol" -version = "0.2.2" +version = "0.3.0" dependencies = [ "bincode", "bytes", @@ -1176,7 +1196,7 @@ dependencies = [ [[package]] name = "crowdb-rpc-ffi" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "bytes", @@ -1191,7 +1211,7 @@ dependencies = [ [[package]] name = "crowdb-test-harness" -version = "0.2.2" +version = "0.3.0" dependencies = [ "crowdb-chunkdb-client", "crowdb-diskdb-client", @@ -1208,7 +1228,7 @@ dependencies = [ [[package]] name = "crowdb-tree-ffi" -version = "0.2.2" +version = "0.3.0" dependencies = [ "bytes", "cc", @@ -1220,7 +1240,7 @@ dependencies = [ [[package]] name = "crowdb-web" -version = "0.2.2" +version = "0.3.0" dependencies = [ "arc-swap", "async-trait", @@ -3485,6 +3505,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/Cargo.toml b/Cargo.toml index 53e99b910..2dd63499b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "container/crowdb-monitor", "lib/crowdb-access-s3", "lib/crowdb-access-iceberg", + "lib/crowdb-access-dataset", "lib/crowdb-access-multipart", ] exclude = ["third-party/hyper"] @@ -40,7 +41,7 @@ exclude = ["third-party/hyper"] # `unsafe_code = "deny"`. [workspace.package] -version = "0.2.2" +version = "0.3.0" edition = "2021" rust-version = "1.75" license = "Apache-2.0" diff --git a/SECURITY.md b/SECURITY.md index 383bc307c..90e08ba65 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,7 +14,7 @@ If you discover a security vulnerability in CROWDB, please report it responsibly ## Scope -CROWDB `0.2.2` is a development version for evaluation with disposable data. +CROWDB `0.3.0` is a development version for evaluation with disposable data. There is no production support commitment, supported stable release series, or guaranteed response time. Security reports are reviewed by the maintainers. diff --git a/VERSION b/VERSION index ee1372d33..0d91a54c7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.2 +0.3.0 diff --git a/app/crowdb-access-server/Cargo.toml b/app/crowdb-access-server/Cargo.toml index 30c4252c2..73b45ee89 100644 --- a/app/crowdb-access-server/Cargo.toml +++ b/app/crowdb-access-server/Cargo.toml @@ -23,8 +23,10 @@ s3 = [ hex = "0.4" hmac = "0.12" async-trait = "0.1" +bincode = "1" chrono = { version = "0.4", default-features = false, features = ["std"] } crowdb-access-iceberg = { path = "../../lib/crowdb-access-iceberg" } +crowdb-access-dataset = { path = "../../lib/crowdb-access-dataset" } crowdb-access-s3 = { path = "../../lib/crowdb-access-s3" } crowdb-chunk-client = { path = "../../lib/crowdb-chunk-client" } crowdb-chunk-kv-client = { path = "../../lib/crowdb-chunk-kv-client" } @@ -46,7 +48,7 @@ serde = { version = "1", features = ["derive"] } sha2 = "0.10" sha1 = "0.10" crc = "3.3" -tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync"] } +tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } thiserror = { workspace = true } diff --git a/app/crowdb-access-server/src/config.rs b/app/crowdb-access-server/src/config.rs index 33ae99657..779cacbbe 100644 --- a/app/crowdb-access-server/src/config.rs +++ b/app/crowdb-access-server/src/config.rs @@ -18,6 +18,7 @@ pub struct AccessConfig { pub small_write: SmallWriteConfig, pub s3: S3Config, pub iceberg: IcebergConfig, + pub dataset: DatasetConfig, pub health: HealthConfig, } @@ -222,6 +223,30 @@ pub struct IcebergConfig { pub gc: IcebergGcConfig, } +/// Dataset's listener and read-admission limits. Dataset is optional so an +/// existing S3/Iceberg deployment keeps its current startup behavior. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default)] +pub struct DatasetConfig { + pub listen: Option, + pub max_in_flight: usize, + pub max_batch_size: usize, + pub request_timeout_seconds: u64, + pub lease_ttl_seconds: u64, +} + +impl Default for DatasetConfig { + fn default() -> Self { + Self { + listen: None, + max_in_flight: 3, + max_batch_size: 10_000, + request_timeout_seconds: 60, + lease_ttl_seconds: 60, + } + } +} + impl AccessConfig { #[must_use] pub fn s3_small_write(&self) -> &SmallWriteConfig { @@ -256,6 +281,7 @@ pub struct IcebergGcConfig { } impl BaseConfig for AccessConfig { + #[allow(clippy::too_many_lines)] fn validate(&self) -> Result<(), String> { if self.common.diskio_connections_per_endpoint == 0 || self.common.diskio_rpc_workers == 0 { return Err("common DiskIO connections and RPC workers must be nonzero".into()); @@ -284,6 +310,7 @@ impl BaseConfig for AccessConfig { if self.iceberg.native_budget_bytes == Some(0) { return Err("Iceberg native budget must be nonzero".into()); } + self.validate_dataset()?; if self.iceberg.ec_data == Some(0) || self.iceberg.ec_code == Some(0) || self.iceberg.max_chunk_size == Some(0) @@ -345,6 +372,7 @@ impl BaseConfig for AccessConfig { self.s3.listen.as_deref(), self.iceberg.listen.as_deref(), self.health.listen.as_deref(), + self.dataset.listen.as_deref(), ]; let mut ports = std::collections::HashSet::new(); for listen in listeners.into_iter().flatten() { @@ -360,6 +388,24 @@ impl BaseConfig for AccessConfig { } impl AccessConfig { + fn validate_dataset(&self) -> Result<(), String> { + if self.dataset.max_in_flight == 0 || self.dataset.max_in_flight > 64 { + return Err("dataset.max_in_flight must be between 1 and 64".into()); + } + if self.dataset.max_batch_size == 0 + || self.dataset.max_batch_size > crowdb_access_dataset::MAX_BATCH_SIZE + { + return Err("dataset.max_batch_size is outside Dataset bounds".into()); + } + if self.dataset.request_timeout_seconds == 0 || self.dataset.request_timeout_seconds > 300 { + return Err("dataset.request_timeout_seconds must be between 1 and 300".into()); + } + if self.dataset.lease_ttl_seconds == 0 || self.dataset.lease_ttl_seconds > 3600 { + return Err("dataset.lease_ttl_seconds must be between 1 and 3600".into()); + } + Ok(()) + } + fn validate_small_writes(&self) -> Result<(), String> { for config in [ &self.small_write, diff --git a/app/crowdb-access-server/src/dataset.rs b/app/crowdb-access-server/src/dataset.rs new file mode 100644 index 000000000..ba4521552 --- /dev/null +++ b/app/crowdb-access-server/src/dataset.rs @@ -0,0 +1,751 @@ +//! Dataset HTTP listener. Framing and authentication live here; read +//! semantics are delegated to `crowdb-access-dataset`. + +use std::convert::Infallible; +use std::sync::Arc; +use std::time::Duration; + +use crowdb_access_dataset::{ + AuthorityError, ChunkKvDatasetStore, DatasetAuthority, DatasetManifestResponse, DatasetPublishRequest, + DatasetPublishResponse, DatasetReadRequest, DatasetReadService, DatasetReclaimResponse, + DatasetRetentionResponse, DatasetScanRequest, DatasetSnapshotRequest, DatasetSnapshotResponse, + DatasetSnapshotsResponse, OperationId, ReadTransportError, +}; +use crowdb_chunk_kv_client::{ + ChunkKvClient, ChunkKvRpcTransport, ClientConfig as ChunkKvConfig, Group0ChunkKvRangeCatalogSource, +}; +use crowdb_kv_client::{ClientConfig as KvConfig, CrowdbKvClient}; +use http_body_util::{BodyExt, Full}; +use hyper::body::{Bytes, Incoming}; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Method, Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use tokio::net::TcpListener; +use tokio::task::JoinSet; + +const MAX_REQUEST_BYTES: usize = 8 * 1024 * 1024; + +type BoxError = Box; + +pub struct DatasetHttpService { + reads: Arc, + bearer: Option>, + request_timeout: Duration, +} + +impl DatasetHttpService { + #[must_use] + pub fn new(reads: Arc, request_timeout: Duration) -> Self { + Self { + reads, + bearer: None, + request_timeout, + } + } + + /// Protects every Dataset route with one bearer token, including + /// publication, reads, retention, reclamation, and cancellation. + #[must_use] + pub fn with_bearer(mut self, token: impl Into>) -> Self { + self.bearer = Some(token.into()); + self + } + + async fn handle(&self, request: Request) -> Response> { + if !self.authorized(&request) { + return response(StatusCode::UNAUTHORIZED, b"unauthorized".to_vec()); + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/read" { + return self.read(request).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/scan" { + return self.scan(request).await; + } + if request.method() == Method::POST + && matches!( + request.uri().path(), + "/v1/dataset/start-read-plan" + | "/v1/dataset/next-batch" + | "/v1/dataset/save-progress" + | "/v1/dataset/resume" + ) + { + return self.scan(request).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/create" { + return self.create().await; + } + if request.method() == Method::GET && request.uri().path() == "/v1/dataset/open" { + return self.open().await; + } + if request.method() == Method::GET && request.uri().path() == "/v1/dataset/latest" { + return self.latest().await; + } + if request.method() == Method::GET && request.uri().path() == "/v1/dataset/snapshots" { + return self.snapshots().await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/publish" { + return self.publish(request).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/prepare" { + return self.publish(request).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/retain" { + return self.retain(request, true).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/release" { + return self.retain(request, false).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/snapshot" { + return self.snapshot(request).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/manifest" { + return self.manifest(request).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/stable" { + return self.stable(request).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/reclaim" { + return self.reclaim(request).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/reclaim/status" { + return self.reclaim_status(request).await; + } + if request.method() == Method::POST && request.uri().path() == "/v1/dataset/cancel" { + self.reads.cancel(); + return response(StatusCode::NO_CONTENT, Vec::new()); + } + response(StatusCode::NOT_FOUND, b"not found".to_vec()) + } + + async fn open(&self) -> Response> { + match self.reads.authority().get_dataset(self.reads.identity()).await { + Ok(record) => encode(StatusCode::OK, &record), + Err(_) => response(StatusCode::NOT_FOUND, b"dataset not found".to_vec()), + } + } + + async fn create(&self) -> Response> { + match self + .reads + .authority() + .create_dataset(self.reads.identity().clone()) + .await + { + Ok(id) => encode(StatusCode::OK, &id), + Err(crowdb_access_dataset::AuthorityError::AlreadyExists) => { + response(StatusCode::CONFLICT, b"dataset already exists".to_vec()) + } + Err(_) => response(StatusCode::SERVICE_UNAVAILABLE, b"dataset unavailable".to_vec()), + } + } + + async fn latest(&self) -> Response> { + match self.reads.authority().latest(self.reads.identity()).await { + Ok(snapshot) => encode(StatusCode::OK, &snapshot), + Err(_) => response(StatusCode::SERVICE_UNAVAILABLE, b"dataset unavailable".to_vec()), + } + } + + async fn snapshots(&self) -> Response> { + match self.reads.authority().list_snapshots(self.reads.identity()).await { + Ok(snapshots) => encode(StatusCode::OK, &DatasetSnapshotsResponse { snapshots }), + Err(error) => response(status_for_authority(&error), error.to_string().into_bytes()), + } + } + + async fn publish(&self, request: Request) -> Response> { + let body = match tokio::time::timeout(self.request_timeout, request.into_body().collect()).await { + Ok(Ok(body)) => body.to_bytes(), + _ => return response(StatusCode::REQUEST_TIMEOUT, b"request timeout".to_vec()), + }; + if body.len() > MAX_REQUEST_BYTES { + return response(StatusCode::PAYLOAD_TOO_LARGE, b"request too large".to_vec()); + } + let request: DatasetPublishRequest = match bincode::deserialize(&body) { + Ok(request) => request, + Err(_) => return response(StatusCode::BAD_REQUEST, b"invalid publication request".to_vec()), + }; + if request.validate().is_err() { + return response(StatusCode::BAD_REQUEST, b"invalid publication request".to_vec()); + } + let operation = request.operation.unwrap_or_else(OperationId::random); + match self + .reads + .authority() + .publish_snapshot_with_operation( + self.reads.identity(), + request.parent, + request.manifest, + operation, + ) + .await + { + Ok(snapshot) => encode(StatusCode::OK, &DatasetPublishResponse { snapshot }), + Err(error) => response(status_for_authority(&error), error.to_string().into_bytes()), + } + } + + async fn retain(&self, request: Request, retain: bool) -> Response> { + let body = match tokio::time::timeout(self.request_timeout, request.into_body().collect()).await { + Ok(Ok(body)) => body.to_bytes(), + _ => return response(StatusCode::REQUEST_TIMEOUT, b"request timeout".to_vec()), + }; + let request: DatasetSnapshotRequest = match bincode::deserialize(&body) { + Ok(request) => request, + Err(_) => return response(StatusCode::BAD_REQUEST, b"invalid snapshot request".to_vec()), + }; + let result = if retain { + self.reads + .authority() + .retain_snapshot(self.reads.identity(), request.snapshot) + .await + } else { + self.reads + .authority() + .release_snapshot(self.reads.identity(), request.snapshot) + .await + }; + match result { + Ok(()) => encode( + StatusCode::OK, + &DatasetRetentionResponse { + snapshot: request.snapshot, + retained: retain, + }, + ), + Err(error) => response(status_for_authority(&error), error.to_string().into_bytes()), + } + } + + async fn snapshot(&self, request: Request) -> Response> { + let Some(request) = decode_snapshot_request(request, self.request_timeout).await else { + return response(StatusCode::BAD_REQUEST, b"invalid snapshot request".to_vec()); + }; + match self + .reads + .authority() + .get_snapshot(self.reads.identity(), request.snapshot) + .await + { + Ok(snapshot) => encode(StatusCode::OK, &DatasetSnapshotResponse { snapshot }), + Err(error) => response(status_for_authority(&error), error.to_string().into_bytes()), + } + } + + async fn manifest(&self, request: Request) -> Response> { + let Some(request) = decode_snapshot_request(request, self.request_timeout).await else { + return response(StatusCode::BAD_REQUEST, b"invalid snapshot request".to_vec()); + }; + match self + .reads + .authority() + .get_manifest(self.reads.identity(), request.snapshot) + .await + { + Ok(manifest) => encode(StatusCode::OK, &DatasetManifestResponse { manifest }), + Err(error) => response(status_for_authority(&error), error.to_string().into_bytes()), + } + } + + async fn stable(&self, request: Request) -> Response> { + let Some(request) = decode_snapshot_request(request, self.request_timeout).await else { + return response(StatusCode::BAD_REQUEST, b"invalid snapshot request".to_vec()); + }; + match self + .reads + .authority() + .set_stable(self.reads.identity(), request.snapshot) + .await + { + Ok(()) => encode( + StatusCode::OK, + &DatasetRetentionResponse { + snapshot: request.snapshot, + retained: true, + }, + ), + Err(error) => response(status_for_authority(&error), error.to_string().into_bytes()), + } + } + + async fn reclaim(&self, request: Request) -> Response> { + let Some(request) = decode_snapshot_request(request, self.request_timeout).await else { + return response(StatusCode::BAD_REQUEST, b"invalid snapshot request".to_vec()); + }; + match self + .reads + .authority() + .reclaim_snapshot_metadata(self.reads.identity(), request.snapshot) + .await + { + Ok(()) => encode( + StatusCode::OK, + &DatasetReclaimResponse { + snapshot: request.snapshot, + reclaimed: true, + }, + ), + Err(error) => response(status_for_authority(&error), error.to_string().into_bytes()), + } + } + + async fn reclaim_status(&self, request: Request) -> Response> { + let Some(request) = decode_snapshot_request(request, self.request_timeout).await else { + return response(StatusCode::BAD_REQUEST, b"invalid snapshot request".to_vec()); + }; + match self + .reads + .authority() + .reclaim_progress(self.reads.identity(), request.snapshot) + .await + { + Ok(progress) => encode(StatusCode::OK, &progress), + Err(error) => response(status_for_authority(&error), error.to_string().into_bytes()), + } + } + + fn authorized(&self, request: &Request) -> bool { + let Some(expected) = &self.bearer else { return true }; + request + .headers() + .get(hyper::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .is_some_and(|value| value.as_bytes() == expected.as_bytes()) + } + + async fn read(&self, request: Request) -> Response> { + let body = match tokio::time::timeout(self.request_timeout, request.into_body().collect()).await { + Ok(Ok(body)) => body.to_bytes(), + _ => return response(StatusCode::REQUEST_TIMEOUT, b"request timeout".to_vec()), + }; + if body.len() > MAX_REQUEST_BYTES { + return response(StatusCode::PAYLOAD_TOO_LARGE, b"request too large".to_vec()); + } + let request: DatasetReadRequest = match bincode::deserialize(&body) { + Ok(request) => request, + Err(_) => return response(StatusCode::BAD_REQUEST, b"invalid dataset request".to_vec()), + }; + match self.reads.read(request, unix_seconds()).await { + Ok(result) => encode(StatusCode::OK, &result), + Err(ReadTransportError::WindowFull) => { + response(StatusCode::TOO_MANY_REQUESTS, b"read window full".to_vec()) + } + Err(ReadTransportError::Invalid(crowdb_access_dataset::DatasetError::ReadCancelled)) => { + response(cancelled_status(), b"dataset read cancelled".to_vec()) + } + Err(ReadTransportError::Invalid(_)) => { + response(StatusCode::BAD_REQUEST, b"invalid dataset request".to_vec()) + } + Err(ReadTransportError::Authority(error)) => { + response(status_for_authority(&error), error.to_string().into_bytes()) + } + } + } + + async fn scan(&self, request: Request) -> Response> { + let body = match tokio::time::timeout(self.request_timeout, request.into_body().collect()).await { + Ok(Ok(body)) => body.to_bytes(), + _ => return response(StatusCode::REQUEST_TIMEOUT, b"request timeout".to_vec()), + }; + if body.len() > MAX_REQUEST_BYTES { + return response(StatusCode::PAYLOAD_TOO_LARGE, b"request too large".to_vec()); + } + let request: DatasetScanRequest = match bincode::deserialize(&body) { + Ok(request) => request, + Err(_) => return response(StatusCode::BAD_REQUEST, b"invalid scan request".to_vec()), + }; + match self.reads.scan(request, unix_seconds()).await { + Ok(result) => encode(StatusCode::OK, &result), + Err(ReadTransportError::WindowFull) => { + response(StatusCode::TOO_MANY_REQUESTS, b"read window full".to_vec()) + } + Err(ReadTransportError::Invalid(crowdb_access_dataset::DatasetError::ReadCancelled)) => { + response(cancelled_status(), b"dataset read cancelled".to_vec()) + } + Err(ReadTransportError::Invalid(_)) => { + response(StatusCode::BAD_REQUEST, b"invalid scan request".to_vec()) + } + Err(ReadTransportError::Authority(error)) => { + response(status_for_authority(&error), error.to_string().into_bytes()) + } + } + } +} + +fn status_for_authority(error: &AuthorityError) -> StatusCode { + match error { + AuthorityError::Invalid(_) => StatusCode::BAD_REQUEST, + AuthorityError::AlreadyExists + | AuthorityError::ParentConflict + | AuthorityError::HeadConflict + | AuthorityError::CursorConflict + | AuthorityError::SnapshotProtected => StatusCode::CONFLICT, + AuthorityError::NotFound | AuthorityError::SnapshotNotFound => StatusCode::NOT_FOUND, + AuthorityError::AncestryCycle | AuthorityError::Corrupt | AuthorityError::Store(_) => { + StatusCode::SERVICE_UNAVAILABLE + } + } +} + +async fn decode_snapshot_request( + request: Request, + timeout: Duration, +) -> Option { + let body = tokio::time::timeout(timeout, request.into_body().collect()) + .await + .ok()? + .ok()? + .to_bytes(); + bincode::deserialize(&body).ok() +} + +/// Runs Dataset as a standalone Access Server command. +/// +/// # Errors +/// Returns configuration, Chunk-KV discovery, authentication, or listener +/// failures. +pub async fn run(arguments: Vec) -> Result<(), BoxError> { + run_with_shutdown(arguments, async { + let _ = tokio::signal::ctrl_c().await; + }) + .await +} + +/// Runs Dataset with a caller-owned shutdown signal for combined processes. +/// +/// # Errors +/// Returns configuration, Chunk-KV discovery, authentication, or listener +/// failures. +pub async fn run_with_shutdown( + arguments: Vec, + shutdown: impl std::future::Future, +) -> Result<(), BoxError> { + let (access, arguments) = crate::config::load_args(arguments)?; + if !arguments.is_empty() && arguments != ["serve"] { + return Err("unexpected Dataset server arguments".into()); + } + let address = access + .dataset + .listen + .clone() + .or_else(|| std::env::var("CROWDB_DATASET_LISTEN").ok()) + .unwrap_or_else(|| "127.0.0.1:9093".into()); + let _: std::net::SocketAddr = address.parse()?; + let seeds = if access.common.management_seeds.is_empty() { + std::env::var("CROWDB_MANAGEMENT_SEEDS")? + .split(',') + .map(str::trim) + .filter(|seed| !seed.is_empty()) + .map(str::to_owned) + .collect::>() + } else { + access.common.management_seeds.clone() + }; + if seeds.is_empty() { + return Err("one or more management seeds are required".into()); + } + let control = Arc::new(CrowdbKvClient::new(KvConfig::new(seeds))); + let source = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(Arc::clone(&control))); + let chunk_config = ChunkKvConfig::default(); + let transport = Arc::new(ChunkKvRpcTransport::new(chunk_config.max_owner_connections, 1, 2)); + let client = Arc::new(ChunkKvClient::new(chunk_config, source, transport)?); + client.refresh_catalog().await?; + let store = Arc::new(ChunkKvDatasetStore::new(client)); + let authority = Arc::new(DatasetAuthority::new(store)); + let identity = crowdb_access_dataset::DatasetIdentity::new( + crowdb_access_dataset::NamespacePath::root(), + std::env::var("CROWDB_DATASET_NAME").unwrap_or_else(|_| "default".into()), + )?; + let reads = Arc::new(DatasetReadService::with_ttl( + authority, + identity, + access.dataset.max_in_flight, + unix_seconds(), + access.dataset.lease_ttl_seconds, + )?); + let mut service = + DatasetHttpService::new(reads, Duration::from_secs(access.dataset.request_timeout_seconds)); + if let Ok(token) = std::env::var("CROWDB_DATASET_READ_TOKEN") { + if token.is_empty() { + return Err("CROWDB_DATASET_READ_TOKEN must not be empty".into()); + } + service = service.with_bearer(Arc::::from(token)); + } + let listener = TcpListener::bind(address).await?; + serve(listener, Arc::new(service), shutdown).await?; + Ok(()) +} + +/// Runs the Dataset listener until shutdown resolves. +/// +/// # Errors +/// Returns listener or connection acceptance failures. +pub async fn serve( + listener: TcpListener, + service: Arc, + shutdown: impl std::future::Future, +) -> std::io::Result<()> { + tokio::pin!(shutdown); + let mut connections = JoinSet::new(); + loop { + tokio::select! { + () = &mut shutdown => break, + Some(_) = connections.join_next(), if !connections.is_empty() => {}, + accepted = listener.accept(), if connections.len() < 128 => { + let (stream, _) = accepted?; + let service = Arc::clone(&service); + connections.spawn(async move { + let handler = service_fn(move |request| { + let service = Arc::clone(&service); + async move { Ok::<_, Infallible>(service.handle(request).await) } + }); + let _ = http1::Builder::new().keep_alive(false) + .serve_connection(TokioIo::new(stream), handler).await; + }); + } + } + } + drop(listener); + while connections.join_next().await.is_some() {} + Ok(()) +} + +fn response(status: StatusCode, body: Vec) -> Response> { + Response::builder() + .status(status) + .body(Full::new(Bytes::from(body))) + .unwrap() +} + +fn cancelled_status() -> StatusCode { + StatusCode::from_u16(499).expect("499 is a valid HTTP status") +} + +fn encode(status: StatusCode, value: &T) -> Response> { + match bincode::serialize(value) { + Ok(body) => response(status, body), + Err(_) => response( + StatusCode::INTERNAL_SERVER_ERROR, + b"response encoding failed".to_vec(), + ), + } +} + +fn unix_seconds() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |value| value.as_secs()) +} + +#[cfg(test)] +mod tests { + use super::*; + use async_trait::async_trait; + use crowdb_access_dataset::{ + CasOutcome, DatasetPublishRequest, DatasetPublishResponse, DatasetReadService, DatasetScanRequest, + DatasetScanResponse, DatasetSnapshotRequest, DatasetStore, FieldDefinition, FieldLocator, + FieldRecord, ManifestRecord, NamespacePath, Ordering, ReadPlan, SchemaRecord, Selection, SnapshotId, + StoredValue, + }; + use md5::{Digest, Md5}; + use std::collections::HashMap; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::sync::Mutex; + + struct MemoryStore(Mutex, StoredValue>>); + + #[async_trait] + impl DatasetStore for MemoryStore { + async fn get(&self, key: &[u8]) -> Result, crowdb_access_dataset::StoreError> { + Ok(self.0.lock().await.get(key).cloned()) + } + + async fn compare_exchange( + &self, + key: &[u8], + expected: Option<&[u8]>, + value: &[u8], + ) -> Result { + let mut values = self.0.lock().await; + let current = values.get(key).cloned(); + if current.as_ref().map(|item| item.bytes.as_slice()) != expected { + return Ok(CasOutcome::Conflict(current)); + } + let revision = current.map_or(1, |item| item.revision + 1); + values.insert( + key.to_vec(), + StoredValue { + bytes: value.to_vec(), + revision, + }, + ); + Ok(CasOutcome::Applied(revision)) + } + + async fn delete(&self, key: &[u8]) -> Result<(), crowdb_access_dataset::StoreError> { + self.0.lock().await.remove(key); + Ok(()) + } + } + + #[tokio::test] + async fn dataset_listener_authenticates_and_serves_create_open() { + let identity = crowdb_access_dataset::DatasetIdentity::new(NamespacePath::root(), "e2e").unwrap(); + let authority = Arc::new(DatasetAuthority::new(Arc::new(MemoryStore(Mutex::new( + HashMap::new(), + ))))); + let reads = Arc::new(DatasetReadService::new(authority, identity, 2, 10).unwrap()); + let service = Arc::new( + DatasetHttpService::new(reads, Duration::from_secs(2)).with_bearer(Arc::::from("token")), + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(serve(listener, Arc::clone(&service), async { + let _ = shutdown_rx.await; + })); + + let mut unauthenticated = tokio::net::TcpStream::connect(address).await.unwrap(); + unauthenticated + .write_all(b"GET /v1/dataset/open HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut body = Vec::new(); + unauthenticated.read_to_end(&mut body).await.unwrap(); + assert!(body.starts_with(b"HTTP/1.1 401")); + + let mut client = tokio::net::TcpStream::connect(address).await.unwrap(); + client.write_all(b"POST /v1/dataset/create HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer token\r\nConnection: close\r\nContent-Length: 0\r\n\r\n").await.unwrap(); + let mut response = Vec::new(); + client.read_to_end(&mut response).await.unwrap(); + assert!(response.starts_with(b"HTTP/1.1 200")); + + let _ = shutdown_tx.send(()); + task.await.unwrap().unwrap(); + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn dataset_http_publish_persist_and_scan_round_trip() { + let identity = + crowdb_access_dataset::DatasetIdentity::new(NamespacePath::root(), "round-trip").unwrap(); + let store = Arc::new(MemoryStore(Mutex::new(HashMap::new()))); + let authority = Arc::new(DatasetAuthority::new(store)); + let reads = + Arc::new(DatasetReadService::new(Arc::clone(&authority), identity, 2, unix_seconds()).unwrap()); + let service = Arc::new(DatasetHttpService::new(reads, Duration::from_secs(2))); + authority + .create_dataset(service.reads.identity().clone()) + .await + .unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(serve(listener, Arc::clone(&service), async { + let _ = shutdown_rx.await; + })); + + let value = b"payload".to_vec(); + let mut digest = Md5::new(); + digest.update(&value); + let manifest = ManifestRecord { + version: 1, + snapshot: SnapshotId::random(), + schema: SchemaRecord { + version: 1, + fields: vec![FieldDefinition { + name: "value".into(), + required: true, + }], + }, + samples: vec![crowdb_access_dataset::SampleRecord { + sample_id: b"sample-1".to_vec(), + fields: vec![FieldRecord { + name: "value".into(), + value: FieldLocator::Inline { + value, + md5: digest.finalize().into(), + }, + }], + }], + }; + let publication = DatasetPublishRequest { + parent: None, + manifest: bincode::serialize(&manifest).unwrap(), + operation: None, + }; + let publish_body = bincode::serialize(&publication).unwrap(); + let mut publish = tokio::net::TcpStream::connect(address).await.unwrap(); + publish.write_all(format!("POST /v1/dataset/publish HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nContent-Length: {}\r\n\r\n", publish_body.len()).as_bytes()).await.unwrap(); + publish.write_all(&publish_body).await.unwrap(); + let mut publish_response = Vec::new(); + publish.read_to_end(&mut publish_response).await.unwrap(); + assert!(publish_response.starts_with(b"HTTP/1.1 200")); + let header_end = publish_response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .unwrap() + + 4; + let published: DatasetPublishResponse = + bincode::deserialize(&publish_response[header_end..]).unwrap(); + let manifest_request = DatasetSnapshotRequest { + snapshot: published.snapshot, + }; + let manifest_body = bincode::serialize(&manifest_request).unwrap(); + let mut manifest_stream = tokio::net::TcpStream::connect(address).await.unwrap(); + manifest_stream + .write_all(format!("POST /v1/dataset/manifest HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nContent-Length: {}\r\n\r\n", manifest_body.len()).as_bytes()) + .await + .unwrap(); + manifest_stream.write_all(&manifest_body).await.unwrap(); + let mut manifest_response = Vec::new(); + manifest_stream.read_to_end(&mut manifest_response).await.unwrap(); + assert!(manifest_response.starts_with(b"HTTP/1.1 200")); + let manifest_header_end = manifest_response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .unwrap() + + 4; + let persisted: crowdb_access_dataset::DatasetManifestResponse = + bincode::deserialize(&manifest_response[manifest_header_end..]).unwrap(); + assert_eq!(persisted.manifest.snapshot, published.snapshot); + assert_eq!(persisted.manifest.samples.len(), 1); + + let request = DatasetScanRequest { + plan: ReadPlan { + snapshot: published.snapshot, + selection: Selection::Prefix(b"sample".to_vec()), + ordering: Ordering::SampleId, + projection: vec!["value".into()], + batch_size: 1, + }, + shuffle: None, + cursor: None, + }; + let body = bincode::serialize(&request).unwrap(); + let mut scan = tokio::net::TcpStream::connect(address).await.unwrap(); + scan.write_all(format!("POST /v1/dataset/scan HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nContent-Length: {}\r\n\r\n", body.len()).as_bytes()).await.unwrap(); + scan.write_all(&body).await.unwrap(); + let mut scan_response = Vec::new(); + scan.read_to_end(&mut scan_response).await.unwrap(); + assert!( + scan_response.starts_with(b"HTTP/1.1 200"), + "{}", + String::from_utf8_lossy(&scan_response) + ); + let header_end = scan_response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .unwrap() + + 4; + let scanned: DatasetScanResponse = bincode::deserialize(&scan_response[header_end..]).unwrap(); + assert!(scanned.end); + assert_eq!(scanned.samples.len(), 1); + assert_eq!(scanned.snapshot, published.snapshot); + let _ = shutdown_tx.send(()); + task.await.unwrap().unwrap(); + } +} diff --git a/app/crowdb-access-server/src/lib.rs b/app/crowdb-access-server/src/lib.rs index 24ddedf12..ba1afbb07 100644 --- a/app/crowdb-access-server/src/lib.rs +++ b/app/crowdb-access-server/src/lib.rs @@ -4,6 +4,7 @@ //! Independent listener lifecycle for external access protocols. pub mod config; +pub mod dataset; mod http_receive; pub mod iceberg; mod multipart_complete; diff --git a/app/crowdb-access-server/src/main.rs b/app/crowdb-access-server/src/main.rs index 2cd35021a..3a5263b53 100644 --- a/app/crowdb-access-server/src/main.rs +++ b/app/crowdb-access-server/src/main.rs @@ -39,8 +39,16 @@ use tokio::net::TcpListener; use tokio::sync::watch; #[tokio::main] +#[allow(clippy::too_many_lines)] async fn main() -> Result<(), Box> { let mut args: Vec = std::env::args().skip(1).collect(); + if args.first().is_some_and(|arg| arg == "dataset") { + args.remove(0); + init_access_logging()?; + return crowdb_access_server::dataset::run(args) + .await + .map_err(|error| -> Box { error }); + } if args.first().is_some_and(|arg| arg == "iceberg") { args.remove(0); init_access_logging()?; @@ -78,20 +86,54 @@ async fn main() -> Result<(), Box> { } else { let (shutdown_tx, shutdown_rx) = watch::channel(false); let s3 = run_s3(&access_config, Some(shutdown_rx.clone())); - let iceberg = crowdb_access_server::iceberg::run_with_shutdown(args, Some(shutdown_rx)); - tokio::pin!(s3, iceberg); - tokio::select! { - result = &mut s3 => { - let _ = shutdown_tx.send(true); - let other = iceberg.await; - result?; - other.map_err(|error| -> Box { error })?; + let iceberg = crowdb_access_server::iceberg::run_with_shutdown(args, Some(shutdown_rx.clone())); + let dataset_enabled = + access_config.dataset.listen.is_some() || std::env::var_os("CROWDB_DATASET_LISTEN").is_some(); + if dataset_enabled { + let dataset = crowdb_access_server::dataset::run_with_shutdown( + std::env::args().skip(1).filter(|arg| arg != "dataset").collect(), + wait_for_shutdown(Some(shutdown_rx)), + ); + tokio::pin!(s3, iceberg, dataset); + tokio::select! { + result = &mut s3 => { + let _ = shutdown_tx.send(true); + let _ = dataset.await; + let other = iceberg.await; + result.map_err(|error| -> Box { error })?; + other.map_err(|error| -> Box { error })?; + } + result = &mut iceberg => { + let _ = shutdown_tx.send(true); + let _ = dataset.await; + let other = s3.await; + result.map_err(|error| -> Box { error })?; + other?; + } + result = &mut dataset => { + let _ = shutdown_tx.send(true); + let other_s3 = s3.await; + let other_iceberg = iceberg.await; + result.map_err(|error| -> Box { error })?; + other_s3?; + other_iceberg.map_err(|error| -> Box { error })?; + } } - result = &mut iceberg => { - let _ = shutdown_tx.send(true); - let other = s3.await; - result.map_err(|error| -> Box { error })?; - other?; + } else { + tokio::pin!(s3, iceberg); + tokio::select! { + result = &mut s3 => { + let _ = shutdown_tx.send(true); + let other = iceberg.await; + result?; + other.map_err(|error| -> Box { error })?; + } + result = &mut iceberg => { + let _ = shutdown_tx.send(true); + let other = s3.await; + result.map_err(|error| -> Box { error })?; + other?; + } } } } @@ -238,7 +280,7 @@ async fn bind_health_listener(access_config: &AccessConfig) -> std::io::Result) .transpose() .map_err(|_| S3ErrorCode::InvalidRequest)?; - if declared.is_some_and(|length| length > MAX_DELETE_BODY) { - return Err(S3ErrorCode::InvalidRequest); - } let headers = request.headers().clone(); let mut bytes = Vec::new(); + let mut oversized = declared.is_some_and(|length| length > MAX_DELETE_BODY); let mut body = request.into_body(); while let Some(frame) = body.frame().await { let data = frame .map_err(|_| S3ErrorCode::InvalidRequest)? .into_data() .map_err(|_| S3ErrorCode::InvalidRequest)?; + if oversized { + continue; + } if bytes.len().saturating_add(data.len()) > MAX_DELETE_BODY { - return Err(S3ErrorCode::InvalidRequest); + oversized = true; + bytes.clear(); + continue; } bytes.extend_from_slice(&data); } + if oversized { + return Err(S3ErrorCode::InvalidRequest); + } if declared.is_some_and(|length| length != bytes.len()) { return Err(S3ErrorCode::InvalidRequest); } diff --git a/app/crowdb-access-server/tests/common/iceberg_rust/Cargo.lock b/app/crowdb-access-server/tests/common/iceberg_rust/Cargo.lock index 761e5761a..fc8ca78d1 100644 --- a/app/crowdb-access-server/tests/common/iceberg_rust/Cargo.lock +++ b/app/crowdb-access-server/tests/common/iceberg_rust/Cargo.lock @@ -583,7 +583,7 @@ checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crowdb-iceberg-rust-client-fixture" -version = "0.2.2" +version = "0.3.0" dependencies = [ "iceberg", "iceberg-catalog-rest", diff --git a/app/crowdb-access-server/tests/common/iceberg_rust/Cargo.toml b/app/crowdb-access-server/tests/common/iceberg_rust/Cargo.toml index 887e51900..11375182a 100644 --- a/app/crowdb-access-server/tests/common/iceberg_rust/Cargo.toml +++ b/app/crowdb-access-server/tests/common/iceberg_rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "crowdb-iceberg-rust-client-fixture" -version = "0.2.2" +version = "0.3.0" edition = "2021" publish = false diff --git a/app/crowdb-access-server/tests/config_test.rs b/app/crowdb-access-server/tests/config_test.rs index 7cbbb4c64..ba2f0aa76 100644 --- a/app/crowdb-access-server/tests/config_test.rs +++ b/app/crowdb-access-server/tests/config_test.rs @@ -36,6 +36,8 @@ fn tracked_access_configs_load_and_set_bounded_read_resources() { assert_eq!(config.small_write.policy().max_pipelines, 32); assert!(config.s3.listen.is_some()); assert!(config.iceberg.listen.is_some()); + assert_eq!(config.dataset.max_in_flight, 3); + assert_eq!(config.dataset.max_batch_size, 10_000); assert_eq!(config.iceberg.native_budget_bytes, Some(256 * 1024 * 1024)); assert_eq!(config.s3.list_scan_bytes, Some(4 * 1024 * 1024)); assert_eq!(config.iceberg.gc.kv_bytes, Some(64 * 1024 * 1024)); @@ -76,6 +78,19 @@ fn invalid_read_budget_is_rejected() { assert!(config.validate().is_err()); } +#[test] +fn dataset_listener_limits_are_bounded() { + let mut config = AccessConfig::default(); + config.dataset.max_in_flight = 0; + assert!(config.validate().is_err()); + config.dataset.max_in_flight = 1; + config.dataset.max_batch_size = 0; + assert!(config.validate().is_err()); + config.dataset.max_batch_size = 10_000; + config.dataset.listen = Some("not-an-address".into()); + assert!(config.validate().is_err()); +} + #[test] fn small_threshold_uses_strip_data_capacity_for_ec_and_mirror() { let mut config = SmallWriteConfig { @@ -140,7 +155,7 @@ fn independent_health_listener_rejects_data_listener_ports() { let mut config = crowdb_access_server::config::AccessConfig::default(); config.s3.listen = Some("0.0.0.0:9091".into()); config.iceberg.listen = Some("0.0.0.0:9092".into()); - config.health.listen = Some("127.0.0.1:9093".into()); + config.health.listen = Some("127.0.0.1:9094".into()); config.validate().unwrap(); for value in ["127.0.0.1:9091", "127.0.0.1:9092", "127.0.0.1:0", "invalid"] { config.health.listen = Some(value.into()); diff --git a/app/crowdb-access-server/tests/iceberg_file_http_test.rs b/app/crowdb-access-server/tests/iceberg_file_http_test.rs index 48e4b1c7f..82771e24e 100644 --- a/app/crowdb-access-server/tests/iceberg_file_http_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_http_test.rs @@ -360,18 +360,19 @@ async fn thirty_two_concurrent_direct_puts() { #[ignore = "manual TPC-H and TPC-DS loader stress on the small cluster"] async fn tpc_loader_parallel_stress() { let (_stack, _process, client, _table) = setup().await; - let python = "/cpp/crowdb-tpc-loader/.venv/bin/python"; + let python = std::env::var("CROWDB_TPC_LOADER_PYTHON") + .expect("set CROWDB_TPC_LOADER_PYTHON to the installed TPC loader environment"); let endpoint = format!("http://{}", client.address); let mut commands = Vec::new(); for (benchmark, workers) in [("tpch", "8"), ("tpcds", "24")] { - let mut command = tokio::process::Command::new("/home/cj/.pixi/bin/pixi"); + let mut command = tokio::process::Command::new("pixi"); command .args([ "run", "-e", "iceberg-e2e", "--", - python, + &python, "-m", "crowdb_tpc_loader", "load", diff --git a/app/crowdb-access-server/tests/iceberg_namespace_sdk_test.rs b/app/crowdb-access-server/tests/iceberg_namespace_sdk_test.rs index 0c54ebb0a..8752e24a5 100644 --- a/app/crowdb-access-server/tests/iceberg_namespace_sdk_test.rs +++ b/app/crowdb-access-server/tests/iceberg_namespace_sdk_test.rs @@ -79,9 +79,14 @@ async fn python(test: &TestTableHttp, mode: &str, count: usize) { let mode = mode.to_owned(); let status = tokio::task::spawn_blocking(move || { let python = std::env::var_os("CROWDB_ICEBERG_E2E_PYTHON").expect("set pinned Python path"); - std::process::Command::new("timeout") - .arg("60") - .arg(python) + let mut command = if cfg!(target_os = "macos") { + std::process::Command::new(python) + } else { + let mut command = std::process::Command::new("timeout"); + command.arg("60").arg(python); + command + }; + command .arg(concat!( env!("CARGO_MANIFEST_DIR"), "/tests/common/iceberg_namespace_client.py" @@ -143,9 +148,14 @@ async fn official_catalog_continues_through_empty_namespace_pages() { let endpoint = test.endpoint(); let status = tokio::task::spawn_blocking(move || { let maven = std::env::var_os("CROWDB_ICEBERG_E2E_MVN").expect("set pinned Maven path"); - std::process::Command::new("timeout") - .arg("60") - .arg(maven) + let mut command = if cfg!(target_os = "macos") { + std::process::Command::new(maven) + } else { + let mut command = std::process::Command::new("timeout"); + command.arg("60").arg(maven); + command + }; + command .args(["-o", "--batch-mode", "--no-transfer-progress", "-f"]) .arg(concat!( env!("CARGO_MANIFEST_DIR"), diff --git a/app/crowdb-access-server/tests/iceberg_table_http_test.rs b/app/crowdb-access-server/tests/iceberg_table_http_test.rs index 0ae0e2056..5235a0aba 100644 --- a/app/crowdb-access-server/tests/iceberg_table_http_test.rs +++ b/app/crowdb-access-server/tests/iceberg_table_http_test.rs @@ -300,7 +300,12 @@ async fn read_admission_is_shared_by_complete_and_paged_lists_and_releases_after fixture.request(Method::GET, path, "r", None).await.status() })); } - tokio::time::timeout(Duration::from_secs(1), async { + let scan_start_timeout = if cfg!(target_os = "macos") { + Duration::from_secs(3) + } else { + Duration::from_secs(1) + }; + tokio::time::timeout(scan_start_timeout, async { while fixture.store.scans.load(Ordering::SeqCst) < 32 { tokio::task::yield_now().await; } diff --git a/app/crowdb-access-server/tests/protocol_http_policy_test.rs b/app/crowdb-access-server/tests/protocol_http_policy_test.rs index 3dc2d4376..a560b90a9 100644 --- a/app/crowdb-access-server/tests/protocol_http_policy_test.rs +++ b/app/crowdb-access-server/tests/protocol_http_policy_test.rs @@ -192,7 +192,10 @@ async fn start_access_with_fault( } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[ignore = "starts a complete simulated three-rack production storage stack"] +#[cfg_attr( + target_os = "macos", + ignore = "starts a complete simulated three-rack production storage stack" +)] async fn combined_http_listeners_keep_protocol_chunk_policies_separate() { let dir = TestDir::new("access-protected-http-policy").unwrap(); s3::start_protected_test_cluster(dir.path()).await.unwrap(); @@ -220,7 +223,10 @@ async fn combined_http_listeners_keep_protocol_chunk_policies_separate() { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[ignore = "starts a complete simulated three-rack production storage stack"] +#[cfg_attr( + target_os = "macos", + ignore = "starts a complete simulated three-rack production storage stack" +)] async fn terminal_s3_storage_failure_stops_both_access_listeners() { let dir = TestDir::new("access-storage-failure").unwrap(); s3::start_protected_test_cluster(dir.path()).await.unwrap(); @@ -433,7 +439,10 @@ async fn assert_chunk_layouts(cluster: &ConsoleConfig) { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[ignore = "stops all real DiskIO processes in a simulated three-rack production cluster"] +#[cfg_attr( + target_os = "macos", + ignore = "stops all real DiskIO processes in a simulated three-rack production cluster" +)] async fn protected_two_copy_write_stops_after_repair_and_chunk_rotation_fail() { let dir = TestDir::new("access-protected-mirror-failure").unwrap(); s3::start_protected_test_cluster(dir.path()).await.unwrap(); diff --git a/app/crowdb-access-server/tests/protocol_production_policy_test.rs b/app/crowdb-access-server/tests/protocol_production_policy_test.rs index 7711bfe6d..c07ba236e 100644 --- a/app/crowdb-access-server/tests/protocol_production_policy_test.rs +++ b/app/crowdb-access-server/tests/protocol_production_policy_test.rs @@ -191,7 +191,10 @@ async fn assert_system_chunk_purposes( } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "starts a complete simulated three-rack production storage stack"] +#[cfg_attr( + target_os = "macos", + ignore = "starts a complete simulated three-rack production storage stack" +)] async fn s3_and_iceberg_keep_distinct_policies_on_protected_storage() { let dir = TestDir::new("access-production-protocol-policy").unwrap(); s3::start_protected_test_cluster(dir.path()).await.unwrap(); diff --git a/app/crowdb-access-server/tests/s3_copy_body_test.rs b/app/crowdb-access-server/tests/s3_copy_body_test.rs index 35cc05489..0d13539f6 100644 --- a/app/crowdb-access-server/tests/s3_copy_body_test.rs +++ b/app/crowdb-access-server/tests/s3_copy_body_test.rs @@ -39,16 +39,19 @@ async fn official_boto3_recognizes_a_copy_error_after_http_200_and_keepalives() }, )); let status = tokio::task::spawn_blocking(move || { - std::process::Command::new("timeout") - .arg("60") - .arg(std::env::var_os("CROWDB_S3_E2E_PYTHON").expect("run the pinned boto3 task")) - .arg(concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/s3_e2e/embedded_copy_error.py" - )) - .arg(format!("http://{address}")) - .status() - .unwrap() + let python = std::env::var_os("CROWDB_S3_E2E_PYTHON").expect("run the pinned boto3 task"); + let script = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/s3_e2e/embedded_copy_error.py"); + let mut command = if cfg!(target_os = "macos") { + let mut command = std::process::Command::new(python); + command.arg(script); + command + } else { + let mut command = std::process::Command::new("timeout"); + command.args(["60"]); + command.arg(python).arg(script); + command + }; + command.arg(format!("http://{address}")).status().unwrap() }) .await .unwrap(); diff --git a/app/crowdb-access-server/tests/s3_full_stack_test.rs b/app/crowdb-access-server/tests/s3_full_stack_test.rs index 4a114b95c..7be127374 100644 --- a/app/crowdb-access-server/tests/s3_full_stack_test.rs +++ b/app/crowdb-access-server/tests/s3_full_stack_test.rs @@ -351,7 +351,11 @@ async fn start_full_stack() -> FullStackSetup { .expect("create durable DiskIO directory") .join("data") .join("disk.dat"); - let capacity = 16_384_u64 * 1024 * 1024; + // The compatibility suite intentionally keeps objects from earlier cases + // alive while exercising retries and overwrite semantics. A sparse 64 GiB + // backing file avoids exhausting the allocator on macOS without consuming + // that space on disk. + let capacity = 65_536_u64 * 1024 * 1024; std::fs::File::create(&disk_path) .expect("create block disk") .set_len(capacity) @@ -365,7 +369,7 @@ async fn start_full_stack() -> FullStackSetup { let diskdb_started_at = unix_time_ms(); let seeds = cluster.mgmt_endpoints.clone(); - let diskdb = DiskdbProcess::start_for_instance_in(cluster.runtime_mut(), &seeds, 999, Some(16_384)); + let diskdb = DiskdbProcess::start_for_instance_in(cluster.runtime_mut(), &seeds, 999, Some(65_536)); diskdb.wait_for_ready().await; diskdb .wait_for_registry_ready( @@ -724,10 +728,12 @@ fn run_benchmark( access_server.log_content() ); assert!(result.stdout.starts_with(b"{\n"), "benchmark did not emit JSON"); - assert!( - String::from_utf8_lossy(&result.stdout).contains("\"server_process\": {"), - "benchmark did not capture access-server CPU/RSS" - ); + if !cfg!(target_os = "macos") { + assert!( + String::from_utf8_lossy(&result.stdout).contains("\"server_process\": {"), + "benchmark did not capture access-server CPU/RSS" + ); + } let artifact = artifacts_dir.join("s3-request-path.json"); std::fs::write(&artifact, result.stdout).expect("write S3 baseline samples"); eprintln!("S3 benchmark samples: {}", artifact.display()); @@ -847,7 +853,7 @@ fn metric_value(response: &str, name: &str) -> u64 { async fn seed_compact_hardware(hardware: &HardwareClient) -> Vec { const RACK_ID: u64 = 1; const UNIT_BYTES: u32 = 1024 * 1024; - const ZONE_UNITS: u64 = 16_384; + const ZONE_UNITS: u64 = 65_536; let node_ids = vec![10]; hardware .add_rack( diff --git a/app/crowdb-chunk-kv-server/Cargo.toml b/app/crowdb-chunk-kv-server/Cargo.toml index 0c0eb16a4..2b81aab44 100644 --- a/app/crowdb-chunk-kv-server/Cargo.toml +++ b/app/crowdb-chunk-kv-server/Cargo.toml @@ -10,6 +10,9 @@ description = "Range-partitioned chunk-backed KV service." [lints] workspace = true +[features] +test-util = [] + [dependencies] arc-swap = "1.7" async-trait = "0.1" @@ -32,6 +35,8 @@ toml = "0.8" tracing = { workspace = true } [dev-dependencies] +crowdb-chunk-kv-server = { path = ".", features = ["test-util"] } +crowdb-test-harness = { path = "../../lib/crowdb-test-harness", features = ["kv-client"] } crowdb-chunk-kv = { path = "../../lib/crowdb-chunk-kv", features = ["test-util"] } crowdb-chunk-stream = { path = "../../lib/crowdb-chunk-stream", features = ["test-util"] } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi", features = ["test-util"] } diff --git a/app/crowdb-chunk-kv-server/src/catalog.rs b/app/crowdb-chunk-kv-server/src/catalog.rs index ca80295b2..27cd6a63c 100644 --- a/app/crowdb-chunk-kv-server/src/catalog.rs +++ b/app/crowdb-chunk-kv-server/src/catalog.rs @@ -3,6 +3,7 @@ //! Catalog store and scan validation. +mod recovery; mod scan; mod store; mod transition; diff --git a/app/crowdb-chunk-kv-server/src/catalog/recovery.rs b/app/crowdb-chunk-kv-server/src/catalog/recovery.rs new file mode 100644 index 000000000..85a8feb60 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/catalog/recovery.rs @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Recovery reconciled with publication that advances while artifacts reopen. + +use super::{ChunkKvRangeCatalogError, ChunkKvRangeCatalogPublisher}; +use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage}; +use std::{fmt::Display, future::Future, time::Instant}; + +impl ChunkKvRangeCatalogPublisher { + /// Recover a published catalog, discarding obsolete recovery results only + /// when a newer authoritative generation proves publication advanced. + /// + /// # Errors + /// Returns unchanged-generation recovery errors, catalog read errors, or + /// generation conflicts when publication does not settle by the deadline. + pub async fn recover_current( + &self, + deadline: Instant, + mut recover: F, + ) -> Result, T)>, ChunkKvRangeCatalogError> + where + F: FnMut(Vec) -> R, + R: Future>, + { + let mut current = self.load_current().await?; + while let Some((head, pages)) = current { + let result = recover(pages.clone()).await; + let latest = self.load_current().await?; + if latest + .as_ref() + .is_some_and(|(next, _)| next.generation > head.generation) + { + if Instant::now() >= deadline { + return Err(ChunkKvRangeCatalogError::GenerationConflict); + } + current = latest; + continue; + } + if latest.as_ref().map(|(next, _)| next) != Some(&head) { + return Err(ChunkKvRangeCatalogError::GenerationConflict); + } + return result + .map(|recovered| Some((head, pages, recovered))) + .map_err(|error| ChunkKvRangeCatalogError::Unavailable(error.to_string())); + } + Ok(None) + } +} diff --git a/app/crowdb-chunk-kv-server/src/config.rs b/app/crowdb-chunk-kv-server/src/config.rs index 854b6ad56..57390909f 100644 --- a/app/crowdb-chunk-kv-server/src/config.rs +++ b/app/crowdb-chunk-kv-server/src/config.rs @@ -136,7 +136,9 @@ impl ChunkKvServerConfig { self.monitor .validate() .map_err(|error| ConfigError::Invalid(error.to_string()))?; - if self.balance.target_partitions_per_owner == 0 + if self.balance.byte_weight_percent != 0 + || self.balance.imbalance_tolerance_percent > 100 + || self.balance.target_partitions_per_owner == 0 || self.balance.target_partition_bytes == 0 || self.balance.minimum_weighted_improvement_percent > 100 || self.balance.cooldown_ms == 0 @@ -253,6 +255,8 @@ fn default_monitor() -> DomainMonitorDescriptor { fn balance_policy(config: &BalanceConfig) -> ChunkKvRangeBalancePolicy { ChunkKvRangeBalancePolicy { + byte_weight_percent: config.byte_weight_percent, + imbalance_tolerance_percent: config.imbalance_tolerance_percent, target_partitions_per_owner: u32::try_from(config.target_partitions_per_owner).unwrap_or(u32::MAX), target_partition_bytes: config.target_partition_bytes, minimum_weighted_improvement_percent: config.minimum_weighted_improvement_percent, diff --git a/app/crowdb-chunk-kv-server/src/control_store.rs b/app/crowdb-chunk-kv-server/src/control_store.rs index 662e269fc..6b25ad1f8 100644 --- a/app/crowdb-chunk-kv-server/src/control_store.rs +++ b/app/crowdb-chunk-kv-server/src/control_store.rs @@ -389,6 +389,11 @@ impl Group0ControlStore { if current == *transition { return Ok(revision); } + if !current.preserves_handoff(transition) { + return Err(MonitorError::PlanFailed( + "committed split handoff cannot be replaced or cleared".into(), + )); + } if revision != expected_revision { return Err(MonitorError::PlanFailed( "split transition revision conflict".into(), diff --git a/app/crowdb-chunk-kv-server/src/main.rs b/app/crowdb-chunk-kv-server/src/main.rs index 04a730aa7..092f67a29 100644 --- a/app/crowdb-chunk-kv-server/src/main.rs +++ b/app/crowdb-chunk-kv-server/src/main.rs @@ -181,28 +181,28 @@ async fn main() -> std::process::ExitCode { } let catalog = Arc::new(ChunkKvRangeCatalogPublisher::new(control_store.clone())); - match catalog.load_current().await { - Ok(Some((head, pages))) => { - match recover_assigned_partitions(&storage, &service, &pages, config.instance_id).await { - Ok(recovered) => { - if let Err(error) = service.install_catalog_and_reconcile(&head, &pages, &recovered) { - error!(%error, "failed to install initial chunk KV catalog and partitions"); - return std::process::ExitCode::FAILURE; - } - info!( - generation = head.generation, - partitions = recovered.len(), - "installed initial chunk KV catalog and replayed assigned partitions" - ); - } - Err(error) => { - // A persisted assignment can reference a tree chunk that - // was reclaimed by an older finalizer. Keep the service - // alive so the monitor and catalog refresh loop can retry - // recovery instead of taking down the whole container. - error!(%error, "initial chunk KV recovery deferred; retaining the persisted catalog"); - } + let recovery_storage = &storage; + let recovery_service = &service; + let instance_id = config.instance_id; + match catalog + .recover_current( + std::time::Instant::now() + std::time::Duration::from_secs(60), + |pages| async move { + recover_assigned_partitions(recovery_storage, recovery_service, &pages, instance_id).await + }, + ) + .await + { + Ok(Some((head, pages, recovered))) => { + if let Err(error) = service.install_catalog_and_reconcile(&head, &pages, &recovered) { + error!(%error, "failed to install initial chunk KV catalog and partitions"); + return std::process::ExitCode::FAILURE; } + info!( + generation = head.generation, + partitions = recovered.len(), + "installed initial chunk KV catalog and replayed assigned partitions" + ); } Ok(None) => { if let Some(bootstrap) = &config.bootstrap_partition { @@ -235,6 +235,9 @@ async fn main() -> std::process::ExitCode { warn!("chunk KV catalog is not published; service remains unready"); } } + Err(crowdb_chunk_kv_server::ChunkKvRangeCatalogError::Unavailable(error)) => { + error!(%error, "initial chunk KV recovery deferred; retaining the persisted catalog"); + } Err(error) => { error!(%error, "failed to load initial chunk KV catalog"); return std::process::ExitCode::FAILURE; @@ -290,8 +293,18 @@ async fn main() -> std::process::ExitCode { } }; if grant_only { - install_latest_grant(&refresh_store, &refresh_service, &refresh_config).await; - continue; + let grant_generation = + install_latest_grant(&refresh_store, &refresh_service, &refresh_config).await; + if grant_generation.map_or(true, |generation| { + generation + == refresh_service + .health(refresh_service.monotonic_ms()) + .catalog_generation + }) { + continue; + } + // A grant for a new catalog fences the old generation. Catch up + // immediately rather than leaving requests fenced until the poll. } match refresh_catalog.load_current().await { Ok(Some((head, pages))) => { @@ -591,7 +604,7 @@ async fn install_latest_grant( store: &Group0ControlStore, service: &ChunkKvService, config: &ChunkKvServerConfig, -) { +) -> Option { match store.load_serving_grant(config.instance_id).await { Ok(Some(grant)) => { let catalog_generation = grant.catalog_generation; @@ -616,9 +629,16 @@ async fn install_latest_grant( } } } + Some(catalog_generation) + } + Ok(None) => { + service.authority().clear(); + None + } + Err(error) => { + warn!(%error, "serving-grant refresh failed; retaining local lease deadline"); + None } - Ok(None) => service.authority().clear(), - Err(error) => warn!(%error, "serving-grant refresh failed; retaining local lease deadline"), } } @@ -627,7 +647,25 @@ async fn activate_granted_assignment( service: &ChunkKvService, assignment: &crowdb_protocol::chunk_kv::ServingAssignment, ) -> Result<(), String> { - let Some(transition_id) = service.catalog_transition_id(assignment.partition_id) else { + let Some(transition_id) = service.catalog_overlay_transition_id(assignment.partition_id) else { + if service.needs_split_handoff_recovery(assignment.partition_id) { + let transitions = store + .list_split_transitions() + .await + .map_err(|error| error.to_string())?; + if transitions.iter().any(|(split, _)| { + split.parent_id == assignment.partition_id + && split.parent_epoch == assignment.owner_epoch + && split.handoff_proof.is_some() + && matches!( + split.phase, + crowdb_protocol::chunk_kv::SplitPhase::ParentPreparing + | crowdb_protocol::chunk_kv::SplitPhase::ChildPrepared + ) + }) { + return Ok(()); + } + } return service .activate_recovered_partition(assignment.partition_id, assignment.owner_epoch) .map_err(|error| error.to_string()); diff --git a/app/crowdb-chunk-kv-server/src/server.rs b/app/crowdb-chunk-kv-server/src/server.rs index b5de74d3f..39d27ee12 100644 --- a/app/crowdb-chunk-kv-server/src/server.rs +++ b/app/crowdb-chunk-kv-server/src/server.rs @@ -34,6 +34,7 @@ mod load_sampling; mod observation; mod prepared_transfer; mod reconcile; +mod routing; pub(crate) use observation::PageQuery; const DEFAULT_SCAN_RESPONSE_BYTES: usize = 17 * 1024 * 1024; @@ -404,7 +405,9 @@ impl ChunkKvService { } for (id, partition) in partitions.iter() { let snapshot = partition.snapshot(); - let durable_bytes = partition.estimated_bytes().unwrap_or_default(); + let Ok(durable_bytes) = partition.estimated_bytes() else { + continue; + }; let live_byte_samples = if max_samples >= 2 && snapshot.lifecycle == crowdb_chunk_kv::PartitionLifecycle::Serving { self.load_sampling.samples(*id, snapshot.ownership_epoch) @@ -1465,20 +1468,6 @@ impl ChunkKvService { }) } - fn partition_for_request( - &self, - routing: &RequestRouting, - key: &[u8], - entry: &ChunkKvRangeCatalogEntry, - ) -> Option { - self.local_split_sessions - .load() - .get(&routing.partition_id) - .map(|session| session.dispatcher.clone()) - .filter(|dispatcher| dispatcher.snapshot().range.contains(key)) - .or_else(|| self.partition_for_catalog_entry(entry)) - } - fn authorize_resolved_writers( &self, catalog: &CatalogSnapshot, @@ -1542,6 +1531,12 @@ impl ChunkKvService { fn local_split_writer(&self, entry: &ChunkKvRangeCatalogEntry) -> Option { self.local_split_sessions.load().values().find_map(|session| { let dispatcher = &session.dispatcher; + // Until catalog publication the full-range parent assignment is + // still served by this dispatcher. Reopening it would create a + // second publisher for the retained original tree. + if partition_matches_entry(dispatcher, entry) { + return Some(dispatcher.clone()); + } let ingress = dispatcher.split_ingress()?; [ingress.retained_parent(), ingress.child()] .into_iter() diff --git a/app/crowdb-chunk-kv-server/src/server/activation.rs b/app/crowdb-chunk-kv-server/src/server/activation.rs index 5a883a606..386c543bd 100644 --- a/app/crowdb-chunk-kv-server/src/server/activation.rs +++ b/app/crowdb-chunk-kv-server/src/server/activation.rs @@ -10,6 +10,28 @@ use crowdb_protocol::chunk_kv::{ }; impl ChunkKvService { + pub(crate) fn split_child_is_independent(&self, transition: &SplitTransition) -> bool { + self.catalog + .load() + .entry_for_partition(transition.child.partition_id) + .is_some_and(|entry| { + entry.owner_epoch >= transition.child.owner_epoch + && entry.transition_id != Some(transition.transition_id) + && entry.artifact.tail_overlay.as_ref().map_or(true, |overlay| { + overlay.source_stream_name != transition.parent_artifact.stream_name + }) + }) + } + + /// Returns whether cold parent recovery still needs durable handoff dispatch. + #[must_use] + pub fn needs_split_handoff_recovery(&self, partition_id: Id128) -> bool { + self.hosted_partition(partition_id).is_some_and(|parent| { + parent.lifecycle() == crowdb_chunk_kv::PartitionLifecycle::Prepared + && parent.split_ingress().is_none() + }) + } + /// Activates a recovered split writer only with exact committed catalog evidence. /// /// # Errors @@ -62,11 +84,13 @@ impl ChunkKvService { "committed split does not prove this serving assignment".into(), )); } - self.partitions - .load() - .get(&partition_id) - .ok_or(ChunkKvError::OutOfRange)? - .activate_recovered_overlay(owner_epoch) + let partitions = self.partitions.load(); + let partition = partitions.get(&partition_id).ok_or(ChunkKvError::OutOfRange)?; + if artifact.tail_overlay.is_none() { + partition.activate_recovered(owner_epoch) + } else { + partition.activate_recovered_overlay(owner_epoch) + } } /// Activates one replayed assignment after a matching catalog and serving @@ -93,7 +117,7 @@ impl ChunkKvService { "catalog does not publish this serving assignment".into(), )); } - // A local split dispatcher owns both replacement writers. Its old + // A local split dispatcher retains the parent and routes to its child. Its old // parent identity remains a compatibility route for g1 requests, not // a partition that a later grant may reactivate at its obsolete epoch. if self.local_split_sessions.load().contains_key(&partition_id) { @@ -106,12 +130,15 @@ impl ChunkKvService { .activate_recovered(owner_epoch) } - /// Returns the catalog transition attached to one hosted assignment. + /// Returns the transition needed to prove a recovered overlay assignment. + /// A materialized split half can retain the pair's transition marker until + /// its sibling is materialized, but already uses independent recovery. #[must_use] - pub fn catalog_transition_id(&self, partition_id: Id128) -> Option { + pub fn catalog_overlay_transition_id(&self, partition_id: Id128) -> Option { self.catalog .load() .entry_for_partition(partition_id) + .filter(|entry| entry.artifact.tail_overlay.is_some()) .and_then(|entry| entry.transition_id) } diff --git a/app/crowdb-chunk-kv-server/src/server/load_sampling.rs b/app/crowdb-chunk-kv-server/src/server/load_sampling.rs index 4a52509a3..04fc2e84a 100644 --- a/app/crowdb-chunk-kv-server/src/server/load_sampling.rs +++ b/app/crowdb-chunk-kv-server/src/server/load_sampling.rs @@ -90,10 +90,17 @@ async fn sample(partition: &Partition, epoch: u64, limit: usize) -> Result= 2 { + let right = partition + .scan_forward_after(epoch, &page.entries[0].key, None, 1, PAGE_BYTES, None) + .await?; + page.entries.extend(right.entries); + } Ok(page .entries .into_iter() diff --git a/app/crowdb-chunk-kv-server/src/server/routing.rs b/app/crowdb-chunk-kv-server/src/server/routing.rs new file mode 100644 index 000000000..5eee52618 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/server/routing.rs @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Current assignment selection and bounded historical split compatibility. + +use super::{partition_matches_entry, ChunkKvService}; +use crowdb_chunk_kv::Partition; +use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogEntry, RequestRouting}; + +impl ChunkKvService { + pub(super) fn partition_for_request( + &self, + routing: &RequestRouting, + key: &[u8], + entry: &ChunkKvRangeCatalogEntry, + ) -> Option { + let current = self + .partitions + .load() + .get(&entry.partition_id) + .filter(|partition| partition_matches_entry(partition, entry)) + .cloned(); + if routing.partition_id == entry.partition_id && routing.owner_epoch == entry.owner_epoch { + if let Some(partition) = current.as_ref() { + return Some(partition.clone()); + } + } + self.local_split_sessions + .load() + .get(&routing.partition_id) + .map(|session| session.dispatcher.clone()) + .filter(|dispatcher| { + dispatcher.snapshot().range.contains(key) + && (partition_matches_entry(dispatcher, entry) + || dispatcher.split_ingress().is_some_and(|ingress| { + partition_matches_entry(&ingress.writer_for_key(key), entry) + })) + }) + .or(current) + .or_else(|| self.partition_for_catalog_entry(entry)) + } +} diff --git a/app/crowdb-chunk-kv-server/src/serving/balance.rs b/app/crowdb-chunk-kv-server/src/serving/balance.rs index 160da420a..951125f55 100644 --- a/app/crowdb-chunk-kv-server/src/serving/balance.rs +++ b/app/crowdb-chunk-kv-server/src/serving/balance.rs @@ -1,11 +1,15 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_protocol::chunk_kv::{Id128, KeyRange}; +use crowdb_protocol::chunk_kv::{balance::BalanceWeights, ChunkKvRangeBalancePolicy, Id128, KeyRange}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct BalanceConfig { + #[serde(default = "crowdb_protocol::chunk_kv::balance::default_byte_weight_percent")] + pub byte_weight_percent: u32, + #[serde(default = "crowdb_protocol::chunk_kv::balance::default_tolerance_percent")] + pub imbalance_tolerance_percent: u32, #[serde(default = "default_enabled")] pub enabled: bool, pub target_partitions_per_owner: usize, @@ -19,6 +23,8 @@ impl Default for BalanceConfig { fn default() -> Self { Self { enabled: true, + byte_weight_percent: 0, + imbalance_tolerance_percent: 20, target_partitions_per_owner: 4, target_partition_bytes: 1 << 30, minimum_weighted_improvement_percent: 25, @@ -112,22 +118,32 @@ pub fn choose_transfer( now_ms: u64, config: &BalanceConfig, ) -> Option { - let sources = owners - .iter() - .filter(|owner| owner.healthy && !owner.transfer_active); - let targets: Vec<&OwnerLoad> = owners + if !config.enabled || config.byte_weight_percent != 0 { + return None; + } + let live: Vec<_> = owners.iter().filter(|owner| owner.healthy).collect(); + let loads: Vec<_> = live .iter() - .filter(|owner| owner.healthy && !owner.transfer_active) + .map(|owner| (owner.partition_count as u64, owner.durable_bytes)) .collect(); - let mut best: Option<(bool, u128, &PartitionLoad, &OwnerLoad)> = None; - for source in sources { + let weights = BalanceWeights::new(&loads, config.byte_weight_percent)?; + let policy = ChunkKvRangeBalancePolicy { + byte_weight_percent: config.byte_weight_percent, + imbalance_tolerance_percent: config.imbalance_tolerance_percent, + minimum_weighted_improvement_percent: config.minimum_weighted_improvement_percent, + ..ChunkKvRangeBalancePolicy::default() + }; + if policy.validate().is_err() || weights.within_tolerance(policy.imbalance_tolerance_percent) { + return None; + } + let mut best = None; + for source in live.iter().filter(|owner| !owner.transfer_active) { for partition in partitions.iter().filter(|partition| { partition.owner_instance_id == source.instance_id && eligible_partition(partition, now_ms, config) }) { - for target in targets + for target in live .iter() - .copied() - .filter(|target| target.instance_id != source.instance_id) + .filter(|target| target.instance_id != source.instance_id && !target.transfer_active) { if target.headroom_bytes < partition.durable_bytes || config.max_owner_request_rate != 0 @@ -136,32 +152,34 @@ pub fn choose_transfer( { continue; } - let fixes_count = source.partition_count > target.partition_count.saturating_add(1); - let old_spread = source.durable_bytes.abs_diff(target.durable_bytes); - let new_source = source.durable_bytes.saturating_sub(partition.durable_bytes); - let new_target = target.durable_bytes.saturating_add(partition.durable_bytes); - let new_spread = new_source.abs_diff(new_target); - let improvement = old_spread.saturating_sub(new_spread); - let weighted_qualifies = old_spread != 0 - && u128::from(improvement) * 100 - >= u128::from(old_spread) * u128::from(config.minimum_weighted_improvement_percent); - if !fixes_count && !weighted_qualifies { + let Some(improvement) = weights.improvement( + (source.partition_count as u64, source.durable_bytes), + (target.partition_count as u64, target.durable_bytes), + partition.durable_bytes, + ) else { continue; - } - let score = u128::from(improvement); - if best.as_ref().map_or(true, |(best_count, best_score, _, _)| { - (fixes_count, score) > (*best_count, *best_score) - }) { - best = Some((fixes_count, score, partition, target)); + }; + let score = ( + improvement, + std::cmp::Reverse(partition.partition_id), + std::cmp::Reverse(target.instance_id), + ); + if weights.qualifies(improvement, &policy) + && best.as_ref().map_or(true, |(prior, _)| score > *prior) + { + best = Some(( + score, + TransferProposal { + partition_id: partition.partition_id, + source_instance_id: source.instance_id, + target_instance_id: target.instance_id, + }, + )); } } } } - best.map(|(_, _, partition, target)| TransferProposal { - partition_id: partition.partition_id, - source_instance_id: partition.owner_instance_id, - target_instance_id: target.instance_id, - }) + best.map(|(_, proposal)| proposal) } fn eligible_partition(partition: &PartitionLoad, now_ms: u64, config: &BalanceConfig) -> bool { diff --git a/app/crowdb-chunk-kv-server/src/serving/split.rs b/app/crowdb-chunk-kv-server/src/serving/split.rs index 1b1a8cf85..fa4a6cd49 100644 --- a/app/crowdb-chunk-kv-server/src/serving/split.rs +++ b/app/crowdb-chunk-kv-server/src/serving/split.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_protocol::chunk_kv::{SplitPhase, SplitReadinessProof, SplitTransition}; +use crowdb_protocol::chunk_kv::{SplitPhase, SplitReadinessProof, SplitTransition, TailOverlayArtifact}; use crate::MonitorError; @@ -66,6 +66,30 @@ impl SplitStateMachine { } } + /// Records the proof to persist before installing local child dispatch. + /// + /// # Errors + /// + /// Returns an error for a conflicting proof or outside parent preparation. + pub fn record_handoff(&mut self, proof: TailOverlayArtifact) -> Result<(), MonitorError> { + if let Some(existing) = &self.transition.handoff_proof { + return if existing == &proof { + Ok(()) + } else { + Err(MonitorError::PlanFailed("split handoff proof conflicts".into())) + }; + } + if self.transition.phase != SplitPhase::ParentPreparing { + return Err(MonitorError::PlanFailed("split parent was not preparing".into())); + } + let mut next = self.transition.clone(); + next.handoff_proof = Some(proof); + next.validate() + .map_err(|error| MonitorError::PlanFailed(error.to_string()))?; + self.transition = next; + Ok(()) + } + /// Records the exact common cutover frontier reported by the child. /// /// # Errors @@ -82,12 +106,21 @@ impl SplitStateMachine { if self.transition.phase != SplitPhase::ParentPreparing { return Err(MonitorError::PlanFailed("split parent was not preparing".into())); } - self.transition.retained_parent_artifact.tail_overlay = - Some(proof.retained_parent_tail_overlay.clone()); - self.transition.child.artifact.tail_overlay = Some(proof.child_tail_overlay.clone()); - self.transition.readiness_proof = Some(proof); - self.transition.phase = SplitPhase::ChildPrepared; - self.validate_current() + let mut next = self.transition.clone(); + if next.retained_parent_artifact.tree_id != next.parent_artifact.tree_id + || next.retained_parent_artifact.stream_name != next.parent_artifact.stream_name + { + next.retained_parent_artifact + .tail_overlay + .clone_from(&proof.retained_parent_tail_overlay); + } + next.child.artifact.tail_overlay = Some(proof.child_tail_overlay.clone()); + next.readiness_proof = Some(proof); + next.phase = SplitPhase::ChildPrepared; + next.validate() + .map_err(|error| MonitorError::PlanFailed(error.to_string()))?; + self.transition = next; + Ok(()) } /// Marks the atomic catalog update after the retained parent and child are ready. @@ -114,8 +147,13 @@ impl SplitStateMachine { /// /// Returns an error after commit or for an empty/conflicting reason. pub fn abort(&mut self, reason: &str) -> Result<(), MonitorError> { - if reason.is_empty() || self.transition.phase == SplitPhase::CatalogCommitted { - return Err(MonitorError::PlanFailed("committed split cannot abort".into())); + if reason.is_empty() + || self.transition.phase == SplitPhase::CatalogCommitted + || self.transition.handoff_proof.is_some() + { + return Err(MonitorError::PlanFailed( + "committed split handoff cannot abort".into(), + )); } if self.transition.phase == SplitPhase::Aborted { return if self.transition.failure.as_deref() == Some(reason) { diff --git a/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs b/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs index 3f0656984..e09edb80c 100644 --- a/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs +++ b/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs @@ -37,13 +37,18 @@ impl TransitionProcessor { pub async fn tick(&self) -> Result<(), MonitorError> { let transfers = self.store.list_transfer_transitions().await?; let splits = self.store.list_split_transitions().await?; + let mut first_error = None; for (transition, revision) in transfers { - self.process_transfer(transition, revision).await?; + if let Err(error) = self.process_transfer(transition, revision).await { + first_error.get_or_insert(error); + } } for (transition, revision) in splits { - self.process_split(transition, revision).await?; + if let Err(error) = self.process_split(transition, revision).await { + first_error.get_or_insert(error); + } } - Ok(()) + first_error.map_or(Ok(()), Err) } async fn process_transfer( @@ -160,6 +165,17 @@ impl TransitionProcessor { } if machine.transition().phase == SplitPhase::ParentPreparing { let proof = self.executor.prepare_split_parent(machine.transition()).await?; + let (latest, latest_revision) = self + .store + .load_split_transition(machine.transition().transition_id) + .await? + .ok_or_else(|| { + MonitorError::PlanFailed("split transition disappeared during preparation".into()) + })?; + if latest.handoff_proof.is_some() { + machine = SplitStateMachine::restore(latest)?; + revision = latest_revision; + } machine.record_child_ready(proof)?; self.store .persist_split_transition(machine.transition(), revision) diff --git a/app/crowdb-chunk-kv-server/src/serving/worker.rs b/app/crowdb-chunk-kv-server/src/serving/worker.rs index a5a7d35ae..22d7ee731 100644 --- a/app/crowdb-chunk-kv-server/src/serving/worker.rs +++ b/app/crowdb-chunk-kv-server/src/serving/worker.rs @@ -193,15 +193,22 @@ impl TransitionExecutor { /// /// Returns an error when the hosted child cannot delete the durable pin. pub fn release_split_generation_pin(&self, transition: &SplitTransition) -> Result<(), MonitorError> { - let Some(partition) = self.service.hosted_partition(transition.child.partition_id) else { + if transition.phase == SplitPhase::CatalogCommitted + && !self.service.split_child_is_independent(transition) + { return Ok(()); - }; - partition - .release_generation_pin(crowdb_chunk_kv::TransitionId { - high: transition.transition_id.high, - low: transition.transition_id.low, - }) - .map_err(|error| plan_error(&error.to_string())) + } + for id in [transition.parent_id, transition.child.partition_id] { + if let Some(partition) = self.service.hosted_partition(id) { + partition + .release_generation_pin(crowdb_chunk_kv::TransitionId { + high: transition.transition_id.high, + low: transition.transition_id.low, + }) + .map_err(|error| plan_error(&error.to_string()))?; + } + } + Ok(()) } /// Aborts local split ingress and removes the unpublished child using the @@ -250,7 +257,11 @@ impl TransitionExecutor { range: transition.range.clone(), owner: transition.target.clone(), owner_epoch: transition.target_epoch, - state: ChunkKvRangeCatalogPartitionState::Prepared, + state: if transition.phase == TransferPhase::CatchupPublished { + ChunkKvRangeCatalogPartitionState::TargetCatchingUp + } else { + ChunkKvRangeCatalogPartitionState::Prepared + }, artifact: transition.target_artifact.clone(), transition_id: Some(transition.transition_id), }; @@ -355,9 +366,16 @@ impl TransitionExecutor { if artifact.child.applied_seq != artifact.cutover_seq { return Err(plan_error("split child does not share the cutover frontier")); } - let retained_parent_tail_overlay = split_tail_overlay(&artifact, &artifact.retained_parent); + let mut retained_parent_tail_overlay = None; let mut retained_parent_artifact = transition.retained_parent_artifact.clone(); - retained_parent_artifact.tail_overlay = Some(retained_parent_tail_overlay.clone()); + if retained_parent_artifact.tree_id != transition.parent_artifact.tree_id + || retained_parent_artifact.stream_name != transition.parent_artifact.stream_name + { + retained_parent_tail_overlay = Some(split_tail_overlay(&artifact, &artifact.retained_parent)); + retained_parent_artifact + .tail_overlay + .clone_from(&retained_parent_tail_overlay); + } info!( transition_id_high = transition.transition_id.high, transition_id_low = transition.transition_id.low, @@ -366,8 +384,8 @@ impl TransitionExecutor { parent_epoch = transition.parent_epoch, expected_source_stream_high = transition.parent_artifact.stream_name.high, expected_source_stream_low = transition.parent_artifact.stream_name.low, - observed_source_stream_high = retained_parent_tail_overlay.source_stream_name.high, - observed_source_stream_low = retained_parent_tail_overlay.source_stream_name.low, + observed_source_stream_high = artifact.retained_parent.parent_stream_name.high, + observed_source_stream_low = artifact.retained_parent.parent_stream_name.low, cutover_seq = artifact.cutover_seq, "local split readiness prepared" ); @@ -443,25 +461,14 @@ impl TransitionStorage for ChunkKvStorage { let prepared = ChunkKvStorage::prepare_split(self, parent, transition, max_catchup_lag_records).await?; let artifact = prepared.artifact.clone(); - let retained_parent = prepared - .retained_parent - .ok_or_else(|| plan_error("split session did not create a retained parent writer"))? - .open_warmed(PartitionConfig::default()) - .map_err(|error| plan_error(&error.to_string()))?; let child = prepared .child .open_warmed(PartitionConfig::default()) .map_err(|error| plan_error(&error.to_string()))?; - retained_parent - .activate_local_split_writer(&artifact) - .map_err(|error| plan_error(&error.to_string()))?; - child - .activate_local_split_writer(&artifact) - .map_err(|error| plan_error(&error.to_string()))?; - parent - .install_split_ingress(retained_parent.clone(), child.clone()) - .await - .map_err(|error| plan_error(&error.to_string()))?; + let retained_parent = parent + .split_ingress() + .ok_or_else(|| plan_error("split child dispatch is absent"))? + .retained_parent(); Ok(PreparedLocalSplit { artifact, retained_parent, diff --git a/app/crowdb-chunk-kv-server/src/storage.rs b/app/crowdb-chunk-kv-server/src/storage.rs index 31bf79d10..9e3d65229 100644 --- a/app/crowdb-chunk-kv-server/src/storage.rs +++ b/app/crowdb-chunk-kv-server/src/storage.rs @@ -3,19 +3,16 @@ //! Production chunk-stream dependency assembly. -use std::collections::HashMap; use std::sync::Arc; -use arc_swap::ArcSwap; -use bytes::Bytes; use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, ChunkReadPolicy, SmallWritePolicy}; use crowdb_chunk_kv::{ MutationOperation, Partition, PartitionConfig, PartitionId, PartitionJournal, PartitionRange, - PreparedSplit, PreparedSplitWriterArtifact, RequestId, SplitArtifact, SplitChild, SplitPlan, - SplitWriterTarget, StreamPartitionJournal, TransitionId, + PartitionTree, PreparedSplit, PreparedSplitWriterArtifact, RequestId, SplitArtifact, SplitChild, + SplitPlan, SplitWriterTarget, StreamPartitionJournal, TransitionId, }; use crowdb_chunk_stream::{ChunkStream, ProductionStreamRuntime, StreamConfig, StreamName, StreamRegistry}; -use crowdb_kv_client::{BatchOp, ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, PartitionArtifact, SplitChildAssignment, SplitTransition, TailOverlayArtifact, TransferTransition, @@ -23,12 +20,16 @@ use crowdb_protocol::chunk_kv::{ use crowdb_protocol::chunk_stream::{StreamBinding, StreamBindingState}; use crowdb_tree_ffi::{ ChunkPageStoreOptions, ChunkRootCatalog, ChunkTransport, OwnedChunkRpcTransportOptions, PageStore, - RootCatalogObject, RootCatalogStore, }; use thiserror::Error; use crate::{BootstrapPartitionConfig, ChunkKvServerConfig}; +mod root_catalog; +mod split_handoff; +mod split_recovery; +use root_catalog::KvRootCatalogStore; + #[derive(Debug, Error)] pub enum StorageRuntimeError { #[error("failed to connect chunk IO: {0}")] @@ -316,30 +317,35 @@ impl ChunkKvStorage { .tail_overlay .as_ref() .map_or(0, |overlay| overlay.base_root_manifest_generation); - let page_store = self - .open_durable_tree_page_store( - ChunkPageStoreOptions { - tree_id: entry.artifact.tree_id, - owner_epoch: entry.owner_epoch, - open_generation, - pack_bytes: 0, - iu_size: 0, - max_concurrent_packs: 0, - materialization_bytes_per_pass: 0, - mirror_copies: 0, - max_chunk_bytes: 0, - }, - binding.metadata_group_id, - ) - .await?; + let options = ChunkPageStoreOptions { + tree_id: entry.artifact.tree_id, + owner_epoch: entry.owner_epoch, + open_generation, + pack_bytes: 0, + iu_size: 0, + max_concurrent_packs: 0, + materialization_bytes_per_pass: 0, + mirror_copies: 0, + max_chunk_bytes: 0, + }; + let catalog = KvRootCatalogStore::for_assignment( + self.kv.clone(), + self.metadata_store_id, + binding.metadata_group_id, + entry, + ) + .await?; + let page_store = self.open_tree_page_store( + options, + Arc::new( + ChunkRootCatalog::open_callback(Arc::new(catalog)) + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?, + ), + )?; if let Some(overlay) = &entry.artifact.tail_overlay { let parent_stream = self .streams - .open_read_only( - overlay.source_stream_name, - self.metadata_store_id, - overlay.source_epoch, - ) + .open_read_only_current(overlay.source_stream_name, self.metadata_store_id) .await .map_err(|error| { StorageRuntimeError::Stream(format!( @@ -391,6 +397,22 @@ impl ChunkKvStorage { target: &Partition, entry: &ChunkKvRangeCatalogEntry, ) -> Result<(), crate::MonitorError> { + KvRootCatalogStore::prepare( + self.kv.clone(), + self.metadata_store_id, + self.streams + .registry() + .load(entry.artifact.stream_name) + .await + .map_err(|error| storage_plan_error(&error.to_string()))? + .ok_or_else(|| storage_plan_error("target stream binding is absent"))? + .metadata_group_id, + entry, + ) + .map_err(|error| storage_plan_error(&error.to_string()))? + .claim_published(true) + .await + .map_err(|error| storage_plan_error(&error.to_string()))?; let overlay = entry .artifact .tail_overlay @@ -398,11 +420,7 @@ impl ChunkKvStorage { .ok_or_else(|| storage_plan_error("transfer target overlay is absent"))?; let parent_stream = self .streams - .open_read_only( - overlay.source_stream_name, - self.metadata_store_id, - overlay.source_epoch, - ) + .open_read_only_current(overlay.source_stream_name, self.metadata_store_id) .await .map_err(|error| storage_plan_error(&error.to_string()))?; let parent_journal: Arc = Arc::new( @@ -517,9 +535,8 @@ impl ChunkKvStorage { /// Rebuilds one durable split child from the retained authoritative parent. /// - /// Existing child streams and tree roots are reopened under the same - /// stable identities, so a `ParentPreparing` retry after restart replaces - /// incomplete preparation with a newly recorded common frontier. + /// A live retry reuses its exact pending base and WAL, including when its + /// handoff is now confirmed. Cold recovery reopens the recorded artifacts. /// /// # Errors /// @@ -531,51 +548,55 @@ impl ChunkKvStorage { transition: &SplitTransition, max_catchup_lag_records: u64, ) -> Result { - let parent_binding = self - .streams - .registry() - .load(transition.parent_artifact.stream_name) - .await - .map_err(|error| storage_plan_error(&error.to_string()))? - .ok_or_else(|| storage_plan_error("split parent stream binding does not exist"))?; - let retained_parent = self - .split_target_artifact( - &transition.retained_parent_artifact, - transition.parent_next_epoch, - parent_binding.metadata_group_id, - ) - .await - .map_err(|error| { - storage_plan_error(&format!( - "retained split writer setup failed (tree_id={}, owner_epoch={}): {error}", - transition.retained_parent_artifact.tree_id, transition.parent_next_epoch - )) - })?; - let child = self - .split_target(&transition.child, parent_binding.metadata_group_id) - .await - .map_err(|error| { - storage_plan_error(&format!( - "child split writer setup failed (tree_id={}, owner_epoch={}): {error}", - transition.child.artifact.tree_id, transition.child.owner_epoch - )) - })?; + let cached_child = parent + .pending_split_child_target(&split_plan(transition)) + .map_err(|error| storage_plan_error(&error.to_string()))?; + if transition.handoff_proof.is_some() && cached_child.is_none() { + return self.resume_split_handoff(parent, transition).await; + } + let store = Arc::new(crate::Group0ControlStore::from_client(self.kv.clone())); + let (current, _) = store + .load_split_transition(transition.transition_id) + .await? + .ok_or_else(|| storage_plan_error("split transition disappeared before preparation"))?; + if ¤t != transition { + return Err(storage_plan_error("split changed before preparation retry")); + } + let child = if let Some(child) = cached_child { + child + } else { + parent + .release_generation_pin(split_plan(transition).transition_id) + .map_err(|error| storage_plan_error(&error.to_string()))?; + let parent_binding = self + .streams + .registry() + .load(transition.parent_artifact.stream_name) + .await + .map_err(|error| storage_plan_error(&error.to_string()))? + .ok_or_else(|| storage_plan_error("split parent stream binding does not exist"))?; + let child = self + .split_target(&transition.child, parent_binding.metadata_group_id) + .await?; + if child.journal.tail() != 0 { + return Err(storage_plan_error( + "nonempty child WAL has no committed handoff proof", + )); + } + crowdb_chunk_kv::CrowdbPartitionTree::open(child.tree_id, &child.tree_config) + .map_err(|error| storage_plan_error(&error.to_string()))? + .unpin_generation(split_plan(transition).transition_id) + .map_err(|error| storage_plan_error(&error.to_string()))?; + child + }; + let handoff = Arc::new(split_handoff::SplitHandoffCommit { + store, + expected: transition.clone(), + }); let prepared = parent - .prepare_split_session( - split_plan(transition), - crowdb_chunk_kv::SplitSessionTargets { - retained_parent, - child, - }, - max_catchup_lag_records, - ) + .prepare_split_child_session(split_plan(transition), child, max_catchup_lag_records, handoff) .await - .map_err(|error| { - storage_plan_error(&format!( - "split session build failed (parent_tree_id={}, parent_stream={:?}): {error}", - transition.parent_artifact.tree_id, transition.parent_artifact.stream_name, - )) - })?; + .map_err(|error| storage_plan_error(&error.to_string()))?; validate_prepared_split(transition, &prepared.artifact)?; Ok(prepared) } @@ -911,409 +932,18 @@ fn assignment_recovery_error( )) } -struct KvRootCatalogStore { +/// Opens the production root authority adapter without a chunk IO fixture. +/// +/// # Errors +/// +/// Returns a root identity, KV availability or stale authority error. +#[cfg(feature = "test-util")] +pub async fn open_root_catalog_for_tests( kv: Arc, - runtime: tokio::runtime::Handle, store_id: u64, group_id: u64, - tree_id: u64, - owner_epoch: u64, - published_objects: ArcSwap, Vec>>, -} - -impl KvRootCatalogStore { - async fn claim( - kv: Arc, - store_id: u64, - group_id: u64, - tree_id: u64, - owner_epoch: u64, - ) -> Result { - if group_id == 0 || tree_id == 0 || owner_epoch == 0 { - return Err(StorageRuntimeError::Tree( - "root catalog group, tree, and owner epoch must be nonzero".into(), - )); - } - let key = catalog_key(tree_id, b"authority", 0); - loop { - let (current_epoch, generation, revision) = match kv - .get(store_id, group_id, &key, ReadMode::Linearizable, None) - .await - .map_err(|error| StorageRuntimeError::Tree(error.to_string()))? - { - GetOutcome::NotFound => (0, 0, 0), - GetOutcome::Found { value, revision } => { - let (epoch, generation) = decode_authority(&value) - .ok_or_else(|| StorageRuntimeError::Tree("invalid root authority record".into()))?; - (epoch, generation, revision) - } - }; - if current_epoch > owner_epoch { - return Err(StorageRuntimeError::Tree(format!( - "tree root owner epoch is stale: tree_id={tree_id}, current_epoch={current_epoch}, requested_epoch={owner_epoch}" - ))); - } - if current_epoch == owner_epoch { - break; - } - match kv - .put_cas( - store_id, - group_id, - &key, - &encode_authority(owner_epoch, generation), - revision, - ) - .await - { - Ok(_) => break, - Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => {} - Err(error) => return Err(StorageRuntimeError::Tree(error.to_string())), - } - } - Ok(Self { - kv, - runtime: tokio::runtime::Handle::current(), - store_id, - group_id, - tree_id, - owner_epoch, - published_objects: ArcSwap::from_pointee(HashMap::new()), - }) - } - - fn wait(&self, future: F) -> F::Output { - if tokio::runtime::Handle::try_current().is_ok() { - tokio::task::block_in_place(|| self.runtime.block_on(future)) - } else { - self.runtime.block_on(future) - } - } - - async fn get(&self, key: &[u8]) -> Result, crowdb_kv_client::Error> { - self.kv - .get(self.store_id, self.group_id, key, ReadMode::Linearizable, None) - .await - .map(|outcome| match outcome { - GetOutcome::Found { value, revision } => Some((value, revision)), - GetOutcome::NotFound => None, - }) - } - - fn remember_published(&self, key: &[u8], value: &[u8]) { - self.published_objects.rcu(|current| { - let mut next = HashMap::clone(current); - next.insert(key.to_vec(), value.to_vec()); - next - }); - } -} - -impl RootCatalogStore for KvRootCatalogStore { - fn load( - &self, - tree_id: u64, - object: RootCatalogObject, - ) -> Result>, crowdb_tree_ffi::CtError> { - if tree_id != self.tree_id { - return Err(crowdb_tree_ffi::CtError::InvalidArgument); - } - let key = object_key(tree_id, object); - if let Some(value) = self.published_objects.load().get(&key) { - return Ok(Some(value.clone())); - } - let value = self - .wait(self.get(&key)) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)?; - if value.is_none() && object == RootCatalogObject::CurrentManifest { - let authority_key = catalog_key(tree_id, b"authority", 0); - if let Some((authority, _)) = self - .wait(self.get(&authority_key)) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? - { - let (_, generation) = - decode_authority(&authority).ok_or(crowdb_tree_ffi::CtError::Corruption)?; - if generation != 0 { - return Err(crowdb_tree_ffi::CtError::Corruption); - } - } - } - Ok(value.map(|(bytes, _)| bytes.to_vec())) - } - - fn store( - &self, - tree_id: u64, - object: RootCatalogObject, - data: &[u8], - ) -> Result<(), crowdb_tree_ffi::CtError> { - if tree_id != self.tree_id || !matches!(object, RootCatalogObject::ReferenceSegment(_)) { - return Err(crowdb_tree_ffi::CtError::InvalidArgument); - } - let key = object_key(tree_id, object); - self.wait(self.kv.put(self.store_id, self.group_id, &key, data, None)) - .map(|_| ()) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)?; - self.remember_published(&key, data); - Ok(()) - } - - fn publish( - &self, - tree_id: u64, - expected_generation: u64, - owner_epoch: u64, - generation: u64, - manifest: &[u8], - ) -> Result<(), crowdb_tree_ffi::CtError> { - if tree_id != self.tree_id || owner_epoch != self.owner_epoch || generation != expected_generation + 1 - { - return Err(crowdb_tree_ffi::CtError::InvalidArgument); - } - let authority_key = catalog_key(tree_id, b"authority", 0); - let Some((authority, revision)) = self - .wait(self.get(&authority_key)) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? - else { - return Err(crowdb_tree_ffi::CtError::Unavailable); - }; - if decode_authority(&authority) != Some((owner_epoch, expected_generation)) { - return Err(crowdb_tree_ffi::CtError::Unavailable); - } - let ops = [ - BatchOp::Put { - key: Bytes::from(authority_key.clone()), - value: Bytes::copy_from_slice(&encode_authority(owner_epoch, generation)), - }, - BatchOp::Put { - key: Bytes::from(catalog_key(tree_id, b"current", 0)), - value: Bytes::copy_from_slice(manifest), - }, - BatchOp::Put { - key: Bytes::from(catalog_key(tree_id, b"manifest", generation)), - value: Bytes::copy_from_slice(manifest), - }, - ]; - self.wait( - self.kv - .batch_write_cas(self.store_id, self.group_id, &ops, &authority_key, revision), - ) - .map(|_| ()) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)?; - self.remember_published(&catalog_key(tree_id, b"current", 0), manifest); - self.remember_published(&catalog_key(tree_id, b"manifest", generation), manifest); - Ok(()) - } - - fn allocate_reference_segment_id(&self, tree_id: u64) -> Result { - if tree_id != self.tree_id { - return Err(crowdb_tree_ffi::CtError::InvalidArgument); - } - let key = catalog_key(tree_id, b"next-reference", 0); - loop { - let (current, revision) = match self - .wait(self.get(&key)) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? - { - Some((value, revision)) => ( - decode_u64(&value).ok_or(crowdb_tree_ffi::CtError::Corruption)?, - revision, - ), - None => (1, 0), - }; - let next = current - .checked_add(1) - .ok_or(crowdb_tree_ffi::CtError::ResourceExhausted)?; - match self.wait(self.kv.put_cas( - self.store_id, - self.group_id, - &key, - &next.to_be_bytes(), - revision, - )) { - Ok(_) => return Ok(current), - Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => {} - Err(_) => return Err(crowdb_tree_ffi::CtError::Unavailable), - } - } - } - - fn discard_reference_segments(&self, tree_id: u64, object_ids: &[u64]) -> u64 { - if tree_id != self.tree_id || object_ids.is_empty() { - return 0; - } - let ops = object_ids - .iter() - .map(|object_id| BatchOp::Delete { - key: Bytes::from(catalog_key(tree_id, b"reference", *object_id)), - }) - .collect::>(); - self.wait(self.kv.batch_write(self.store_id, self.group_id, &ops)) - .map_or(0, |_| object_ids.len() as u64) - } - - fn reclaim_before(&self, tree_id: u64, generation: u64) -> u64 { - if tree_id != self.tree_id || generation <= 2 { - return 0; - } - let pin_prefix = catalog_pin_prefix(tree_id); - let pinned_generation = match self.wait(self.kv.scan( - self.store_id, - self.group_id, - &pin_prefix, - &[], - &[], - 0, - ReadMode::Linearizable, - None, - false, - None, - )) { - Ok(outcome) => { - let mut oldest = None; - for (_, value) in outcome.items { - let Some(pin) = decode_u64(&value) else { - return 0; - }; - oldest = Some(oldest.map_or(pin, |current: u64| current.min(pin))); - } - oldest - } - Err(_) => return 0, - }; - let generation = pinned_generation.map_or(generation, |pin| generation.min(pin)); - if generation <= 2 { - return 0; - } - let floor_key = catalog_key(tree_id, b"reclaim-floor", 0); - let (floor, revision) = match self.wait(self.get(&floor_key)) { - Ok(Some((value, revision))) => match decode_u64(&value) { - Some(floor) => (floor, revision), - None => return 0, - }, - Ok(None) => (1, 0), - Err(_) => return 0, - }; - let end = generation.saturating_sub(1).min(floor.saturating_add(128)); - if floor >= end { - return 0; - } - let mut ops = (floor..end) - .map(|old_generation| BatchOp::Delete { - key: Bytes::from(catalog_key(tree_id, b"manifest", old_generation)), - }) - .collect::>(); - ops.push(BatchOp::Put { - key: Bytes::from(floor_key.clone()), - value: Bytes::copy_from_slice(&end.to_be_bytes()), - }); - self.wait( - self.kv - .batch_write_cas(self.store_id, self.group_id, &ops, &floor_key, revision), - ) - .map_or(0, |_| end - floor) - } - - fn pin_generation( - &self, - tree_id: u64, - transition_high: u64, - transition_low: u64, - generation: u64, - ) -> Result<(), crowdb_tree_ffi::CtError> { - if tree_id != self.tree_id || (transition_high == 0 && transition_low == 0) || generation == 0 { - return Err(crowdb_tree_ffi::CtError::InvalidArgument); - } - if self - .wait(self.get(&catalog_key(tree_id, b"manifest", generation))) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? - .is_none() - { - return Err(crowdb_tree_ffi::CtError::NotFound); - } - let key = catalog_pin_key(tree_id, transition_high, transition_low); - for _ in 0..3 { - if let Some((value, _)) = self - .wait(self.get(&key)) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? - { - return if decode_u64(&value) == Some(generation) { - Ok(()) - } else { - Err(crowdb_tree_ffi::CtError::InvalidArgument) - }; - } - match self.wait( - self.kv - .put_cas(self.store_id, self.group_id, &key, &generation.to_be_bytes(), 0), - ) { - Ok(_) => return Ok(()), - Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => {} - Err(_) => return Err(crowdb_tree_ffi::CtError::Unavailable), - } - } - Err(crowdb_tree_ffi::CtError::Unavailable) - } - - fn unpin_generation( - &self, - tree_id: u64, - transition_high: u64, - transition_low: u64, - ) -> Result<(), crowdb_tree_ffi::CtError> { - if tree_id != self.tree_id || (transition_high == 0 && transition_low == 0) { - return Err(crowdb_tree_ffi::CtError::InvalidArgument); - } - let key = catalog_pin_key(tree_id, transition_high, transition_low); - self.wait(self.kv.delete(self.store_id, self.group_id, &key, None)) - .map(|_| ()) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable) - } -} - -fn catalog_key(tree_id: u64, kind: &[u8], object_id: u64) -> Vec { - let mut key = b"\0crowdb/chunk-kv/root/v1/".to_vec(); - key.extend_from_slice(&tree_id.to_be_bytes()); - key.push(b'/'); - key.extend_from_slice(kind); - key.push(b'/'); - key.extend_from_slice(&object_id.to_be_bytes()); - key -} - -fn catalog_pin_prefix(tree_id: u64) -> Vec { - let mut key = b"\0crowdb/chunk-kv/root/v1/".to_vec(); - key.extend_from_slice(&tree_id.to_be_bytes()); - key.extend_from_slice(b"/pin/"); - key -} - -fn catalog_pin_key(tree_id: u64, transition_high: u64, transition_low: u64) -> Vec { - let mut key = catalog_pin_prefix(tree_id); - key.extend_from_slice(&transition_high.to_be_bytes()); - key.extend_from_slice(&transition_low.to_be_bytes()); - key -} - -fn object_key(tree_id: u64, object: RootCatalogObject) -> Vec { - match object { - RootCatalogObject::CurrentManifest => catalog_key(tree_id, b"current", 0), - RootCatalogObject::Manifest(generation) => catalog_key(tree_id, b"manifest", generation), - RootCatalogObject::ReferenceSegment(object_id) => catalog_key(tree_id, b"reference", object_id), - } -} - -fn encode_authority(owner_epoch: u64, generation: u64) -> [u8; 16] { - let mut value = [0; 16]; - value[..8].copy_from_slice(&owner_epoch.to_be_bytes()); - value[8..].copy_from_slice(&generation.to_be_bytes()); - value -} - -fn decode_authority(value: &[u8]) -> Option<(u64, u64)> { - (value.len() == 16).then(|| (decode_u64(&value[..8]).unwrap(), decode_u64(&value[8..]).unwrap())) -} - -fn decode_u64(value: &[u8]) -> Option { - value.try_into().ok().map(u64::from_be_bytes) + entry: &ChunkKvRangeCatalogEntry, +) -> Result, StorageRuntimeError> { + let catalog = KvRootCatalogStore::for_assignment(kv, store_id, group_id, entry).await?; + Ok(Arc::new(catalog)) } diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs new file mode 100644 index 000000000..95198a078 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs @@ -0,0 +1,543 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Durable tree root authority and generation publication. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use arc_swap::ArcSwap; +use bytes::Bytes; +use crowdb_kv_client::{BatchOp, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_tree_ffi::{RootCatalogObject, RootCatalogStore}; + +use super::StorageRuntimeError; +use crate::{ChunkKvRangeCatalogPublisher, Group0ControlStore}; +use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState}; + +pub(super) struct KvRootCatalogStore { + kv: Arc, + runtime: tokio::runtime::Handle, + store_id: u64, + group_id: u64, + tree_id: u64, + owner_epoch: u64, + publication_authorized: AtomicBool, + prepared_assignment: Option, + published_objects: ArcSwap, Vec>>, +} + +impl KvRootCatalogStore { + pub(super) async fn claim( + kv: Arc, + store_id: u64, + group_id: u64, + tree_id: u64, + owner_epoch: u64, + ) -> Result { + if group_id == 0 || tree_id == 0 || owner_epoch == 0 { + return Err(StorageRuntimeError::Tree( + "root catalog group, tree, and owner epoch must be nonzero".into(), + )); + } + let key = catalog_key(tree_id, b"authority", 0); + loop { + let (current_epoch, generation, revision) = match kv + .get(store_id, group_id, &key, ReadMode::Linearizable, None) + .await + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))? + { + GetOutcome::NotFound => (0, 0, 0), + GetOutcome::Found { value, revision } => { + let (epoch, generation) = decode_authority(&value) + .ok_or_else(|| StorageRuntimeError::Tree("invalid root authority record".into()))?; + (epoch, generation, revision) + } + }; + if current_epoch > owner_epoch { + return Err(StorageRuntimeError::Tree(format!( + "tree root owner epoch is stale: tree_id={tree_id}, current_epoch={current_epoch}, requested_epoch={owner_epoch}" + ))); + } + if current_epoch == owner_epoch { + break; + } + match kv + .put_cas( + store_id, + group_id, + &key, + &encode_authority(owner_epoch, generation), + revision, + ) + .await + { + Ok(_) => break, + Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => {} + Err(error) => return Err(StorageRuntimeError::Tree(error.to_string())), + } + } + Ok(Self { + kv, + runtime: tokio::runtime::Handle::current(), + store_id, + group_id, + tree_id, + owner_epoch, + publication_authorized: AtomicBool::new(true), + prepared_assignment: None, + published_objects: ArcSwap::from_pointee(HashMap::new()), + }) + } + + pub(super) async fn for_assignment( + kv: Arc, + store_id: u64, + group_id: u64, + entry: &ChunkKvRangeCatalogEntry, + ) -> Result { + if matches!( + entry.state, + ChunkKvRangeCatalogPartitionState::Prepared | ChunkKvRangeCatalogPartitionState::TargetCatchingUp + ) { + let catalog = Self::prepare(kv, store_id, group_id, entry)?; + if entry.state == ChunkKvRangeCatalogPartitionState::TargetCatchingUp { + catalog.claim_published(true).await?; + } + Ok(catalog) + } else { + Self::claim(kv, store_id, group_id, entry.artifact.tree_id, entry.owner_epoch).await + } + } + + pub(super) fn prepare( + kv: Arc, + store_id: u64, + group_id: u64, + entry: &ChunkKvRangeCatalogEntry, + ) -> Result { + if group_id == 0 || entry.artifact.tree_id == 0 || entry.owner_epoch == 0 { + return Err(StorageRuntimeError::Tree("invalid prepared root identity".into())); + } + Ok(Self { + kv, + runtime: tokio::runtime::Handle::current(), + store_id, + group_id, + tree_id: entry.artifact.tree_id, + owner_epoch: entry.owner_epoch, + publication_authorized: AtomicBool::new(false), + prepared_assignment: Some(entry.clone()), + published_objects: ArcSwap::from_pointee(HashMap::new()), + }) + } + + async fn authorize_publication(&self) -> Result<(), crowdb_tree_ffi::CtError> { + self.claim_published(false) + .await + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable) + } + + pub(super) async fn claim_published(&self, allow_catching_up: bool) -> Result<(), StorageRuntimeError> { + if self.publication_authorized.load(Ordering::Acquire) { + return Ok(()); + } + let Some(prepared) = &self.prepared_assignment else { + return Ok(()); + }; + let publisher = + ChunkKvRangeCatalogPublisher::new(Arc::new(Group0ControlStore::from_client(self.kv.clone()))); + let (_, pages) = publisher + .load_current() + .await + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))? + .ok_or_else(|| StorageRuntimeError::Tree("range catalog is absent".into()))?; + let assigned = pages.iter().flat_map(|page| &page.entries).any(|entry| { + entry.partition_id == prepared.partition_id + && entry.range == prepared.range + && entry.owner == prepared.owner + && entry.owner_epoch == prepared.owner_epoch + && entry.artifact.tree_id == prepared.artifact.tree_id + && entry.artifact.stream_name == prepared.artifact.stream_name + && entry.transition_id == prepared.transition_id + && same_recovery_base(entry, prepared) + && (entry.state == ChunkKvRangeCatalogPartitionState::Serving + || (allow_catching_up + && entry.state == ChunkKvRangeCatalogPartitionState::TargetCatchingUp)) + }); + if !assigned { + return Err(StorageRuntimeError::Tree( + "prepared target is not the published tree owner".into(), + )); + } + Self::claim( + self.kv.clone(), + self.store_id, + self.group_id, + self.tree_id, + self.owner_epoch, + ) + .await + .map(|_| { + if !allow_catching_up { + self.publication_authorized.store(true, Ordering::Release); + } + }) + } + + fn wait(&self, future: F) -> F::Output { + if tokio::runtime::Handle::try_current().is_ok() { + tokio::task::block_in_place(|| self.runtime.block_on(future)) + } else { + self.runtime.block_on(future) + } + } + + async fn get(&self, key: &[u8]) -> Result, crowdb_kv_client::Error> { + self.kv + .get(self.store_id, self.group_id, key, ReadMode::Linearizable, None) + .await + .map(|outcome| match outcome { + GetOutcome::Found { value, revision } => Some((value, revision)), + GetOutcome::NotFound => None, + }) + } + + fn remember_published(&self, key: &[u8], value: &[u8]) { + self.published_objects.rcu(|current| { + let mut next = HashMap::clone(current); + next.insert(key.to_vec(), value.to_vec()); + next + }); + } +} + +impl RootCatalogStore for KvRootCatalogStore { + fn load( + &self, + tree_id: u64, + object: RootCatalogObject, + ) -> Result>, crowdb_tree_ffi::CtError> { + if tree_id != self.tree_id { + return Err(crowdb_tree_ffi::CtError::InvalidArgument); + } + let key = object_key(tree_id, object); + if let Some(value) = self.published_objects.load().get(&key) { + return Ok(Some(value.clone())); + } + let value = self + .wait(self.get(&key)) + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)?; + if value.is_none() && object == RootCatalogObject::CurrentManifest { + let authority_key = catalog_key(tree_id, b"authority", 0); + if let Some((authority, _)) = self + .wait(self.get(&authority_key)) + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? + { + let (_, generation) = + decode_authority(&authority).ok_or(crowdb_tree_ffi::CtError::Corruption)?; + if generation != 0 { + return Err(crowdb_tree_ffi::CtError::Corruption); + } + } + } + Ok(value.map(|(bytes, _)| bytes.to_vec())) + } + + fn store( + &self, + tree_id: u64, + object: RootCatalogObject, + data: &[u8], + ) -> Result<(), crowdb_tree_ffi::CtError> { + if tree_id != self.tree_id || !matches!(object, RootCatalogObject::ReferenceSegment(_)) { + return Err(crowdb_tree_ffi::CtError::InvalidArgument); + } + let key = object_key(tree_id, object); + self.wait(self.kv.put(self.store_id, self.group_id, &key, data, None)) + .map(|_| ()) + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)?; + self.remember_published(&key, data); + Ok(()) + } + + fn publish( + &self, + tree_id: u64, + expected_generation: u64, + owner_epoch: u64, + generation: u64, + manifest: &[u8], + ) -> Result<(), crowdb_tree_ffi::CtError> { + if tree_id != self.tree_id || owner_epoch != self.owner_epoch || generation != expected_generation + 1 + { + return Err(crowdb_tree_ffi::CtError::InvalidArgument); + } + self.wait(self.authorize_publication())?; + let authority_key = catalog_key(tree_id, b"authority", 0); + let Some((authority, revision)) = self + .wait(self.get(&authority_key)) + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? + else { + return Err(crowdb_tree_ffi::CtError::Unavailable); + }; + if decode_authority(&authority) != Some((owner_epoch, expected_generation)) { + return Err(crowdb_tree_ffi::CtError::Unavailable); + } + let ops = [ + BatchOp::Put { + key: Bytes::from(authority_key.clone()), + value: Bytes::copy_from_slice(&encode_authority(owner_epoch, generation)), + }, + BatchOp::Put { + key: Bytes::from(catalog_key(tree_id, b"current", 0)), + value: Bytes::copy_from_slice(manifest), + }, + BatchOp::Put { + key: Bytes::from(catalog_key(tree_id, b"manifest", generation)), + value: Bytes::copy_from_slice(manifest), + }, + ]; + self.wait( + self.kv + .batch_write_cas(self.store_id, self.group_id, &ops, &authority_key, revision), + ) + .map(|_| ()) + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)?; + self.remember_published(&catalog_key(tree_id, b"current", 0), manifest); + self.remember_published(&catalog_key(tree_id, b"manifest", generation), manifest); + Ok(()) + } + + fn allocate_reference_segment_id(&self, tree_id: u64) -> Result { + if tree_id != self.tree_id { + return Err(crowdb_tree_ffi::CtError::InvalidArgument); + } + let key = catalog_key(tree_id, b"next-reference", 0); + loop { + let (current, revision) = match self + .wait(self.get(&key)) + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? + { + Some((value, revision)) => ( + decode_u64(&value).ok_or(crowdb_tree_ffi::CtError::Corruption)?, + revision, + ), + None => (1, 0), + }; + let next = current + .checked_add(1) + .ok_or(crowdb_tree_ffi::CtError::ResourceExhausted)?; + match self.wait(self.kv.put_cas( + self.store_id, + self.group_id, + &key, + &next.to_be_bytes(), + revision, + )) { + Ok(_) => return Ok(current), + Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => {} + Err(_) => return Err(crowdb_tree_ffi::CtError::Unavailable), + } + } + } + + fn discard_reference_segments(&self, tree_id: u64, object_ids: &[u64]) -> u64 { + if tree_id != self.tree_id || object_ids.is_empty() { + return 0; + } + let ops = object_ids + .iter() + .map(|object_id| BatchOp::Delete { + key: Bytes::from(catalog_key(tree_id, b"reference", *object_id)), + }) + .collect::>(); + self.wait(self.kv.batch_write(self.store_id, self.group_id, &ops)) + .map_or(0, |_| object_ids.len() as u64) + } + + fn reclaim_before(&self, tree_id: u64, generation: u64) -> u64 { + if tree_id != self.tree_id || generation <= 2 { + return 0; + } + let pin_prefix = catalog_pin_prefix(tree_id); + let pinned_generation = match self.wait(self.kv.scan( + self.store_id, + self.group_id, + &pin_prefix, + &[], + &[], + 0, + ReadMode::Linearizable, + None, + false, + None, + )) { + Ok(outcome) => { + let mut oldest = None; + for (_, value) in outcome.items { + let Some(pin) = decode_u64(&value) else { + return 0; + }; + oldest = Some(oldest.map_or(pin, |current: u64| current.min(pin))); + } + oldest + } + Err(_) => return 0, + }; + let generation = pinned_generation.map_or(generation, |pin| generation.min(pin)); + if generation <= 2 { + return 0; + } + let floor_key = catalog_key(tree_id, b"reclaim-floor", 0); + let (floor, revision) = match self.wait(self.get(&floor_key)) { + Ok(Some((value, revision))) => match decode_u64(&value) { + Some(floor) => (floor, revision), + None => return 0, + }, + Ok(None) => (1, 0), + Err(_) => return 0, + }; + let end = generation.saturating_sub(1).min(floor.saturating_add(128)); + if floor >= end { + return 0; + } + let mut ops = (floor..end) + .map(|old_generation| BatchOp::Delete { + key: Bytes::from(catalog_key(tree_id, b"manifest", old_generation)), + }) + .collect::>(); + ops.push(BatchOp::Put { + key: Bytes::from(floor_key.clone()), + value: Bytes::copy_from_slice(&end.to_be_bytes()), + }); + self.wait( + self.kv + .batch_write_cas(self.store_id, self.group_id, &ops, &floor_key, revision), + ) + .map_or(0, |_| end - floor) + } + + fn pin_generation( + &self, + tree_id: u64, + transition_high: u64, + transition_low: u64, + generation: u64, + ) -> Result<(), crowdb_tree_ffi::CtError> { + if tree_id != self.tree_id || (transition_high == 0 && transition_low == 0) || generation == 0 { + return Err(crowdb_tree_ffi::CtError::InvalidArgument); + } + if self + .wait(self.get(&catalog_key(tree_id, b"manifest", generation))) + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? + .is_none() + { + return Err(crowdb_tree_ffi::CtError::NotFound); + } + let key = catalog_pin_key(tree_id, transition_high, transition_low); + for _ in 0..3 { + if let Some((value, _)) = self + .wait(self.get(&key)) + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? + { + return if decode_u64(&value) == Some(generation) { + Ok(()) + } else { + Err(crowdb_tree_ffi::CtError::InvalidArgument) + }; + } + match self.wait( + self.kv + .put_cas(self.store_id, self.group_id, &key, &generation.to_be_bytes(), 0), + ) { + Ok(_) => return Ok(()), + Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => {} + Err(_) => return Err(crowdb_tree_ffi::CtError::Unavailable), + } + } + Err(crowdb_tree_ffi::CtError::Unavailable) + } + + fn unpin_generation( + &self, + tree_id: u64, + transition_high: u64, + transition_low: u64, + ) -> Result<(), crowdb_tree_ffi::CtError> { + if tree_id != self.tree_id || (transition_high == 0 && transition_low == 0) { + return Err(crowdb_tree_ffi::CtError::InvalidArgument); + } + let key = catalog_pin_key(tree_id, transition_high, transition_low); + self.wait(self.kv.delete(self.store_id, self.group_id, &key, None)) + .map(|_| ()) + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable) + } +} + +fn catalog_key(tree_id: u64, kind: &[u8], object_id: u64) -> Vec { + let mut key = b"\0crowdb/chunk-kv/root/v1/".to_vec(); + key.extend_from_slice(&tree_id.to_be_bytes()); + key.push(b'/'); + key.extend_from_slice(kind); + key.push(b'/'); + key.extend_from_slice(&object_id.to_be_bytes()); + key +} + +fn catalog_pin_prefix(tree_id: u64) -> Vec { + let mut key = b"\0crowdb/chunk-kv/root/v1/".to_vec(); + key.extend_from_slice(&tree_id.to_be_bytes()); + key.extend_from_slice(b"/pin/"); + key +} + +fn catalog_pin_key(tree_id: u64, transition_high: u64, transition_low: u64) -> Vec { + let mut key = catalog_pin_prefix(tree_id); + key.extend_from_slice(&transition_high.to_be_bytes()); + key.extend_from_slice(&transition_low.to_be_bytes()); + key +} + +fn object_key(tree_id: u64, object: RootCatalogObject) -> Vec { + match object { + RootCatalogObject::CurrentManifest => catalog_key(tree_id, b"current", 0), + RootCatalogObject::Manifest(generation) => catalog_key(tree_id, b"manifest", generation), + RootCatalogObject::ReferenceSegment(object_id) => catalog_key(tree_id, b"reference", object_id), + } +} + +fn encode_authority(owner_epoch: u64, generation: u64) -> [u8; 16] { + let mut value = [0; 16]; + value[..8].copy_from_slice(&owner_epoch.to_be_bytes()); + value[8..].copy_from_slice(&generation.to_be_bytes()); + value +} + +fn decode_authority(value: &[u8]) -> Option<(u64, u64)> { + (value.len() == 16).then(|| (decode_u64(&value[..8]).unwrap(), decode_u64(&value[8..]).unwrap())) +} + +fn decode_u64(value: &[u8]) -> Option { + value.try_into().ok().map(u64::from_be_bytes) +} + +fn same_recovery_base(entry: &ChunkKvRangeCatalogEntry, prepared: &ChunkKvRangeCatalogEntry) -> bool { + match (&entry.artifact.tail_overlay, &prepared.artifact.tail_overlay) { + (None, None) => true, + (Some(current), Some(base)) => { + current.source_partition_id == base.source_partition_id + && current.source_epoch == base.source_epoch + && current.source_stream_name == base.source_stream_name + && current.source_stream_manifest_generation == base.source_stream_manifest_generation + && current.replay_offset == base.replay_offset + && current.base_root_manifest_generation == base.base_root_manifest_generation + && current.base_tree_manifest == base.base_tree_manifest + && current.base_applied_seq == base.base_applied_seq + && current.cutover_seq >= base.cutover_seq + && current.cutover_offset >= base.cutover_offset + } + _ => false, + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/split_handoff.rs b/app/crowdb-chunk-kv-server/src/storage/split_handoff.rs new file mode 100644 index 000000000..72abc5af5 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/split_handoff.rs @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Revision-fenced local split handoff persistence. + +use crate::{Group0ControlStore, SplitStateMachine}; +use async_trait::async_trait; +use crowdb_chunk_kv::{ChunkKvError, PreparedSplitWriterArtifact, SplitHandoffStore}; +use crowdb_protocol::chunk_kv::{SplitTransition, TailOverlayArtifact}; +use std::sync::Arc; + +pub(super) struct SplitHandoffCommit { + pub store: Arc, + pub expected: SplitTransition, +} + +#[async_trait] +impl SplitHandoffStore for SplitHandoffCommit { + async fn commit(&self, artifact: &PreparedSplitWriterArtifact) -> crowdb_chunk_kv::Result<()> { + let (current, revision) = self + .store + .load_split_transition(self.expected.transition_id) + .await + .map_err(|error| ChunkKvError::SplitRetry(error.to_string()))? + .ok_or_else(|| ChunkKvError::SplitRetry("split transition disappeared before handoff".into()))?; + if current != self.expected { + return Err(ChunkKvError::SplitRetry( + "split transition changed before handoff".into(), + )); + } + let proof = TailOverlayArtifact { + source_partition_id: current.parent_id, + source_epoch: current.parent_epoch, + source_stream_name: artifact.parent_stream_name, + source_stream_manifest_generation: artifact.parent_stream_manifest_generation, + replay_offset: artifact.parent_replay_offset, + cutover_offset: artifact.parent_cutover_offset, + base_tree_manifest: artifact.tree_manifest, + base_root_manifest_generation: artifact.root_manifest_generation, + base_applied_seq: artifact.base_applied_seq, + cutover_seq: artifact.applied_seq, + target_stream_start_seq: artifact.child_stream_start_seq, + }; + let mut machine = SplitStateMachine::restore(current) + .map_err(|error| ChunkKvError::SplitRetry(error.to_string()))?; + machine + .record_handoff(proof) + .map_err(|error| ChunkKvError::SplitRetry(error.to_string()))?; + self.store + .persist_split_transition(machine.transition(), revision) + .await + .map_err(|error| ChunkKvError::SplitRetry(error.to_string()))?; + Ok(()) + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/split_recovery.rs b/app/crowdb-chunk-kv-server/src/storage/split_recovery.rs new file mode 100644 index 000000000..74213f740 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/split_recovery.rs @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Recovery of the exact child base recorded before local dispatch. + +use super::{prepared_overlay_artifact, split_plan, storage_plan_error, ChunkKvStorage}; +use crate::MonitorError; +use crowdb_chunk_kv::{Partition, PartitionConfig, PreparedSplit}; +use crowdb_chunk_stream::StreamRegistry; +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, SplitTransition, +}; +use crowdb_tree_ffi::ChunkPageStoreOptions; + +impl ChunkKvStorage { + pub(super) async fn resume_split_handoff( + &self, + parent: &Partition, + transition: &SplitTransition, + ) -> Result { + if parent.split_ingress().is_some() { + return parent + .finalize_installed_split_child_session(split_plan(transition)) + .await + .map_err(|error| storage_plan_error(&error.to_string())); + } + let proof = transition + .handoff_proof + .as_ref() + .ok_or_else(|| storage_plan_error("split handoff proof is absent"))?; + let binding = self + .streams + .registry() + .load(transition.child.artifact.stream_name) + .await + .map_err(|error| storage_plan_error(&error.to_string()))? + .ok_or_else(|| storage_plan_error("handoff child stream is absent"))?; + let child_stream = self + .open_stream( + transition.child.artifact.stream_name, + transition.child.owner_epoch, + ) + .await + .map_err(|error| storage_plan_error(&error.to_string()))?; + let source_stream = self + .streams + .open_read_only_current(proof.source_stream_name, self.metadata_store_id) + .await + .map_err(|error| storage_plan_error(&error.to_string()))?; + let page_store = self + .open_durable_tree_page_store( + ChunkPageStoreOptions { + tree_id: transition.child.artifact.tree_id, + owner_epoch: transition.child.owner_epoch, + open_generation: proof.base_root_manifest_generation, + pack_bytes: 0, + iu_size: 0, + max_concurrent_packs: 0, + materialization_bytes_per_pass: 0, + mirror_copies: 0, + max_chunk_bytes: 0, + }, + binding.metadata_group_id, + ) + .await + .map_err(|error| storage_plan_error(&error.to_string()))?; + let entry = ChunkKvRangeCatalogEntry { + partition_id: transition.child.partition_id, + range: transition.child.range.clone(), + owner: transition.child.owner.clone(), + owner_epoch: transition.child.owner_epoch, + state: ChunkKvRangeCatalogPartitionState::Prepared, + artifact: transition.child.artifact.clone(), + transition_id: Some(transition.transition_id), + }; + let (child, artifact) = Partition::recover_native_split_handoff( + prepared_overlay_artifact(&entry, proof), + PartitionConfig::default(), + crowdb_tree_ffi::Config::default(), + page_store, + child_stream, + source_stream, + ) + .await + .map_err(|error| storage_plan_error(&error.to_string()))?; + parent + .resume_split_child_session(split_plan(transition), child, artifact) + .await + .map_err(|error| storage_plan_error(&error.to_string())) + } +} diff --git a/app/crowdb-chunk-kv-server/tests/balance_test.rs b/app/crowdb-chunk-kv-server/tests/balance_test.rs index ffa7990c6..3a7eb4858 100644 --- a/app/crowdb-chunk-kv-server/tests/balance_test.rs +++ b/app/crowdb-chunk-kv-server/tests/balance_test.rs @@ -94,14 +94,14 @@ fn count_imbalance_precedes_bytes_and_respects_cooldown() { } #[test] -fn balanced_counts_require_weighted_improvement_threshold() { +fn balanced_counts_do_not_move_for_unequal_pack_bytes() { let config = BalanceConfig { cooldown_ms: 0, ..BalanceConfig::default() }; let owners = vec![owner(1, 2, 1_000), owner(2, 2, 100)]; let useful = partition(1, 1, 300, 0); - assert!(choose_transfer(&owners, &[useful], 1, &config).is_some()); + assert!(choose_transfer(&owners, &[useful], 1, &config).is_none()); let harmful = partition(2, 1, 800, 0); assert!(choose_transfer(&owners, &[harmful], 1, &config).is_none()); } diff --git a/app/crowdb-chunk-kv-server/tests/catalog_test.rs b/app/crowdb-chunk-kv-server/tests/catalog_test.rs index eeaad5e3b..ddd9a64fa 100644 --- a/app/crowdb-chunk-kv-server/tests/catalog_test.rs +++ b/app/crowdb-chunk-kv-server/tests/catalog_test.rs @@ -62,6 +62,70 @@ fn head( head } +#[tokio::test] +async fn recovery_discards_obsolete_results_only_after_authoritative_publication_advances() { + for failed in [false, true] { + let store = Arc::new(MemoryChunkKvRangeCatalogStore::default()); + let publisher = ChunkKvRangeCatalogPublisher::new(store); + let first = page(1, 1); + publisher + .publish(head(1, None, &first), vec![first]) + .await + .unwrap(); + let mut calls = 0; + let recovered = publisher + .recover_current( + std::time::Instant::now() + std::time::Duration::from_secs(1), + |pages| { + calls += 1; + let publisher = &publisher; + async move { + let epoch = pages[0].entries[0].owner_epoch; + if epoch == 1 { + let next = page(2, 2); + publisher + .publish(head(2, Some(1), &next), vec![next]) + .await + .unwrap(); + if failed { + return Err("old epoch fenced"); + } + } + Ok(epoch) + } + }, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(calls, 2); + assert_eq!(recovered.0.generation, 2); + assert_eq!(recovered.2, 2); + } +} + +#[tokio::test] +async fn recovery_reports_unchanged_catalog_errors_without_repeating_recovery() { + let publisher = ChunkKvRangeCatalogPublisher::new(Arc::new(MemoryChunkKvRangeCatalogStore::default())); + let first = page(1, 1); + publisher + .publish(head(1, None, &first), vec![first]) + .await + .unwrap(); + let mut calls = 0; + let result = publisher + .recover_current(std::time::Instant::now(), |_| { + calls += 1; + async { Err::<(), _>("corrupt artifact") } + }) + .await; + assert_eq!(calls, 1); + assert_eq!( + result.unwrap_err(), + ChunkKvRangeCatalogError::Unavailable("corrupt artifact".into()) + ); +} + #[tokio::test] async fn publisher_writes_pages_before_head_and_reuses_unchanged_pages() { let store = Arc::new(MemoryChunkKvRangeCatalogStore::default()); diff --git a/app/crowdb-chunk-kv-server/tests/catalog_transition_test.rs b/app/crowdb-chunk-kv-server/tests/catalog_transition_test.rs index 039e58c12..ae31e5cb1 100644 --- a/app/crowdb-chunk-kv-server/tests/catalog_transition_test.rs +++ b/app/crowdb-chunk-kv-server/tests/catalog_transition_test.rs @@ -243,6 +243,7 @@ fn prepared_split() -> SplitTransition { }, planned_at_ms: 0, phase: SplitPhase::ChildPrepared, + handoff_proof: None, readiness_proof: Some(SplitReadinessProof { cutover_seq: 55, parent_next_epoch: 4, @@ -253,7 +254,7 @@ fn prepared_split() -> SplitTransition { child_applied_seq: 55, child_tree_manifest: 1, child_root_manifest_generation: 1, - retained_parent_tail_overlay: overlay.clone(), + retained_parent_tail_overlay: Some(overlay.clone()), child_tail_overlay: overlay, }), failure: None, @@ -281,7 +282,7 @@ async fn split_shrinks_parent_and_adds_exact_child_in_one_generation() { transition .readiness_proof .as_ref() - .map(|proof| proof.retained_parent_tail_overlay.clone()) + .and_then(|proof| proof.retained_parent_tail_overlay.clone()) ); assert_eq!( pages[0].entries[1].artifact.tail_overlay, diff --git a/app/crowdb-chunk-kv-server/tests/common/split_storage.rs b/app/crowdb-chunk-kv-server/tests/common/split_storage.rs new file mode 100644 index 000000000..35125a9e1 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/split_storage.rs @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunk_kv::{ + Partition, PartitionConfig, PartitionId, PartitionJournal, PartitionRange, PartitionTree, + StreamPartitionJournal, +}; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamBinding, StreamBindingState, StreamConfig, StreamName, +}; +use std::sync::Arc; + +pub struct TestSplitStorage { + pub store: Arc, +} + +impl TestSplitStorage { + pub fn new() -> Self { + Self { + store: Arc::new(MemoryStreamStore::new(4096)), + } + } + + pub async fn journal(&self, stream_name: StreamName, epoch: u64) -> Arc { + let stream = ChunkStream::create( + StreamBinding { + purpose: crowdb_chunk_stream::StreamPurpose::Wal, + stream_name, + metadata_group_id: 7, + binding_generation: 1, + state: StreamBindingState::Active, + owner_kind: Some("chunk-kv-partition".into()), + }, + epoch, + StreamConfig::default(), + self.store.clone(), + self.store.clone(), + self.store.clone(), + ) + .await + .unwrap(); + Arc::new(StreamPartitionJournal::new(stream, stream_name).unwrap()) + } + + pub async fn partition( + &self, + id: PartitionId, + range: PartitionRange, + epoch: u64, + tree: Arc, + ) -> (Partition, Arc) { + let journal = self + .journal( + StreamName { + high: id.high, + low: id.low, + }, + epoch, + ) + .await; + let partition = Partition::open( + id, + range, + epoch, + PartitionConfig::default(), + tree, + journal.clone(), + ) + .unwrap(); + (partition, journal) + } +} diff --git a/app/crowdb-chunk-kv-server/tests/config_test.rs b/app/crowdb-chunk-kv-server/tests/config_test.rs index df1ee71a4..cfae7cfcd 100644 --- a/app/crowdb-chunk-kv-server/tests/config_test.rs +++ b/app/crowdb-chunk-kv-server/tests/config_test.rs @@ -16,6 +16,8 @@ fn defaults_close_the_documented_timing_contract() { assert_eq!(config.monitor.lease_duration_ms, 12_000); assert!(config.balance.enabled); assert!(config.monitor.chunk_kv_range_balance.is_some()); + assert_eq!(config.balance.byte_weight_percent, 0); + assert_eq!(config.balance.imbalance_tolerance_percent, 20); assert_eq!(config.balance.target_partitions_per_owner, 4); assert_eq!(config.balance.minimum_weighted_improvement_percent, 25); assert_eq!(config.balance.cooldown_ms, 60_000); @@ -46,6 +48,8 @@ max_owner_request_rate = 0", .unwrap(); assert!(config.enabled); + assert_eq!(config.byte_weight_percent, 0); + assert_eq!(config.imbalance_tolerance_percent, 20); } #[test] @@ -116,3 +120,14 @@ fn stream_directory_geometry_survives_config_round_trip() { config.storage.stream_extent_page_entries = 0; assert!(config.validate().is_err()); } + +#[test] +fn data_weight_is_reserved_until_tree_range_metrics_are_available() { + let mut config = ChunkKvServerConfig { + instance_id: 1, + ..ChunkKvServerConfig::default() + }; + config.validate().unwrap(); + config.balance.byte_weight_percent = 80; + assert!(config.validate().is_err()); +} diff --git a/app/crowdb-chunk-kv-server/tests/control_store_test.rs b/app/crowdb-chunk-kv-server/tests/control_store_test.rs index 26c0275e8..f955bcc53 100644 --- a/app/crowdb-chunk-kv-server/tests/control_store_test.rs +++ b/app/crowdb-chunk-kv-server/tests/control_store_test.rs @@ -260,6 +260,7 @@ fn split() -> SplitTransition { }, planned_at_ms: 0, phase: SplitPhase::Planned, + handoff_proof: None, readiness_proof: None, failure: None, } @@ -281,6 +282,151 @@ fn split_overlay(cutover_seq: u64) -> TailOverlayArtifact { } } +#[tokio::test] +async fn split_handoff_survives_unknown_commit_and_cannot_be_discarded() { + let kv = Arc::new(TestKv::default()); + let store = Group0ControlStore::new(kv.clone()); + let mut machine = SplitStateMachine::restore(split()).unwrap(); + machine.begin_parent_prepare().unwrap(); + let preparing = machine.transition().clone(); + let revision = store.persist_split_transition(&preparing, 0).await.unwrap(); + let proof = split_overlay(41); + machine.record_handoff(proof.clone()).unwrap(); + let committed = machine.transition().clone(); + kv.inject(InjectedPut { + path: crowdb_protocol::key::ChunkKvSplitKey { + transition_id: committed.transition_id, + } + .to_path(), + error: Group0KvError::OutcomeUnknown, + commit: true, + }) + .await; + let revision = store + .persist_split_transition(&committed, revision) + .await + .unwrap(); + let (loaded, _) = store + .load_split_transition(committed.transition_id) + .await + .unwrap() + .unwrap(); + assert_eq!(loaded.handoff_proof.as_ref(), Some(&proof)); + assert!(loaded.readiness_proof.is_none()); + assert_eq!(loaded.phase, SplitPhase::ParentPreparing); + let mut recovered = SplitStateMachine::restore(loaded).unwrap(); + recovered.record_handoff(proof.clone()).unwrap(); + assert!(recovered.abort("restart before catalog publication").is_err()); + assert_eq!(recovered.transition(), &committed); + assert!(store + .persist_split_transition(&preparing, revision) + .await + .is_err()); + let mut wrong_plan = committed.clone(); + wrong_plan.child.artifact.stream_name.low += 1; + wrong_plan.validate().unwrap(); + assert!(store + .persist_split_transition(&wrong_plan, revision) + .await + .is_err()); + assert_eq!( + store + .load_split_transition(committed.transition_id) + .await + .unwrap() + .unwrap() + .0, + committed + ); + let mut conflicting = proof; + conflicting.base_root_manifest_generation += 1; + assert!(recovered.record_handoff(conflicting).is_err()); +} + +#[tokio::test] +async fn split_handoff_unknown_without_commit_does_not_create_evidence() { + let kv = Arc::new(TestKv::default()); + let store = Group0ControlStore::new(kv.clone()); + let mut machine = SplitStateMachine::restore(split()).unwrap(); + machine.begin_parent_prepare().unwrap(); + let preparing = machine.transition().clone(); + let revision = store.persist_split_transition(&preparing, 0).await.unwrap(); + machine.record_handoff(split_overlay(41)).unwrap(); + kv.inject(InjectedPut { + path: crowdb_protocol::key::ChunkKvSplitKey { + transition_id: preparing.transition_id, + } + .to_path(), + error: Group0KvError::OutcomeUnknown, + commit: false, + }) + .await; + assert!(store + .persist_split_transition(machine.transition(), revision) + .await + .is_err()); + let (loaded, _) = store + .load_split_transition(preparing.transition_id) + .await + .unwrap() + .unwrap(); + assert_eq!(loaded, preparing); + let mut recovered = SplitStateMachine::restore(loaded).unwrap(); + recovered.abort("handoff was not committed").unwrap(); +} + +#[test] +fn split_handoff_rejects_wrong_source_without_changing_machine() { + let mut machine = SplitStateMachine::restore(split()).unwrap(); + machine.begin_parent_prepare().unwrap(); + let preparing = machine.transition().clone(); + let mut proof = split_overlay(41); + proof.source_epoch += 1; + assert!(machine.record_handoff(proof).is_err()); + assert_eq!(machine.transition(), &preparing); + let mut legacy = serde_json::to_value(&preparing).unwrap(); + legacy.as_object_mut().unwrap().remove("handoff_proof"); + let decoded: SplitTransition = serde_json::from_value(legacy).unwrap(); + assert_eq!(decoded, preparing); +} + +#[test] +fn split_readiness_extends_handoff_tail_without_replacing_base() { + let mut machine = SplitStateMachine::restore(split()).unwrap(); + machine.begin_parent_prepare().unwrap(); + machine.record_handoff(split_overlay(41)).unwrap(); + let committed = machine.transition().clone(); + let mut overlay = split_overlay(41); + overlay.cutover_seq = 43; + overlay.cutover_offset = 43; + overlay.target_stream_start_seq = 44; + let mut retained = committed.retained_parent_artifact.clone(); + retained.tail_overlay = Some(overlay.clone()); + let proof = SplitReadinessProof { + cutover_seq: 43, + parent_next_epoch: committed.parent_next_epoch, + retained_parent_artifact: retained, + retained_parent_tree_manifest: 1, + retained_parent_root_manifest_generation: 1, + retained_parent_applied_seq: 43, + child_applied_seq: 43, + child_tree_manifest: 1, + child_root_manifest_generation: 1, + retained_parent_tail_overlay: Some(overlay.clone()), + child_tail_overlay: overlay, + }; + let mut wrong = proof.clone(); + wrong.child_tail_overlay.base_root_manifest_generation += 1; + assert!(machine.record_child_ready(wrong).is_err()); + assert_eq!(machine.transition(), &committed); + machine.record_child_ready(proof).unwrap(); + assert_eq!(machine.transition().phase, SplitPhase::ChildPrepared); + assert!(committed.preserves_handoff(machine.transition())); + assert!(!machine.transition().preserves_handoff(&committed)); + let recovered = SplitStateMachine::restore(machine.transition().clone()).unwrap(); + assert_eq!(recovered.next_action(), SplitAction::PublishCatalog); +} + #[tokio::test] async fn group0_catalog_reconciles_an_ambiguous_committed_head() { let kv = Arc::new(TestKv::default()); @@ -468,7 +614,7 @@ async fn group0_split_store_resumes_prepared_child_before_catalog_cutover() { child_applied_seq: 41, child_tree_manifest: 1, child_root_manifest_generation: 1, - retained_parent_tail_overlay, + retained_parent_tail_overlay: Some(retained_parent_tail_overlay), child_tail_overlay: split_overlay(41), }) .unwrap(); diff --git a/app/crowdb-chunk-kv-server/tests/load_sampling_test.rs b/app/crowdb-chunk-kv-server/tests/load_sampling_test.rs index 3457e683d..5dd527ca9 100644 --- a/app/crowdb-chunk-kv-server/tests/load_sampling_test.rs +++ b/app/crowdb-chunk-kv-server/tests/load_sampling_test.rs @@ -15,7 +15,13 @@ use std::{ }; async fn partition(id: u64, count: u64) -> Partition { - let store = Arc::new(MemoryStreamStore::new(4096)); + partition_with_value_size(id, count, 16).await +} + +async fn partition_with_value_size(id: u64, count: u64, value_size: usize) -> Partition { + let store = Arc::new(MemoryStreamStore::new( + u64::try_from(value_size.saturating_mul(4).max(4096)).unwrap(), + )); let name = StreamName { high: 1, low: id }; let stream = ChunkStream::create( StreamBinding { @@ -54,7 +60,7 @@ async fn partition(id: u64, count: u64) -> Partition { }, MutationOperation::Put { key: format!("key-{index:05}").into_bytes(), - value: vec![1; 16], + value: vec![1; value_size], }, ) .await @@ -63,6 +69,29 @@ async fn partition(id: u64, count: u64) -> Partition { partition } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn large_values_still_supply_two_live_split_witnesses() { + let service = ChunkKvService::new(1, 8).unwrap(); + let partition = partition_with_value_size(3, 4, 64 * 1024).await; + service.install_partition(&partition).unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + let observation = service + .registry_observation_with_load_samples(1024 * 1024, 0, 2) + .await + .unwrap(); + let samples = &observation.partition_loads[0].live_byte_samples; + if samples.len() == 2 { + assert!(samples[0].0 < samples[1].0); + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn observation_bounds_samples_and_does_not_starve_other_partitions() { let service = ChunkKvService::new(1, 8).unwrap(); diff --git a/app/crowdb-chunk-kv-server/tests/split_current_assignment_test.rs b/app/crowdb-chunk-kv-server/tests/split_current_assignment_test.rs new file mode 100644 index 000000000..ffb58143e --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/split_current_assignment_test.rs @@ -0,0 +1,230 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/split_storage.rs"] +mod split_storage; + +use crowdb_chunk_kv::{ + memory::MemoryPartitionTree, MutationOperation, Partition, PartitionConfig, PartitionId, PartitionRange, + RequestId, SplitChild, SplitHandoffStore, SplitPlan, SplitWriterTarget, TransitionId, +}; +use crowdb_chunk_kv_server::ChunkKvService; +use crowdb_protocol::chunk_kv::*; +use split_storage::TestSplitStorage; +use std::sync::Arc; + +struct TestCommittedHandoff; +#[async_trait::async_trait] +impl SplitHandoffStore for TestCommittedHandoff { + async fn commit(&self, _: &crowdb_chunk_kv::PreparedSplitWriterArtifact) -> crowdb_chunk_kv::Result<()> { + Ok(()) + } +} + +fn entry(partition: &Partition) -> ChunkKvRangeCatalogEntry { + let snapshot = partition.snapshot(); + ChunkKvRangeCatalogEntry { + partition_id: Id128 { + high: snapshot.partition_id.high, + low: snapshot.partition_id.low, + }, + range: KeyRange { + start: snapshot.range.start.unwrap_or_default(), + end: snapshot.range.end, + }, + owner: OwnerDescriptor { + instance_id: 1, + rpc_endpoint: "127.0.0.1:9900".into(), + }, + owner_epoch: snapshot.ownership_epoch, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: partition.tree_id(), + stream_name: snapshot.stream_name, + tail_overlay: None, + }, + transition_id: None, + } +} + +fn publish(service: &ChunkKvService, generation: u64, partitions: &[Partition]) { + let mut page = ChunkKvRangeCatalogPage { + generation, + page_index: 0, + entries: partitions.iter().map(entry).collect(), + checksum: [0; 32], + }; + page.seal().unwrap(); + let mut head = ChunkKvRangeCatalogHead { + generation, + previous_generation: (generation > 1).then_some(generation - 1), + pages: vec![ChunkKvRangeCatalogPageRef { + page_generation: generation, + page_index: 0, + first_key: Vec::new(), + page_checksum: page.checksum, + }], + checksum: [0; 32], + }; + head.seal().unwrap(); + service + .install_catalog_and_reconcile(&head, &[page], partitions) + .unwrap(); + let mut grant = ServingGrant { + instance_id: 1, + lease_sequence: generation, + catalog_generation: generation, + issued_at_ms: 1000, + expires_at_ms: 13000, + assignments: partitions + .iter() + .map(|p| ServingAssignment { + partition_id: entry(p).partition_id, + owner_epoch: p.snapshot().ownership_epoch, + }) + .collect(), + assignment_digest: [0; 32], + }; + grant.seal(); + let descriptor = DomainMonitorDescriptor { + domain: "chunk-kv".into(), + service_registry_name: "chunk-kv".into(), + driver_version: 1, + capability_version: 1, + heartbeat_interval_ms: 2000, + suspect_after_ms: 6000, + dead_after_ms: 10000, + lease_duration_ms: 12000, + max_clock_skew_ms: 1000, + self_fence_margin_ms: 1000, + failure_policy: DomainFailurePolicy::AutomaticSharedStorage, + balance_policy: "count-first-v1".into(), + chunk_kv_range_balance: Some(ChunkKvRangeBalancePolicy::default()), + }; + service + .authority() + .install(grant, &descriptor, 1000, 50000) + .unwrap(); +} + +async fn write(partition: &Partition, sequence: u64, value: &[u8]) { + partition + .mutate( + partition.snapshot().ownership_epoch, + RequestId { + client_high: 1, + client_low: 1, + client_sequence: sequence, + }, + MutationOperation::Put { + key: b"b".to_vec(), + value: value.to_vec(), + }, + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn current_assignment_supersedes_historical_split_dispatcher() { + let storage = TestSplitStorage::new(); + let id = PartitionId { high: 90, low: 1 }; + let (parent, _) = storage + .partition( + id, + PartitionRange { + start: Some(Vec::new()), + end: None, + }, + 1, + Arc::new(MemoryPartitionTree::with_tree_id(90)), + ) + .await; + write(&parent, 1, b"old-view").await; + let service = ChunkKvService::new(1, 8).unwrap(); + publish(&service, 1, std::slice::from_ref(&parent)); + let plan = SplitPlan { + transition_id: TransitionId { high: 90, low: 2 }, + parent_id: id, + parent_range: parent.snapshot().range, + parent_epoch: 1, + parent_next_epoch: 2, + split_key: b"m".to_vec(), + child: SplitChild { + partition_id: PartitionId { high: 91, low: 1 }, + ownership_epoch: 1, + range: PartitionRange { + start: Some(b"m".to_vec()), + end: None, + }, + }, + }; + let prepared = parent + .prepare_split_child_session( + plan, + SplitWriterTarget { + tree_id: 91, + tree_config: crowdb_tree_ffi::Config::default(), + journal: storage + .journal(crowdb_chunk_stream::StreamName { high: 91, low: 1 }, 1) + .await, + }, + 8, + Arc::new(TestCommittedHandoff), + ) + .await + .unwrap(); + let child = prepared.child.open_warmed(PartitionConfig::default()).unwrap(); + service.install_partition(&child).unwrap(); + service + .record_local_split_ready(&prepared.artifact) + .await + .unwrap(); + assert!( + service.hosts_catalog_assignment(&entry(&parent)), + "the unpublished parent assignment must reuse its live split dispatcher" + ); + publish(&service, 2, std::slice::from_ref(&parent)); + let retained = parent.split_ingress().unwrap().retained_parent(); + publish(&service, 3, &[retained.clone(), child.clone()]); + let journal = storage + .journal(crowdb_chunk_stream::StreamName { high: 92, low: 1 }, 3) + .await; + let current = Partition::open( + id, + retained.snapshot().range, + 3, + PartitionConfig::default(), + Arc::new(MemoryPartitionTree::with_tree_id(90)), + journal, + ) + .unwrap(); + write(¤t, 2, b"current-view").await; + publish(&service, 4, &[current.clone(), child]); + for (map_revision, owner_epoch) in [(1, 1), (4, 3)] { + let response = service + .handle_point( + PointRequest { + routing: RequestRouting { + request_id: ClientRequestId { + client_instance_id: Id128 { high: 93, low: 1 }, + client_sequence: map_revision, + }, + map_revision, + partition_id: entry(¤t).partition_id, + owner_epoch, + min_journal_position: None, + deadline_ms: Some(2000), + }, + operation: PointOperation::Get { key: b"b".to_vec() }, + }, + 1500, + 50100, + ) + .await; + let OperationResult::Value(Some(value)) = response.result.unwrap() else { + panic!("expected current record") + }; + assert_eq!(value.value, b"current-view"); + } +} diff --git a/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs b/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs index eae2e2287..7c5dde832 100644 --- a/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs +++ b/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs @@ -77,6 +77,31 @@ async fn committed_split_proves_both_recovered_writers_and_rejects_mismatches() } } +#[tokio::test] +async fn materialized_split_half_recovers_while_sibling_keeps_the_transition_marker() { + let transition = transition(); + for retained in [true, false] { + let mut clean = entry(&transition, retained); + clean.artifact.tail_overlay = None; + let partition = recovered(&clean).await; + let service = ChunkKvService::new(1, 4).unwrap(); + install(&service, &clean, &partition); + assert_eq!(service.catalog_overlay_transition_id(clean.partition_id), None); + let sibling = entry(&transition, !retained); + assert_eq!( + service.catalog_overlay_transition_id(sibling.partition_id), + Some(transition.transition_id), + ); + service + .activate_recovered_partition(clean.partition_id, clean.owner_epoch) + .unwrap(); + assert_eq!(partition.lifecycle(), PartitionLifecycle::Serving); + assert!(service + .activate_recovered_partition(clean.partition_id, clean.owner_epoch - 1) + .is_err()); + } +} + #[tokio::test] async fn materialized_catalog_replaces_unactivated_overlay_with_independent_recovery() { let transition = transition(); @@ -184,6 +209,7 @@ fn transition() -> SplitTransition { }, planned_at_ms: 0, phase: SplitPhase::CatalogCommitted, + handoff_proof: None, readiness_proof: Some(SplitReadinessProof { cutover_seq: 1, parent_next_epoch: 2, @@ -194,7 +220,7 @@ fn transition() -> SplitTransition { child_applied_seq: 1, child_tree_manifest: 1, child_root_manifest_generation: 1, - retained_parent_tail_overlay: overlay.clone(), + retained_parent_tail_overlay: Some(overlay.clone()), child_tail_overlay: overlay, }), failure: None, @@ -352,6 +378,11 @@ fn install(service: &ChunkKvService, entry: &ChunkKvRangeCatalogEntry, partition entries: vec![self::entry(&transition, true), self::entry(&transition, false)], checksum: [0; 32], }; + *page + .entries + .iter_mut() + .find(|current| current.partition_id == entry.partition_id) + .unwrap() = entry.clone(); page.seal().unwrap(); let mut head = ChunkKvRangeCatalogHead { generation: 1, diff --git a/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs b/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs new file mode 100644 index 000000000..2600f7a6e --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs @@ -0,0 +1,167 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_chunk_kv_server::storage::open_root_catalog_for_tests; +use crowdb_chunk_kv_server::{ChunkKvRangeCatalogPublisher, Group0ControlStore}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, + ChunkKvRangeCatalogPartitionState, Id128, KeyRange, OwnerDescriptor, PartitionArtifact, +}; +use crowdb_protocol::chunk_stream::StreamName; +use crowdb_test_harness::cluster::KvCluster; +use crowdb_tree_ffi::{CtError, RootCatalogObject}; + +fn source() -> ChunkKvRangeCatalogEntry { + ChunkKvRangeCatalogEntry { + partition_id: Id128 { high: 1, low: 1 }, + range: KeyRange { + start: Vec::new(), + end: None, + }, + owner: OwnerDescriptor { + instance_id: 1, + rpc_endpoint: "127.0.0.1:1".into(), + }, + owner_epoch: 1, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 42, + stream_name: StreamName { high: 1, low: 2 }, + tail_overlay: None, + }, + transition_id: None, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn preparing_target_does_not_fence_source_or_mutate_shared_root() { + let cluster = KvCluster::start().await; + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + cluster.mgmt_endpoints.clone(), + ))); + let source_entry = source(); + let source = open_root_catalog_for_tests(kv.clone(), 0, 1, &source_entry) + .await + .unwrap(); + publish_assignment(kv.clone(), source_entry.clone(), 1).await; + source.publish(42, 0, 1, 1, b"source-checkpoint-1").unwrap(); + let mut target_entry = source_entry.clone(); + target_entry.owner.instance_id = 2; + target_entry.owner_epoch = 2; + target_entry.state = ChunkKvRangeCatalogPartitionState::Prepared; + target_entry.artifact.stream_name.low = 3; + target_entry.transition_id = Some(Id128 { high: 2, low: 2 }); + let target = open_root_catalog_for_tests(kv.clone(), 0, 1, &target_entry) + .await + .unwrap(); + assert_eq!( + target.load(42, RootCatalogObject::Manifest(1)).unwrap(), + Some(b"source-checkpoint-1".to_vec()) + ); + assert_eq!( + target.publish(42, 1, 2, 2, b"unpublished-target"), + Err(CtError::Unavailable) + ); + // Private preparation metadata and exact pin cleanup do not publish a root. + let reference = target.allocate_reference_segment_id(42).unwrap(); + target + .store( + 42, + RootCatalogObject::ReferenceSegment(reference), + b"prepared-reference", + ) + .unwrap(); + source.pin_generation(42, 2, 2, 1).unwrap(); + target.unpin_generation(42, 2, 2).unwrap(); + source.publish(42, 1, 1, 2, b"source-checkpoint-2").unwrap(); + drop(source); + let recovered = open_root_catalog_for_tests(kv, 0, 1, &source_entry) + .await + .unwrap(); + assert_eq!( + recovered.load(42, RootCatalogObject::CurrentManifest).unwrap(), + Some(b"source-checkpoint-2".to_vec()) + ); + recovered.publish(42, 2, 1, 3, b"source-checkpoint-3").unwrap(); +} + +async fn publish_assignment(kv: Arc, entry: ChunkKvRangeCatalogEntry, generation: u64) { + let mut page = ChunkKvRangeCatalogPage { + generation, + page_index: 0, + entries: vec![entry], + checksum: [0; 32], + }; + page.seal().unwrap(); + let mut head = ChunkKvRangeCatalogHead { + generation, + previous_generation: (generation > 1).then_some(generation - 1), + pages: vec![ChunkKvRangeCatalogPageRef { + page_generation: generation, + page_index: 0, + first_key: Vec::new(), + page_checksum: page.checksum, + }], + checksum: [0; 32], + }; + head.seal().unwrap(); + ChunkKvRangeCatalogPublisher::new(Arc::new(Group0ControlStore::from_client(kv))) + .publish(head, vec![page]) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn only_published_matching_target_can_claim_mutable_root() { + let cluster = KvCluster::start().await; + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + cluster.mgmt_endpoints.clone(), + ))); + let source_entry = source(); + let source = open_root_catalog_for_tests(kv.clone(), 0, 1, &source_entry) + .await + .unwrap(); + source.publish(42, 0, 1, 1, b"source-root").unwrap(); + publish_assignment(kv.clone(), source_entry.clone(), 1).await; + let mut target_entry = source_entry.clone(); + target_entry.owner.instance_id = 2; + target_entry.owner_epoch = 2; + target_entry.artifact.stream_name.low = 3; + target_entry.transition_id = Some(Id128 { high: 3, low: 4 }); + target_entry.state = ChunkKvRangeCatalogPartitionState::Prepared; + let target = open_root_catalog_for_tests(kv.clone(), 0, 1, &target_entry) + .await + .unwrap(); + assert_eq!( + target.publish(42, 1, 2, 2, b"target-root"), + Err(CtError::Unavailable) + ); + let mut wrong = target_entry.clone(); + wrong.state = ChunkKvRangeCatalogPartitionState::TargetCatchingUp; + wrong.transition_id = Some(Id128 { high: 3, low: 5 }); + publish_assignment(kv.clone(), wrong, 2).await; + assert_eq!( + target.publish(42, 1, 2, 2, b"target-root"), + Err(CtError::Unavailable) + ); + source.publish(42, 1, 1, 2, b"source-root-2").unwrap(); + target_entry.state = ChunkKvRangeCatalogPartitionState::TargetCatchingUp; + publish_assignment(kv.clone(), target_entry.clone(), 3).await; + assert_eq!( + target.publish(42, 2, 2, 3, b"target-root"), + Err(CtError::Unavailable) + ); + target_entry.state = ChunkKvRangeCatalogPartitionState::Serving; + publish_assignment(kv.clone(), target_entry, 4).await; + target.publish(42, 2, 2, 3, b"target-root").unwrap(); + assert_eq!( + source.publish(42, 3, 1, 4, b"stale-source"), + Err(CtError::Unavailable) + ); + assert!(open_root_catalog_for_tests(kv, 0, 1, &source_entry) + .await + .is_err()); +} diff --git a/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs b/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs index 07195a306..2be74865e 100644 --- a/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs +++ b/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs @@ -334,6 +334,7 @@ fn split_transition() -> SplitTransition { }, planned_at_ms: 0, phase: SplitPhase::ParentPreparing, + handoff_proof: None, readiness_proof: None, failure: None, } @@ -733,6 +734,7 @@ async fn repeated_local_split_uses_current_retained_writer() { }, planned_at_ms: 0, phase: SplitPhase::ParentPreparing, + handoff_proof: None, readiness_proof: None, failure: None, }; @@ -880,3 +882,49 @@ async fn processor_resumes_planned_split_through_durable_readiness() { assert_eq!(stored.readiness_proof.unwrap().cutover_seq, 8); assert_eq!(revision, 3); } + +#[tokio::test] +async fn failed_transfer_does_not_starve_independent_split_processing() { + let store = Arc::new(Group0ControlStore::new(Arc::new(MemoryKv::default()))); + let mut failing = transfer(TransferPhase::SourcePreparing); + failing.partition_id = id(404); + store.persist_transfer_transition(&failing, 0).await.unwrap(); + let mut split = split_transition(); + split.phase = SplitPhase::Planned; + store.persist_split_transition(&split, 0).await.unwrap(); + let parent = partition( + split.parent_id, + split.parent_range.clone(), + split.parent_epoch, + &split.parent_artifact, + false, + ) + .await; + let service = Arc::new(ChunkKvService::new(1, 4).unwrap()); + service.install_partition(&parent).unwrap(); + let executor = Arc::new( + TransitionExecutor::with_storage( + 1, + service, + Arc::new(FakeStorage { + recovered: partition(id(9), KeyRange::default(), 1, &artifact(99), true).await, + split: split_artifact(&split), + expected_split_parent_range: None, + }), + 8, + ) + .unwrap(), + ); + let processor = TransitionProcessor::new(1, store.clone(), executor); + let failure = processor.tick().await.unwrap_err(); + assert!(failure + .to_string() + .contains("transfer source partition is not hosted")); + let (stored, _) = store + .load_split_transition(split.transition_id) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.phase, SplitPhase::ChildPrepared); + assert_eq!(stored.readiness_proof.unwrap().cutover_seq, 8); +} diff --git a/app/crowdb-chunkdb/src/allocator/pool.rs b/app/crowdb-chunkdb/src/allocator/pool.rs index 890dd2471..2a4709942 100644 --- a/app/crowdb-chunkdb/src/allocator/pool.rs +++ b/app/crowdb-chunkdb/src/allocator/pool.rs @@ -157,7 +157,9 @@ impl DiskdbClientPool { ) -> Result { if matches!( result, - Err(DiskdbClientError::Unreachable(_) | DiskdbClientError::NotOwner(_)) + Err(DiskdbClientError::ConnectFailed(_) + | DiskdbClientError::Unreachable(_) + | DiskdbClientError::NotOwner(_)) ) { // Allocation may already have succeeded when the reply was lost. // Refresh for the next supervised attempt, but never replay this @@ -169,14 +171,14 @@ impl DiskdbClientPool { /// Allocate blocks on the diskdb instance owning `disk_group_id`. /// - /// The mutation is sent once. A transport failure is ambiguous because - /// DiskDB may already have persisted the tentative blocks, so retrying - /// here could allocate a second physical set for the same chunk. + /// A connection failure before submission refreshes discovery and permits + /// one attempt at a different endpoint. Once submitted, the mutation is + /// never replayed: DiskDB may have persisted blocks before losing its reply. /// /// # Errors - /// Returns `DiskdbClientError::Unreachable` if the endpoint is not cached - /// or the RPC transport fails, or `DiskdbClientError::Rpc` for a server - /// error. + /// Returns `DiskdbClientError::ConnectFailed` when connection setup fails, + /// `DiskdbClientError::Unreachable` for missing routing or an ambiguous + /// transport failure, or `DiskdbClientError::Rpc` for a server error. pub async fn allocate_blocks( &self, dg_id: u64, @@ -205,11 +207,7 @@ impl DiskdbClientPool { allow_disk_reuse: false, }; - let endpoint = self.endpoint_for_dg(dg_id).await.map_err(|e| { - DiskdbClientError::Unreachable(format!("no endpoint for disk_group {dg_id}: {e}")) - })?; - self.refresh_after_owner_failure(self.transport.allocate_blocks(&endpoint, &req).await) - .await + self.allocate_at_owner(&req).await } pub async fn allocate_blocks_reusing_disks( @@ -227,11 +225,31 @@ impl DiskdbClientPool { owner_chunk: Some(*owner_chunk), allow_disk_reuse: true, }; + self.allocate_at_owner(&req).await + } + + async fn allocate_at_owner( + &self, + request: &AllocateBlocksRequest, + ) -> Result { + let dg_id = request.disk_group_id; let endpoint = self.endpoint_for_dg(dg_id).await.map_err(|error| { DiskdbClientError::Unreachable(format!("no endpoint for disk_group {dg_id}: {error}")) })?; - self.refresh_after_owner_failure(self.transport.allocate_blocks(&endpoint, &req).await) - .await + let result = self.transport.allocate_blocks(&endpoint, request).await; + if matches!(result, Err(DiskdbClientError::ConnectFailed(_))) { + if self.refresh_endpoints().await.is_err() { + return result; + } + let replacement = self.endpoints.load().get(&dg_id).cloned(); + if let Some(replacement) = replacement.filter(|replacement| *replacement != endpoint) { + return self + .refresh_after_owner_failure(self.transport.allocate_blocks(&replacement, request).await) + .await; + } + return result; + } + self.refresh_after_owner_failure(result).await } /// Deliver an already-reserved target to its owning `DiskDB` for durable @@ -367,6 +385,8 @@ impl DiskdbClientPool { /// Replace the endpoint snapshot atomically in integration tests. pub fn replace_endpoints_for_tests(&self, endpoints: Vec<(u64, String)>) { self.endpoints.store(Arc::new(endpoints.into_iter().collect())); + self.last_registry_refresh_ms + .store(unix_time_ms(), Ordering::Release); } /// Return one complete endpoint snapshot in stable order. diff --git a/app/crowdb-chunkdb/tests/allocator_pool_test.rs b/app/crowdb-chunkdb/tests/allocator_pool_test.rs index e61858e0d..b028e94dc 100644 --- a/app/crowdb-chunkdb/tests/allocator_pool_test.rs +++ b/app/crowdb-chunkdb/tests/allocator_pool_test.rs @@ -37,3 +37,79 @@ fn endpoint_refresh_is_one_complete_publication() { } writer.join().expect("refresh thread panicked"); } + +#[tokio::test] +async fn allocation_connect_failure_reroutes_but_lost_reply_does_not() { + use crowdb_diskdb_client::{DiskdbClientError, DiskdbRpcTransport}; + use crowdb_test_harness::cluster::KvCluster; + use crowdb_test_harness::diskdb::{make_chunk_id, require_binaries, DiskdbProcess}; + use crowdb_test_harness::hardware::{seed_hardware, standard_disk_ids_3, DG_ID, UNIT_SIZE_BYTES}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + require_binaries(); + let cluster = KvCluster::start().await; + seed_hardware(&cluster.make_hardware_client(), &standard_disk_ids_3()).await; + let diskdb = DiskdbProcess::start(&cluster.mgmt_endpoints, true); + diskdb.wait_for_ready().await; + let pool = DiskdbClientPool::new(cluster.make_service_registry_client()); + pool.refresh_endpoints().await.expect("discover live owner"); + let endpoint = pool + .endpoint_snapshot_for_tests() + .into_iter() + .find(|(group, _)| *group == DG_ID) + .expect("DiskDB advertises group") + .1; + + let unavailable = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let stale = unavailable.local_addr().unwrap().to_string(); + drop(unavailable); + pool.replace_endpoints_for_tests(vec![(DG_ID, stale)]); + let allocated = pool + .allocate_blocks(DG_ID, 1, 1, &make_chunk_id(0, 41)) + .await + .expect("unsent request reroutes to discovered owner"); + assert_eq!(allocated.segments.len(), 1); + + // The proxy forwards the allocation, then closes without its response. + // Group-0 still advertises the direct endpoint, so an unsafe replay would + // allocate another set there. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy = listener.local_addr().unwrap().to_string(); + let upstream_endpoint = endpoint.clone(); + let forward = tokio::spawn(async move { + let (downstream, _) = listener.accept().await.unwrap(); + let upstream = tokio::net::TcpStream::connect(upstream_endpoint).await.unwrap(); + let (mut input, mut output) = downstream.into_split(); + let (mut responses, mut requests) = upstream.into_split(); + let send = tokio::io::copy(&mut input, &mut requests); + let discard = async { + let mut buffer = [0; 8192]; + assert!(responses.read(&mut buffer).await.unwrap() > 0); + output.shutdown().await.unwrap(); + }; + tokio::select! { + _ = send => panic!("request stream ended before response"), + () = discard => {} + } + }); + pool.replace_endpoints_for_tests(vec![(DG_ID, proxy)]); + let result = pool + .allocate_blocks_reusing_disks(DG_ID, 1, 1, &make_chunk_id(0, 42)) + .await; + assert!( + matches!(result, Err(DiskdbClientError::Unreachable(_))), + "{result:?}" + ); + forward.await.unwrap(); + + let transport = DiskdbRpcTransport::new(); + let info = transport + .get_disk_group_info(&endpoint, DG_ID) + .await + .expect("query durable allocations"); + assert_eq!( + info.group.expect("group exists").busy_bytes, + 2 * u64::from(UNIT_SIZE_BYTES), + "one rerouted allocation and one allocation with a lost reply" + ); +} diff --git a/app/crowdb-chunkdb/tests/common/cluster.rs b/app/crowdb-chunkdb/tests/common/cluster.rs index db02fd17a..e8ebb27a5 100644 --- a/app/crowdb-chunkdb/tests/common/cluster.rs +++ b/app/crowdb-chunkdb/tests/common/cluster.rs @@ -369,6 +369,10 @@ async fn start_kv_node_with_groups( &listen_port.to_string(), "--election-profile", "e2e", + "--kv-backend", + "block", + "--wal-backend", + "block-device", ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/app/crowdb-cli/src/commands/cluster.rs b/app/crowdb-cli/src/commands/cluster.rs index d21066d7e..3b4e63fa2 100644 --- a/app/crowdb-cli/src/commands/cluster.rs +++ b/app/crowdb-cli/src/commands/cluster.rs @@ -338,6 +338,7 @@ pub async fn run_cluster_verb(cli: &Cli, verb: ClusterVerb) -> ExitCode { no_fsync: no_fsync.then_some(true), }; let disk = crowdb_console_shared::ops::cluster::LocalDiskdbDeployConfig { + keepalive_interval_secs: None, disk_groups_per_node, disks_per_group, capacity_bytes: disk_capacity_bytes, @@ -353,6 +354,7 @@ pub async fn run_cluster_verb(cli: &Cli, verb: ClusterVerb) -> ExitCode { let chunk = crowdb_console_shared::ops::cluster::LocalChunkdbDeployConfig { instance_count: chunkdb_instances, storage_groups: vec![1, 2, 3], + dynamic_ownership: false, allow_unsafe_ec, rpc_workers: nonzero(rpc_workers), diskio_rpc_workers: nonzero(diskio_rpc_workers), @@ -490,6 +492,7 @@ pub async fn run_cluster_verb(cli: &Cli, verb: ClusterVerb) -> ExitCode { let workspace = deploy_workspace(cli).unwrap_or_else(|| std::path::PathBuf::from("cli-deploy")); let config = crowdb_console_shared::ops::cluster::LocalDiskdbDeployConfig { + keepalive_interval_secs: None, disk_groups_per_node, disks_per_group, capacity_bytes: disk_capacity_bytes, @@ -531,6 +534,7 @@ pub async fn run_cluster_verb(cli: &Cli, verb: ClusterVerb) -> ExitCode { let chunk = crowdb_console_shared::ops::cluster::LocalChunkdbDeployConfig { instance_count: chunkdb_instances, storage_groups: vec![1, 2, 3], + dynamic_ownership: false, allow_unsafe_ec, rpc_workers: nonzero(rpc_workers), diskio_rpc_workers: nonzero(diskio_rpc_workers), diff --git a/app/crowdb-cli/tests/s3_cli_test.rs b/app/crowdb-cli/tests/s3_cli_test.rs index 1ee9da093..45996bfcf 100644 --- a/app/crowdb-cli/tests/s3_cli_test.rs +++ b/app/crowdb-cli/tests/s3_cli_test.rs @@ -13,7 +13,7 @@ fn cli() -> Command { } #[test] -#[ignore = "starts the complete local storage stack twice"] +#[cfg_attr(target_os = "macos", ignore = "starts the complete local storage stack twice")] fn interrupted_s3_launch_resumes_confirmed_group_zero_without_topology_file() { let directory = TestDir::new("s3-bootstrap-replay-cli").expect("test directory"); let root = directory.path(); @@ -58,7 +58,7 @@ fn interrupted_s3_launch_resumes_confirmed_group_zero_without_topology_file() { } #[test] -#[ignore = "starts the complete local storage stack twice"] +#[cfg_attr(target_os = "macos", ignore = "starts the complete local storage stack twice")] fn interrupted_s3_storage_launch_replays_without_local_topology() { let directory = TestDir::new("s3-storage-replay-cli").expect("test directory"); let root = directory.path(); diff --git a/app/crowdb-diskdb/src/ddb_config.rs b/app/crowdb-diskdb/src/ddb_config.rs index b4761495d..9bc85a74d 100644 --- a/app/crowdb-diskdb/src/ddb_config.rs +++ b/app/crowdb-diskdb/src/ddb_config.rs @@ -513,3 +513,27 @@ pub fn validate(config: &DdbConfig) -> Result<(), String> { fn is_power_of_two(n: u32) -> bool { n > 0 && n.is_power_of_two() } + +impl DdbConfig { + /// Keep ownership observation at least as frequent as configured heartbeats. + /// The ordinary controller retains its five-second observation cadence. + #[must_use] + pub fn ownership_monitor_descriptor(&self) -> crowdb_protocol::chunk_kv::DomainMonitorDescriptor { + use crowdb_protocol::chunk_kv::{DomainFailurePolicy, DomainMonitorDescriptor}; + DomainMonitorDescriptor { + domain: "diskdb-ownership".into(), + service_registry_name: "diskdb".into(), + driver_version: 1, + capability_version: 1, + heartbeat_interval_ms: u64::from(self.heartbeat.interval_secs.min(5)) * 1_000, + suspect_after_ms: 10_000, + dead_after_ms: 15_000, + lease_duration_ms: 20_000, + max_clock_skew_ms: 1_000, + self_fence_margin_ms: 1_000, + failure_policy: DomainFailurePolicy::AutomaticSharedStorage, + balance_policy: "disk-group-count-v1".into(), + chunk_kv_range_balance: None, + } + } +} diff --git a/app/crowdb-diskdb/src/main.rs b/app/crowdb-diskdb/src/main.rs index d024473e0..692eddf31 100644 --- a/app/crowdb-diskdb/src/main.rs +++ b/app/crowdb-diskdb/src/main.rs @@ -28,9 +28,7 @@ use crowdb_kv_client::{ ClientConfig, CrowdbKvClient, DomainMonitorClient, HardwareClient, RangeBindingClient as KvRangeBindingClient, ServiceRegistryClient, WatchNotifyClient, }; -use crowdb_protocol::chunk_kv::{ - DomainFailurePolicy, DomainMonitorDescriptor, EnsureDomainMonitorOutcome, EnsureDomainMonitorRequest, -}; +use crowdb_protocol::chunk_kv::{EnsureDomainMonitorOutcome, EnsureDomainMonitorRequest}; use tracing::{error, info, warn}; /// CROWDB diskdb server CLI. @@ -229,21 +227,7 @@ async fn main() { Arc::new(DiskdbRpcTransport::new()), )); let monitor_request = EnsureDomainMonitorRequest { - descriptor: DomainMonitorDescriptor { - domain: "diskdb-ownership".into(), - service_registry_name: "diskdb".into(), - driver_version: 1, - capability_version: 1, - heartbeat_interval_ms: 5_000, - suspect_after_ms: 10_000, - dead_after_ms: 15_000, - lease_duration_ms: 20_000, - max_clock_skew_ms: 1_000, - self_fence_margin_ms: 1_000, - failure_policy: DomainFailurePolicy::AutomaticSharedStorage, - balance_policy: "disk-group-count-v1".into(), - chunk_kv_range_balance: None, - }, + descriptor: config.load().ownership_monitor_descriptor(), }; // Domain monitor registration is retried in the background so the // diskdb can start serving even when the KV server (group-0) is not diff --git a/app/crowdb-diskdb/tests/common/cluster.rs b/app/crowdb-diskdb/tests/common/cluster.rs index e91f10b22..18b5cc3cd 100644 --- a/app/crowdb-diskdb/tests/common/cluster.rs +++ b/app/crowdb-diskdb/tests/common/cluster.rs @@ -355,6 +355,10 @@ async fn start_kv_node_with_groups( &listen_port.to_string(), "--election-profile", "e2e", + "--kv-backend", + "block", + "--wal-backend", + "block-device", ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/app/crowdb-diskdb/tests/ddb_config_test.rs b/app/crowdb-diskdb/tests/ddb_config_test.rs index 84d19010f..1678c8521 100644 --- a/app/crowdb-diskdb/tests/ddb_config_test.rs +++ b/app/crowdb-diskdb/tests/ddb_config_test.rs @@ -246,3 +246,16 @@ fn partial_heartbeat_override_retains_other_liveness_defaults() { ); validate(&config).unwrap(); } + +#[test] +fn ownership_monitor_observes_fast_heartbeats_without_changing_failure_thresholds() { + let mut config = crowdb_diskdb::ddb_config::DdbConfig::default(); + let ordinary = config.ownership_monitor_descriptor(); + assert_eq!(ordinary.heartbeat_interval_ms, 5_000); + config.heartbeat.interval_secs = 1; + let accelerated = config.ownership_monitor_descriptor(); + assert_eq!(accelerated.heartbeat_interval_ms, 1_000); + assert_eq!(accelerated.suspect_after_ms, ordinary.suspect_after_ms); + assert_eq!(accelerated.dead_after_ms, ordinary.dead_after_ms); + assert_eq!(accelerated.lease_duration_ms, ordinary.lease_duration_ms); +} diff --git a/app/crowdb-diskio/CMakeLists.txt b/app/crowdb-diskio/CMakeLists.txt index dafd2682e..534fc83b6 100644 --- a/app/crowdb-diskio/CMakeLists.txt +++ b/app/crowdb-diskio/CMakeLists.txt @@ -78,6 +78,12 @@ enable_testing() include(GoogleTest) find_package(GTest REQUIRED) file(GLOB_RECURSE CROWDB_DISKIO_TEST_SRC CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/tests/*.cpp) +if(APPLE) + list(REMOVE_ITEM CROWDB_DISKIO_TEST_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/tests/uring_engine_test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/sq_full_test.cpp + ) +endif() if(CROWDB_DISKIO_TEST_SRC) list(APPEND CROWDB_DISKIO_TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/crowdb-common/cpp/tests/test_main.cpp) add_executable(crowdb_diskio_tests ${CROWDB_DISKIO_TEST_SRC}) @@ -87,5 +93,5 @@ if(CROWDB_DISKIO_TEST_SRC) ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/crowdb-common/cpp/include ) target_link_libraries(crowdb_diskio_tests PRIVATE crowdb-diskio-lib GTest::gtest) - gtest_discover_tests(crowdb_diskio_tests DISCOVERY_MODE PRE_TEST) + gtest_discover_tests(crowdb_diskio_tests DISCOVERY_MODE PRE_TEST DISCOVERY_TIMEOUT 60) endif() diff --git a/app/crowdb-diskio/src/disk/mem_disk.cpp b/app/crowdb-diskio/src/disk/mem_disk.cpp index 84b6f2803..cbe16e311 100644 --- a/app/crowdb-diskio/src/disk/mem_disk.cpp +++ b/app/crowdb-diskio/src/disk/mem_disk.cpp @@ -32,8 +32,17 @@ int create_memfd(int64_t capacity) } return fd; #else - (void)capacity; - return -1; + char path[] = "/tmp/crowdb-mem-disk-XXXXXX"; + int fd = ::mkstemp(path); + if (fd < 0) { + return -1; + } + ::unlink(path); + if (::ftruncate(fd, capacity) < 0) { + ::close(fd); + return -1; + } + return fd; #endif } diff --git a/app/crowdb-diskio/tests/dio_startup_test.cpp b/app/crowdb-diskio/tests/dio_startup_test.cpp index 393540406..ea5d093a3 100644 --- a/app/crowdb-diskio/tests/dio_startup_test.cpp +++ b/app/crowdb-diskio/tests/dio_startup_test.cpp @@ -18,6 +18,9 @@ #include #include #include +#ifdef __APPLE__ +# include +#endif #include #include @@ -45,11 +48,18 @@ std::string find_binary() // The test executable is at app/crowdb-diskio/build/crowdb_diskio_tests. // The binary is at app/crowdb-diskio/build/crowdb-diskio. char buf[4096]; +#ifdef __APPLE__ + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) != 0) { + return ""; + } +#else ssize_t n = readlink("/proc/self/exe", buf, sizeof(buf) - 1); if (n <= 0) { return ""; } buf[n] = '\0'; +#endif std::string path(buf); // Replace "crowdb_diskio_tests" with "crowdb-diskio". size_t pos = path.rfind("crowdb_diskio_tests"); diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs index 9eed60f8e..bbf79862a 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Persisted automatic split and count-first placement planning. +//! Persisted automatic split and unified-weight placement planning. use std::collections::{HashMap, HashSet}; @@ -11,7 +11,7 @@ use crowdb_protocol::chunk_kv::{ ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, DomainMonitorDescriptor, Id128, OwnerDescriptor, SplitPhase, TransferPhase, TransferTransition, }; -use crowdb_protocol::common::{ChunkKvExtra, InstanceValue}; +use crowdb_protocol::common::{ChunkKvExtra, ChunkKvPartitionLoad, InstanceValue}; use crowdb_protocol::key::{ChunkKvSplitKey, ChunkKvTransferKey, TextKey}; use tracing::info; @@ -21,6 +21,7 @@ use super::{ catalog, operation_error, read_instances, read_splits, read_transfers, transfer_id, wall_time_ms, }; +mod observation; mod selection; mod split; @@ -29,6 +30,9 @@ use split::split_transition; struct PlanningState { healthy: HashMap, + partition_loads: HashMap<(u64, Id128), ChunkKvPartitionLoad>, + partition_bytes: HashMap<(u64, Id128), u64>, + hosted_epochs: HashMap<(u64, Id128), u64>, active_partitions: HashSet, busy_owners: HashSet, last_changed_ms: HashMap, @@ -86,12 +90,20 @@ pub async fn plan(control: &Group0ControlPlane, descriptor: &DomainMonitorDescri return Ok(()); }; policy.validate().map_err(|error| error.to_string())?; - if plan_transfer(control, &entries, &state, policy, now_ms, true).await? - || plan_split(control, &entries, &state, policy, now_ms).await? - { - return Ok(()); + let selection = choose_transfer( + &entries, + &state, + policy, + now_ms, + catalog.head.generation, + descriptor.suspect_after_ms, + ); + if let Some((entry, target_id)) = selection.candidate { + plan_transfer(control, entry, target_id, &state, now_ms).await?; + } else { + plan_split(control, &entries, &state, policy, now_ms).await?; } - plan_transfer(control, &entries, &state, policy, now_ms, false).await?; + observation::publish(control, &selection.observation).await; Ok(()) } @@ -114,8 +126,39 @@ async fn planning_state( }; healthy.insert(instance.instance_id, (instance, extra)); } + let partition_loads = healthy + .iter() + .flat_map(|(id, (_, extra))| { + extra + .partition_loads + .iter() + .map(move |load| ((*id, load.partition_id), load.clone())) + }) + .collect(); + let partition_bytes = healthy + .iter() + .flat_map(|(id, (_, extra))| { + extra + .partition_loads + .iter() + .map(move |load| ((*id, load.partition_id), effective_bytes(load))) + }) + .collect(); + let hosted_epochs = healthy + .iter() + .flat_map(|(id, (_, extra))| { + extra + .hosted + .iter() + .filter(|hosted| !hosted.recovering) + .map(move |hosted| ((*id, hosted.partition_id), hosted.owner_epoch)) + }) + .collect(); let mut state = PlanningState { healthy, + partition_loads, + partition_bytes, + hosted_epochs, active_partitions: HashSet::new(), busy_owners: HashSet::new(), last_changed_ms: HashMap::new(), @@ -186,7 +229,7 @@ async fn plan_split( let Some(load) = partition_load(state, entry) else { continue; }; - let effective_bytes = effective_bytes(load); + let effective_bytes = state.partition_bytes[&(entry.owner.instance_id, entry.partition_id)]; if (count_shortfall || effective_bytes > policy.target_partition_bytes) && eligible(entry, state) && cooled_down(entry, &state.last_changed_ms, policy, now_ms) @@ -230,18 +273,11 @@ async fn plan_split( async fn plan_transfer( control: &Group0ControlPlane, - entries: &[&ChunkKvRangeCatalogEntry], + entry: &ChunkKvRangeCatalogEntry, + target_id: u64, state: &PlanningState, - policy: &ChunkKvRangeBalancePolicy, now_ms: u64, - count_only: bool, ) -> Result { - let (counts, bytes) = owner_loads(entries, state); - let Some((entry, target_id)) = - choose_transfer(entries, state, policy, now_ms, &counts, &bytes, count_only) - else { - return Ok(false); - }; let (target, _) = state .healthy .get(&target_id) @@ -301,31 +337,6 @@ async fn plan_transfer( // Preparation and forwarding bounds are independent of placement pacing. const TRANSFER_SAFETY_WINDOW_MS: u64 = 10 * 60 * 1_000; -fn owner_loads( - entries: &[&ChunkKvRangeCatalogEntry], - state: &PlanningState, -) -> (HashMap, HashMap) { - let mut counts = state - .healthy - .keys() - .map(|owner| (*owner, 0_usize)) - .collect::>(); - let mut bytes = state - .healthy - .keys() - .map(|owner| (*owner, 0_u64)) - .collect::>(); - for entry in entries { - if counts.contains_key(&entry.owner.instance_id) { - *counts.entry(entry.owner.instance_id).or_default() += 1; - let load = partition_load(state, entry).map_or(0, effective_bytes); - let owner_bytes = bytes.entry(entry.owner.instance_id).or_default(); - *owner_bytes = owner_bytes.saturating_add(load); - } - } - (counts, bytes) -} - async fn persist_new( control: &Group0ControlPlane, path: String, diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/observation.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/observation.rs new file mode 100644 index 000000000..85add281e --- /dev/null +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/observation.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Diagnostic publication is separate from topology and cannot prevent progress. + +use crate::group0_control_plane::Group0ControlPlane; +use bytes::Bytes; +use crowdb_protocol::chunk_kv::balance::{BalanceObservation, MAX_OBSERVATION_BYTES, OBSERVATION_KEY}; + +pub(super) async fn publish(control: &Group0ControlPlane, observation: &BalanceObservation) { + if let Err(error) = persist(control, observation).await { + tracing::warn!(%error, "balance diagnostic observation unavailable"); + } +} + +async fn persist(control: &Group0ControlPlane, observation: &BalanceObservation) -> Result<(), String> { + let encoded = serde_json::to_vec(observation).map_err(|error| error.to_string())?; + if encoded.len() > MAX_OBSERVATION_BYTES { + return Err("balance diagnostic byte budget exceeded".into()); + } + let current = control + .get(OBSERVATION_KEY.as_bytes()) + .await + .map_err(|error| error.to_string())?; + if let Some(prior) = current + .value + .as_deref() + .and_then(|value| serde_json::from_slice::(value).ok()) + { + let mut comparable = observation.clone(); + comparable.observed_at_ms = prior.observed_at_ms; + if comparable == prior + && observation.observed_at_ms.saturating_sub(prior.observed_at_ms) < observation.valid_for_ms / 2 + { + return Ok(()); + } + } + control + .compare_and_put( + Bytes::from_static(OBSERVATION_KEY.as_bytes()), + Bytes::from(encoded), + current.revision, + ) + .await + .map_err(|error| error.to_string())?; + Ok(()) +} diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection.rs index 14ccb4590..76b90b092 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection.rs @@ -8,57 +8,142 @@ use crowdb_protocol::chunk_kv::{ use crowdb_protocol::common::{ChunkKvExtra, ChunkKvPartitionLoad}; use std::collections::HashMap; +mod candidates; +mod diagnostics; +use candidates::CandidateContext; + +pub(super) struct Selection<'a> { + pub candidate: Option<(&'a ChunkKvRangeCatalogEntry, u64)>, + pub observation: crowdb_protocol::chunk_kv::balance::BalanceObservation, +} + pub(super) fn choose_transfer<'a>( entries: &[&'a ChunkKvRangeCatalogEntry], state: &PlanningState, policy: &ChunkKvRangeBalancePolicy, now_ms: u64, - counts: &HashMap, - bytes: &HashMap, - count_only: bool, -) -> Option<(&'a ChunkKvRangeCatalogEntry, u64)> { + generation: u64, + valid_for_ms: u64, +) -> Selection<'a> { + use crowdb_protocol::chunk_kv::balance::{ + BalancePartitionObservation, BalanceWeights, MAX_OBSERVED_OWNERS, MAX_OBSERVED_PARTITIONS, + WEIGHT_SCALE, + }; + let mut result = Selection { + candidate: None, + observation: diagnostics::empty(policy, now_ms, generation, valid_for_ms), + }; + let Some((counts, bytes)) = owner_loads(entries, state) else { + return result; + }; + let loads: Vec<_> = counts.iter().map(|(id, count)| (*count, bytes[id])).collect(); + let Some(weights) = BalanceWeights::new(&loads, policy.byte_weight_percent) else { + return result; + }; + result.observation.deviation_percent_millionths = + u64::try_from(weights.deviation_units * 100_000_000 / u128::from(WEIGHT_SCALE)).unwrap_or(u64::MAX); + result.observation.loss_millionths = + Some(u64::try_from(weights.loss / 1_000_000_000_000).unwrap_or(u64::MAX)); + let within = weights.within_tolerance(policy.imbalance_tolerance_percent); + result.observation.reason = if within { + "within tolerance" + } else { + "no useful safe move" + } + .into(); + if counts.len() <= MAX_OBSERVED_OWNERS { + result.observation.owners = diagnostics::owners(&counts, &bytes, &weights, state); + } + let context = CandidateContext { + state, + policy, + weights: &weights, + counts: &counts, + bytes: &bytes, + now_ms, + }; let mut best = None; - for entry in entries.iter().copied().filter(|entry| { - eligible(entry, state) && cooled_down(entry, &state.last_transferred_ms, policy, now_ms) - }) { - let source_id = entry.owner.instance_id; - let partition_bytes = partition_load(state, entry).map_or(0, effective_bytes); - for (&target_id, (_, target)) in &state.healthy { - if !target_accepts(state, policy, source_id, target_id, target, partition_bytes) { - continue; - } - let source_count = counts.get(&source_id).copied().unwrap_or_default(); - let target_count = counts.get(&target_id).copied().unwrap_or_default(); - let fixes_count = source_count > target_count.saturating_add(1); - if count_only && !fixes_count { - continue; - } - let source_bytes = bytes.get(&source_id).copied().unwrap_or_default(); - let target_bytes = bytes.get(&target_id).copied().unwrap_or_default(); - let old_spread = source_bytes.abs_diff(target_bytes); - let new_spread = source_bytes - .saturating_sub(partition_bytes) - .abs_diff(target_bytes.saturating_add(partition_bytes)); - let improvement = old_spread.saturating_sub(new_spread); - let weighted = old_spread != 0 - && u128::from(improvement) * 100 - >= u128::from(old_spread) * u128::from(policy.minimum_weighted_improvement_percent); - let score = ( - fixes_count, - improvement, - std::cmp::Reverse(entry.partition_id), - std::cmp::Reverse(target_id), - ); - if (fixes_count || weighted) - && best - .as_ref() - .map_or(true, |(best_score, _, _)| score > *best_score) - { - best = Some((score, entry, target_id)); - } + let mut best_rejected = None; + for entry in entries.iter().copied() { + let partition_bytes = state.partition_bytes[&(entry.owner.instance_id, entry.partition_id)]; + let (accepted, rejected, partition_reason) = context.assess(entry, within); + candidates::retain_best(&mut best, accepted); + candidates::retain_best(&mut best_rejected, rejected); + if entries.len() <= MAX_OBSERVED_PARTITIONS { + result.observation.partitions.push(BalancePartitionObservation { + partition_id: entry.partition_id, + instance_id: entry.owner.instance_id, + owner_epoch: entry.owner_epoch, + estimated_bytes: partition_bytes, + weight: weights.weight(1, partition_bytes), + reason: partition_reason.into(), + }); + } + } + if let Some((_, entry, target_id)) = best { + result.candidate = Some((entry, target_id)); + result.observation.reason = "move selected".into(); + if let Some(partition) = result + .observation + .partitions + .iter_mut() + .find(|partition| partition.partition_id == entry.partition_id) + { + partition.reason = "selected move".into(); + } + } + result.observation.candidate = best + .or(best_rejected) + .map(|choice| diagnostics::candidate(choice, &context)); + result +} + +fn exclusion_reason( + entry: &ChunkKvRangeCatalogEntry, + state: &PlanningState, + policy: &ChunkKvRangeBalancePolicy, + now_ms: u64, +) -> Option<&'static str> { + if entry.artifact.tail_overlay.is_some() { + Some("inherited overlay") + } else if state.busy_owners.contains(&entry.owner.instance_id) + || state.active_partitions.contains(&entry.partition_id) + || entry.transition_id.is_some() + { + Some("active transition") + } else if !eligible(entry, state) { + Some("not independently serving") + } else if !cooled_down(entry, &state.last_transferred_ms, policy, now_ms) { + Some("cooldown") + } else { + None + } +} + +fn owner_loads( + entries: &[&ChunkKvRangeCatalogEntry], + state: &PlanningState, +) -> Option<(HashMap, HashMap)> { + let mut counts: HashMap<_, _> = state.healthy.keys().map(|id| (*id, 0_u64)).collect(); + let mut bytes = counts.clone(); + for entry in entries { + if state + .hosted_epochs + .get(&(entry.owner.instance_id, entry.partition_id)) + != Some(&entry.owner_epoch) + { + return None; } + let count = counts.get_mut(&entry.owner.instance_id)?; + *count = count.checked_add(1)?; + let total_bytes = bytes.get_mut(&entry.owner.instance_id)?; + *total_bytes = total_bytes.checked_add( + *state + .partition_bytes + .get(&(entry.owner.instance_id, entry.partition_id))?, + )?; } - best.map(|(_, entry, target_id)| (entry, target_id)) + Some((counts, bytes)) } fn target_accepts( @@ -80,17 +165,16 @@ pub(super) fn partition_load<'a>( entry: &ChunkKvRangeCatalogEntry, ) -> Option<&'a ChunkKvPartitionLoad> { state - .healthy - .get(&entry.owner.instance_id)? - .1 .partition_loads - .iter() - .find(|load| load.partition_id == entry.partition_id) + .get(&(entry.owner.instance_id, entry.partition_id)) } pub(super) fn effective_bytes(load: &ChunkKvPartitionLoad) -> u64 { - load.durable_bytes - .max(load.live_byte_samples.iter().map(|(_, bytes)| *bytes).sum()) + load.durable_bytes.max( + load.live_byte_samples + .iter() + .fold(0_u64, |sum, (_, bytes)| sum.saturating_add(*bytes)), + ) } pub(super) fn eligible(entry: &ChunkKvRangeCatalogEntry, state: &PlanningState) -> bool { diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/candidates.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/candidates.rs new file mode 100644 index 000000000..ae181f612 --- /dev/null +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/candidates.rs @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::super::PlanningState; +use crowdb_protocol::chunk_kv::{ + balance::BalanceWeights, ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogEntry, Id128, +}; +use std::{cmp::Reverse, collections::HashMap}; + +type Score = (u128, Reverse, Reverse); +pub(super) type Candidate<'a> = (Score, &'a ChunkKvRangeCatalogEntry, u64); + +pub(super) struct CandidateContext<'a> { + pub state: &'a PlanningState, + pub policy: &'a ChunkKvRangeBalancePolicy, + pub weights: &'a BalanceWeights, + pub counts: &'a HashMap, + pub bytes: &'a HashMap, + pub now_ms: u64, +} + +pub(super) fn retain_best<'a>(best: &mut Option>, next: Option>) { + if let Some(next) = next { + if best.as_ref().map_or(true, |previous| next.0 > previous.0) { + *best = Some(next); + } + } +} + +impl CandidateContext<'_> { + pub(super) fn assess<'a>( + &self, + entry: &'a ChunkKvRangeCatalogEntry, + within: bool, + ) -> (Option>, Option>, &'static str) { + if let Some(reason) = super::exclusion_reason(entry, self.state, self.policy, self.now_ms) { + return (None, None, reason); + } + if within { + return (None, None, "within tolerance"); + } + let source_id = entry.owner.instance_id; + let partition_bytes = self.state.partition_bytes[&(source_id, entry.partition_id)]; + let source = (self.counts[&source_id], self.bytes[&source_id]); + let mut accepted = None; + let mut rejected = None; + let mut safe_target = false; + for (&target_id, (_, target)) in &self.state.healthy { + if !super::target_accepts( + self.state, + self.policy, + source_id, + target_id, + target, + partition_bytes, + ) { + continue; + } + safe_target = true; + let target_load = (self.counts[&target_id], self.bytes[&target_id]); + let Some(improvement) = self.weights.improvement(source, target_load, partition_bytes) else { + continue; + }; + let candidate = Some(( + (improvement, Reverse(entry.partition_id), Reverse(target_id)), + entry, + target_id, + )); + if self.weights.qualifies(improvement, self.policy) { + retain_best(&mut accepted, candidate); + } else { + retain_best(&mut rejected, candidate); + } + } + let reason = if accepted.is_some() { + "eligible improving move" + } else if rejected.is_some() { + "insufficient benefit" + } else if safe_target { + "no improving target" + } else { + "no safe target: health, active transition, capacity or rate limit" + }; + (accepted, rejected, reason) + } +} diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/diagnostics.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/diagnostics.rs new file mode 100644 index 000000000..e88bfa873 --- /dev/null +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/diagnostics.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + candidates::{Candidate, CandidateContext}, + PlanningState, +}; +use crowdb_protocol::chunk_kv::{ + balance::{ + BalanceCandidateObservation, BalanceObservation, BalanceOwnerObservation, BalanceWeights, + POLICY_VERSION, + }, + ChunkKvRangeBalancePolicy, +}; +use std::collections::HashMap; + +pub(super) fn empty( + policy: &ChunkKvRangeBalancePolicy, + now_ms: u64, + generation: u64, + valid_for_ms: u64, +) -> BalanceObservation { + BalanceObservation { + policy_version: POLICY_VERSION, + catalog_generation: generation, + observed_at_ms: now_ms, + valid_for_ms, + policy: policy.clone(), + reason: "unavailable observation".into(), + deviation_percent_millionths: 0, + loss_millionths: None, + owners: Vec::new(), + partitions: Vec::new(), + candidate: None, + } +} + +pub(super) fn owners( + counts: &HashMap, + bytes: &HashMap, + weights: &BalanceWeights, + state: &PlanningState, +) -> Vec { + let mut owners: Vec<_> = counts + .iter() + .map(|(id, count)| BalanceOwnerObservation { + instance_id: *id, + rpc_endpoint: state.healthy[id].0.rpc_endpoint.clone(), + partition_count: *count, + estimated_bytes: bytes[id], + weight: weights.weight(*count, bytes[id]), + }) + .collect(); + owners.sort_by_key(|owner| owner.instance_id); + owners +} + +pub(super) fn candidate( + choice: Candidate<'_>, + context: &CandidateContext<'_>, +) -> BalanceCandidateObservation { + let ((improvement, _, _), entry, target_id) = choice; + let source_id = entry.owner.instance_id; + let bytes = context.state.partition_bytes[&(source_id, entry.partition_id)]; + BalanceCandidateObservation { + partition_id: entry.partition_id, + source_id, + target_id, + source_after: context + .weights + .weight(context.counts[&source_id] - 1, context.bytes[&source_id] - bytes), + target_after: context + .weights + .weight(context.counts[&target_id] + 1, context.bytes[&target_id] + bytes), + improvement_percent_millionths: u64::try_from( + improvement.saturating_mul(100_000_000) / context.weights.loss, + ) + .unwrap_or(u64::MAX), + } +} diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/split.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/split.rs index 6e9cbdaa2..cffa68c34 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/split.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/split.rs @@ -36,12 +36,13 @@ pub(super) fn split_transition( parent_owner: parent.owner.clone(), parent_epoch: parent.owner_epoch, parent_artifact: parent.artifact.clone(), - retained_parent_artifact: split_artifact(parent, &split_key, b"left"), + retained_parent_artifact: parent.artifact.clone(), parent_next_epoch, split_key, child, planned_at_ms: now_ms, phase: SplitPhase::Planned, + handoff_proof: None, readiness_proof: None, failure: None, }; diff --git a/app/crowdb-kv-server/src/background/keepalive.rs b/app/crowdb-kv-server/src/background/keepalive.rs index f297e680f..166e7f1ff 100644 --- a/app/crowdb-kv-server/src/background/keepalive.rs +++ b/app/crowdb-kv-server/src/background/keepalive.rs @@ -50,7 +50,7 @@ impl KeepAliveLoop { group0_management_seeds }; let handle = tokio::spawn(async move { - let kv_client = CrowdbKvClient::new(ClientConfig::new(mgmt_seeds)); + let kv_client = CrowdbKvClient::new(ClientConfig::new(mgmt_seeds.clone())); if bootstrap_local { kv_client.seed_leader(0, 0, ep); } @@ -58,7 +58,7 @@ impl KeepAliveLoop { let mut discovered_seeds = Vec::new(); let discovery_ready = !bootstrap_local - || refresh_discovery(®istry, &svc, &mut discovered_seeds).await; + || refresh_discovery(®istry, &svc, &mgmt_seeds, &mut discovered_seeds).await; // Initial registration. let (stores, groups) = hosted_summary(®istry); @@ -88,7 +88,7 @@ impl KeepAliveLoop { tokio::select! { _ = ticker.tick() => { if bootstrap_local - && !refresh_discovery(®istry, &svc, &mut discovered_seeds).await + && !refresh_discovery(®istry, &svc, &mgmt_seeds, &mut discovered_seeds).await { continue; } @@ -158,6 +158,7 @@ impl KeepAliveLoop { async fn refresh_discovery( registry: &KvStoreRegistry, service: &ServiceRegistryClient, + bootstrap_seeds: &[String], current: &mut Vec, ) -> bool { match super::discovery::load(®istry.config.config_root).await { @@ -171,9 +172,24 @@ async fn refresh_discovery( true } Ok(Some(_)) => true, - Ok(None) => registry - .get_store(0) - .is_some_and(|store| store.get_group(0).is_some()), + Ok(None) => { + let Some(group) = registry.get_store(0).and_then(|store| store.get_group(0)) else { + return false; + }; + // Bootstrap membership is wired before election. Resolve its + // elected leader before sending the first registration write. + if group.leader_id() == 0 { + return false; + } + if current.is_empty() { + if let Err(error) = service.kv().refresh_topology().await { + debug!(%error, "keep-alive: bootstrap leader unavailable; deferring registration"); + return false; + } + *current = bootstrap_seeds.to_vec(); + } + true + } Err(error) => { warn!(%error, "keep-alive: discovery hints unreadable; deferring registration"); false diff --git a/app/crowdb-kv-server/src/cli.rs b/app/crowdb-kv-server/src/cli.rs index d378c8209..dbb7048de 100644 --- a/app/crowdb-kv-server/src/cli.rs +++ b/app/crowdb-kv-server/src/cli.rs @@ -80,11 +80,11 @@ pub struct Cli { #[arg(long, default_value = "block", value_parser = ["file", "block", "mem-block"])] pub kv_backend: String, - /// WAL storage backend. `file` (default) uses durable `tokio::fs` I/O; + /// WAL storage backend. `file` uses durable `tokio::fs` I/O; /// `uring` explicitly requests buffered-file `io_uring`; `mem-block` uses an in-memory block device (no /// alignment); `block-device` uses an aligned block device model - /// (SSD/NVMe, 4K I/O unit). - #[arg(long, default_value = "file", value_parser = ["file", "uring", "mem-block", "block-device"])] + /// (SSD/NVMe, 4K I/O unit, default). + #[arg(long, default_value = "block-device", value_parser = ["file", "uring", "mem-block", "block-device"])] pub wal_backend: String, /// Metrics flush interval in seconds. 0 disables metrics logging. diff --git a/app/crowdb-kv-server/src/mgmt.rs b/app/crowdb-kv-server/src/mgmt.rs index 596f0ce78..423b39262 100644 --- a/app/crowdb-kv-server/src/mgmt.rs +++ b/app/crowdb-kv-server/src/mgmt.rs @@ -63,9 +63,19 @@ pub(crate) async fn resolve_store_port(state: &RegistryArc, explicit: Option // Licensed under the Apache License, Version 2.0. +#[path = "common/balance_distribution.rs"] +mod balance_distribution; #[path = "common/balance_planning.rs"] mod balance_planning; @@ -157,3 +159,73 @@ async fn idle_owner_receives_a_partition_before_another_count_driven_split() { 600_000 ); } + +#[tokio::test] +async fn equal_counts_do_not_move_for_unequal_shared_pack_estimates() { + use crowdb_protocol::chunk_kv::balance::{BalanceObservation, OBSERVATION_KEY}; + let fixture = TestBalancePlanning::new().await; + balance_distribution::seed( + &fixture.control, + &[(1, 60), (1, 30), (1, 30), (1, 20), (2, 5), (2, 5), (2, 5), (2, 5)], + 2, + ) + .await; + fixture.tick().await; + assert!(fixture + .control + .scan_all_prefix(Bytes::from(ChunkKvTransferKey::text_prefix_all()), 16) + .await + .unwrap() + .is_empty()); + let row = fixture.control.get(OBSERVATION_KEY.as_bytes()).await.unwrap(); + let observation: BalanceObservation = serde_json::from_slice(&row.value.unwrap()).unwrap(); + assert_eq!(observation.policy.byte_weight_percent, 0); + assert_eq!(observation.reason, "within tolerance"); + assert!(observation + .owners + .iter() + .all(|owner| owner.weight.byte_units == 0)); +} + +#[tokio::test] +async fn unavailable_assignment_observation_defers_moves_instead_of_counting_zero() { + use crowdb_protocol::chunk_kv::balance::{BalanceObservation, OBSERVATION_KEY}; + use crowdb_protocol::common::InstanceValue; + use crowdb_protocol::key::InstanceKey; + let fixture = TestBalancePlanning::new().await; + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 1, + } + .to_path(); + let row = fixture.control.get(path.as_bytes()).await.unwrap(); + let mut instance: InstanceValue = serde_json::from_slice(row.value.as_ref().unwrap()).unwrap(); + instance + .extra + .as_mut() + .unwrap() + .chunk_kv + .as_mut() + .unwrap() + .partition_loads + .clear(); + fixture + .control + .put_batch(vec![( + Bytes::from(path), + Bytes::from(serde_json::to_vec(&instance).unwrap()), + )]) + .await + .unwrap(); + fixture.tick().await; + assert!(fixture + .control + .scan_all_prefix(Bytes::from(ChunkKvTransferKey::text_prefix_all()), 16) + .await + .unwrap() + .is_empty()); + let row = fixture.control.get(OBSERVATION_KEY.as_bytes()).await.unwrap(); + let observation: BalanceObservation = serde_json::from_slice(row.value.as_ref().unwrap()).unwrap(); + assert_eq!(observation.reason, "unavailable observation"); + assert!(observation.owners.is_empty()); +} diff --git a/app/crowdb-kv-server/tests/cli_parse_test.rs b/app/crowdb-kv-server/tests/cli_parse_test.rs index 4d074f81b..711ad4d8b 100644 --- a/app/crowdb-kv-server/tests/cli_parse_test.rs +++ b/app/crowdb-kv-server/tests/cli_parse_test.rs @@ -75,6 +75,13 @@ fn parse_root_cli_option() { // --config is now optional. assert!(cli.config.is_none()); assert_eq!(cli.binding_monitor_interval, 1); + assert_eq!(cli.kv_backend, "block"); + assert_eq!(cli.wal_backend, "block-device"); + assert!(!cli.no_fsync); + let defaults = CrowDBConfig::default(); + assert_eq!(defaults.crowtree_backend, cli.kv_backend); + assert_eq!(defaults.wal_backend, cli.wal_backend); + assert!(!defaults.wal_skip_fsync); } #[test] @@ -84,6 +91,17 @@ fn parse_root_is_required() { assert!(result.is_err()); } +#[test] +fn loaded_configuration_uses_durable_block_defaults() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("block-default-config"); + let path = root.path().join("server.toml"); + std::fs::write(&path, "").unwrap(); + let loaded = CrowDBConfig::load_from_file(&path).unwrap(); + assert_eq!(loaded.crowtree_backend, "block"); + assert_eq!(loaded.wal_backend, "block-device"); + assert!(!loaded.wal_skip_fsync); +} + #[test] fn management_port_rejects_zero() { let result = Cli::try_parse_from([ diff --git a/app/crowdb-kv-server/tests/common/balance_distribution.rs b/app/crowdb-kv-server/tests/common/balance_distribution.rs new file mode 100644 index 000000000..f9438de2c --- /dev/null +++ b/app/crowdb-kv-server/tests/common/balance_distribution.rs @@ -0,0 +1,137 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_protocol::{ + chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, + ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, HostedPartition, Id128, KeyRange, + OwnerDescriptor, PartitionArtifact, + }, + chunk_stream::StreamName, + common::{ChunkKvExtra, ChunkKvPartitionLoad, InstanceValue, ServiceExtra}, + key::{ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, InstanceKey, TextKey}, +}; + +pub async fn seed(control: &Group0ControlPlane, distribution: &[(u64, u64)], generation: u64) { + let mut page = ChunkKvRangeCatalogPage { + generation, + page_index: 0, + checksum: [0; 32], + entries: distribution + .iter() + .enumerate() + .map(|(index, (owner, _))| entry(index, *owner, distribution.len(), generation)) + .collect(), + }; + page.seal().unwrap(); + let mut head = ChunkKvRangeCatalogHead { + generation, + previous_generation: (generation > 1).then_some(generation - 1), + checksum: [0; 32], + pages: vec![ChunkKvRangeCatalogPageRef { + page_generation: generation, + page_index: 0, + first_key: vec![], + page_checksum: page.checksum, + }], + }; + head.seal().unwrap(); + let mut records = vec![ + ( + Bytes::from(ChunkKvRangeCatalogHeadKey.to_path()), + Bytes::from(serde_json::to_vec(&head).unwrap()), + ), + ( + Bytes::from( + ChunkKvRangeCatalogPageKey { + generation, + page_index: 0, + } + .to_path(), + ), + Bytes::from(serde_json::to_vec(&page).unwrap()), + ), + ]; + for owner in [1, 2] { + let mut extra = ChunkKvExtra { + capacity_bytes: 10_000, + ..Default::default() + }; + for (index, (_, bytes)) in distribution + .iter() + .enumerate() + .filter(|(_, (id, _))| *id == owner) + { + let partition_id = Id128 { + high: 41, + low: u64::try_from(index).unwrap() + 1, + }; + extra.durable_bytes += bytes; + extra.hosted.push(HostedPartition { + partition_id, + owner_epoch: generation, + recovering: false, + }); + extra.partition_loads.push(ChunkKvPartitionLoad { + partition_id, + durable_bytes: *bytes, + live_byte_samples: vec![], + independently_recoverable: true, + }); + } + let instance = InstanceValue { + instance_id: owner, + rpc_endpoint: format!("127.0.0.1:{}", 17000 + owner), + last_heartbeat_ms: u64::MAX, + extra: Some(ServiceExtra { + chunk_kv: Some(extra), + ..Default::default() + }), + }; + records.push(( + Bytes::from( + InstanceKey { + service: "chunk-kv".into(), + instance_id: owner, + } + .to_path(), + ), + Bytes::from(serde_json::to_vec(&instance).unwrap()), + )); + } + control.put_batch(records).await.unwrap(); +} + +fn entry(index: usize, owner: u64, length: usize, generation: u64) -> ChunkKvRangeCatalogEntry { + ChunkKvRangeCatalogEntry { + partition_id: Id128 { + high: 41, + low: u64::try_from(index).unwrap() + 1, + }, + range: KeyRange { + start: if index == 0 { + vec![] + } else { + vec![u8::try_from(index).unwrap()] + }, + end: (index + 1 < length).then(|| vec![u8::try_from(index + 1).unwrap()]), + }, + owner: OwnerDescriptor { + instance_id: owner, + rpc_endpoint: format!("127.0.0.1:{}", 17000 + owner), + }, + owner_epoch: generation, + state: ChunkKvRangeCatalogPartitionState::Serving, + transition_id: None, + artifact: PartitionArtifact { + tree_id: 50 + u64::try_from(index).unwrap(), + stream_name: StreamName { + high: 51, + low: u64::try_from(index).unwrap() + 1, + }, + tail_overlay: None, + }, + } +} diff --git a/app/crowdb-kv-server/tests/common/balance_planning.rs b/app/crowdb-kv-server/tests/common/balance_planning.rs index 74bf79831..04310c96d 100644 --- a/app/crowdb-kv-server/tests/common/balance_planning.rs +++ b/app/crowdb-kv-server/tests/common/balance_planning.rs @@ -192,6 +192,7 @@ pub async fn recent_split(control: &Group0ControlPlane) { .try_into() .unwrap(), phase: SplitPhase::Aborted, + handoff_proof: None, readiness_proof: None, failure: Some("split preparation aborted while the original source remains serving".into()), }; diff --git a/app/crowdb-kv-server/tests/common/process.rs b/app/crowdb-kv-server/tests/common/process.rs index ed7e68077..9b70c996f 100644 --- a/app/crowdb-kv-server/tests/common/process.rs +++ b/app/crowdb-kv-server/tests/common/process.rs @@ -101,6 +101,15 @@ pub async fn start_test_server_with_ports(args: &[&str], ports: &[u16]) -> std_i Ok(handle) } +fn configure_storage_backends(cmd: &mut Command, args: &[&str]) { + if !args.iter().any(|arg| arg.starts_with("--kv-backend")) { + cmd.args(["--kv-backend", "block"]); + } + if !args.iter().any(|arg| arg.starts_with("--wal-backend")) { + cmd.args(["--wal-backend", "block-device"]); + } +} + /// Start a server at a caller-owned `root` path. The caller is /// responsible for keeping `root` alive for the process's lifetime /// (e.g. holding the `TestDir`). Used by restart/restore @@ -131,6 +140,7 @@ pub async fn start_test_server_at( let bin = crowdb_kv_server_bin(); let mut cmd = Command::new(bin); + configure_storage_backends(&mut cmd, args); cmd.args(args) .arg("--root") .arg(root) diff --git a/app/crowdb-kv-server/tests/domain_monitor_test.rs b/app/crowdb-kv-server/tests/domain_monitor_test.rs index c85cf3c17..8df2f5374 100644 --- a/app/crowdb-kv-server/tests/domain_monitor_test.rs +++ b/app/crowdb-kv-server/tests/domain_monitor_test.rs @@ -673,7 +673,6 @@ async fn assert_chunk_kv_split_plan(target_partitions_per_owner: u32, target_par cutover_seq: 11, target_stream_start_seq: 12, }; - prepared.retained_parent_artifact.tail_overlay = Some(overlay.clone()); prepared.child.artifact.tail_overlay = Some(overlay.clone()); prepared.phase = SplitPhase::ChildPrepared; prepared.readiness_proof = Some(crowdb_protocol::chunk_kv::SplitReadinessProof { @@ -686,7 +685,7 @@ async fn assert_chunk_kv_split_plan(target_partitions_per_owner: u32, target_par child_applied_seq: 11, child_tree_manifest: 1, child_root_manifest_generation: 1, - retained_parent_tail_overlay: overlay.clone(), + retained_parent_tail_overlay: None, child_tail_overlay: overlay, }); prepared.validate().unwrap(); diff --git a/app/crowdb-kv-server/tests/group0_discovery_test.rs b/app/crowdb-kv-server/tests/group0_discovery_test.rs index b098a376d..b7abab678 100644 --- a/app/crowdb-kv-server/tests/group0_discovery_test.rs +++ b/app/crowdb-kv-server/tests/group0_discovery_test.rs @@ -11,6 +11,43 @@ use serde_json::json; use common::process::{start_test_server, start_test_server_at}; +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn prebootstrap_registration_waits_for_election_before_admitting_writes() { + let server = start_test_server(&["--node-id", "2", "--keepalive-interval", "1"]) + .await + .unwrap(); + let http = reqwest::Client::new(); + let response = http + .post(format!("{}/system/init", server.base_url())) + .json(&json!({"replica_id": 2, "start_election": false})) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 201); + // Leave bootstrap paused across two registration ticks. No write may + // enter the unwired group before its first election. + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + loop { + let topology: serde_json::Value = http + .get(format!("{}/topology", server.base_url())) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let group = &topology["stores"][0]["groups"][0]; + assert_eq!(group["leader_id"], 0); + assert_eq!(group["local_replica"]["role"], "follower"); + assert_eq!(group["local_replica"]["election"]["current_term"], 0); + assert_eq!(group["inflight"]["total_enqueued"], 0); + if tokio::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + async fn wait_registered(service: &ServiceRegistryClient, endpoint: &str) { let deadline = tokio::time::Instant::now() + Duration::from_secs(10); loop { diff --git a/app/crowdb-web/src/chunk_kv.rs b/app/crowdb-web/src/chunk_kv.rs index b830a932d..43a516434 100644 --- a/app/crowdb-web/src/chunk_kv.rs +++ b/app/crowdb-web/src/chunk_kv.rs @@ -3,6 +3,7 @@ //! Bounded catalog observations for the Chunk-KV workbench. +mod balance; mod runtime; pub(crate) use runtime::runtime; @@ -145,6 +146,7 @@ async fn observe(state: &AppState, query: &CatalogQuery) -> Result, if query.offset >= page.entries.len() { return Err(err_400("Catalog offset is outside the referenced page")); } + let balance = balance::observe(&kv, head.generation).await; let entries: Vec<_> = page .entries .iter() @@ -158,6 +160,7 @@ async fn observe(state: &AppState, query: &CatalogQuery) -> Result, "start": hex::encode(&entry.range.start), "end": entry.range.end.as_ref().map(hex::encode), "owner_id": entry.owner.instance_id.to_string(), "endpoint": entry.owner.rpc_endpoint, "epoch": entry.owner_epoch.to_string(), "state": entry.state, "artifact": artifact, + "balance": balance::partition(&balance, entry), "transition_id": entry.transition_id.map(|id| format!("{:016x}{:016x}", id.high, id.low)), }) }) @@ -178,7 +181,8 @@ async fn observe(state: &AppState, query: &CatalogQuery) -> Result, Ok(Json( json!({"generation":head.generation.to_string(),"page":query.page, "offset":query.offset,"catalog_pages":head.pages.len(),"entries":entries,"next":next, - "source":"group0","coverage":"referenced page","runtime_available":false}), + "source":"group0","coverage":"referenced page","runtime_available":false, + "balance":balance::summary(&balance)}), )) } diff --git a/app/crowdb-web/src/chunk_kv/balance.rs b/app/crowdb-web/src/chunk_kv/balance.rs new file mode 100644 index 000000000..8aac86b6f --- /dev/null +++ b/app/crowdb-web/src/chunk_kv/balance.rs @@ -0,0 +1,94 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Read the monitor's bounded diagnostic snapshot without scanning catalog/data. + +use crowdb_kv_client::{CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_protocol::chunk_kv::{ + balance::{BalanceObservation, MAX_OBSERVATION_BYTES, OBSERVATION_KEY}, + ChunkKvRangeCatalogEntry, +}; +use serde_json::{json, Value}; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub(super) struct Observation { + value: Option, + reason: String, +} + +pub(super) async fn observe(kv: &CrowdbKvClient, generation: u64) -> Observation { + match load(kv, generation).await { + Ok(value) => Observation { + reason: value.reason.clone(), + value: Some(value), + }, + Err(reason) => Observation { value: None, reason }, + } +} + +async fn load(kv: &CrowdbKvClient, generation: u64) -> Result { + let outcome = kv + .get(0, 0, OBSERVATION_KEY.as_bytes(), ReadMode::Linearizable, None) + .await + .map_err(|error| error.to_string())?; + let GetOutcome::Found { value, .. } = outcome else { + return Err("Balance observation unavailable".into()); + }; + if value.len() > MAX_OBSERVATION_BYTES { + return Err("Balance observation exceeds byte budget".into()); + } + let result: BalanceObservation = + serde_json::from_slice(&value).map_err(|_| "Invalid balance observation".to_string())?; + let now_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |time| u64::try_from(time.as_millis()).unwrap_or(u64::MAX)); + if result.catalog_generation != generation { + return Err("Stale balance catalog generation".into()); + } + if result.observed_at_ms > now_ms.saturating_add(1_000) + || now_ms.saturating_sub(result.observed_at_ms) > result.valid_for_ms + { + return Err("Stale balance observation".into()); + } + if result.policy_version != crowdb_protocol::chunk_kv::balance::POLICY_VERSION + || result.policy.validate().is_err() + { + return Err("Invalid balance policy observation".into()); + } + Ok(result) +} + +pub(super) fn partition(observation: &Observation, entry: &ChunkKvRangeCatalogEntry) -> Value { + let Some(value) = &observation.value else { + return Value::Null; + }; + let Some(partition) = value.partitions.iter().find(|partition| { + partition.partition_id == entry.partition_id + && partition.instance_id == entry.owner.instance_id + && partition.owner_epoch == entry.owner_epoch + }) else { + return Value::Null; + }; + let Some(owner) = value + .owners + .iter() + .find(|owner| owner.instance_id == entry.owner.instance_id) + else { + return Value::Null; + }; + let mut result = json!({"partition":partition,"owner":owner}); + super::exact_integers(&mut result); + result +} + +pub(super) fn summary(observation: &Observation) -> Value { + let Some(value) = &observation.value else { + return json!({"reason":observation.reason}); + }; + let mut result = json!({"reason":value.reason,"policy_version":value.policy_version,"generation":value.catalog_generation, + "observed_at_ms":value.observed_at_ms,"valid_for_ms":value.valid_for_ms,"policy":value.policy, + "deviation_percent_millionths":value.deviation_percent_millionths,"loss_millionths":value.loss_millionths,"candidate":value.candidate, + "owners":value.owners}); + super::exact_integers(&mut result); + result +} diff --git a/app/crowdb-web/src/diskdb/lifecycle.rs b/app/crowdb-web/src/diskdb/lifecycle.rs index a8125b12b..bc003a991 100644 --- a/app/crowdb-web/src/diskdb/lifecycle.rs +++ b/app/crowdb-web/src/diskdb/lifecycle.rs @@ -42,6 +42,16 @@ fn validate_diskdb_ports(body: &DeployDiskdbBody) -> Result<(u16, u16, u16), (St Ok((listen_port, http_port, rpc_listen_port)) } +#[cfg(target_os = "macos")] +fn ensure_local_ports_free(ports: [u16; 3]) -> Result<(), (StatusCode, Json)> { + for port in ports { + if std::net::TcpListener::bind(("127.0.0.1", port)).is_err() { + return Err(err_409(format!("DiskDB listener port {port} is unavailable"))); + } + } + Ok(()) +} + /// `POST /api/nodes/:id/diskdb/deploy` — spawn `crowdb-diskdb` on the /// node's workspace. Registers a `ServerEntry` with /// `service_type: Diskdb` and tracks the PID. @@ -83,6 +93,8 @@ async fn deploy_diskdb( )); } let (listen_port, http_port, rpc_listen_port) = validate_diskdb_ports(&body)?; + #[cfg(target_os = "macos")] + ensure_local_ports_free([listen_port, http_port, rpc_listen_port])?; let _ports = crate::services::defaults::claim_ports(&state, &[listen_port, http_port, rpc_listen_port])?; let node = { let cfg = state.config.read().unwrap(); diff --git a/app/crowdb-web/src/lifecycle.rs b/app/crowdb-web/src/lifecycle.rs index 0573ebb44..438125eab 100644 --- a/app/crowdb-web/src/lifecycle.rs +++ b/app/crowdb-web/src/lifecycle.rs @@ -1453,7 +1453,7 @@ pub async fn http_add_disk( let entry = ops::hardware::add_disk(&ctx, node_id, dg_id, &input) .await .map_err(map_config_err)?; - state.commit_op_context(&ctx).map_err(map_persist_err)?; + crate::physical::disk::persist_added(&state, std::slice::from_ref(&entry))?; Ok((StatusCode::CREATED, Json(entry))) } @@ -1492,7 +1492,7 @@ pub async fn http_add_disks_batch( let added = ops::hardware::add_disks_batch(&ctx, node_id, dg_id, &inputs) .await .map_err(map_config_err)?; - state.commit_op_context(&ctx).map_err(map_persist_err)?; + crate::physical::disk::persist_added(&state, &added)?; Ok(( StatusCode::CREATED, Json(AddDisksBatchResult { diff --git a/app/crowdb-web/src/physical.rs b/app/crowdb-web/src/physical.rs index e23d39501..2fa78fdd5 100644 --- a/app/crowdb-web/src/physical.rs +++ b/app/crowdb-web/src/physical.rs @@ -3,6 +3,7 @@ //! Physical-tree per-node primitives and expanded views. +pub(crate) mod disk; pub(crate) mod disk_group; pub mod node; pub mod view; diff --git a/app/crowdb-web/src/physical/disk.rs b/app/crowdb-web/src/physical/disk.rs new file mode 100644 index 000000000..f8b0d22bf --- /dev/null +++ b/app/crowdb-web/src/physical/disk.rs @@ -0,0 +1,27 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Merge newly provisioned disks without replacing concurrent service updates. + +use axum::{http::StatusCode, Json}; +use crowdb_console_shared::config::DiskEntry; + +use crate::{ + error::{map_config_err, map_persist_err, ErrorBody}, + state::AppState, +}; + +pub(crate) fn persist_added( + state: &AppState, + entries: &[DiskEntry], +) -> Result<(), (StatusCode, Json)> { + { + let mut config = state.config.write().unwrap(); + let mut updated = config.clone(); + for entry in entries { + updated.add_disk(entry.clone()).map_err(map_config_err)?; + } + *config = updated; + } + state.persist().map_err(map_persist_err) +} diff --git a/app/crowdb-web/src/physical/disk_group.rs b/app/crowdb-web/src/physical/disk_group.rs index a77e3d6ee..cf1ef0198 100644 --- a/app/crowdb-web/src/physical/disk_group.rs +++ b/app/crowdb-web/src/physical/disk_group.rs @@ -44,17 +44,19 @@ async fn provision( let entry = ops::hardware::add_disk_group_bound_to_group0(&ctx, node_id, id, &name, Some(binding)) .await .map_err(map_config_err)?; - if !ctx - .config() - .disk_groups - .iter() - .any(|g| g.node_id == node_id && g.id == id) { - ctx.config_mut() - .add_disk_group(entry.clone()) - .map_err(map_config_err)?; + // Group-0 publication can overlap service registration. Merge only + // this entry so the older operation snapshot cannot erase services. + let mut config = state.config.write().unwrap(); + if !config + .disk_groups + .iter() + .any(|g| g.node_id == node_id && g.id == id) + { + config.add_disk_group(entry.clone()).map_err(map_config_err)?; + } } - state.commit_op_context(&ctx).map_err(map_persist_err)?; + state.persist().map_err(map_persist_err)?; Ok((StatusCode::CREATED, Json(entry))) } diff --git a/app/crowdb-web/src/services/lifecycle.rs b/app/crowdb-web/src/services/lifecycle.rs index 615834778..6f0d7e8af 100644 --- a/app/crowdb-web/src/services/lifecycle.rs +++ b/app/crowdb-web/src/services/lifecycle.rs @@ -162,39 +162,67 @@ fn matching_pid(entry: &ServerEntry, launch: &LocalLaunchSpec) -> Result = command - .split(|byte| *byte == 0) - .filter(|part| !part.is_empty()) - .collect(); - let program = std::fs::canonicalize(&launch.program).map_err(|error| err_409(error.to_string()))?; - let executable = - std::fs::read_link(format!("/proc/{pid}/exe")).map_err(|error| err_409(error.to_string()))?; - // Linux retains the original executable mapping after an atomic upgrade. - // Keep verifying the exact workspace and arguments before signalling it. - let replaced_program = executable - .to_str() - .and_then(|path| path.strip_suffix(" (deleted)")) - .is_some_and(|path| std::path::Path::new(path) == program); - if (executable != program && !replaced_program) - || arguments.get(1..).map_or(true, |args| { - args.len() != launch.args.len() - || args - .iter() - .zip(&launch.args) - .any(|(actual, expected)| *actual != expected.as_bytes()) - }) + #[cfg(not(target_os = "macos"))] { - return Err(err_409("Recorded PID command does not match the retained launch")); + let cwd = std::fs::read_link(format!("/proc/{pid}/cwd")) + .map_err(|error| err_409(format!("Cannot verify service process: {error}")))?; + let expected = std::fs::canonicalize(&launch.workdir).map_err(|error| err_409(error.to_string()))?; + if cwd != expected { + return Err(err_409( + "Recorded PID belongs to a different workspace; refusing to signal it", + )); + } + let command = + std::fs::read(format!("/proc/{pid}/cmdline")).map_err(|error| err_409(error.to_string()))?; + let arguments: Vec<_> = command + .split(|byte| *byte == 0) + .filter(|part| !part.is_empty()) + .collect(); + let program = std::fs::canonicalize(&launch.program).map_err(|error| err_409(error.to_string()))?; + let executable = + std::fs::read_link(format!("/proc/{pid}/exe")).map_err(|error| err_409(error.to_string()))?; + // Linux retains the original executable mapping after an atomic upgrade. + // Keep verifying the exact workspace and arguments before signalling it. + let replaced_program = executable + .to_str() + .and_then(|path| path.strip_suffix(" (deleted)")) + .is_some_and(|path| std::path::Path::new(path) == program); + if (executable != program && !replaced_program) + || arguments.get(1..).map_or(true, |args| { + args.len() != launch.args.len() + || args + .iter() + .zip(&launch.args) + .any(|(actual, expected)| *actual != expected.as_bytes()) + }) + { + return Err(err_409("Recorded PID command does not match the retained launch")); + } + Ok(Some(pid)) } - Ok(Some(pid)) } diff --git a/app/crowdb-web/tests/chunk_kv_catalog_test.rs b/app/crowdb-web/tests/chunk_kv_catalog_test.rs index 6d675712a..2bc4f9196 100644 --- a/app/crowdb-web/tests/chunk_kv_catalog_test.rs +++ b/app/crowdb-web/tests/chunk_kv_catalog_test.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/catalog_balance.rs"] +mod catalog_balance; + use std::sync::Arc; use axum::{ @@ -93,6 +96,7 @@ async fn catalog_windows_pin_generation_validate_checksums_and_keep_exact_ids() get(app.clone(), "?generation=9&offset=105").await.0, StatusCode::BAD_REQUEST ); + verify_balance(&kv, &app, &page).await; verify_runtime(&state, &app).await; page.entries[0].owner_epoch = 1; kv.put( @@ -338,3 +342,60 @@ fn catalog_fixture() -> (ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage) { head.seal().unwrap(); (head, page) } + +async fn verify_balance(kv: &CrowdbKvClient, app: &axum::Router, page: &ChunkKvRangeCatalogPage) { + use crowdb_protocol::chunk_kv::balance::OBSERVATION_KEY; + catalog_balance::seed(kv, page).await; + let (status, first) = get(app.clone(), "").await; + assert_eq!(status, StatusCode::OK); + let (_, tail) = get(app.clone(), "?generation=9&offset=100").await; + assert_eq!( + first["entries"][0]["balance"]["partition"]["weight"], + tail["entries"][0]["balance"]["partition"]["weight"] + ); + assert_eq!(first["balance"]["owners"][0]["partition_count"], "105"); + assert_eq!(first["balance"]["owners"][0]["weight"]["byte_units"], "800000000"); + assert_eq!( + first["entries"][0]["balance"]["partition"]["weight"]["byte_units"], + "7619047" + ); + let crowdb_kv_client::GetOutcome::Found { value, .. } = kv + .get( + 0, + 0, + OBSERVATION_KEY.as_bytes(), + crowdb_kv_client::ReadMode::Linearizable, + None, + ) + .await + .unwrap() + else { + panic!("Missing seeded balance"); + }; + let original: Value = serde_json::from_slice(&value).unwrap(); + for (field, expected) in [ + ("catalog_generation", "Stale balance catalog generation"), + ("observed_at_ms", "Stale balance observation"), + ("owner_epoch", "within tolerance"), + ] { + let mut value = original.clone(); + if field == "owner_epoch" { + value["partitions"][0][field] = 1.into(); + } else { + value[field] = 0.into(); + } + kv.put( + 0, + 0, + OBSERVATION_KEY.as_bytes(), + &serde_json::to_vec(&value).unwrap(), + None, + ) + .await + .unwrap(); + let (status, observed) = get(app.clone(), "").await; + assert_eq!(status, StatusCode::OK); + assert!(observed["entries"][0]["balance"].is_null()); + assert_eq!(observed["balance"]["reason"], expected); + } +} diff --git a/app/crowdb-web/tests/common/catalog_balance.rs b/app/crowdb-web/tests/common/catalog_balance.rs new file mode 100644 index 000000000..41c3bbfe1 --- /dev/null +++ b/app/crowdb-web/tests/common/catalog_balance.rs @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv_client::CrowdbKvClient; +use crowdb_protocol::chunk_kv::{ + balance::{ + BalanceObservation, BalanceOwnerObservation, BalancePartitionObservation, BalanceWeights, + OBSERVATION_KEY, + }, + ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogPage, +}; + +pub async fn seed(kv: &CrowdbKvClient, page: &ChunkKvRangeCatalogPage) { + let weights = BalanceWeights::new(&[(105, 1_050)], 80).unwrap(); + let first = &page.entries[0]; + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + let observation = BalanceObservation { + policy_version: crowdb_protocol::chunk_kv::balance::POLICY_VERSION, + catalog_generation: 9, + observed_at_ms: now, + valid_for_ms: 6_000, + policy: ChunkKvRangeBalancePolicy::default(), + reason: "within tolerance".into(), + deviation_percent_millionths: 0, + loss_millionths: Some(0), + owners: vec![BalanceOwnerObservation { + instance_id: first.owner.instance_id, + rpc_endpoint: first.owner.rpc_endpoint.clone(), + partition_count: 105, + estimated_bytes: 1_050, + weight: weights.weight(105, 1_050), + }], + partitions: page + .entries + .iter() + .map(|entry| BalancePartitionObservation { + partition_id: entry.partition_id, + instance_id: entry.owner.instance_id, + owner_epoch: entry.owner_epoch, + estimated_bytes: 10, + weight: weights.weight(1, 10), + reason: "within tolerance".into(), + }) + .collect(), + candidate: None, + }; + kv.put( + 0, + 0, + OBSERVATION_KEY.as_bytes(), + &serde_json::to_vec(&observation).unwrap(), + None, + ) + .await + .unwrap(); +} diff --git a/app/crowdb-web/tests/common/native_balance.rs b/app/crowdb-web/tests/common/native_balance.rs index 6f6a20f5f..cfe2011bc 100644 --- a/app/crowdb-web/tests/common/native_balance.rs +++ b/app/crowdb-web/tests/common/native_balance.rs @@ -4,11 +4,14 @@ //! Slow acceptance of the persisted production balance policy and real data. use crowdb_chunk_kv_client::{ - ChunkKvClient, ChunkKvRangeCatalogMap, ChunkKvRangeCatalogSource, ChunkKvRpcTransport, ClientConfig, - Group0ChunkKvRangeCatalogSource, + BatchItem, ChunkKvClient, ChunkKvRangeCatalogMap, ChunkKvRangeCatalogSource, ChunkKvRpcTransport, + ClientConfig, Group0ChunkKvRangeCatalogSource, }; use crowdb_kv_client::ServiceRegistryClient; -use crowdb_protocol::{chunk_kv::ChunkKvRangeCatalogPartitionState, common::ChunkKvExtra}; +use crowdb_protocol::{ + chunk_kv::{ChunkKvRangeCatalogPartitionState, OperationResult, PointOperation}, + common::ChunkKvExtra, +}; use crowdb_web::AppState; use std::{ collections::BTreeMap, @@ -19,100 +22,78 @@ use std::{ pub(super) struct TestNativeBalance; impl TestNativeBalance { + pub(super) async fn settle_for_inspection(state: &AppState) { + let kv = state.kv_client().await; + let source = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(kv.clone())); + let registry = ServiceRegistryClient::from_shared(kv.clone()); + let client = ChunkKvClient::new( + ClientConfig::default(), + source.clone(), + Arc::new(ChunkKvRpcTransport::new(64, 1, 2)), + ) + .unwrap(); + seed_values(&client).await; + wait_balanced(&source, ®istry, &kv, Instant::now()).await; + verify_values(&client).await; + } + pub(super) async fn verify(state: &AppState) { let kv = state.kv_client().await; let source = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(kv.clone())); - let registry = ServiceRegistryClient::from_shared(kv); + let registry = ServiceRegistryClient::from_shared(kv.clone()); let client = ChunkKvClient::new( ClientConfig::default(), source.clone(), Arc::new(ChunkKvRpcTransport::new(64, 1, 2)), ) .unwrap(); - for index in 0..512 { - let result = client - .put(super::native_load::key(index), super::native_load::value(index)) - .await - .unwrap(); - assert!(result.result.is_ok()); - } + seed_values(&client).await; // The normal policy has a one-minute cooldown. This slow acceptance // observes that real horizon; it does not replace any request/lease budget. let started = Instant::now(); - let catalog = wait_balanced(&source, ®istry, started).await; - let mut interval = tokio::time::interval(Duration::from_millis(500)); - let mut report = Instant::now(); - let hot_owner = catalog.entries()[0].owner.instance_id; - let hot_keys: Vec<_> = (0..512) - .filter(|index| { - catalog - .route(&super::native_load::key(*index)) - .unwrap() - .owner - .instance_id - == hot_owner + wait_balanced(&source, ®istry, &kv, started).await; + verify_values(&client).await; + } +} + +async fn seed_values(client: &ChunkKvClient) { + for batch in 0..32 { + let items = (batch * 16..(batch + 1) * 16) + .map(|index| BatchItem { + request_id: None, + operation: PointOperation::Put { + key: super::native_load::key(index), + value: super::native_load::value(index), + }, }) .collect(); - assert!(!hot_keys.is_empty()); - let mut previous = observe(®istry).await; - let mut transferred = false; - for index in &hot_keys { - assert!(client - .put(super::native_load::key(*index), hot_value(*index)) - .await - .unwrap() - .result - .is_ok()); - let current = load(&source).await; - if !transferred { - transferred = verify_transfer(&catalog, ¤t, &previous); - } - previous = observe(®istry).await; - } - while !transferred { - interval.tick().await; - let current = load(&source).await; - transferred = verify_transfer(&catalog, ¤t, &previous); - previous = observe(®istry).await; - if report.elapsed() >= Duration::from_secs(30) { - eprintln!( - "[PHASE] native weighted balance {}s: counts={:?}, bytes={:?}", - started.elapsed().as_secs(), - counts(¤t), - previous - .iter() - .map(|(owner, extra)| (*owner, extra.durable_bytes)) - .collect::>() - ); - report = Instant::now(); - } - assert!( - started.elapsed() < Duration::from_secs(15 * 60), - "unequal retained data did not redistribute" - ); + let outcomes = client.batch_mutate(items).await.unwrap(); + assert_eq!(outcomes.len(), 16); + for outcome in outcomes { + assert!(matches!( + outcome.unwrap().result.unwrap(), + OperationResult::Mutation { applied: true, .. } + )); } - verify_values(&client, &hot_keys).await; } } -async fn verify_values(client: &ChunkKvClient, hot_keys: &[u64]) { +async fn verify_values(client: &ChunkKvClient) { for index in 0..512 { let result = client.get(super::native_load::key(index), None).await.unwrap(); - let crowdb_protocol::chunk_kv::OperationResult::Value(Some(record)) = result.result.unwrap() else { - panic!("native balance lost a record"); + let operation = result.result.unwrap(); + let crowdb_protocol::chunk_kv::OperationResult::Value(Some(record)) = operation else { + panic!("native balance lost record {index}: {operation:?}"); }; - let expected = if hot_keys.contains(&index) { - hot_value(index) - } else { - super::native_load::value(index) - }; - assert_eq!(record.value, expected); + let expected = super::native_load::value(index); + assert_eq!(record.value, expected, "native balance record {index}"); } } async fn wait_balanced( source: &Group0ChunkKvRangeCatalogSource, registry: &ServiceRegistryClient, + kv: &crowdb_kv_client::CrowdbKvClient, started: Instant, ) -> ChunkKvRangeCatalogMap { let mut interval = tokio::time::interval(Duration::from_millis(500)); @@ -164,7 +145,8 @@ async fn wait_balanced( report = Instant::now(); } if counts.len() == 3 - && counts.values().all(|count| *count == 4) + && catalog.entries().len() >= 12 + && placement_ready(kv, catalog.generation()).await && catalog.entries().iter().all(|entry| { entry.state == ChunkKvRangeCatalogPartitionState::Serving && entry.transition_id.is_none() @@ -175,38 +157,42 @@ async fn wait_balanced( } assert!( started.elapsed() < Duration::from_secs(10 * 60), - "normal count balance did not converge: {counts:?}" + "normal count-based placement did not converge: {counts:?}" ); previous = Some(catalog); } } -fn verify_transfer( - before: &ChunkKvRangeCatalogMap, - after: &ChunkKvRangeCatalogMap, - observed: &BTreeMap, -) -> bool { - let Some((old, moved)) = before.entries().iter().find_map(|old| { - after - .entries() - .iter() - .find(|entry| { - entry.partition_id == old.partition_id && entry.owner.instance_id != old.owner.instance_id - }) - .map(|entry| (old, entry)) - }) else { +async fn placement_ready(kv: &crowdb_kv_client::CrowdbKvClient, generation: u64) -> bool { + use crowdb_protocol::chunk_kv::balance::{BalanceObservation, OBSERVATION_KEY}; + let value = kv + .get( + 0, + 0, + OBSERVATION_KEY.as_bytes(), + crowdb_kv_client::ReadMode::Linearizable, + None, + ) + .await + .unwrap(); + let crowdb_kv_client::GetOutcome::Found { value, .. } = value else { return false; }; - // Every owner had four partitions; a first transfer cannot fix count spread. - assert!(counts(before).values().all(|count| *count == 4)); - let source_bytes = observed[&old.owner.instance_id].durable_bytes; - let target_bytes = observed[&moved.owner.instance_id].durable_bytes; - assert!(source_bytes > target_bytes, "actual transfer must leave the owner with more retained data: source={source_bytes}, target={target_bytes}"); - eprintln!( - "[PHASE] native weighted transfer {:?}: {} ({source_bytes} bytes) -> {} ({target_bytes} bytes)", - old.partition_id, old.owner.instance_id, moved.owner.instance_id - ); - true + let observation: BalanceObservation = serde_json::from_slice(&value).unwrap(); + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + observation.catalog_generation == generation + && now.saturating_sub(observation.observed_at_ms) <= observation.valid_for_ms + && (observation.reason == "within tolerance" + || observation.reason == "no useful safe move" + && observation.partitions.iter().all(|partition| { + partition.reason == "no improving target" || partition.reason == "insufficient benefit" + })) } async fn load(source: &Group0ChunkKvRangeCatalogSource) -> ChunkKvRangeCatalogMap { @@ -237,10 +223,3 @@ async fn observe(registry: &ServiceRegistryClient) -> BTreeMap Vec { - let mut value = super::native_load::value(index); - value.extend(super::native_load::value(index + 512)); - value.extend(super::native_load::value(index + 1024)); - value.extend(super::native_load::value(index + 1536)); - value -} diff --git a/app/crowdb-web/tests/native_access_deployment_test.rs b/app/crowdb-web/tests/native_access_deployment_test.rs index 71b04b44d..0fa6bacdb 100644 --- a/app/crowdb-web/tests/native_access_deployment_test.rs +++ b/app/crowdb-web/tests/native_access_deployment_test.rs @@ -4,10 +4,7 @@ use std::os::unix::fs::PermissionsExt; use axum::{body::Body, http::Request}; -use crowdb_console_shared::{ - config::{NodeEntry, RackEntry, ServerEntry}, - ConsoleConfig, -}; +use crowdb_console_shared::config::{NodeEntry, RackEntry}; use crowdb_web::{router, AppState}; use serde_json::{json, Value}; use tower::ServiceExt; @@ -41,24 +38,27 @@ impl Drop for TestProcess { } } +struct TestCluster(std::path::PathBuf); + +impl Drop for TestCluster { + fn drop(&mut self) { + crowdb_console_shared::ops::s3::stop(&self.0).unwrap(); + } +} + #[tokio::test] -#[ignore = "Requires an isolated native storage chain and its private credential directory"] +#[cfg_attr(target_os = "macos", ignore = "Requires a complete native storage chain")] async fn native_access_deployment_provisions_private_credentials_and_reuses_them() { - let source = std::path::PathBuf::from(std::env::var("CROWDB_NATIVE_DEPLOYMENT_DATA").unwrap()); - let seed = std::env::var("CROWDB_NATIVE_DEPLOYMENT_SEED").unwrap(); + let fixture = crowdb_test_harness::test_dirs::tempdir_in_test_data("native-access-storage"); + let source = fixture.path().to_owned(); + let _cluster = TestCluster(source.clone()); + crowdb_console_shared::ops::s3::start(&source).await.unwrap(); + let (cluster, _) = crowdb_console_shared::ops::s3::load(&source).unwrap(); let workspace = crowdb_test_harness::test_dirs::tempdir_in_test_data("native-access-deploy"); - std::fs::create_dir(workspace.path().join("secrets")).unwrap(); - std::fs::set_permissions( - workspace.path().join("secrets"), - std::fs::Permissions::from_mode(0o700), - ) - .unwrap(); - std::fs::copy( - source.join("secrets/server.env"), - workspace.path().join("secrets/server.env"), - ) - .unwrap(); - let mut config = ConsoleConfig::default(); + let mut config = cluster; + config + .servers + .retain(|server| server.service_type != crowdb_console_shared::config::ServiceType::AccessServer); config .add_rack(RackEntry { id: 1, @@ -77,17 +77,24 @@ async fn native_access_deployment_provisions_private_credentials_and_reuses_them ssh_credential_ref: None, }) .unwrap(); - config.servers.push(ServerEntry::new("kv-native", seed)); - let state = AppState::with_runtime_root(config, workspace.path().to_owned()); + let seeds = config + .servers + .iter() + .filter(|server| server.service_type == crowdb_console_shared::config::ServiceType::PaxosKv) + .map(|server| server.url.clone()) + .collect(); + let mut state = AppState::with_runtime_root(config, workspace.path().to_owned()); + state.authority_seeds = std::sync::Arc::new(seeds); let app = router(state.clone()); let http = crowdb_protocol::port::alloc::alloc_test_port(crowdb_protocol::ServicePort::Web); let s3 = crowdb_protocol::port::alloc::alloc_test_port(crowdb_protocol::ServicePort::Web); + let health = crowdb_protocol::port::alloc::alloc_test_port(crowdb_protocol::ServicePort::Web); let (status, body) = call( &app, "POST", "/api/nodes/1/services/deploy", json!({ - "kind":"access-server", "instance_id":"902", "http_port":http, "s3_port":s3, + "kind":"access-server", "instance_id":"902", "http_port":http, "s3_port":s3, "health_port":health, "test_single_node":true }), ) .await; diff --git a/app/crowdb-web/tests/native_cluster_provisioning_test.rs b/app/crowdb-web/tests/native_cluster_provisioning_test.rs index f49954af9..85877bb95 100644 --- a/app/crowdb-web/tests/native_cluster_provisioning_test.rs +++ b/app/crowdb-web/tests/native_cluster_provisioning_test.rs @@ -85,9 +85,21 @@ async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> Valu async fn deploy(app: &axum::Router, node: u64, kind: &str) { let defaults = call(app, "GET", "/api/deployment-defaults", Value::Null).await; let mut body = defaults[kind].clone(); + // Each concurrent fixture reserves its listeners through the shared test allocator. + for field in ["http_port", "rpc_port", "s3_port", "health_port"] { + if body.get(field).is_some() { + let port = if kind == "diskdb" { + crowdb_protocol::port::alloc::alloc_test_port_range(crowdb_protocol::ServicePort::Web, 3)[2] + } else { + crowdb_protocol::port::alloc::alloc_test_port(crowdb_protocol::ServicePort::Web) + }; + body[field] = json!(port); + } + } let path = match kind { "paxos-kv" => { - body = json!({"rest_port": body["http_port"], "rpc_port": body["rpc_port"]}); + body = json!({"rest_port": body["http_port"], "rpc_port": body["rpc_port"], + "kv_backend": "block", "wal_backend": "block-device"}); format!("/api/nodes/{node}/server/deploy") } "diskdb" => { @@ -155,15 +167,17 @@ async fn assert_mixed_geometry_rejected(app: &axum::Router, state: &AppState) { ); } -async fn add_native_disk(app: &axum::Router, node: u64, root: &std::path::Path) { +async fn add_native_disk(app: &axum::Router, node: u64, root: &std::path::Path, inspection_only: bool) { let device = root.join(format!("disk-{node}.img")); // Repeated production splits reserve whole 256-MiB mirrored Chunks. // Keep the ordinary browser geometry, but provision the slow fixture for // every retained writer and its split/transfer preparation artifacts. - let gib = if std::env::var_os("CROWDB_NATIVE_WEIGHTED_ACCEPTANCE").is_some() { + let gib = if std::env::var_os("CROWDB_NATIVE_COUNT_ACCEPTANCE").is_some() { 256 } else if node == 1 { 80 + } else if inspection_only { + 256 } else { 8 }; @@ -272,7 +286,11 @@ async fn upload_native_multipart(app: &axum::Router, object: &str) -> Vec { .skip(1) .map(|part| part.split_once("").unwrap().0) .collect(); - assert_eq!(etags.len(), 2); + assert_eq!( + etags.len(), + 2, + "ListParts response after two successful uploads: {parts}" + ); let completion = format!("1{}2{}", etags[0], etags[1]); s3_request( app, @@ -399,7 +417,10 @@ async fn assert_native_browser_diagnostics(app: axum::Router, chunks: Option bool { for node in 1..=3 { deploy(app, node, kind).await; } - if kind == "diskio" && std::env::var_os("CROWDB_NATIVE_PLAN_PREREQUISITES").is_some() { + if kind == "chunkdb" && std::env::var_os("CROWDB_NATIVE_PLAN_PREREQUISITES").is_some() { assert_native_browser_diagnostics(app.clone(), None).await; return false; } diff --git a/app/crowdb-web/tests/replica_leader_removal_test.rs b/app/crowdb-web/tests/replica_leader_removal_test.rs index e88da45e5..c5f3a6ff3 100644 --- a/app/crowdb-web/tests/replica_leader_removal_test.rs +++ b/app/crowdb-web/tests/replica_leader_removal_test.rs @@ -82,11 +82,9 @@ fn tempdir(tag: &str) -> PathBuf { } #[allow(clippy::unused_async)] -async fn spawn_upstream(node_id: u64, workspace: &std::path::Path) -> Option { - let bin = crowdb_kv_server_bin()?; - if !bin.exists() { - return None; - } +async fn spawn_upstream(node_id: u64, workspace: &std::path::Path) -> Upstream { + let bin = crowdb_kv_server_bin().expect("build crowdb-kv-server before running this test"); + assert!(bin.exists(), "KV server binary missing: {}", bin.display()); let node = NodeEntry { id: node_id, rack_id: 1, @@ -116,12 +114,12 @@ async fn spawn_upstream(node_id: u64, workspace: &std::path::Path) -> Option) -> SocketAddr { @@ -192,21 +190,15 @@ async fn spawn_web(upstreams: &BTreeMap) -> SocketAddr { } #[allow(clippy::unused_async)] -async fn spawn_five_node_cluster(test_name: &str) -> Option { +async fn spawn_five_node_cluster(test_name: &str) -> Cluster { let workspace = tempdir(test_name); let mut nodes: BTreeMap = BTreeMap::new(); for id in 1u64..=5 { - let Some(node) = spawn_upstream(id, &workspace).await else { - for n in nodes.into_values() { - let _ = lifecycle::stop_pid_with_timeout(n.pid, Duration::from_secs(5)); - } - eprintln!("skipping: crowdb-kv-server binary not built"); - return None; - }; + let node = spawn_upstream(id, &workspace).await; nodes.insert(id, node); } let web = spawn_web(&nodes).await; - Some(Cluster { nodes, web }) + Cluster { nodes, web } } async fn create_five_node_group(cluster: &Cluster) { @@ -386,11 +378,7 @@ async fn role_on_node(cluster: &Cluster, node_id: u64, rid: u64) -> Option { const api = await apiContext(baseURL); try { - const response = await api.post(path, { data }); - expect(response.status(), `${label}: ${await response.text()}`).toBe(201); + const attempts = process.platform === 'darwin' ? 60 : 1; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await api.post(path, { data }); + if (response.status() === 201) return; + const body = await response.text(); + if (process.platform !== 'darwin' || response.status() !== 404 || attempt === attempts) { + expect(response.status(), `${label}: ${body}`).toBe(201); + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + } } finally { await api.dispose(); } @@ -164,7 +172,8 @@ export const COMPLEX: TopologyDescriptor = { // ── Low-level API helpers (kept for backward compat) ───────────────── export async function apiContext(baseURL: string) { - return request.newContext({ baseURL }); + const timeout = process.platform === 'darwin' ? 30_000 : undefined; + return request.newContext({ baseURL, timeout }); } // Debug helper: dump API state to console.log. @@ -348,12 +357,13 @@ export async function waitForLeader(baseURL: string, storeId: number, groupId: n export async function clusterInit(baseURL: string, nodeIds: number[]) { const api = await apiContext(baseURL); try { - const maxAttempts = 5; + const maxAttempts = process.platform === 'darwin' ? 10 : 5; + const timeout = process.platform === 'darwin' ? 30_000 : 10_000; for (let attempt = 1; attempt <= maxAttempts; attempt++) { - const response = await api.post('/api/cluster/init', { data: { nodes: nodeIds }, timeout: 10_000 }); + const response = await api.post('/api/cluster/init', { data: { nodes: nodeIds }, timeout }); if (response.status() === 201 || response.status() === 409) return; const text = await response.text(); - if (response.status() === 500 && text.includes('not leader') && attempt < maxAttempts) { + if (response.status() === 500 && (text.includes('not leader') || process.platform === 'darwin') && attempt < maxAttempts) { console.log(`cluster_init attempt ${attempt} got transient not-leader response; retrying...`); await new Promise((resolve) => setTimeout(resolve, 1_000)); continue; diff --git a/app/crowdb-web/ui/e2e/fixtures/defaultDeployments.ts b/app/crowdb-web/ui/e2e/fixtures/defaultDeployments.ts index 91d43e349..683c3aa90 100644 --- a/app/crowdb-web/ui/e2e/fixtures/defaultDeployments.ts +++ b/app/crowdb-web/ui/e2e/fixtures/defaultDeployments.ts @@ -6,7 +6,7 @@ import { expect } from './realBackend'; /** Observe actual automatic deployment mutations before checking durable progress. */ export function observeDefaultDeployments(page: Page) { - const replies = new Map(); + const replies = new Map(); function key(response: Response) { if (response.request().method() !== 'POST') return undefined; const path = new URL(response.url()).pathname; @@ -17,12 +17,29 @@ export function observeDefaultDeployments(page: Page) { } page.on('response', response => { const identity = key(response); - if (identity && !replies.has(identity)) replies.set(identity, response); + if (identity) replies.set(identity, [...(replies.get(identity) ?? []), response]); }); return { + async nextCompleted(nodes: number[], kind: string) { + const pending = new Set(nodes.map(node => `${node}:${kind}`)); + const completed = nodes.find(node => replies.get(`${node}:${kind}`)?.some(response => response.status() === 201)); + if (completed !== undefined) return completed; + const timeout = process.platform === 'darwin' ? 30_000 : 3_000; + const response = await page.waitForResponse(response => pending.has(key(response) ?? '') && response.status() === 201, { timeout }); + return Number(key(response)!.split(':')[0]); + }, async verify(node: number, kind: string) { const identity = `${node}:${kind}`; - const response = replies.get(identity) ?? await page.waitForResponse(response => key(response) === identity); + const timeout = process.platform === 'darwin' ? 30_000 : 3_000; + const response = replies.get(identity)?.find(response => response.status() === 201) + ?? await page.waitForResponse(response => key(response) === identity && response.status() === 201, { timeout }); + for (const rejected of replies.get(identity) ?? []) { + if (rejected === response) break; + const body = await rejected.json(); + expect(rejected.status(), JSON.stringify(body)).toBe(409); + expect(['chunkdb', 'chunk-kv']).toContain(kind); + expect(body.error).toMatch(/^Waiting: /); + } expect(response.status(), `Node ${node} ${kind}: ${await response.text()}`).toBe(201); }, }; diff --git a/app/crowdb-web/ui/e2e/fixtures/realBackend.ts b/app/crowdb-web/ui/e2e/fixtures/realBackend.ts index 4e9819727..de377d4ce 100644 --- a/app/crowdb-web/ui/e2e/fixtures/realBackend.ts +++ b/app/crowdb-web/ui/e2e/fixtures/realBackend.ts @@ -8,6 +8,20 @@ const rejectInterception = async () => { }; export const test = base.extend({ + request: async ({ request, playwright, baseURL }, use, testInfo) => { + if (testInfo.project.metadata.fileBackedSync !== true) { + await use(request); + return; + } + // File-backed native devices can spend tens of milliseconds in each + // durable sync. Data preparation has a separate budget from UI actions. + const durableRequest = await playwright.request.newContext({ baseURL, timeout: 10_000 }); + try { + await use(durableRequest); + } finally { + await durableRequest.dispose(); + } + }, context: async ({ context }, use) => { context.route = rejectInterception; context.routeFromHAR = rejectInterception; diff --git a/app/crowdb-web/ui/e2e/flows/01-shell-ui-behaviors.spec.ts b/app/crowdb-web/ui/e2e/flows/01-shell-ui-behaviors.spec.ts index d20684617..cde988f46 100644 --- a/app/crowdb-web/ui/e2e/flows/01-shell-ui-behaviors.spec.ts +++ b/app/crowdb-web/ui/e2e/flows/01-shell-ui-behaviors.spec.ts @@ -14,6 +14,7 @@ import { resetAll, seedRackAndNode, stopNodeServer, + stopNodeServerAndReleasePorts, waitForLeader, } from '../fixtures/consoleSetup'; import { step } from '../fixtures/stepTimer'; @@ -151,9 +152,9 @@ test.describe('shell · UI behaviors', () => { } finally { await api.dispose(); await step('shell: stop servers', () => Promise.all([ - stopNodeServer(baseURL!, 201), - stopNodeServer(baseURL!, 202), - stopNodeServer(baseURL!, 203), + stopNodeServerAndReleasePorts(baseURL!, 201), + stopNodeServerAndReleasePorts(baseURL!, 202), + stopNodeServerAndReleasePorts(baseURL!, 203), ])); } diff --git a/app/crowdb-web/ui/e2e/flows/11-cluster-server-lifecycle.spec.ts b/app/crowdb-web/ui/e2e/flows/11-cluster-server-lifecycle.spec.ts index 3a1d23299..3155d8202 100644 --- a/app/crowdb-web/ui/e2e/flows/11-cluster-server-lifecycle.spec.ts +++ b/app/crowdb-web/ui/e2e/flows/11-cluster-server-lifecycle.spec.ts @@ -5,6 +5,7 @@ import { test, expect } from '../fixtures/realBackend'; import { apiContext, clusterInit, createNode, createRack, deployNodeServer, freePort, resetAll, seedRackAndNode, stopNodeServer } from '../fixtures/consoleSetup'; import { step } from '../fixtures/stepTimer'; +import { observeDefaultDeployments } from '../fixtures/defaultDeployments'; import { readFileSync } from 'node:fs'; test('native diagnostics: auxiliary menus stop and restart the actual typed process', async ({ page, request }) => { @@ -23,7 +24,7 @@ test('native diagnostics: auxiliary menus stop and restart the actual typed proc }; await page.goto('/?domain=Cluster'); const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); - for (const [kind, label, prefix] of [['chunkdb', 'CDB (ChunkDB)', 'CDB'], ['diskio', 'DiskIO', 'DIO'], ['chunk-kv', 'Chunk-KV', 'CKV'], ['access-server', 'Access Server', 'AS']]) { + for (const [kind, label, prefix] of [['chunkdb', 'crowdb-chunk-db', 'CDB'], ['diskio', 'crowdb-disk-io', 'DIO'], ['chunk-kv', 'crowdb-chunk-kv', 'CKV'], ['access-server', 'crowdb-access-server', 'AS']]) { const original = (await list()).find((row: { node_id: number; service_type: string }) => row.node_id === 1 && row.service_type === kind); expect(original.pid).toBeGreaterThan(0); expect(alive(original.pid)).toBe(true); const node = sidebar.getByRole('button', { name: 'N-1', exact: true }); @@ -31,24 +32,44 @@ test('native diagnostics: auxiliary menus stop and restart the actual typed proc await expect(item).toBeVisible(); await node.click({ button: 'right' }); await page.getByRole('menuitem', { name: `${prefix}-1 Running`, exact: true }).hover(); + const stoppedObservation = page.waitForResponse(async response => { + if (new URL(response.url()).pathname !== '/api/servers' || !response.ok()) return false; + const rows = await response.json(); + return rows.some((row: { id: string; pid?: number }) => row.id === original.id && !row.pid); + }); await page.getByRole('menuitem', { name: `Stop ${label}`, exact: true }).click(); + await stoppedObservation; + await expect(page.getByRole('button', { name: 'Refresh', exact: true }).locator('svg')).not.toHaveClass(/tw-animate-spin/); await expect.poll(() => alive(original.pid), { intervals: [100] }).toBe(false); await expect.poll(async () => { const stopped = (await list()).find((row: { id: string }) => row.id === original.id); return { id: stopped?.id, pid: stopped?.pid ?? null }; }, { intervals: [100] }).toEqual({ id: original.id, pid: null }); + let startSubmitted = false; try { await node.click({ button: 'right' }); await page.getByRole('menuitem', { name: `${prefix}-1 Stopped`, exact: true }).hover(); - await page.getByRole('menuitem', { name: `Start ${label}`, exact: true }).click(); - await expect.poll(async () => { + // Native recovery previously reached its listener after 3.15s; keep a + // modest response budget while process and DOM assertions remain 3s. + const started = page.waitForResponse(response => + new URL(response.url()).pathname === `/api/services/${original.id}/restart` + && response.request().method() === 'POST', { timeout: 5_000 }); + const startAt = performance.now(); + startSubmitted = true; + const response = await step(`native lifecycle: ${original.id} Start response`, async () => { + await page.getByRole('menuitem', { name: `Start ${label}`, exact: true }).click(); + return started; + }); + console.log(`[START] ${original.id}: ${(performance.now() - startAt).toFixed(0)}ms`); + expect(response.status(), await response.text()).toBe(200); + await step(`native lifecycle: ${original.id} new process`, () => expect.poll(async () => { const current = (await list()).find((row: { id: string }) => row.id === original.id); return current?.pid && current.pid !== original.pid && alive(current.pid); - }, { intervals: [100] }).toBe(true); + }, { intervals: [100] }).toBe(true)); expect((await list()).filter((row: { id: string }) => row.id === original.id)).toHaveLength(1); } finally { const current = (await list()).find((row: { id: string }) => row.id === original.id); - if (!current?.pid) expect((await request.post(`/api/services/${original.id}/restart`, { data: {} })).ok()).toBe(true); + if (!startSubmitted && !current?.pid) expect((await request.post(`/api/services/${original.id}/restart`, { data: {} })).ok()).toBe(true); } } expect(await list()).toHaveLength(18); @@ -167,8 +188,8 @@ test.describe('cluster · server lifecycle', () => { test('deploys and stops a real crowdb-kv-server through the UI', async ({ page, baseURL }) => { await step('deploy-ui: seedRackAndNode', () => seedRackAndNode(baseURL!, 4, 4)); - const restPort = freePort(); - const rpcPort = freePort(); + const restPort = freePort('kv-mgmt'); + const rpcPort = freePort('kv-listen'); const api = await apiContext(baseURL!); try { await step('deploy-ui: deploy dialog', async () => { @@ -183,7 +204,9 @@ test.describe('cluster · server lifecycle', () => { await expect(page.getByRole('dialog', { name: /deploy CrowDB Storage on 4/i })).toBeVisible(); await page.getByLabel('REST Port').fill(String(restPort)); await page.getByLabel('RPC Port').fill(String(rpcPort)); - await page.getByRole('button', { name: 'Deploy' }).click(); + const deploy = page.getByRole('button', { name: 'Deploy' }); + await expect(deploy).toBeEnabled({ timeout: process.platform === 'darwin' ? 30_000 : 3_000 }); + await deploy.click(); }); await step('deploy-ui: poll server', () => expect.poll(async () => { @@ -387,7 +410,7 @@ test('native diagnostics: auxiliary dialogs reopen with unused IDs ports and val const servers = await serversResponse.json(); await page.goto('/?domain=Cluster'); const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); - for (const [kind, label] of [['chunkdb', 'CDB (ChunkDB)'], ['diskio', 'DiskIO'], ['chunk-kv', 'Chunk-KV'], ['access-server', 'Access Server']]) { + for (const [kind, label] of [['chunkdb', 'crowdb-chunk-db'], ['diskio', 'crowdb-disk-io'], ['chunk-kv', 'crowdb-chunk-kv'], ['access-server', 'crowdb-access-server']]) { for (let reopen = 0; reopen < 2; reopen++) { await sidebar.getByRole('button', { name: 'N-1', exact: true }).click({ button: 'right' }); await page.getByRole('menuitem', { name: `Deploy ${label}`, exact: true }).click(); @@ -415,7 +438,7 @@ test('native diagnostics: auxiliary dialogs reopen with unused IDs ports and val // Baseline: 1.3s (2026-10-04); existing instances come from the owned fixture. test('native diagnostics: interrupted six-service plan reconciles without duplicate deployment', async ({ page, request }) => { test.skip(!!process.env.CROWDB_NATIVE_PLAN_PREREQUISITES, 'Requires the complete six-service fixture phase'); - const kinds = ['kv', 'diskdb', 'chunkdb', 'diskio', 'chunk-kv', 'access-server']; + const kinds = ['paxos-kv', 'diskdb', 'chunkdb', 'diskio', 'chunk-kv', 'access-server']; const servers = await request.get('/api/servers'); expect(servers.ok()).toBe(true); const original = await servers.json(); @@ -425,9 +448,9 @@ test('native diagnostics: interrupted six-service plan reconciles without duplic for (const kind of kinds) expect(original.filter((row: { node_id: number; service_type: string }) => row.node_id === 1 && row.service_type === kind)).toHaveLength(1); const saved = await request.put('/api/nodes/1/service-plan', { data: { revision: 0, - steps: Object.fromEntries(kinds.map(kind => [kind, { state: kind === 'kv' ? 'deployed' : kind === 'chunkdb' ? 'deploying' : 'pending' }])), + steps: Object.fromEntries(kinds.map(kind => [kind, { state: kind === 'paxos-kv' ? 'deployed' : kind === 'chunkdb' ? 'deploying' : 'pending' }])), } }); - expect(saved.ok()).toBe(true); + expect(saved.ok(), await saved.text()).toBe(true); let deploymentRequests = 0; page.on('request', current => { if (current.method() === 'POST' && /\/api\/nodes\/\d+\/(server|diskdb|services)\/deploy$/.test(new URL(current.url()).pathname)) deploymentRequests++; @@ -472,7 +495,8 @@ test('native diagnostics: interrupted six-service plan reconciles without duplic // Baseline: 7.6s (2026-10-04); prerequisite services are controlled through real lifecycle APIs. test('native diagnostics: waiting plan resumes when DiskIO prerequisites arrive', async ({ page, request }) => { test.skip(!process.env.CROWDB_NATIVE_PLAN_PREREQUISITES, 'Requires the partial deployment fixture phase'); - const kinds = ['kv', 'diskdb', 'chunkdb', 'diskio', 'chunk-kv', 'access-server']; + const kinds = ['paxos-kv', 'diskdb', 'chunkdb', 'diskio', 'chunk-kv', 'access-server']; + const deployments = observeDefaultDeployments(page); const list = async () => { const response = await request.get('/api/servers'); expect(response.ok()).toBe(true); @@ -503,7 +527,7 @@ test('native diagnostics: waiting plan resumes when DiskIO prerequisites arrive' revision: 0, steps: Object.fromEntries(kinds.map(kind => [kind, { state: ['chunk-kv', 'access-server'].includes(kind) ? 'pending' : 'deployed' }])), } }); - expect(saved.ok()).toBe(true); + expect(saved.ok(), await saved.text()).toBe(true); await page.goto('/?domain=Cluster'); await open(); await expect(dialog).toContainText('restart registered DiskIO services'); @@ -524,6 +548,8 @@ test('native diagnostics: waiting plan resumes when DiskIO prerequisites arrive' expect(deploys).toEqual([]); expect((await request.post(`/api/services/${diskioIds[1]}/restart`, { data: {} })).ok()).toBe(true); await expect(dialog.getByRole('listitem').filter({ hasText: 'deploying' })).toHaveCount(1); + await deployments.verify(1, 'chunk-kv'); + await deployments.verify(1, 'access-server'); await expect.poll(async () => { const response = await request.get('/api/service-plans'); expect(response.ok()).toBe(true); diff --git a/app/crowdb-web/ui/e2e/flows/22-kv-topology.spec.ts b/app/crowdb-web/ui/e2e/flows/22-kv-topology.spec.ts index 52d97824b..5f25e9fc6 100644 --- a/app/crowdb-web/ui/e2e/flows/22-kv-topology.spec.ts +++ b/app/crowdb-web/ui/e2e/flows/22-kv-topology.spec.ts @@ -139,6 +139,10 @@ test.describe('kv cluster · multi-rack/multi-store/multi-group topology', () => test.describe.configure({ timeout: 180_000 }); test.beforeAll(async () => { + // The full UI suite leaves many service processes to drain through the + // reset endpoint. Give macOS enough time for those registrations to + // disappear before bootstrapping the next group-0 cluster. + test.setTimeout(180_000); await step('topology: resetAll', () => resetAll(apiBase)); const allNodes = [ @@ -189,13 +193,12 @@ test.describe('kv cluster · multi-rack/multi-store/multi-group topology', () => addGroup(apiBase, 901, 9003, 1, [200, 201, 202]), ])); - // Extend store 199 to nodes 192+193 via addReplica. Both calls - // are independent Paxos writes to group-0 sysdata — run them - // concurrently instead of serially. - await step('topology: addReplica x2', () => Promise.all([ - addReplica(apiBase, 199, 1990, 192, 19901), - addReplica(apiBase, 199, 1990, 193, 19902), - ])); + // Each addition changes the same group's peer configuration. Follow + // the reconfiguration contract: finish one member before adding another. + await step('topology: addReplica x2', async () => { + await addReplica(apiBase, 199, 1990, 192, 19901); + await addReplica(apiBase, 199, 1990, 193, 19902); + }); // Groups 1991/1992 span all 3 nodes — create after addReplica // extends store 199's node set. @@ -244,23 +247,29 @@ test.describe('kv cluster · multi-rack/multi-store/multi-group topology', () => } }); - // Monitor leader election via API polling. - // Three concurrent fresh elections (one per group) need a few - // election deadlines (default 4-8 s each) plus PreVote/RequestVote - // round-trip; 30 s gives headroom on a busy CI machine. + // Observe the current complete membership, not an earlier leader seen + // while a newly added replica was still unknown or catching up. const groups = [1990, 1991, 1992]; const leaders = new Map(); // Poll until all groups have exactly one leader, or timeout. await step('multi-rack: poll leaders', () => expect.poll(async () => { + leaders.clear(); for (const gid of groups) { - if (leaders.has(gid)) continue; const response = await api.get(`/api/stores/199/groups/${gid}`); if (!response.ok()) continue; - const detail: { replicas: Array<{ replica_id: number; role: string }> } = await response.json(); + const detail: { replicas: Array<{ + replica_id: number; role: string; state: string; engine_healthy: boolean; + election?: { current_term: number; lease_remaining_ms?: number; bulk_phase1_in_flight_slots: number }; + }> } = await response.json(); const leaderReplicas = detail.replicas.filter((r) => r.role === 'leader'); - if (leaderReplicas.length === 1) { - leaders.set(gid, leaderReplicas[0].replica_id); + const leader = leaderReplicas.length === 1 ? leaderReplicas[0] : undefined; + if (leader && (leader.election?.lease_remaining_ms ?? 0) > 0 + && detail.replicas.length === 3 && detail.replicas.every(replica => + replica.state === 'running' && replica.engine_healthy + && replica.election?.current_term === leader.election?.current_term + && replica.election?.bulk_phase1_in_flight_slots === 0)) { + leaders.set(gid, leader.replica_id); } } return leaders.size; diff --git a/app/crowdb-web/ui/e2e/flows/54-chunk-layout.spec.ts b/app/crowdb-web/ui/e2e/flows/54-chunk-layout.spec.ts index e683df2d9..87b55f580 100644 --- a/app/crowdb-web/ui/e2e/flows/54-chunk-layout.spec.ts +++ b/app/crowdb-web/ui/e2e/flows/54-chunk-layout.spec.ts @@ -92,13 +92,13 @@ test('native diagnostics: Chunk default ownership and Node type pagination stay test('native diagnostics: Chunk replacement windows and real Mirror EC placement', async ({ page, request }) => { const fixture = JSON.parse(process.env.CROWDB_NATIVE_CHUNK_FIXTURE ?? 'null'); expect(fixture, 'Run with CROWDB_NATIVE_DATA_WINDOWS=1').not.toBeNull(); + let observation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunks'); await page.goto('/?domain=Chunk'); const rows = page.getByRole('table', { name: 'Chunks' }).getByRole('button'); const pages = page.getByRole('navigation', { name: 'Chunk page window' }); - let after: string | null = null; const seen = new Set(); for (let window = 0; window < 20; window++) { - const response = await request.get(`/api/chunks?limit=10${after ? `&after=${after}` : ''}`); + const response = await observation; expect(response.ok(), await response.text()).toBeTruthy(); const data = await response.json(); expect(data.failures).toEqual([]); @@ -108,7 +108,10 @@ test('native diagnostics: Chunk replacement windows and real Mirror EC placement await expect.poll(() => rows.allTextContents(), { intervals: [100] }).toEqual(expected); for (const id of expected) { expect(seen.has(id)).toBe(false); seen.add(id); } if (!data.next) { await expect(pages.getByRole('button', { name: 'Next', exact: true })).toBeDisabled(); break; } - after = data.next; + observation = page.waitForResponse(response => { + const url = new URL(response.url()); + return url.pathname === '/api/chunks' && url.searchParams.get('after') === data.next; + }); await pages.getByRole('button', { name: 'Next', exact: true }).click(); } for (const id of fixture.ids) expect(seen.has(id), id).toBe(true); diff --git a/app/crowdb-web/ui/e2e/flows/55-chunk-kv-catalog.spec.ts b/app/crowdb-web/ui/e2e/flows/55-chunk-kv-catalog.spec.ts index cab99df51..5819627db 100644 --- a/app/crowdb-web/ui/e2e/flows/55-chunk-kv-catalog.spec.ts +++ b/app/crowdb-web/ui/e2e/flows/55-chunk-kv-catalog.spec.ts @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. // Baseline: partition/overlay 1.4s, unavailable 0.208s, journal 0.534s, placement 0.321s (2026-10-03). import { test, expect } from '../fixtures/realBackend'; +import type { APIRequestContext, APIResponse, Page, Response } from '@playwright/test'; import type { CatalogPage, Partition } from '../../src/chunk-kv/catalog'; test('native diagnostics: journal identity survives Chunk navigation and owner interruption', async ({ page, request }) => { @@ -14,22 +15,7 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i await page.goto('/?domain=Chunk-KV'); const response = await catalogObservation; expect(response.ok(), await response.text()).toBe(true); - const catalog = await response.json(); - // Earlier native cases can split the catalog; the first range need not - // contain this write. Select a current range with a real active journal. - let partition = null; - for (const entry of catalog.entries) { - if (entry.state !== 'Serving' || entry.transition_id !== null) continue; - const query = new URLSearchParams({ id: entry.id, epoch: entry.epoch, generation: catalog.generation, page: '0', offset: '0' }); - const response = await request.get(`/api/chunk-kv/runtime?${query}`); - expect(response.ok(), await response.text()).toBe(true); - if ((await response.json()).journal.active) { partition = entry; break; } - } - expect(partition).not.toBeNull(); - const runtimeResponse = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/runtime'); - await page.getByLabel('Partition range map').getByRole('button', { name: `Partition ${partition.id}`, exact: true }).click(); - const observed = await runtimeResponse; - expect(observed.ok(), await observed.text()).toBe(true); + const { partition, observed } = await selectActiveJournal(page, request, await response.json()); const runtime = await observed.json(); expect(runtime.journal.active).not.toBeNull(); await page.getByRole('tab', { name: 'Journal', exact: true }).click(); @@ -43,6 +29,9 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i await expect(page.getByLabel('Exact Chunk ID')).toHaveValue(runtime.journal.active.chunk_id); await page.goBack(); await expect(journal).toBeVisible(); + // Back navigation mounts a fresh runtime observation. Complete it before + // stopping its owner so the unavailable refresh starts from an idle view. + await expect(page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true })).toBeEnabled(); const owner = `chunk-kv-${partition.owner_id}`; expect((await request.post(`/api/services/${owner}/stop`, { data: {} })).ok()).toBe(true); try { @@ -54,8 +43,14 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i } finally { expect((await request.post(`/api/services/${owner}/restart`, { data: {} })).ok()).toBe(true); } + const refreshedMap = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/catalog'); + await page.getByRole('button', { name: 'Refresh catalog', exact: true }).click(); + const mapResponse = await refreshedMap; + expect(mapResponse.ok(), await mapResponse.text()).toBe(true); + const restored = await selectActiveJournal(page, request, await mapResponse.json(), partition.id); + expect(restored.partition.id).toBe(partition.id); + await page.getByRole('tab', { name: 'Journal', exact: true }).click(); const observation = page.getByRole('region', { name: 'Partition runtime', exact: true }); - await observation.getByRole('button', { name: 'Refresh runtime', exact: true }).click(); await expect(observation).not.toContainText('Runtime unavailable'); await expect(journal.getByRole('heading', { name: 'Published extent index', exact: true })).toBeVisible(); const stale = new URL(observed.url()); @@ -82,6 +77,7 @@ test('native diagnostics: Chunk-KV actual Page observation', async ({ page, requ expect(partition, 'inspect a current serving assignment').toBeDefined(); const tree = page.getByTestId('chunk-kv-graph').getByRole('button', { name: `KV Tree for ${partition.id}`, exact: true }); await expect(tree).toHaveAttribute('title', 'Inspect base pages, checkpoint and counters.'); + await expect(page.getByTestId('chunk-kv-graph')).not.toContainText('Weight'); const pageObservation = page.waitForResponse(response => { const url = new URL(response.url()); return url.pathname === '/api/chunk-kv/runtime' && url.searchParams.has('page_path'); @@ -90,6 +86,11 @@ test('native diagnostics: Chunk-KV actual Page observation', async ({ page, requ const response = await pageObservation; expect(response.ok(), await response.text()).toBeTruthy(); const observation = await response.json(); + const properties = page.getByRole('region', { name: 'Split properties', exact: true }); + await expect(properties.locator('dt').filter({ hasText: /^Weight$/ })).toHaveCount(0); + await page.getByRole('tab', { name: 'Overview', exact: true }).click(); + await expect(page.getByRole('region', { name: 'Balance explanation', exact: true })).toHaveCount(0); + await page.getByRole('tab', { name: 'Tree', exact: true }).click(); expect(observation.page.rows.length).toBeLessThanOrEqual(20); const query = new URL(response.url()).searchParams; const explorer = page.getByRole('region', { name: 'KV Page explorer', exact: true }); @@ -125,7 +126,7 @@ test('native diagnostics: Chunk-KV graph survives tab changes, refresh and resiz }); const graph = page.getByTestId('chunk-kv-graph'); const verify = async () => { - await expect(graph.getByRole('button', { name: 'Chunk-KV', exact: true })).toBeVisible(); + await expect(graph.getByRole('button', { name: 'ChunkKV', exact: true })).toBeVisible(); await expect(graph.getByRole('button', { name: /^CKV-/ })).toHaveCount(3); await expect(graph.getByRole('button', { name: /^Partition / })).toHaveCount(catalog.entries.length); await expect(graph.getByRole('button', { name: /^KV Tree for / })).toHaveCount(catalog.entries.length); @@ -143,7 +144,7 @@ test('native diagnostics: Chunk-KV graph survives tab changes, refresh and resiz await step('native graph tab return', async () => { for (let index = 0; index < 3; index++) { await page.getByRole('button', { name: 'PaxosKV', exact: true }).click(); - await observeCatalog(() => page.getByRole('button', { name: 'Chunk-KV', exact: true }).click()); + await observeCatalog(() => page.getByRole('navigation', { name: 'Console domains' }).getByRole('button', { name: 'ChunkKV', exact: true }).click()); await verify(); } }); @@ -215,6 +216,10 @@ test('native diagnostics: production split displays actual inherited and current let current = catalog; await expect.poll(async () => { const response = await request.get('/api/chunk-kv/catalog?page=0&offset=0'); + if (response.status() === 409) { + expect(await response.json()).toEqual({ error: 'Chunk-KV catalog changed during observation; refresh the range map' }); + return catalog.generation; + } expect(response.ok(), await response.text()).toBe(true); current = await response.json(); if (!current) throw new Error('Expected an observed catalog generation'); @@ -280,3 +285,49 @@ test('native diagnostics: large Journal replaces 100 extent fences with its rema await expect(fences).toHaveCount(100); await expect(journal).toContainText('offset 0'); }); + +async function selectActiveJournal(page: Page, request: APIRequestContext, initial: CatalogPage, partitionId?: string) { + let catalog = initial; + let selected: { partition: Partition; observed: Response } | null = null; + const refresh = async (action: () => Promise) => { + const observation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/catalog'); + await action(); + const response = await observation; + expect(response.ok(), await response.text()).toBe(true); + catalog = await response.json(); + }; + const changed = async (response: APIResponse | Response) => { + if (response.status() !== 409) return false; + expect((await response.json()).error).toMatch(/^(Chunk-KV catalog changed; refresh the range map|Catalog changed during runtime observation|Owner, catalog or tree page observation changed; refresh the catalog or page root)$/); + await refresh(() => page.getByRole('button', { name: 'Refresh catalog', exact: true }).click()); + return true; + }; + // Observe a live journal from a coherent current map. A concurrent catalog + // publication explicitly invalidates the old selection and requires the UI refresh. + await expect.poll(async () => { + for (const entry of catalog.entries) { + if (partitionId && entry.id !== partitionId) continue; + if (entry.state !== 'Serving') continue; + const query = new URLSearchParams({ id: entry.id, epoch: entry.epoch, generation: catalog.generation, page: String(catalog.page), offset: String(catalog.offset) }); + const response = await request.get(`/api/chunk-kv/runtime?${query}`); + if (await changed(response)) return false; + expect(response.ok(), await response.text()).toBe(true); + if (!(await response.json()).journal.active) continue; + await page.getByLabel('Partition range map').getByRole('button', { name: `Partition ${entry.id}`, exact: true }).click(); + // Selecting the same partition leaves its identity unchanged. Explicitly + // refresh runtime so owner restart is observed even without a new catalog. + const observation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/runtime'); + await page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true }).click(); + const observed = await observation; + if (await changed(observed)) return false; + expect(observed.ok(), await observed.text()).toBe(true); + selected = { partition: entry, observed }; + return true; + } + if (!catalog.next) throw new Error('No current catalog range has an active journal'); + await refresh(() => page.getByRole('button', { name: 'Next partitions', exact: true }).click()); + return false; + }, { intervals: [100] }).toBe(true); + if (!selected) throw new Error('No coherent journal observation'); + return selected as { partition: Partition; observed: Response }; +} diff --git a/app/crowdb-web/ui/e2e/flows/92-three-node-data.spec.ts b/app/crowdb-web/ui/e2e/flows/92-three-node-data.spec.ts index f0718a774..0b39bd80e 100644 --- a/app/crowdb-web/ui/e2e/flows/92-three-node-data.spec.ts +++ b/app/crowdb-web/ui/e2e/flows/92-three-node-data.spec.ts @@ -106,6 +106,10 @@ async function addDisks({ page }: UIContext) { const branch = sidebar.getByTestId(`tree-node-N-${id}`); const expand = branch.getByRole('button', { name: 'Expand', exact: true }); if (await expand.count()) await expand.click(); + await expect.poll( + () => sidebar.getByRole('button', { name: `DG-${id}`, exact: true }).count(), + { timeout: 5_000, intervals: [100], message: `Disk group DG-${id} appears in the refreshed tree` }, + ).toBeGreaterThan(0); await sidebar.getByRole('button', { name: `DG-${id}`, exact: true }).click({ button: 'right' }); await page.getByRole('menuitem', { name: 'Add Disk', exact: true }).click(); const disks = page.getByRole('dialog', { name: 'Add Disks', exact: true }); @@ -119,12 +123,18 @@ async function verifyServices({ page, request, deployments }: UIContext) { await step('three-node: all six services resume without Retry', async () => { await expect.poll(async () => (await (await request.get('/api/chunk-storage-readiness')).json()), { intervals: [100] }).toMatchObject({ ready: true }); for (const kind of ['paxos-kv', 'diskdb', 'diskio', 'chunkdb', 'chunk-kv', 'access-server']) { - for (const id of [1, 2, 3]) { + const pending = new Set([1, 2, 3]); + while (pending.size) { + // The serial plan may start any ready node first. Verify each actual + // completion before checking its durable progress, without charging + // earlier nodes' startup time to a later node's assertion. + const id = await deployments.nextCompleted([...pending], kind); await deployments.verify(id, kind); await expect.poll(async () => { const plans = await (await request.get('/api/service-plans')).json(); return plans[id].steps[kind]; }, { intervals: [100], message: `Node ${id}: ${kind} automatically deploys` }).toEqual({ state: 'deployed' }); + pending.delete(id); } } const servers = await (await request.get('/api/servers')).json(); @@ -151,12 +161,22 @@ async function s3RoundTrip({ page, request }: UIContext) { await page.getByTestId('domain-s3').click(); await page.getByText('S3 actions', { exact: true }).click(); await page.getByLabel('New bucket', { exact: true }).fill('ui-three-node'); + const created = page.waitForResponse(response => new URL(response.url()).pathname === '/api/access/s3/ui-three-node' + && response.request().method() === 'PUT'); await page.getByRole('button', { name: 'Create bucket', exact: true }).click(); + const bucketResponse = await created; + expect(bucketResponse.status(), bucketResponse.statusText()).toBe(200); await page.getByRole('navigation', { name: 'S3 buckets' }).getByRole('button', { name: 'ui-three-node', exact: true }).click(); await page.getByText('Bucket actions', { exact: true }).click(); await page.getByLabel('Object key', { exact: true }).fill('round-trip.txt'); await page.getByLabel('Object file').setInputFiles({ name: 'round-trip.txt', mimeType: 'text/plain', buffer: Buffer.from(content) }); - await page.getByRole('button', { name: 'Upload', exact: true }).click(); + await step('three-node: S3 upload mutation', async () => { + const uploaded = page.waitForResponse(response => response.request().method() === 'PUT' + && new URL(response.url()).pathname === '/api/access/s3/ui-three-node/round-trip.txt'); + await page.getByRole('button', { name: 'Upload', exact: true }).click(); + const response = await uploaded; + expect(response.status(), response.statusText()).toBe(200); + }); await page.getByRole('button', { name: 'round-trip.txt', exact: true }).click(); await page.getByText('Object actions', { exact: true }).click(); await page.getByRole('button', { name: 'Preview first 4 KiB', exact: true }).click(); @@ -172,7 +192,11 @@ async function icebergRoundTrip({ page, baseURL }: UIContext) { await page.getByTestId('domain-iceberg').click(); await page.getByText('Catalog actions', { exact: true }).click(); await page.getByLabel('Namespace name', { exact: true }).fill('ui_three_node'); + const createdNamespace = page.waitForResponse(response => new URL(response.url()).pathname === '/api/access/iceberg/v1/namespaces' && response.request().method() === 'POST'); await page.getByRole('button', { name: 'Create namespace', exact: true }).click(); + const namespaceResponse = await createdNamespace; + expect(namespaceResponse.status(), await namespaceResponse.text()).toBe(200); + expect((await namespaceResponse.json()).namespace).toEqual(['ui_three_node']); const tree = page.getByRole('navigation', { name: 'Iceberg tree' }); await tree.getByRole('button', { name: 'ui_three_node', exact: true }).click(); await page.getByText('Namespace actions', { exact: true }).click(); @@ -203,6 +227,15 @@ async function teardown({ request, baseURL }: UIContext) { .map((server: { id: string }) => request.delete(`/api/services/${server.id}`))); for (const response of results) expect(response.ok(), await response.text()).toBe(true); } + await step('three-node: stop owned authority services', async () => { + for (const kind of ['diskdb', 'paxos-kv']) { + const stops = await Promise.all(servers.filter((server: { service_type: string; pid?: number }) => server.service_type === kind && server.pid) + .map((server: { node_id: number }) => request.post(`/api/nodes/${server.node_id}/${kind === 'diskdb' ? 'diskdb' : 'server'}/stop`))); + for (const response of stops) expect(response.ok(), await response.text()).toBe(true); + } + await expect.poll(async () => (await (await request.get('/api/servers')).json()) + .filter((server: { pid?: number }) => server.pid).length, { intervals: [100] }).toBe(0); + }); await resetAll(baseURL!); }); } diff --git a/app/crowdb-web/ui/e2e/managedNative.config.ts b/app/crowdb-web/ui/e2e/managedNative.config.ts index e725333c5..b22f0d8ff 100644 --- a/app/crowdb-web/ui/e2e/managedNative.config.ts +++ b/app/crowdb-web/ui/e2e/managedNative.config.ts @@ -3,6 +3,8 @@ import config from './realBackend.config'; export default { ...config, + metadata: { ...config.metadata, fileBackedSync: true }, + timeout: 180_000, globalSetup: undefined, globalTeardown: undefined, webServer: undefined, diff --git a/app/crowdb-web/ui/e2e/realBackend.config.ts b/app/crowdb-web/ui/e2e/realBackend.config.ts index b5354b2d3..f997d05a8 100644 --- a/app/crowdb-web/ui/e2e/realBackend.config.ts +++ b/app/crowdb-web/ui/e2e/realBackend.config.ts @@ -29,6 +29,8 @@ const localBrowsers = [ '/snap/bin/chromium', '/usr/bin/chromium', '/usr/bin/chromium-browser', + '/usr/bin/google-chrome', + '/usr/bin/google-chrome-stable', '/Applications/Chromium.app/Contents/MacOS/Chromium', '/usr/bin/microsoft-edge', '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge', diff --git a/app/crowdb-web/ui/package-lock.json b/app/crowdb-web/ui/package-lock.json index ce4de8077..084e319c7 100644 --- a/app/crowdb-web/ui/package-lock.json +++ b/app/crowdb-web/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "crowdb-console-frontend", - "version": "0.2.2", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "crowdb-console-frontend", - "version": "0.2.2", + "version": "0.3.0", "dependencies": { "clsx": "^2.1.1", "lucide-react": "^0.456.0", diff --git a/app/crowdb-web/ui/package.json b/app/crowdb-web/ui/package.json index 375bfa512..2a7d4f8a5 100644 --- a/app/crowdb-web/ui/package.json +++ b/app/crowdb-web/ui/package.json @@ -1,7 +1,7 @@ { "name": "crowdb-console-frontend", "private": true, - "version": "0.2.2", + "version": "0.3.0", "type": "module", "description": "CrowDB Console SPA. Built with Vite + React + TypeScript + Tailwind. Compiled output in dist/ is served by crowdb-web (Axum) at runtime.", "scripts": { diff --git a/app/crowdb-web/ui/src/chunk-kv/BalanceDetails.test.tsx b/app/crowdb-web/ui/src/chunk-kv/BalanceDetails.test.tsx new file mode 100644 index 000000000..f8cfce433 --- /dev/null +++ b/app/crowdb-web/ui/src/chunk-kv/BalanceDetails.test.tsx @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import { cleanup, render, screen } from '@testing-library/react'; +import { afterEach, expect, it } from 'vitest'; +import { BalanceDetails } from './BalanceDetails'; +import { weightPercent, type Partition, type BalanceSummary } from './catalog'; +afterEach(cleanup); +const owner = { instance_id: '1', partition_count: '5', estimated_bytes: '80', weight: { byte_units: '400000000', count_units: '125000000' } }; +const partition: Partition = { id: 'p', start: 'ff'.repeat(64), end: null, owner_id: '1', endpoint: 'localhost', epoch: '3', state: 'Serving', transition_id: null, + artifact: { tree_id: '1', stream_name: { high: '0', low: '1' } }, + balance: { owner, partition: { estimated_bytes: '60', weight: { byte_units: '300000000', count_units: '25000000' }, reason: 'within tolerance' } } }; +const summary: BalanceSummary = { generation: '3', reason: 'within tolerance', observed_at_ms: '0', deviation_percent_millionths: '5000000', + policy: { byte_weight_percent: '80', imbalance_tolerance_percent: '20', minimum_weighted_improvement_percent: '25', cooldown_ms: '60000' } }; +it('shows the backend contributions and percentage thresholds without using the visible window', () => { + render(); + expect(weightPercent(partition.balance?.partition.weight)).toBe('32.50% (estimated)'); + expect(screen.getByText('52.50% (estimated)')).toBeVisible(); + expect(screen.getByText('30.00 percentage points')).toBeVisible(); + expect(screen.getByText('2.50 percentage points')).toBeVisible(); + expect(screen.getByText('Coefficients: data 80% · count 20%.')).toBeVisible(); + expect(screen.getByText(/Tolerance: 20% relative to equal owner share/)).toBeVisible(); +}); +it('shows unavailable and stale reasons without fabricating zero weight', () => { + const { rerender } = render(); + expect(weightPercent()).toBe('Unavailable'); + expect(screen.queryByText('0.00% (estimated)')).toBeNull(); + rerender(); + expect(screen.getByText(/Stale balance catalog generation/)).toBeVisible(); + expect(screen.queryByText('52.50% (estimated)')).toBeNull(); +}); diff --git a/app/crowdb-web/ui/src/chunk-kv/BalanceDetails.tsx b/app/crowdb-web/ui/src/chunk-kv/BalanceDetails.tsx new file mode 100644 index 000000000..5f83758eb --- /dev/null +++ b/app/crowdb-web/ui/src/chunk-kv/BalanceDetails.tsx @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import { identity, weightPercent, type BalanceSummary, type Partition } from './catalog'; + +export function BalanceDetails({ partition, summary }: { partition: Partition; summary?: BalanceSummary }) { + const value = partition.balance; + const policy = summary?.policy; + const candidate = summary?.candidate; + return
+

Balance

+

{value ? '' : 'Weight unavailable · '}{value?.partition.reason ?? summary?.reason ?? 'Balance observation unavailable'}

+ {value &&
+
Owner weight
{weightPercent(value.owner.weight)}
+
Owner partitions
{value.owner.partition_count}
+
Estimated partition bytes
{value.partition.estimated_bytes}
+
Estimated owner bytes
{value.owner.estimated_bytes}
+
Partition byte contribution
{(Number(value.partition.weight.byte_units) / 10_000_000).toFixed(2)} percentage points
+
Partition count contribution
{(Number(value.partition.weight.count_units) / 10_000_000).toFixed(2)} percentage points
+
} + {policy && <> +

Policy version: {summary?.policy_version ?? 'Unavailable'}.

+

Coefficients: data {policy.byte_weight_percent}% · count {100 - Number(policy.byte_weight_percent)}%.

+ {summary?.owners?.length && summary.owners.every(owner => owner.estimated_bytes === '0') ?

All byte estimates are known zero; this observation uses count-only weight.

: null} +

Tolerance: {policy.imbalance_tolerance_percent}% relative to equal owner share · minimum global loss improvement: {policy.minimum_weighted_improvement_percent}%.

+

Observed maximum deviation: {(Number(summary?.deviation_percent_millionths ?? 0) / 1_000_000).toFixed(2)}%.

+ {summary?.loss_millionths != null &&

Global imbalance score: {(Number(summary.loss_millionths) / 1_000_000).toFixed(4)} (lower is better).

} +

Cooldown: {Number(policy.cooldown_ms) / 1000}s · observation {summary?.observed_at_ms ? new Date(Number(summary.observed_at_ms)).toISOString() : 'Unavailable'} · freshness window {Number(summary?.valid_for_ms ?? 0) / 1000}s.

+ } + {candidate &&

Best candidate split {identity(candidate.partition_id)}: server {candidate.source_id} → {candidate.target_id}, global loss improvement {(Number(candidate.improvement_percent_millionths) / 1_000_000).toFixed(2)}%; predicted owner weights {weightPercent(candidate.source_after)} / {weightPercent(candidate.target_after)}. {summary?.reason}.

} +

Weight uses the complete catalog and cached estimates. Range splitting does not guarantee equal sizes.

+
; +} diff --git a/app/crowdb-web/ui/src/chunk-kv/ChunkKvView.tsx b/app/crowdb-web/ui/src/chunk-kv/ChunkKvView.tsx index d13b979c5..c1899ce06 100644 --- a/app/crowdb-web/ui/src/chunk-kv/ChunkKvView.tsx +++ b/app/crowdb-web/ui/src/chunk-kv/ChunkKvView.tsx @@ -102,12 +102,12 @@ export function ChunkKvView({ active, racks, nodes, servers, onChunk }: { onChun {!page && !busy && error &&

Catalog unavailable. Check Group 0 and the current cluster's ChunkKV deployment in Cluster.

} {page && <>

Generation {page.generation} · catalog page {page.page + 1} / {page.catalog_pages} · {page.entries.length} loaded partitions. Catalog assignment; live serving state is not observed.

- Loading topology…

}>Loading topology…

}> { select(entry); changeQuery({ ...queryRef.current, tab: 'Tree' }); }} /> {!entries.length &&

No splits in this catalog window.

} {page.next && } } - {selected && { checkpoint(); setSelected(null); }} />} + {selected && { checkpoint(); setSelected(null); }} />} ; } diff --git a/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.tsx b/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.tsx index a0f780df8..1ac3801c7 100644 --- a/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.tsx +++ b/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.tsx @@ -4,7 +4,7 @@ import type { SplitQuery } from './query'; import { createPortal } from 'react-dom'; import { buttonClass } from '../access/Workbench'; -import { identity, range, type Partition } from './catalog'; +import { identity, range, type BalanceSummary, type Partition } from './catalog'; import { RuntimeObservation } from './RuntimeObservation'; import { useRuntimeObservation } from './useRuntimeObservation'; import { TreeStorage, JournalStorage } from './StorageObservation'; @@ -16,7 +16,7 @@ function Properties({ values, stacked = false }: { stacked?: boolean; values: Re ; } -export function PartitionDetail({ partition: p, generation, currentGeneration, active, catalogPage, catalogOffset, onBack, onChunk, propertyHost, query, onQuery }: { query: SplitQuery; onQuery: (query: SplitQuery) => void; propertyHost: HTMLDivElement | null; onChunk: (id: string) => void; partition: Partition; generation: string; currentGeneration?: string; active: boolean; catalogPage: number; catalogOffset: number; onBack: () => void }) { +export function PartitionDetail({ partition: p, generation, currentGeneration, active, catalogPage, catalogOffset, onBack, onChunk, propertyHost, query, onQuery }: { balance?: BalanceSummary; query: SplitQuery; onQuery: (query: SplitQuery) => void; propertyHost: HTMLDivElement | null; onChunk: (id: string) => void; partition: Partition; generation: string; currentGeneration?: string; active: boolean; catalogPage: number; catalogOffset: number; onBack: () => void }) { const tab = query.tab; const setTab = (tab: string) => onQuery({ ...query, tab }); const overlay = p.artifact.tail_overlay; diff --git a/app/crowdb-web/ui/src/chunk-kv/PartitionGraph.test.tsx b/app/crowdb-web/ui/src/chunk-kv/PartitionGraph.test.tsx index 168b0a4be..7a195528d 100644 --- a/app/crowdb-web/ui/src/chunk-kv/PartitionGraph.test.tsx +++ b/app/crowdb-web/ui/src/chunk-kv/PartitionGraph.test.tsx @@ -20,6 +20,7 @@ it('replaces bounded split windows and collapses only the selected server', asyn return ; } render(); + expect(screen.queryByText(/Weight/)).toBeNull(); // jsdom has no layout: inspect real graph cards by their accessible labels. // The native browser case separately verifies measured cards are visible. const cards = (prefix: string) => screen.getByTestId('chunk-kv-graph').querySelectorAll(`button[aria-label^="${prefix}"]`); diff --git a/app/crowdb-web/ui/src/chunk-kv/PartitionGraph.tsx b/app/crowdb-web/ui/src/chunk-kv/PartitionGraph.tsx index 72d2027c9..87741fb1e 100644 --- a/app/crowdb-web/ui/src/chunk-kv/PartitionGraph.tsx +++ b/app/crowdb-web/ui/src/chunk-kv/PartitionGraph.tsx @@ -9,7 +9,7 @@ import { Boxes, ChevronDown, ChevronRight, Cog, Database, FolderTree, ScanSearch import type { ServerSummary } from '../api'; import { buttonClass } from '../access/Workbench'; import { serviceInstanceLabel } from '../services/client'; -import { range, type Partition } from './catalog'; +import { range, type BalanceSummary, type Partition } from './catalog'; const endpointKey = (value: string) => value.replace(/^[a-z]+:\/\//, '').replace(/\/$/, ''); const SERVER_LIMIT = 8; @@ -104,7 +104,7 @@ function GraphCanvas({ layout, edges, layoutKey }: { layout: Node[]; edges export function PartitionGraph({ entries, servers, selectedId, disabled, onSelect, onTree, query, onQuery }: { query: GraphQuery; onQuery: (query: GraphQuery) => void; - entries: Partition[]; servers: ServerSummary[]; selectedId?: string; disabled: boolean; + balance?: BalanceSummary; entries: Partition[]; servers: ServerSummary[]; selectedId?: string; disabled: boolean; onSelect: (partition: Partition) => void; onTree: (partition: Partition) => void; }) { const { serverPage, offsets } = query; @@ -113,6 +113,7 @@ export function PartitionGraph({ entries, servers, selectedId, disabled, onSelec const setOffsets = (update: (value: Record) => Record) => onQuery({ ...query, offsets: update(offsets) }); const setCollapsed = (update: (value: Set) => Set) => onQuery({ ...query, collapsed: [...update(collapsed)] }); const groups = servers.map(server => ({ + endpoint: endpointKey(server.rpc_url ?? server.endpoint ?? ''), id: `server-${server.id ?? server.rpc_url ?? server.endpoint}`, label: serviceInstanceLabel('chunk-kv', server.id ?? String(server.node_id)), entries: entries.filter(entry => endpointKey(entry.endpoint) === endpointKey(server.rpc_url ?? server.endpoint ?? '')), @@ -122,7 +123,7 @@ export function PartitionGraph({ entries, servers, selectedId, disabled, onSelec if (matched.has(entry.id)) continue; const id = `owner-${entry.owner_id}`; let group = groups.find(group => group.id === id); - if (!group) { group = { id, label: `CKV-${entry.owner_id}`, entries: [] }; groups.push(group); } + if (!group) { group = { id, endpoint: endpointKey(entry.endpoint), label: `CKV-${entry.owner_id}`, entries: [] }; groups.push(group); } group.entries.push(entry); } const page = Math.min(serverPage, Math.max(0, Math.ceil(groups.length / SERVER_LIMIT) - 1)); diff --git a/app/crowdb-web/ui/src/chunk-kv/catalog.ts b/app/crowdb-web/ui/src/chunk-kv/catalog.ts index e1ac6baf7..34d2eb5f9 100644 --- a/app/crowdb-web/ui/src/chunk-kv/catalog.ts +++ b/app/crowdb-web/ui/src/chunk-kv/catalog.ts @@ -15,7 +15,16 @@ export interface Overlay { cutover_seq: string; target_stream_start_seq: string; } +export interface Weight { byte_units: string; count_units: string } +export interface BalanceOwner { instance_id: string; rpc_endpoint?: string; partition_count: string; estimated_bytes: string; weight: Weight } +export interface BalanceSummary { + reason: string; policy_version?: string; generation?: string; observed_at_ms?: string; valid_for_ms?: string; + policy?: { byte_weight_percent: string; imbalance_tolerance_percent: string; minimum_weighted_improvement_percent: string; cooldown_ms: string }; + deviation_percent_millionths?: string; loss_millionths?: string | null; owners?: BalanceOwner[]; + candidate?: { partition_id: Identity; source_id: string; target_id: string; improvement_percent_millionths: string; source_after: Weight; target_after: Weight } | null; +} export interface Partition { + balance?: { partition: { estimated_bytes: string; weight: Weight; reason: string }; owner: BalanceOwner } | null; id: string; start: string; end: string | null; @@ -28,6 +37,7 @@ export interface Partition { } export interface Cursor { page: number; offset: number; generation?: string } export interface CatalogPage extends Cursor { + balance?: BalanceSummary; generation: string; catalog_pages: number; entries: Partition[]; @@ -36,3 +46,5 @@ export interface CatalogPage extends Cursor { } export const identity = (id: Identity) => BigInt(id.high).toString(16).padStart(16, '0') + BigInt(id.low).toString(16).padStart(16, '0'); export const range = (partition: Partition) => `[${partition.start ? `0x${partition.start}` : '−∞'}, ${partition.end === null ? '+∞' : `0x${partition.end}`})`; + +export const weightPercent = (weight?: Weight) => weight ? `${((Number(weight.byte_units) + Number(weight.count_units)) / 10_000_000).toFixed(2)}% (estimated)` : 'Unavailable'; diff --git a/app/crowdb-web/ui/src/components/dialogs/DeployServerDialog.tsx b/app/crowdb-web/ui/src/components/dialogs/DeployServerDialog.tsx index 65d493e7d..c693cf412 100644 --- a/app/crowdb-web/ui/src/components/dialogs/DeployServerDialog.tsx +++ b/app/crowdb-web/ui/src/components/dialogs/DeployServerDialog.tsx @@ -34,10 +34,12 @@ export function DeployServerDialog({ const [rpcPort, setRpcPort] = useState(defaultRpcPort); const [isLoading, setIsLoading] = useState(false); const wasOpenRef = useRef(false); + const portsEditedRef = useRef(false); const { success, error } = useToast(); useEffect(() => { if (isOpen && !wasOpenRef.current) { + portsEditedRef.current = false; setRestPort(defaultRestPort); setRpcPort(defaultRpcPort); } @@ -46,7 +48,7 @@ export function DeployServerDialog({ const defaults = useDeploymentDefaults(isOpen); useEffect(() => { - if (!defaults.values) return; + if (!defaults.values || portsEditedRef.current) return; setRestPort(String(defaults.values['paxos-kv'].http_port)); setRpcPort(String(defaults.values['paxos-kv'].rpc_port)); }, [defaults.values]); @@ -85,7 +87,7 @@ export function DeployServerDialog({ description="Spawn a CrowDB Storage instance on this node. Required before stores or replicas can be created." confirmLabel="Deploy" onConfirm={handleSubmit} - confirmDisabled={!defaults.values || !valid || isLoading} + confirmDisabled={!valid || isLoading} confirmLoading={isLoading} >
@@ -95,14 +97,20 @@ export function DeployServerDialog({ label="REST Port" inputMode="numeric" value={restPort} - onChange={(e) => setRestPort(e.target.value)} + onChange={(e) => { + portsEditedRef.current = true; + setRestPort(e.target.value); + }} autoFocus /> setRpcPort(e.target.value)} + onChange={(e) => { + portsEditedRef.current = true; + setRpcPort(e.target.value); + }} />
diff --git a/app/crowdb-web/ui/src/components/dialogs/createFlows.test.tsx b/app/crowdb-web/ui/src/components/dialogs/createFlows.test.tsx index e0f86f379..93410ae81 100644 --- a/app/crowdb-web/ui/src/components/dialogs/createFlows.test.tsx +++ b/app/crowdb-web/ui/src/components/dialogs/createFlows.test.tsx @@ -40,7 +40,7 @@ const deploymentDefaults = { diskio: { instance_id: '1', rpc_port: 13010 }, chunkdb: { instance_id: '1', http_port: 12010, rpc_port: 12110 }, 'chunk-kv': { instance_id: '1', http_port: 15010, rpc_port: 15110 }, - 'access-server': { instance_id: '1', http_port: 9092, s3_port: 9091, health_port: 9093 }, + 'access-server': { instance_id: '1', http_port: 9092, s3_port: 9091, health_port: 9094 }, }; function installFetchMock(response: any = {}, status = 200) { diff --git a/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx b/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx index 3a97708dc..4c48ee539 100644 --- a/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx +++ b/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx @@ -23,7 +23,7 @@ export function DeployServiceDialog({ nodeId, kind, servers, stores, diskGroups, const [httpPort, setHttpPort] = useState(String(base)); const [rpcPort, setRpcPort] = useState(String(kind === 'diskio' ? base : base + 100)); const [s3Port, setS3Port] = useState('9091'); - const [healthPort, setHealthPort] = useState('9093'); + const [healthPort, setHealthPort] = useState('9094'); const [diskGroup, setDiskGroup] = useState(String(diskGroups[0]?.id ?? '')); const [store, setStore] = useState(String(stores.find(store => String(store.store_id) !== '0')?.store_id ?? stores[0]?.store_id ?? '')); const firstGroup = stores.find(entry => String(entry.store_id) === store)?.groups.find(entry => String(entry.group_id) !== '0'); diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx b/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx index 1d2533c60..c5b3b4c38 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx @@ -19,6 +19,80 @@ beforeEach(() => { }); afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); describe('node service plans', () => { + it('does not publish failed progress when page unload cancels an observation', async () => { + const steps = newPlan(); + for (const kind of serviceOrder) steps[kind] = { state: 'disabled' }; + steps['chunk-kv'] = { state: 'waiting', detail: 'Waiting: initialize Group 0 in Paxos KV' }; + let cancel!: (error: Error) => void; + vi.mocked(listServers).mockReturnValue(new Promise((_, reject) => { cancel = reject; })); + const request = vi.mocked(serviceRequest).getMockImplementation()!; + vi.mocked(serviceRequest).mockImplementation(async (path, method, body) => path === '/service-plans' + ? { 1: { revision: 8, steps } } : request(path, method, body)); + const { result, unmount } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + await act(async () => { + window.dispatchEvent(new Event('pagehide')); + cancel(new TypeError('Failed to fetch')); + }); + expect(result.current.plans[1]['chunk-kv'].state).toBe('waiting'); + expect(vi.mocked(serviceRequest).mock.calls.some(([, method]) => method === 'PUT')).toBe(false); + unmount(); + }); + it('observes registered DiskIO restart without waiting for Capacity data', async () => { + const steps = newPlan(); + for (const kind of serviceOrder) steps[kind] = { state: 'disabled' }; + steps['chunk-kv'] = { state: 'waiting' }; + let restarted = false; + vi.mocked(listServers).mockImplementation(async () => [ + { node_id: 1, service_type: 'chunkdb', pid: 12 }, + { node_id: 1, service_type: 'diskio', pid: 13 }, + { node_id: 2, service_type: 'diskio', pid: restarted ? 14 : undefined }, + ] as Awaited>); + const request = vi.mocked(serviceRequest).getMockImplementation()!; + vi.mocked(serviceRequest).mockImplementation(async (path, method, body) => { + if (path === '/service-plans') return { 1: { revision: 1, steps } }; + if (path === '/deployment-defaults') return { 'chunk-kv': { instance_id: '1', http_port: 12010, rpc_port: 12110 } }; + return request(path, method, body); + }); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1]['chunk-kv'].detail).toContain('restart registered DiskIO services'); + expect(serviceRequest).not.toHaveBeenCalledWith('/deployment-defaults', 'GET'); + restarted = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['chunk-kv'].state).toBe('deployed'); + expect(vi.mocked(serviceRequest).mock.calls.filter(([path, method]) => path.endsWith('/services/deploy') && method === 'POST')).toHaveLength(1); + }); + it('loads saved revisions before resuming waiting steps after reload', async () => { + let release!: (value: unknown) => void; + const saved = new Promise(resolve => { release = resolve; }); + const request = vi.mocked(serviceRequest).getMockImplementation()!; + vi.mocked(serviceRequest).mockImplementation(async (path, method, body) => + path === '/service-plans' ? saved : request(path, method, body)); + const { result } = renderHook(() => useNodeServicePlans([], {}, async () => {}, true)); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(serviceRequest).toHaveBeenCalledTimes(1); + const steps = newPlan(); + steps['paxos-kv'] = { state: 'deployed' }; + await act(async () => { release({ 1: { revision: 7, steps } }); }); + expect(result.current.plans[1].diskio.state).toBe('waiting'); + expect(serviceRequest).toHaveBeenCalledWith('/nodes/1/service-plan', 'PUT', expect.objectContaining({ revision: 7 })); + }); + it('keeps a deployment queued when ownership changes after its readiness probe', async () => { + vi.mocked(listServers).mockResolvedValue([{ node_id: 1, service_type: 'diskio', pid: 10 }] as Awaited>); + let ownershipPublished = false; + const request = vi.mocked(serviceRequest).getMockImplementation()!; + vi.mocked(serviceRequest).mockImplementation(async (path, method, body) => { + if (path.endsWith('/services/deploy') && !ownershipPublished) throw new Error('HTTP 409: Waiting: every configured disk group needs live DiskIO ownership'); + return request(path, method, body); + }); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await result.current.start(1, ['chunkdb']); }); + expect(result.current.plans[1].chunkdb).toEqual({ state: 'waiting', detail: 'Waiting: every configured disk group needs live DiskIO ownership' }); + ownershipPublished = true; + await act(async () => { await vi.advanceTimersByTimeAsync(10000); }); + expect(result.current.plans[1].chunkdb.state).toBe('deployed'); + }); it('finishes every selected ChunkDB deployment before journal bootstrap can block the serial plan', async () => { const first = newPlan(); const second = newPlan(); diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts index 4a3efa6ab..bda00eb39 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts @@ -109,6 +109,8 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< if (!busy.current && input.current.enabled && !stopped.current) { busy.current = true; try { + await recovery.current; + if (disposed || stopped.current) return; for (const [key, initial] of Object.entries(plansRef.current)) { if (disposed || stopped.current) break; if (!serviceOrder.some(kind => ['pending', 'waiting'].includes(initial[kind].state))) continue; @@ -118,6 +120,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< let existing; try { existing = await listServers(); } catch (error) { + if (disposed || stopped.current) return; for (const kind of serviceOrder) if (!['deployed', 'disabled'].includes(plan[kind].state)) await write(kind, { state: 'failed', detail: String(error) }); continue; } @@ -182,6 +185,14 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< wakePending.current = true; deploymentStarted = true; } catch (error) { + if (disposed || stopped.current) return; + // Hardware can change between the readiness probe and mutation. + // A rejected prerequisite has not launched a service; keep it queued. + const dependency = String(error).match(/^Error: HTTP 409: (Waiting: .+)$/); + if (dependency && (kind === 'chunkdb' || kind === 'chunk-kv')) { + await write(kind, { state: 'waiting', detail: dependency[1] }); + continue; + } const listeners = Object.entries(overrides.current[id]?.[kind] ?? {}).map(([name, port]) => `${name}=${port}`).join(', '); const detail = `${serviceLabels[kind]}${listeners ? ` (${listeners})` : ''}: ${String(error)}`.slice(0, 4096); // A node without a fixed ChunkDB slot is valid. Keep it as a @@ -194,21 +205,33 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< try { await input.current.refresh(); } catch { /* The shared data hooks report refresh errors. */ } } } catch (error) { + if (disposed || stopped.current) return; for (const id of Object.keys(plansRef.current)) failProgress(Number(id), `Progress could not be saved: ${String(error)}. Reload to reconcile before retrying.`); } finally { busy.current = false; } } if (!disposed) { const hasQueuedWork = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].state === 'waiting')); const waitingForAuthority = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail === 'Waiting: initialize Group 0 in Paxos KV')); + const waitingForRestart = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail?.startsWith('Waiting: restart registered DiskIO services'))); const hasDisks = Object.values(input.current.groups).some(node => Object.values(node.disksByDg).some(disks => disks.length > 0)); - const delay = wakePending.current ? 0 : hasQueuedWork && !hasDisks && !waitingForAuthority ? 10000 : 2000; + const delay = wakePending.current ? 0 : hasQueuedWork && !hasDisks && !waitingForAuthority && !waitingForRestart ? 10000 : 2000; wakePending.current = false; timer = setTimeout(() => { running.current = tick(); }, delay); } } wake.current = () => { if (busy.current) wakePending.current = true; else { clearTimeout(timer); running.current = tick(); } }; running.current = tick(); - return () => { disposed = true; clearTimeout(timer); }; + const dispose = () => { disposed = true; clearTimeout(timer); }; + const restore = (event: PageTransitionEvent) => { if (event.persisted) window.location.reload(); }; + window.addEventListener('pagehide', dispose); + window.addEventListener('beforeunload', dispose); + window.addEventListener('pageshow', restore); + return () => { + dispose(); + window.removeEventListener('pagehide', dispose); + window.removeEventListener('beforeunload', dispose); + window.removeEventListener('pageshow', restore); + }; }, [persist, failProgress]); useEffect(() => { wake.current(); }, [prerequisiteKey]); return { plans, start, stop }; diff --git a/container/crowdb-monitor/tests/liveness_test.rs b/container/crowdb-monitor/tests/liveness_test.rs index 0b434fd41..41f223491 100644 --- a/container/crowdb-monitor/tests/liveness_test.rs +++ b/container/crowdb-monitor/tests/liveness_test.rs @@ -9,9 +9,16 @@ struct TestRunRoot(PathBuf); impl TestRunRoot { fn new() -> Self { - let root = std::env::temp_dir().join(format!("cm-live-{}", Uuid::new_v4().simple())); + // macOS limits Unix-domain socket paths to a short fixed buffer. Keep + // this test root under /tmp instead of the long per-user temp path. + let base = if cfg!(target_os = "macos") { + PathBuf::from("/tmp") + } else { + std::env::temp_dir() + }; + let root = base.join(format!("cm-live-{}", Uuid::new_v4().simple())); fs::create_dir_all(&root).unwrap(); - Self(root.canonicalize().unwrap()) + Self(root) } } diff --git a/container/crowdb-monitor/tests/preview_run_test.rs b/container/crowdb-monitor/tests/preview_run_test.rs index d07530558..1b779b67a 100644 --- a/container/crowdb-monitor/tests/preview_run_test.rs +++ b/container/crowdb-monitor/tests/preview_run_test.rs @@ -9,7 +9,12 @@ struct TestRoot(PathBuf); impl TestRoot { fn new() -> Self { - let path = std::env::temp_dir().join(format!("cm-preview-{}", Uuid::new_v4().simple())); + let temp_root = if cfg!(target_os = "macos") { + PathBuf::from("/tmp") + } else { + std::env::temp_dir() + }; + let path = temp_root.join(format!("cm-preview-{}", Uuid::new_v4().simple())); for name in ["bin", "templates", "data", "run"] { fs::create_dir_all(path.join(name)).unwrap(); } @@ -35,7 +40,12 @@ impl TestRoot { let name = service.program.file_name().unwrap(); service.program = self.0.join("bin").join(name); if service.id == "kv" { - symlink("/bin/false", &service.program).unwrap(); + let failing_program = if cfg!(target_os = "macos") { + "/usr/bin/false" + } else { + "/bin/false" + }; + symlink(failing_program, &service.program).unwrap(); for argument in &mut service.args { if argument == "/opt/crowdb/data/kv/node-1" { *argument = self.0.join("data/kv/node-1").to_string_lossy().into_owned(); diff --git a/container/crowdb-monitor/tests/single_node_profile_test.rs b/container/crowdb-monitor/tests/single_node_profile_test.rs index dd1acca5a..879dd7a12 100644 --- a/container/crowdb-monitor/tests/single_node_profile_test.rs +++ b/container/crowdb-monitor/tests/single_node_profile_test.rs @@ -52,7 +52,7 @@ fn single_node_preview_has_exact_topology_and_endpoints() { assert_eq!(access_service.additional_probes.len(), 1); assert_eq!( access_service.additional_probes[0].target, - "http://127.0.0.1:9093/_crowdb/health/ready" + "http://127.0.0.1:9094/_crowdb/health/ready" ); assert_eq!( access_service.additional_probes[0].failure_threshold, diff --git a/container/crowdb-monitor/tests/supervisor_test.rs b/container/crowdb-monitor/tests/supervisor_test.rs index 792edc318..f986bb8fb 100644 --- a/container/crowdb-monitor/tests/supervisor_test.rs +++ b/container/crowdb-monitor/tests/supervisor_test.rs @@ -5,6 +5,7 @@ use std::collections::BTreeMap; use std::fs; use std::os::unix::fs::symlink; use std::path::{Path, PathBuf}; +use std::process::Command; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -74,7 +75,7 @@ async fn exited_service_restarts_with_same_identity_and_event_log() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let marker = roots.0.join("first-exit"); let script = format!( - "if [ ! -e '{}' ]; then : > '{}'; sleep 0.2; exit 0; fi; exec sleep 30", + "if [ ! -e '{}' ]; then : > '{}'; /bin/sleep 0.2; exit 0; fi; exec /bin/sleep 30", marker.display(), marker.display() ); @@ -110,7 +111,7 @@ async fn restarted_service_waits_for_authority_validation() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let marker = roots.0.join("validation-exit"); let script = format!( - "if [ ! -e '{}' ]; then : > '{}'; sleep 0.2; exit 0; fi; exec sleep 30", + "if [ ! -e '{}' ]; then : > '{}'; /bin/sleep 0.2; exit 0; fi; exec /bin/sleep 30", marker.display(), marker.display() ); @@ -144,7 +145,7 @@ async fn repeated_exits_exhaust_budget_and_leave_unready() { let roots = TestRoots::new(); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let profile = roots.profile( - "sleep 0.2; exit 1".into(), + "/bin/sleep 0.2; exit 1".into(), listener.local_addr().unwrap().port(), 1, ); @@ -172,7 +173,11 @@ async fn repeated_exits_exhaust_budget_and_leave_unready() { async fn stable_health_resets_crash_loop_budget() { let roots = TestRoots::new(); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let mut profile = roots.profile("exec sleep 30".into(), listener.local_addr().unwrap().port(), 1); + let mut profile = roots.profile( + "exec /bin/sleep 30".into(), + listener.local_addr().unwrap().port(), + 1, + ); profile.services[0].restart.stable_after_ms = 50; let mut supervisor = Supervisor::new( profile, @@ -190,11 +195,20 @@ async fn stable_health_resets_crash_loop_budget() { rustix::process::kill_process(pid, rustix::process::Signal::KILL).unwrap(); let deadline = tokio::time::Instant::now() + Duration::from_secs(3); loop { - let status = fs::read_to_string(format!("/proc/{}/status", pid.as_raw_pid())).unwrap(); - if status - .lines() - .any(|line| line.starts_with("State:") && line.contains('Z')) - { + let exited = if cfg!(target_os = "linux") { + fs::read_to_string(format!("/proc/{}/status", pid.as_raw_pid())).map_or(true, |status| { + status + .lines() + .any(|line| line.starts_with("State:") && line.contains('Z')) + }) + } else { + let output = Command::new("ps") + .args(["-p", &pid.as_raw_pid().to_string(), "-o", "stat="]) + .output() + .unwrap(); + !output.status.success() || String::from_utf8_lossy(&output.stdout).contains('Z') + }; + if exited { break; } assert!(tokio::time::Instant::now() < deadline); @@ -220,7 +234,7 @@ async fn live_listener_prevents_replacement_after_child_exit() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap().to_string(); let mut profile = roots.profile( - "sleep 0.2; exit 1".into(), + "/bin/sleep 0.2; exit 1".into(), listener.local_addr().unwrap().port(), 1, ); @@ -249,7 +263,7 @@ async fn transient_probe_failure_clears_readiness_without_restarting() { let roots = TestRoots::new(); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); - let mut profile = roots.profile("exec sleep 30".into(), address.port(), 2); + let mut profile = roots.profile("exec /bin/sleep 30".into(), address.port(), 2); profile.services[0].probe.failure_threshold = 2; let mut supervisor = Supervisor::new( profile, @@ -297,7 +311,7 @@ async fn child_exit_after_probe_failure_is_recorded() { stream.write_all(response.as_bytes()).await.unwrap(); } }); - let mut profile = roots.profile("exec sleep 30".into(), address.port(), 2); + let mut profile = roots.profile("exec /bin/sleep 30".into(), address.port(), 2); profile.services[0].probe.kind = ProbeKind::Http; profile.services[0].probe.target = format!("http://{address}/health"); profile.services[0].probe.failure_threshold = 5; @@ -349,7 +363,7 @@ async fn dependency_restart_stops_dependents_before_replacement() { let dependent_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let marker = roots.0.join("first-exit"); let script = format!( - "if [ ! -e '{}' ]; then : > '{}'; sleep 0.2; exit 0; fi; exec sleep 30", + "if [ ! -e '{}' ]; then : > '{}'; /bin/sleep 0.2; exit 0; fi; exec /bin/sleep 30", marker.display(), marker.display() ); @@ -357,7 +371,7 @@ async fn dependency_restart_stops_dependents_before_replacement() { let mut dependent = profile.services[0].clone(); dependent.id = "web".into(); dependent.dependencies = vec!["kv".into()]; - dependent.args = vec!["-c".into(), "exec sleep 30".into()]; + dependent.args = vec!["-c".into(), "exec /bin/sleep 30".into()]; dependent.probe.target = dependent_listener.local_addr().unwrap().to_string(); profile.services.push(dependent); profile.validate().unwrap(); diff --git a/container/single-node-container/README.md b/container/single-node-container/README.md index a224e6fcb..2033c8691 100644 --- a/container/single-node-container/README.md +++ b/container/single-node-container/README.md @@ -35,7 +35,7 @@ pixi run clean-container ``` `start-container` builds the image, starts `crowdb-single-node`, exposes -ports 9090–9093 on the container network, waits for readiness, and prints client +ports 9090–9094 on the container network, waits for readiness, and prints client credentials. Docker creates an anonymous data volume unless configured otherwise. `inject-container` loads TPC-H SF1 into Iceberg and copies its Parquet files to the container's S3 endpoint. The loader retains its data and diff --git a/container/single-node-container/dev-container.sh b/container/single-node-container/dev-container.sh index 8714f339e..e851700e7 100644 --- a/container/single-node-container/dev-container.sh +++ b/container/single-node-container/dev-container.sh @@ -46,7 +46,7 @@ start_container() { fi if ! docker run --detach --name "$name" \ - --expose 9090 --expose 9091 --expose 9092 --expose 9093 \ + --expose 9090 --expose 9091 --expose 9092 --expose 9093 --expose 9094 \ "$image"; then if docker container inspect "$name" >/dev/null 2>&1; then docker rm --force --volumes "$name" >/dev/null diff --git a/container/single-node-container/profile.toml b/container/single-node-container/profile.toml index 883e3e8af..d3ad83850 100644 --- a/container/single-node-container/profile.toml +++ b/container/single-node-container/profile.toml @@ -174,9 +174,9 @@ backoff_max_ms = 5000 id = "access" program = "/opt/crowdb/bin/crowdb-access-server" args = ["--config", "/opt/crowdb/run/config/access.toml"] -env = { CROWDB_S3_PUBLIC_URI = "http://localhost:9091", CROWDB_S3_REGION = "us-east-1", CROWDB_ICEBERG_PUBLIC_URI = "http://localhost:9092", CROWDB_ICEBERG_GC_ENABLED = "0", CROWDB_MANAGEMENT_SEEDS = "http://127.0.0.1:10000", CROWDB_ACCESS_HEALTH_LISTEN = "127.0.0.1:9093", CROWDB_ACCESS_LOG_DIR = "/opt/crowdb/data/log/access" } +env = { CROWDB_S3_PUBLIC_URI = "http://localhost:9091", CROWDB_S3_REGION = "us-east-1", CROWDB_ICEBERG_PUBLIC_URI = "http://localhost:9092", CROWDB_ICEBERG_GC_ENABLED = "0", CROWDB_MANAGEMENT_SEEDS = "http://127.0.0.1:10000", CROWDB_ACCESS_HEALTH_LISTEN = "127.0.0.1:9094", CROWDB_ACCESS_LOG_DIR = "/opt/crowdb/data/log/access" } dependencies = ["kv", "chunk-kv", "chunkdb", "diskio"] -fence_listeners = ["127.0.0.1:9092", "127.0.0.1:9091", "127.0.0.1:9093"] +fence_listeners = ["127.0.0.1:9092", "127.0.0.1:9091", "127.0.0.1:9094"] config_template = "/opt/crowdb/etc/templates/access.toml" [services.probe] kind = "http" @@ -186,7 +186,7 @@ timeout_ms = 1000 failure_threshold = 5 [[services.additional_probes]] kind = "http" -target = "http://127.0.0.1:9093/_crowdb/health/ready" +target = "http://127.0.0.1:9094/_crowdb/health/ready" timeout_ms = 1000 failure_threshold = 5 [services.restart] diff --git a/container/single-node-container/templates/access.toml b/container/single-node-container/templates/access.toml index a0471d36b..fd8b76edc 100644 --- a/container/single-node-container/templates/access.toml +++ b/container/single-node-container/templates/access.toml @@ -84,3 +84,10 @@ retry_max_ms = 60000 corruption_attempts = 3 minimum_retention_ms = 604800000 catalogs = [] + +[dataset] +listen = "0.0.0.0:9093" +max_in_flight = 3 +request_timeout_seconds = 5 +lease_ttl_seconds = 60 +max_batch_size = 10000 diff --git a/container/single-node-container/tests/container-e2e.sh b/container/single-node-container/tests/container-e2e.sh index e23bf0c9b..622346956 100644 --- a/container/single-node-container/tests/container-e2e.sh +++ b/container/single-node-container/tests/container-e2e.sh @@ -45,7 +45,7 @@ start_container() { fi docker run -d --name "$name" \ "${mount_args[@]}" \ - -p 127.0.0.1:9092:9092 -p 127.0.0.1::9091 -p 127.0.0.1::9090 \ + -p 127.0.0.1:9092:9092 -p 127.0.0.1::9091 -p 127.0.0.1::9090 -p 127.0.0.1::9093 -p 127.0.0.1::9094 \ "$image" >/dev/null for attempt in $(seq 1 240); do state=$(docker inspect --format '{{.State.Status}}' "$name") @@ -70,15 +70,19 @@ port() { } verify_public_services() { - local iceberg_port s3_port web_port token + local iceberg_port s3_port web_port dataset_port token iceberg_port=$(port 9092) s3_port=$(port 9091) web_port=$(port 9090) + dataset_port=$(port 9093) docker exec "$name" crowdb-monitor readiness >/dev/null + docker exec "$name" curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:9094/_crowdb/health/ready" >/dev/null curl --fail --silent --show-error --max-time 5 \ "http://127.0.0.1:$web_port/api/authority" | jq -e '.source == "group0" and .available == true' >/dev/null curl --fail --silent --show-error --max-time 5 \ "http://127.0.0.1:$web_port/api/preview" | jq -e '.source == "group0" and (.services | length) > 0' >/dev/null + [[ "$(curl --silent --output /dev/null --write-out '%{http_code}' --max-time 5 "http://127.0.0.1:$dataset_port/v1/dataset/open")" == 404 ]] token=$(printf '%s\n' "$client_env" | sed -n 's/^ICEBERG_TOKEN=//p') [[ -n "$token" ]] printf 'header = "Authorization: Bearer %s"\nurl = "http://127.0.0.1:%s/v1/config"\n' "$token" "$iceberg_port" | diff --git a/doc/backlog/R220-dataset-fat-client-plan.md b/doc/backlog/R220-dataset-fat-client-plan.md new file mode 100644 index 000000000..1fca49109 --- /dev/null +++ b/doc/backlog/R220-dataset-fat-client-plan.md @@ -0,0 +1,66 @@ + + + +### R220: Dataset — Fat-client direct access plan + +Status: Separate Dataset SDK project maintained by this team. Deferred until R219 +is implemented and the CPU HTTP path has measured latency, throughput, memory, +and digest costs. + +Problem: The SDK fat client can avoid HTTP routing and payload relay by connecting +directly to internal Chunk-KV and DiskIO, but it also takes responsibility for +routing freshness, retries, buffer lifetime, backpressure, and worker +partitioning. Implementing it before the HTTP baseline would make optimization +claims hard to measure and could duplicate authority logic. + +Solution: + +- Document the SDK direct-client protocol over the same Dataset authority and Read + Plan contract used by HTTP. +- Compare direct Chunk-KV metadata reads and DiskIO/chunk range reads with the + HTTP baseline, including routing refresh, stale topology, failure retry, and + 60-second active-read TTL behavior. +- Define owner-backed buffer lifetime, bounded prefetch, cache admission, + cancellation, and checksum/MD5 verification without adding client-side global + locks or unbounded queues. +- Analyze which work belongs in the client versus Chunk-KV, DiskIO, and Dataset + library. The client cannot publish placement or bypass Dataset authority. +- Reserve GPU/direct-memory delivery for a later design after CPU direct access + is measured and correct. + +Dependencies: R219 HTTP baseline, R217 CPU E2E/container acceptance, and the +measured Chunk-KV/DiskIO read flows. Unlanded direct-access work remains a design +artifact; no native client is required for R213–R219 acceptance. + +Acceptance: + +1. Given HTTP and direct-client prototypes over the same snapshot, produce a + comparison of p50/p99 latency, throughput, CPU, memory, and digest cost. + Measurement contract. Integration test. +2. Given stale routing or a disconnected DiskIO endpoint, assert the fat client + refreshes or retries without changing Dataset selection or snapshot identity. + Non-authoritative topology invariant. Integration test. +3. Given a slow consumer, assert direct prefetch and in-flight buffers remain + bounded and cancellation releases client buffers. Bounded delivery invariant. + Integration test. + +Resolved decisions: + +- The core remains Rust. The separate SDK exposes a narrow C ABI over opaque + handles first, then builds Python/PyTorch bindings over the same contract. + C++ and RDMA wrappers remain later data-plane work. +- Client metadata is cached by immutable Snapshot ID and explicitly refreshed + when resolving `latest`; no Dataset-specific global watch stream is required + for the first version. + +Open Issues: + +- Define the C ABI ownership, error, cancellation, and buffer lifetime contract + before implementing the fat client. Keep the ABI in the separate SDK project. + +Verification commands: + +- `pixi run rs-fmt-check` +- `pixi run rs-lint` +- `pixi run test-access-server` +- `pixi run test-unit` diff --git a/doc/backlog/R227-console-multi-node-deployment.md b/doc/backlog/R227-console-multi-node-deployment.md new file mode 100644 index 000000000..a796d904c --- /dev/null +++ b/doc/backlog/R227-console-multi-node-deployment.md @@ -0,0 +1,280 @@ + + + +### R227: console — Multi-node discovery and light-container deployment + +## Problem + +CROWDB needs a multi-node deployment flow in which each node runs +`crowdb-monitor` and a Web UI. Operators should not have to enter every peer +address before discovering available nodes. The current monitor and console +surfaces do not establish the LAN discovery and shared bootstrap contract +described here; existing service registration is not node discovery. + +The permanent [console design](../design/console/design-crowdb-console.md) +defines Group 0 as topology authority, and the +[Group-0 design](../design/kv/design-crowdb-kv-group0.md) defines its service +registry. Neither a local discovery cache nor a console-local launch registry +can replace that authority. Existing console node/server coupling must be +reviewed when adding nodes before any server exists. + +Concrete scenarios: + +- Three machines on a home LAN discover each other through their monitors. + An operator opens any node's UI, assigns racks, and initializes Group 0. +- Several containers on one machine simulate independent nodes on one Docker + bridge network, with independent identities, state directories and disks. +- Production Linux nodes use light containers with host networking, host + persistent state, and explicit disk/device access. Containers package binaries + and dependencies while host resources remain explicitly managed. +- A network partition must not allow an isolated UI or monitor to create a + replacement cluster or execute stale deployment commands. + +## Solution + +The target architecture is defined in the +[multi-node deployment design](../design/deploy/design-crowdb-deploy.md). +This requirement tracks its implementation and acceptance; unresolved decisions +in that design must be settled before their dependent implementation begins. + +### Architecture and authority + +```text +browser on any node + -> local crowdb-web / shared console operations + -> Group 0: confirmed topology and management authorization + -> target crowdb-monitor + -> containerd / managed server processes + +before Group 0 exists: +local UI -> local monitor discovery -> peer monitors -> bootstrap preparation +``` + +Every node has a monitor management endpoint and a Web UI. No fixed UI node +is required. Discovery advertises nodes, not each managed server. Group 0 is +the first managed service allowed to start; more complex server deployment is +enabled only after Group 0 is ready and the initial topology is published. + +### Invariants + +- **I1 — Stable node identity:** node identity is persisted outside the image + and container writable layer. IP, container name, restart and rack reassignment + do not change it. Cloned identity directories are detected as conflicts rather + than silently merging two nodes. +- **I2 — Discovery is a candidate view:** discovery never grants cluster + membership, voting rights, rack authority or deployment permission. Records + can expire without deleting confirmed members from Group 0. +- **I3 — Shared authority:** after bootstrap, confirmed node/rack topology and + cluster deployment intents are authoritative in Group 0. UI caches can lag, + but mutations succeed only through committed control-plane state. +- **I4 — One bootstrap identity per node:** a node persists its accepted cluster + identity and initial Group-0 manifest before starting a replica. Conflicting + manifests are rejected; retries of the same operation are idempotent. No + timeout or empty discovery result permits automatic replacement bootstrap. +- **I5 — Quorum-backed authorization:** connectivity to one Group-0 replica is + insufficient. New management mutations require current quorum-backed + authority. Stale commands cannot execute after authority moves or expires. +- **I6 — Explicit placement:** virtual racks are editable logical groups. + Physical host/failure-domain information is recorded separately; multiple + containers on one host do not become independent physical failure domains + because their virtual rack labels differ. +- **I7 — Portable image, explicit resources:** the same OCI image for a supported + OS/CPU architecture is used for single-host Docker tests and production + containerd deployment. Network, disks, persistent mounts and resource limits + are runtime inputs, not baked cluster identity. + +### Numbered work items + +1. Extend `crowdb-monitor` identity/profile/status surfaces and shared types in + `crowdb-protocol` to expose stable node identity, monitor endpoint, protocol + compatibility, optional rack hint, physical-host identity and cluster binding. + Obtain full hardware/rack information through a direct monitor handshake. + A node can exist before any KV or application server is deployed. + +2. Add mDNS + DNS-SD announcement and continuous browsing to `crowdb-monitor`. + Use a CROWDB node service type (proposed `_crowdb-node._tcp.local.`), SRV + address/port resolution and bounded TXT metadata. Select management interfaces + explicitly; do not announce on storage data-plane interfaces by default. + Support record updates, expiry, clean departure, duplicate identity detection + and incompatible-peer reporting. The browser UI reads its backend's candidate + view; it does not perform multicast itself. + + mDNS is link-local UDP discovery, not consensus or a health guarantee. + DNS-SD structures service enumeration and endpoint metadata. These are borrowed + standard discovery mechanisms; CROWDB defines its own node and authorization + semantics. See [RFC 6762](https://www.rfc-editor.org/rfc/rfc6762.html) and + [RFC 6763](https://www.rfc-editor.org/rfc/rfc6763.html). + Provide explicit peer/seed addresses as a fallback when multicast is blocked + or nodes span subnets. Neither fallback nor discovered endpoints bypass the + same handshake and cluster-binding checks. + +3. Extend `crowdb-web` and `crowdb-console-shared` with a pre-bootstrap view: + candidate nodes, editable virtual racks, node-to-rack assignments and Group-0 + initialization. Node-supplied rack information is a hint, not immutable + placement. A pre-bootstrap draft is local and visibly uncommitted; differing + drafts on different UIs do not constitute confirmed topology. Limit this + stage to node/rack/bootstrap operations, excluding other server deployment. + +4. Extend the existing shared `bootstrap_intent` and cluster operations to submit + one explicit initial Group-0 manifest: cluster identity, selected initial + replica members, reachable endpoints and confirmed node/rack draft. Peer + monitors persist compatible preparation before replica startup. Concurrent + incompatible requests must fail safely without constructing overlapping + conflicting groups. Partial preparation and initiator crashes must have an + explicit resume/abort procedure; an abort cannot erase a cluster that has + committed state. The coordination protocol is a human decision below, not an + assumption that multicast or smallest-node-ID election provides agreement. + +5. Wait for a usable Group-0 leader/quorum, then publish the initial topology + idempotently. Gate general deployment on both consensus readiness and + topology publication. Persist cluster binding and Group-0 contact information + at each accepted node. Other UIs attach to that cluster and read Group 0; + incompatible cluster bindings are shown separately and never auto-merged. + Newly discovered nodes remain candidates until a committed admission operation. + Later Group-0 voting membership changes use the existing reconfiguration path. + +6. Route rack changes and deployment mutations from every UI through shared + console operations and Group 0. Use consistent reads where confirmation needs + current authority, with watch/refresh for presentation. Record operation + identity and execution state so UI retries or UI-node loss cannot duplicate a + deployment. Monitor execution validates the current authorization and rejects + stale commands. Group-0 unavailability leaves discovery/diagnostics available + but disables new cluster management mutations. + +7. Add a containerd-backed execution path to `crowdb-monitor` alongside existing + managed-process support. Production uses containerd APIs; nerdctl is an operator + tool, not a required per-command subprocess. Define image digest, CPU/memory + boundaries, host networking, persistent config/state/log mounts and permitted + disk/device inputs. No implicit privileged container or unrestricted host + device access is required. Preserve process-level liveness and explicit + start/stop/upgrade operations; do not add a general scheduler or automatic + global reconciliation loop. + +8. Extend `container/single-node-container` packaging/test infrastructure for + multiple node containers from the same OCI image. Each gets an independent + persistent root and simulated disk, joins one dedicated user-defined Docker + bridge, and advertises its internally reachable monitor endpoint. Publish + distinct host UI ports for browser access. Separate test networks bound the + discovery scope. Production uses host networking and real disks on supported + Linux hosts; discovery and cluster-management semantics stay identical. + OCI compatibility does not claim complete Kubernetes lifecycle integration + or unverified CPU/OS support. + +## Dependencies + +- Existing Group-0 KV operations, consistent reads, service registry and + membership reconfiguration are reused; node discovery does not replace them. +- Existing `crowdb-monitor`, `crowdb-console-shared`, `crowdb-web`, + `crowdb-protocol` and single-node image infrastructure are the implementation + base. Existing working-tree changes must be preserved. +- R139 concerns distributed service configuration. This requirement must define + the minimal deployment-intent/authorization records it needs without silently + assuming R139 is implemented or replacing all local process configuration. +- R193 concerns failure-budget placement; physical failure-domain records must + remain distinguishable from editable virtual racks for that integration. +- The backlog index currently references absent R210/R211/R212 detail files. + Reconcile those workstreams before implementation to avoid duplicating node + lifecycle, health and cluster-scope contracts; do not assume they are available. +- `/nv/cpp/adcp/topic/deploy.md` is discussion context for light-container + deployment, not a portable repository dependency. The applicable resource and + lifecycle choices are captured above. +- Implementation must resolve the bootstrap and authorization decisions below. + Permanent console, Group-0, configuration and container documentation must be + updated to the shipped behavior during implementation cleanup. + +## Acceptance + +1. Setup: a persisted node root. Action: recreate the container, change its IP + and rack. Assertion: node identity remains unchanged; a second live endpoint + using the same identity is reported as a conflict (I1). Integration test. +2. Setup: three monitors on a multicast-capable test network. Action: start, + update and stop nodes. Assertion: every backend discovers reachable peer + identities/endpoints, removes expired candidate records, and does not modify + confirmed membership on expiry (I2). Integration test. +3. Setup: multicast disabled and explicit reachable seeds supplied. Action: + discover peers. Assertion: the same compatibility/binding checks apply; + discovery does not itself authorize admission (I2). Integration test. +4. Setup: multiple network interfaces and an incompatible peer. Action: enable + discovery on the management interface only. Assertion: advertisements remain + scoped there, incompatible nodes are reported and cannot bootstrap (I2). + Integration test. +5. Setup: uninitialized nodes with rack hints. Action: open any node UI, create + virtual racks and move nodes. Assertion: hints can be overridden, drafts are + visibly uncommitted and non-Group-0 deployment is blocked (I3, I6). E2E test. +6. Setup: overlapping candidate nodes and two incompatible bootstrap manifests. + Action: submit concurrently and restart a prepared monitor. Assertion: durable + conflict rejection prevents conflicting replica startup; the same operation + can resume without duplicating a group (I4). Integration test. +7. Setup: bootstrap interrupted before and after Group-0 commit. Action: follow + the specified recovery flow. Assertion: partial setup is resumable, committed + cluster identity is preserved and no timeout creates a replacement (I4). + Integration test. +8. Setup: three selected Group-0 replicas. Action: bootstrap and delay initial + topology publication. Assertion: general deployment remains gated until both + quorum readiness and publication; all UIs then read the same confirmed + node/rack state (I3). E2E test. +9. Setup: an active cluster and a new candidate. Action: discover then admit it + through another UI. Assertion: discovery alone changes no membership; admission + persists in Group 0 and any voting change uses reconfiguration (I2, I3). + Integration test. +10. Setup: two independent initialized clusters on one LAN. Action: discover + each other. Assertion: their identities remain separate and no automatic + merge or rebinding occurs (I4). Integration test. +11. Setup: concurrent UI mutations and retried deployment requests. Action: + update racks/deploy and lose the initiating UI. Assertion: committed outcomes + are shared and each operation executes at most once or resumes through its + recorded idempotent stages (I3, I5). Integration test. +12. Setup: a three-replica Group 0 partitioned two-to-one. Action: issue management + commands on both sides and deliver a previously authorized stale command. + Assertion: only quorum-authorized mutations execute, stale execution is + rejected, and minority monitors retain diagnostics without replacement + bootstrap (I4, I5). Integration test. +13. Setup: several simulated nodes on one physical host in different virtual + racks. Action: inspect placement inputs. Assertion: the shared physical + failure domain remains visible and rack changes do not rewrite it (I6). + Integration test. +14. Setup: one supported-architecture OCI image digest. Action: launch multiple + Docker bridge nodes and Linux containerd host-network nodes with appropriate + disks/mounts. Assertion: both discover/bootstrap/manage successfully, state + survives replacement, internal advertised addresses are reachable and the + simulated-disk run makes no production-performance claim (I1, I7). + Integration test. +15. Setup: containerd deployment with explicit devices/resources. Action: start, + stop, restart and upgrade through monitor. Assertion: requested resource + limits and mounts are applied, process health is observable and restart + preserves persistent state (I7). Integration test. + +Implementation verification commands (these are required future gates, not +claims that the new acceptance coverage already exists): + +```bash +pixi run cargo test -p crowdb-monitor +pixi run cargo test -p crowdb-console-shared +pixi run cargo test -p crowdb-web +pixi run cargo test -p crowdb-protocol +pixi run test-console +pixi run test-console-ui +pixi run rs-fmt-check +pixi run rs-lint +``` + +## Open Questions + +- Bootstrap coordination: use one explicitly selected initiating node with + durable reservations and resumable manifest delivery, or a quorum/prepare + protocol over a fixed selected membership. Choose recovery and conflict rules + before implementation. Neither option guarantees one cluster across arbitrary + disjoint selections without an explicit shared identity/admission boundary. +- Authorization during Group-0 loss: new management writes stop in all cases. + Decide whether existing data services must also stop after bounded authority + expiry, or may continue under their own Paxos/data-plane authority. The former + matches a strict "only Group-0-authorized nodes are active" policy but couples + whole-cluster availability to Group 0. Specify lease timing, renewal, fencing + enforcement points and recovery before claiming that stricter policy. +- Discovery admission trust: explicitly approve discovered candidates in UI, + or provision deployment credentials for automatic admission. Discovery packets + and rack hints alone are not authenticated authorization. +- Production monitor location: host system service controlling light containers, + or a dedicated management container with narrowly scoped runtime access. + Choose privilege and recovery boundaries; do not expose the containerd socket + to every application/UI container by default. diff --git a/doc/backlog/R228-tree-range-metrics.md b/doc/backlog/R228-tree-range-metrics.md new file mode 100644 index 000000000..dd4e121de --- /dev/null +++ b/doc/backlog/R228-tree-range-metrics.md @@ -0,0 +1,151 @@ + + + +### R228: crowdb-tree — Root and range statistics for split and placement + +Status: Deferred implementation by user request. Establish the metrics contract +and review page-format, maintenance and performance costs before implementation. +Data-weighted placement and its console display remain disabled meanwhile. + +## Problem + +[Tree storage design](../design/tree/design-crowdb-tree-storage.md) defines leaf +records and inner separators/child PIDs. Leaves expose local base-record and delta +counts; inner pages do not carry child-subtree page, byte or live-KV aggregates. +The existing tree Stats API reports buffer-pool, flush and snapshot activity, +not current reachable tree size or current live record count. Snapshot pages +written, resident cache pages and cumulative writes are not these metrics. + +[Chunk-KV placement](../design/chunkds/design-crowdb-chunk-kv-server.md) currently +balances assigned split counts. Automatic split supplements the partition-count +target with coarse retained-pack size thresholds/ranking. Those pack estimates +lag ordinary flushes, and split trees share historical chunks. They cannot +reliably determine the current range's data size or justify size-driven split. +Dividing an old estimate by a split ratio and accumulating later writes also +accumulates error without a structural calibration point. + +A growing tree needs continuous, reasonably accurate size metrics to decide +whether another split is needed. Parent and child need independent range +statistics even while they share physical packs. Overwrites, deletes and +compaction must not make a historical estimate permanently drift. + +## Solution + +- **TREE-METRICS-ROOT:** Metrics describe a specific current tree root/version + and its key range. Report observation/root identity and coverage frontier; + do not silently mix roots, epochs or ranges. Memtable and WAL sizes do not + contribute to tree size. Writes enter tree metrics when the normal flush + incorporates them; callers must see this coverage boundary. +- **TREE-METRICS-LOGICAL:** Distinguish reachable inner, leaf, overflow and delta + page counts; reachable page bytes; live KV count; live key/value bytes; and + physical retained/shared pack storage. Logical range data and physical + storage are separate metrics. A key's current visible value counts once; + overwritten versions and tombstones do not count as live KV/data. Overflow + data must contribute even when stored outside a leaf. State any approximation + and its source explicitly rather than claiming exact live bytes. +- **TREE-METRICS-STRUCTURAL:** Carry child-subtree aggregates in index metadata + so a full-tree/root aggregate or a range estimate can be obtained from index + pages without scanning leaf records or values. Leaf summaries originate from + existing rebuild/fold/flush work. Aggregates can be rebuilt from current + structure, not cumulative writes or recursively inherited split ratios. + A range boundary crossing a leaf may produce a documented bounded estimate; + report that uncertainty rather than extrapolating arbitrary key samples. +- **TREE-METRICS-NONBLOCKING:** Maintain summaries through existing single-writer + flush/page rebuild work. Do not add request-path locks, wait queues, per-key + ancestor walks, forced flush/checkpoint, or synchronous remote scans to + heartbeat or balance. Background index traversal is bounded and cancellable; + heartbeat reads cached identity-fenced results and never waits for traversal. + Review additional write amplification, atomic bookkeeping and memory costs. +- **TREE-METRICS-SHARING:** Split evaluates each resulting root/range's logical + data; shared historical chunks are not counted as both ranges' logical data. + Transfer preserves metrics for the same logical tree/range. Cleanup/GC of + historical physical packs does not reduce current live logical data. +- **TREE-METRICS-RECOVERY:** Persist summaries consistently with their page/root + publication. Reopen reconstructs/verifies metrics against the selected root. + Abort or failed publication cannot install candidate statistics on the serving + root. Legacy pages without summaries report unavailable until bounded rebuild; + unknown is never zero and never authorizes a size-driven split or move. + +1. Define metric semantics and coverage in the tree storage design, and review + the persisted index/leaf summary format and legacy compatibility. Name the + range-boundary error model and separate live logical data from page capacity. +2. Extend tree page builders/rebuild/flush and root publication to maintain and + aggregate summaries. Include overwrite, delete, delta folding, overflow, + range narrowing, split, transfer, compaction and reopen. Avoid new hot-path + synchronization; measure cost before accepting the maintenance strategy. +3. Expose cached root/range statistics through the C API, Rust FFI and chunk-KV + runtime/heartbeat. Index traversal must have page/I/O/time budgets and expose + progress, freshness, unavailable state and identity fences. +4. Replace retained-pack split size triggering/ranking with these tree metrics; + retain safe nonempty separator selection and count-target bootstrap. Use + logical-byte distribution to improve boundary selection when summaries allow + it, without promising perfectly equal ranges or scanning leaf values. +5. Re-enable data-weighted placement only after metrics acceptance, with explicit + coefficients/tolerance and no count-only pass that reverses useful data moves. + Update the console spec before restoring Weight, including metric coverage, + uncertainty and the actual split/move decision explanation. + +## Dependencies + +- Existing tree/root publication and chunk-KV parent-plus-child recovery remain + authoritative. This work must not redefine handoff, WAL or catalog cutover. +- Tree storage/engine and Chunk-KV server designs are upstream contracts. +- Shared-pack reclamation is separate from logical statistics; it is not required + to obtain a range estimate and must not be used to force a physical copy. +- Until implemented, placement uses split count only and UI hides Weight. The + count-target split path remains available; coarse pack sizing remains an + explicitly limited interim mechanism, not a reliable current-data metric. + +## Acceptance + +- Empty tree -> read metrics -> known zero live KV/data and documented structural + root-page cost, distinct from unavailable. TREE-METRICS-LOGICAL. Unit test. +- Known records with overflow values -> normal flush and aggregate via indexes + -> expected live count/bytes and reachable page categories without value reads. + TREE-METRICS-STRUCTURAL. Integration test. +- Same keys repeatedly overwritten/deleted -> normal fold/flush -> current live + metrics reflect latest state, not accumulated writes; WAL-only changes leave + tree metrics unchanged and coverage explicit. TREE-METRICS-ROOT. Integration test. +- Uneven range tree -> split while sharing old packs -> parent/child statistics + describe their own ranges within declared boundary error; aggregate does not + duplicate shared physical pack bytes as logical data. TREE-METRICS-SHARING. + Integration test. +- Publish root, crash/reopen and replay/flush -> metrics follow selected root and + coverage; fail/abort candidate publication -> serving statistics unchanged. + TREE-METRICS-RECOVERY. Integration test. +- Legacy/cold tree -> bounded index traversal with cancellation -> no leaf/value + scan or unbounded heartbeat wait; unavailable/freshness and progress explicit. + TREE-METRICS-NONBLOCKING. Integration test. +- Repeated split, overwrite, delete, compaction and reopen -> rebuild aggregates + from current structure -> no history-dependent drift; stale root/range results + rejected. TREE-METRICS-STRUCTURAL. Integration test. +- Known growing tree -> normal flush crosses configured logical-byte threshold + -> eligible split planned; below threshold or unknown -> no size-triggered split; + count-target bootstrap remains independently tested. Integration test. +- Metrics accepted, explicit data-weight policy enabled -> weighted move/split + decisions use matching cached statistics; console shows the backend decision, + coverage and estimate qualification. E2E test. +- Fixed read/write/flush workload -> compare before/after -> report throughput, + latency, page writes, memory, index I/O and traversal budgets; no new request + locks, queues or forced checkpoint. Integration test. + +## Open Questions + +- Review summary encoding and update granularity: exact subtree live aggregates + require maintenance during folding/rebuild; bounded estimates may reduce write + amplification but must state error and recalibration guarantees. +- Review crossing-leaf ranges: report a boundary uncertainty interval using leaf + summaries, or perform a separate bounded background boundary-leaf inspection. + Neither option allows routine full leaf/value scans. + +Verification commands: + +- `pixi run test-cpp` +- `pixi run cargo test -p crowdb-tree-ffi` +- `pixi run cargo test -p crowdb-chunk-kv-server --test balance_test` +- `pixi run cargo test -p crowdb-kv-server --test chunk_kv_balance_planning_test` +- `pixi run cargo test -p crowdb-web --test native_cluster_provisioning_test` +- `pixi run tree-fmt` +- `pixi run tree-lint` +- `pixi run rs-fmt-check` +- `pixi run cargo clippy -p crowdb-tree-ffi -p crowdb-chunk-kv-server -p crowdb-kv-server -p crowdb-web --all-targets -- -D warnings` diff --git a/doc/backlog/R229-kv-membership-conflict.md b/doc/backlog/R229-kv-membership-conflict.md new file mode 100644 index 000000000..8ed052b89 --- /dev/null +++ b/doc/backlog/R229-kv-membership-conflict.md @@ -0,0 +1,123 @@ + + + +### R229: KV membership — Complete members record with epoch CAS + +## Problem + +[Reconfiguration design](../design/kv/design-crowdb-kv-reconfiguration.md) +recommends adding members one at a time. Console currently reads membership, +wires peers, then CAS-creates individual replica records. Different replica +keys do not conflict, so two UI servers can concurrently change the same group. + +KV server remote mutation rebuilds a captured group, persists its configuration, +then replaces the running group. Concurrent replacements can start from the +same snapshot. Checking for conflict after writing disk is too late. + +The original concurrent-add topology case fails with two of three ready groups. +Sequential additions pass the complete five-case spec. Controlled regressions +must establish the server interleaving; no data-loss claim is established. + +## Solution + +The approved model is one authoritative Group-0 record per store/group, +containing the complete members list and a monotonically increasing epoch. +The epoch identifies the configuration change; no separate operation ID, +admission record, or administrative version is introduced. Installation state +is kept with this record so an unfinished change rejects the next submission. + +- **MEMBERSHIP-CAS:** Membership mutation APIs accept an expected epoch and the + complete requested members list. KV client submits through Group-0 CAS before + changing any peer. CAS checks the observed record revision, expected epoch + and Ready state together. The server assigns the successor epoch. A stale + epoch or Installing state returns explicit 409 Conflict. Different groups + remain independent; losing callers do not automatically retry on a new epoch. +- **MEMBERSHIP-INSTALL:** Successful CAS publishes the successor members and + epoch in Installing state. KV client propagates that exact complete + configuration through the existing reconfiguration procedure. Keep enough + installation scope to reconcile removed as well as retained/new participants + after restart. Installation follows existing consensus fencing and catch-up + rules; publishing desired members alone is not proof of completed cutover. +- **MEMBERSHIP-SERVER:** Each KV server checks and atomically guards configuration + submission before persistence, covering persistence through memory publication. + Lower epoch is rejected. Same epoch and same complete configuration is an + idempotent replay; same epoch with different configuration is rejected. + A higher epoch must be the authorized successor, not an arbitrary caller + value. Guard add/remove, batch wiring, replacement and reconciliation. +- **MEMBERSHIP-ORDER:** Validate and acquire the local submission guard, persist + the authorized successor, then publish that same successor. A losing/stale + request must not modify either durable or running configuration. + Checking only at in-memory group replacement is insufficient. +- **MEMBERSHIP-COMPLETE:** After all required installation/fencing confirmations, + CAS the exact Installing record to Ready without changing its epoch/members. + A stale completion cannot clear a newer record. Only Ready permits the next + membership mutation. Same-group UI additions await completion and refresh + membership before another submission. +- **MEMBERSHIP-RECOVERY:** Timeout or lost reply is an unknown outcome, not + permission to clear Installing. Linearizably reread the authoritative record + and confirm the exact epoch/configuration. Partial installation resumes that + epoch; no rollback to older members and no time-based release. Restart after + local persistence restores the durable successor and reconciles with Group 0. + An interrupted completion is confirmed or retried by exact-record CAS. +- **MEMBERSHIP-FORMAT:** Use the new complete-record and local epoch format + directly. No legacy migration, compatibility reads, or old unconditional + mutation path. Endpoint and voting changes participate in the same complete + configuration/epoch checks; consensus integration must be reviewed explicitly. +- **MEMBERSHIP-COST:** Extra CAS, reads and coordination belong to management + operations, which may be slower. Add no data-path locks or wait queues. + Any proposed new lock requires separate contention/ordering review. +- **MEMBERSHIP-UI:** Keep submission sequential for each group, show 409 as a + conflict, refresh current configuration, and never silently resubmit changes. + +1. Define the complete record and epoch/Installing/Ready API contract in + protocol types, and review integration with existing reconfiguration. +2. Implement shared CAS submission and exact-epoch recovery in the corresponding + KV-client operation; route Console membership changes through it. +3. Protect KV server durable configuration submission and publication. +4. Verify races, lost replies, partial installation and restart boundaries. +5. Update permanent reconfiguration/UI documents after implementation; maintain + separate sequential topology and concurrent rejection coverage. + +## Dependencies + +- Existing Group-0 CAS and linearizable reads. +- Existing local configuration durability, catch-up and consensus fencing. +- Independent of Chunk-KV cutover and deferred tree metrics. +- Sequential operator changes are the interim behavior, not proof of + cross-UI protection. Implementation is not yet complete. + +## Acceptance + +- Two UI servers read one Ready epoch -> submit different configurations -> + one CAS succeeds and the other returns 409 before peer writes + (**MEMBERSHIP-CAS**). Integration test +- A successor is Installing -> submit another change even with its current + epoch -> explicit 409 until exact completion (**MEMBERSHIP-COMPLETE**). + Integration test +- Force overlapping local submissions -> losing request changes neither disk + nor running configuration; independent groups progress + (**MEMBERSHIP-SERVER**, **MEMBERSHIP-ORDER**, **MEMBERSHIP-COST**). Integration test +- Replay same epoch/configuration, then same epoch/different configuration and + lower epoch -> respectively idempotent success and explicit conflicts; + authorized higher epoch installs through reconfiguration + (**MEMBERSHIP-SERVER**, **MEMBERSHIP-FORMAT**). Integration test +- Lose CAS/installation/completion replies -> reread and resume exact epoch -> + no duplicate configuration change or premature Ready + (**MEMBERSHIP-RECOVERY**). Integration test +- Crash during partial fan-out or after local persistence before publication -> + recover the durable successor and finish the current epoch; reject delayed + older configuration and stale completion (**MEMBERSHIP-RECOVERY**). Integration test +- Remove a participant -> restart during propagation -> installation scope + preserves the required removal/fencing proof before Ready + (**MEMBERSHIP-INSTALL**, **MEMBERSHIP-COMPLETE**). Integration test +- Two UI sessions change one group -> loser displays conflict; winner waits for + completion and refreshes before its next change (**MEMBERSHIP-UI**). E2E test + +Verification: + +- `pixi run cargo test -p crowdb-kv-server --tests` +- `pixi run cargo test -p crowdb-kv-client --tests` +- `pixi run cargo test -p crowdb-console-shared --tests -- --test-threads=1` +- `pixi run test-console-ui` +- `pixi run rs-fmt-check` +- `pixi run cargo clippy -p crowdb-kv-server -p crowdb-kv-client -p crowdb-console-shared --all-targets -- -D warnings` diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index 8082d8c9f..cd78740f5 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -1,254 +1,125 @@ -# New Requirements — Backlog & Analysis - -Forward-looking implementation items. Each item is classified by priority, -complexity, and dependency. Before implementation, follow the -[Implementation Process](#implementation-process) below. - ---- - -## Item Index - -**Next R number: R222** — Bump this line in the same commit when adding a new item. - -### Next Milestone — Chunk-backed range KV - -R144 is a deferred merge follow-up to the completed chunk-KV server and routed -client split/transfer baseline. R147 is the remaining deferred chunk lifecycle -follow-up. R148 follows the now-measured mirror-only production baseline and -keeps stream metadata scale-out and sealed-chunk EC disabled until that -requirement is implemented. -- **[R144](R144-chunk-kv-partition-merge.md)** — adjacent partition merge — - Area: crowdb-tree / KV / server / group 0 — Deferred follow-up that composes - two adjacent chunk-backed trees, fences both owners, reconciles their WAL - sequences, and atomically replaces both parent ranges with one destination. -- **[R147](R147-tree-chunk-gc.md)** — reclaim B+tree chunk strips — Area: - crowdb-tree / chunkdb / diskdb — Turn tree logical-GC results into durable, - manifest-fenced reclaim candidates. Repack mixed live strips, then use an - idempotent generic in-chunk operation to release whole unreachable strips or - chunks without racing retained manifests, snapshot pins, or layout readers. -- **[R148](R148-chunk-stream-scale-out.md)** — partition metadata scale-out and - sealed-chunk EC — Area: chunk-stream / chunk-kv / KV / chunkdb — Move the - stream namespace and tree root catalog as one fenced binding generation, - optionally shard their indexes, and convert sealed mirror chunks to EC. - -### Planned — S3 data access service - -R152–R166 delivered the limited basic S3 service, including the restart -acceptance baseline. Multipart upload is available; R168–R169 defer -shared-storage GC without blocking basic large-object deletion. R170 adds -optional cuObject/RDMA acceleration after the TCP baseline is correct and measured. -- **[R196](R196-access-upload-benchmark-regression.md)** — S3 and Iceberg HTTP - upload benchmark regression — Area: CLI / access server / benchmark — Add a - shared real-protocol CLI workload and retained local regression scripts for - direct and multipart small/large uploads, with correctness gates, metrics, - and profile-matched performance baselines. -- **[R193](R193-chunkdb-node-failure-budget.md)** — configurable node failure - budget and EC placement — Area: KV / chunkdb / chunk IO / deployment — - Generalize the fixed one-node and three-node profiles to larger clusters. - Validate mirror copies and EC against the worst configured set of failed - nodes. A six-node cluster with a two-node budget admits `4+2` and `8+4` EC, - but rejects `2+1`. Persist degraded placement and restore full protection - after capacity returns. +# CROWDB Backlog Index + +This index contains only requirements that are not complete. Completed +requirements and their temporary execution plans are removed during the final +`implement-requirement` cleanup. Requirement detail and acceptance criteria +live in the linked document; implementation follows +[`implement-requirement`](../../.agents/skills/implement-requirement/SKILL.md). + +**Next R number: R230** — R221 is reserved by another workstream. + +## Dataset and access + +- **[R220](R220-dataset-fat-client-plan.md)** — Dataset SDK fat-client direct + access and optimization contract. **Area:** Dataset SDK. **Complexity:** High. + **Status:** Deferred until the core Dataset contract has workload measurements. + +## Console and service control + +- **[R229](R229-kv-membership-conflict.md)** — complete members record with epoch CAS and + server submission protection; explicit conflicts across UI servers. + **Area:** KV client / KV server / Console. **Complexity:** Medium. + **Status:** Members/epoch CAS model approved; implementation pending. + +- **[R227](R227-console-multi-node-deployment.md)** — multi-node discovery, + Group-0 bootstrap, shared UI authority and light-container deployment. + **Area:** monitor / console / deployment. **Complexity:** High. + **Dependencies:** existing Group 0; bootstrap and authorization decisions. +- **[R210](R210-console-service-configuration-health.md)** — unified node service + configuration and health. **Area:** console / deployment / FlatBuffer RPC. + **Complexity:** High. **Dependencies:** none recorded. +- **[R211](R211-console-access-health-listener.md)** — independent Access Server + health listener. **Area:** console / access server / deployment. + **Complexity:** Medium. **Dependencies:** R210. +- **[R212](R212-console-node-create-and-cluster-scope.md)** — reliable node + creation and Cluster scope. **Area:** console / node lifecycle / UI. + **Complexity:** High. **Dependencies:** R210, R211. + +## Access server and external data + - **[R168](R168-s3-shared-object-reclamation.md)** — shared small-object - reclamation — Area: access server / S3 / chunkdb — **Deferred on R95.** Turn - exact pending shared ranges into qualified, restart-safe physical deletion. -- **[R169](R169-s3-shared-chunk-tree-gc.md)** — B+tree and shared-chunk garbage - collection — Area: access server / S3 / crowdb-tree / chunkdb — **Deferred on - measurement, R147, and R168.** Compact fragmented shared chunks and retire - obsolete S3 metadata without erasing reader or recovery authority. -- **[R170](R170-s3-cuobject-rdma.md)** — optional cuObject RDMA data plane — - Area: access server / S3 / DiskIO / RDMA — **Deferred until basic TCP S3 is - stable.** Keep acceleration in a separate optional library and requirement; - AccessServer coordinates while DiskIO-owned cuObjServer endpoints transfer - parallel logical spans directly to or from client registered memory. - -### Planned — Native Iceberg storage - -The native catalog correctness milestone is complete: catalog/service foundation, -namespace, immutable FileIO, atomic table commits, reclamation and REST/official-SDK -conformance. Its contract and executable profile are retained in -[Native Iceberg Storage](../design/access-server/iceberge/design-crowdb-iceberg.md). -The [container client matrix](../../container/single-node-container/tests/ecosystem/README.md) -has separate passing engine/restart acceptance. Caches and selected ORC remain -independent deferred work. - -- **[R185](R185-access-iceberg-cache-invalidation.md)** — bounded cache and - invalidation — Area: access server / Iceberg / Group 0 / Chunk-KV — **Deferred - pending focused cache measurements on the completed uncached baseline.** Add one budgeted - cache manager, qualified entries, internal-RPC invalidation, and TTL safety nets. - -- **[R186](R186-access-iceberg-orc-validation.md)** — selected ORC validation — - Area: access server / Iceberg — **Independent follow-up retained by user - decision; outside the verified client ecosystem.** Add bounded canonical ORC - schema, row-count and delete validation with official-client fixtures. The Parquet catalog is - complete; ORC does not block container or client-ecosystem acceptance. - -### High Priority - -- **[R207](R207-chunkdb-repo-metadata-chunk-kv.md)** — repo chunk metadata and - tasks on chunk-kv — Area: chunkdb / chunk-kv — High complexity. - **Deferred beyond the completed direct-KV stage.** When selected, migrate - S3/IcebergTable/business Stream metadata and associated tasks to chunk-kv; - system Wal/BtreePage/PageIndex metadata stays in KV groups. Define range-local - publication, key/split rules, isolated tasks, safe conversion and measured - Paxos relief. Future Dataset follows the user-data layer with its own type. + reclamation. **Area:** access server / S3 / chunkdb. **Complexity:** High. + **Status:** Deferred until R95 and the basic S3 delete path are stable. +- **[R169](R169-s3-shared-chunk-tree-gc.md)** — shared-chunk and B+tree garbage + collection. **Area:** access server / S3 / crowdb-tree / chunkdb. + **Complexity:** High. **Status:** Deferred until R147, R168, and workload + measurements are complete. +- **[R170](R170-s3-cuobject-rdma.md)** — optional cuObject RDMA data plane. + **Area:** access server / S3 / DiskIO / RDMA. **Complexity:** High. + **Status:** Deferred until the basic TCP S3 path is stable and measured. +- **[R185](R185-access-iceberg-cache-invalidation.md)** — bounded Iceberg cache + and invalidation. **Area:** access server / Iceberg / Group 0 / Chunk-KV. + **Complexity:** Medium. **Status:** Deferred pending focused cache measurements. +- **[R186](R186-access-iceberg-orc-validation.md)** — selected ORC validation. + **Area:** access server / Iceberg. **Complexity:** Medium. **Status:** + Deferred as an independent follow-up by user decision. +- **[R196](R196-access-upload-benchmark-regression.md)** — S3 and Iceberg HTTP + upload benchmark regression. **Area:** access server / CLI / benchmarks. + **Complexity:** Medium. **Dependencies:** completed upload correctness paths. + +## Chunk, Chunk-KV, and recovery + +- **[R83](R83-chunkdb-complete-recovery-flow.md)** — complete data recovery and + recovery-speed control. **Area:** chunkdb / diskdb / DiskIO. + **Complexity:** High. **Dependencies:** chunkdb server and DiskIO recovery path. +- **[R84](R84-chunkdb-post-disk-move-placement-scanner.md)** — post-move chunk + placement scanner. **Area:** chunkdb / diskdb. **Complexity:** Medium. + **Dependencies:** R81 Part 2 and the chunkdb server. +- **[R92](R92-chunkdb-in-chunk-gc.md)** — in-chunk garbage-collection operations. + **Area:** chunkdb / chunk IO. **Complexity:** High. **Dependencies:** chunk + lifecycle and qualified range ownership. +- **[R95](R95-chunkdb-chunk-range-delete.md)** — qualified chunk-range deletion + and orphan scanning. **Area:** chunkdb / chunk IO / S3. + **Complexity:** High. **Dependencies:** chunk-range ownership records. +- **[R96](R96-chunkdb-console-cli-integration.md)** — chunkdb CLI management + integration. **Area:** chunkdb / console / CLI. **Complexity:** Medium. + **Dependencies:** stable chunkdb management surface. +- **[R147](R147-tree-chunk-gc.md)** — reclaim B+tree chunk strips. **Area:** + crowdb-tree / chunkdb / diskdb. **Complexity:** High. **Status:** Deferred + until immutable page packs and durable reclaim candidates are available. +- **[R148](R148-chunk-stream-scale-out.md)** — stream metadata scale-out and + sealed-chunk EC. **Area:** chunk-stream / chunk-kv / KV / chunkdb. + **Complexity:** High. **Status:** Deferred until the measured single-group + mirror path is complete. +- **[R193](R193-chunkdb-node-failure-budget.md)** — configurable node-failure + budget and EC placement. **Area:** KV / chunkdb / chunk IO / deployment. + **Complexity:** High. **Status:** Planned. +- **[R207](R207-chunkdb-repo-metadata-chunk-kv.md)** — repo metadata and tasks on + chunk-kv. **Area:** chunkdb / chunk-kv. **Complexity:** High. + **Status:** Deferred beyond the direct-KV stage. + +## KV, DiskDB, and topology + +- **[R80](R80-diskdb-rebalance.md)** — disk space rebalance convergence. + **Area:** diskdb / DiskIO. **Complexity:** Medium. +- **[R82](R82-kv-watch-notify-coalescing.md)** — watch/notify coalescing. + **Area:** KV / diskdb. **Complexity:** Medium. +- **[R102](R102-diskdb-dynamic-binding-migration.md)** — dynamic disk-group + binding migration. **Area:** diskdb / KV. **Complexity:** High. - **[R103](R103-chunkdb-range-migration.md)** — dynamic slot ownership and - KV-group expansion/shrink — Area: chunkdb / kv — **Deferred until dynamic - changes are requested.** The fixed-topology stage is implemented. Independently - support fenced server-slot handoff without metadata copy, and storage-slot - migration of complete chunk/task/index/reservation state when groups are - added, drained or rebalanced. Include durable recovery and safe cleanup; - preserve independent bitmap maps, fixed slot identity and DiskIO payload. - R207 owns future backend conversion; legacy conversion is not a prerequisite. -- **[R102](R102-diskdb-dynamic-binding-migration.md)** — diskdb dynamic - disk-group binding migration — Area: diskdb / kv — Reuse the common - `BindingStrategy` framework - (`doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md` §5) to - dynamically rebind diskdb disk-groups to paxos groups, replacing the - operator-manual `BindMapValue` write with automatic monitoring + - rebinding. Monitor detects instance join/leave, rebalances disk-group - assignments, migrates data during rebinding. -- **[R80](R80-diskdb-rebalance.md)** — diskdb space rebalance across - disks — Area: diskdb — Complete the existing passive allocator and durable - relocation baseline with sustained-skew admission, projected-improvement and - target-headroom gates, bounded repeated moves, truthful stalled/balanced - status, and a real DiskIO/ChunkDB convergence test for newly added or - recovered empty disks. -- **[R82](R82-kv-watch-notify-coalescing.md)** — watch/notify - coalescing (debounce) — Area: kv / diskdb — the watch/notify - extension ships without coalescing: one notify per changed key per - matching prefix. Burst writes to a watched prefix (e.g. diskdb - `batch_write` touching 10 disks) generate 10 separate notifies, - amplifying subscriber wakeups + re-read load. Add a per-prefix - debounce coalescer with timer-task flush between the apply-path hook - and `WatchRegistry::emit`. The original coalescer was removed because - the timer task captured no registry/coalescer refs (buffered keys - were silently dropped); R82 must wire the `Weak` refs properly. Load - optimization, not correctness — the safety-net poller covers missed - notifies. - -### Data Path (diskio + chunk object writers + read flow) - -Chunk reads, read repair, mirror-to-EC conversion, write error handling, and -end-to-end Chunk IO performance workloads are landed. The RPC migration items -(R115, R116, R117) are in a separate area (see RPC Migration section below). - -### Medium Priority - -- **[R83](R83-chunkdb-complete-recovery-flow.md)** — chunkdb - complete recovery flow (real data recovery + speed control) — - Area: chunkdb / diskdb / diskio — diskdb's recovery is disk-layer - only: the R76 `RecoveryScanTask` lists impacted busy blocks + - `owner_chunk` but the repair step is a placeholder - (`RecoveryAction::LogOnly`, no data rebuild). There is no chunkdb - yet (only a reserved proto surface), so when a disk goes `Bad` the - impacted blocks are handed to a "future recovery/relocation path" - (§8) that does not exist — no surviving replica/parity is read, no - rebuilt data is written, no strip is updated. Full data recovery - needs chunkdb (the chunk→strip→segment owner) to rebuild lost - mirror replicas / EC data+parity from surviving strips via the - `diskio` service, `UpdateChunkStrip` to new segments, and free the - old `Bad`-disk segments. Recovery speed must be throttled at the - chunkdb layer (configurable bandwidth/IOps/concurrency) so - foreground traffic is not starved. Blocked on the chunkdb server - component + the `diskio` service (both unlanded; must be filed as - their own backlog items first). Replaces R76's `LogOnly` with - `Relocate` / `RebuildFromEc`. -- **[R84](R84-chunkdb-post-disk-move-placement-scanner.md)** — - chunkdb post-disk-move placement scanner — Area: chunkdb / diskdb — - R81 Part 2 adds disk move with a stable `DiskId` (record copy - during Maintenance, no full scan). The move is placement-only and - the data is intact, but there is no verification that chunk - placement is still consistent after a move: chunks reference blocks - via `Segment { disk_id, ... }` (in `MirrorStrip` / `EcStrip`), and - every chunk with a segment on the moved disk must still reach that - segment via the disk's new placement. Add a placement-integrity - scanner (chunkdb-side, following diskdb's `ScannerTask` / - `BgRunner` pattern, §10) that walks chunk→strip→segment after a - move (and periodically), resolves each segment's `DiskId` to its - current group-0 placement, and reports unreachable / orphaned - segments — handing `Bad`/`Missing`-disk segments to R83 for - rebuild. Triggered on move via watch/notify (R78) with a periodic - safety net. Blocked on the chunkdb server component (unlanded) and - R81 Part 2. -### RPC Migration (legacy → crowdb-rpc) - -Historical migration order: R115 → R116 (unary); R117 (streaming) followed -R114 plus the original R32 consensus migration. R115 first validated the -migration pattern (schema, server, client, and error mapping) before the -streaming services. All four migrations follow the -zero-copy wrapper convention (`design-crowdb-rpc.md` §6): `FB`-prefixed -flatbuffer types, wrapper classes in `crowdb-protocol`, no owned -intermediate structs, no per-field copy. The four transport migrations (R115 -diskdb, the original R32 KV consensus scope, R117 KV client-facing, and R116 -chunkdb) are DONE. The post-migration KV server/library review is also complete. - -- **[R33](R33-crowdb-tree-rename.md)** — Extract crowdb-tree to separate repo and rename — Area: - workspace — Move `crowdbtree/` into its own git repository (preserving - history), wire `crowdb-kv` to depend on `crowdb-tree-ffi` as an external - dependency, and rename the crate/namespace/macros from `crowdbtree` to - `crowdb-tree` / `crow::tree` / `CROWDB_TREE_*`. Establishes the `crowdb-kv` → - `crowdb-tree` dependency boundary analogous to `crowdb-kv` → `crowdb-common`. - Most naturally done after R12. -- **[R50](R50-epoch-protected-memtable.md)** — Epoch-protected - lock-free MemTable — Area: scan / get / crowdb-tree engine — - **Done.** `MemTable::snapshot()` deep-copied every live L0 entry - (key + full cell payload) on every scan regardless of range or - `limit`, and an L0 `get` hit copied twice. Root cause: L0 was the - only reader-visible structure outside the engine's EBR scheme. - Replaced the `absl::btree_map` under `mu_` with a - `ConcurrentSkipList` (inline keys, versioned cell pointers, - epoch-deferred reclamation). Readers now traverse L0 lock-free - under their existing epoch guard with zero copy; the cursor seeks - directly (no `upper_bound` skip pass); `get_view` borrows the - cell directly off the node. Closes the known gap at - `crowdb-tree.h:81`. All 383 `test-tree-ct` tests pass. - -### Low Priority - -**Complexity — Low (placeholder):** -- **[R5](R5-rdma-alloc.md)** — RDMA-pinned allocation — Blocked by: RDMA backend — Area: crowdbtree - engine — `buffer::allocate` seam is designed for RDMA-pinned memory but no - RDMA backend exists yet; placeholder only. + KV-group expansion/shrink. **Area:** chunkdb / KV. **Complexity:** High. + **Status:** Deferred by user decision until dynamic ownership is requested. - **[R139](R139-group0-service-config.md)** — Group-0 distributed service - configuration — Area: config / control plane — Publish versioned, - scoped config through group 0; each service validates revisions, applies - dynamic fields atomically, and reports fields that require restart. - -**Complexity — Medium:** -- **[R4](R4-bounded-mempool.md)** — Bounded memory pool — Area: crowdbtree engine — `buffer::allocate` uses - unbounded `std::malloc`; a burst of large writes can spike RSS without - backpressure. -- **[R60](R60-tree-scan-sibling-leaf-readahead.md)** — Sibling-leaf - readahead on cold scans — **Deferred pending cold file/block-backed - measurement.** Area: scan / crowdb-tree engine — the scan - path demand-loads each L1 leaf inline (sync) or one pending page per - reactor round trip (async), so a cold multi-leaf range pays one - stall/round-trip per leaf, serialized with merge work on prior - leaves. The scan knows `right_sibling` before finishing the current - leaf — issue a readahead for the next - leaf to overlap I/O with merging. Sync path: prefetch the - right-sibling page id via a page-cache async-resolve seam. Async - path: use a fixed one-leaf lookahead to overlap the next leaf read with - merge and packing of the current resident leaf. Readahead is scan-only and - conditional on the range, remaining limits, deadline, residency, and async - disk backend. Win is zero on mem-mode (leaves resident); implement only if - a cold benchmark with eviction shows a material latency or throughput - improvement. Medium complexity. ---- - -## Implementation Process - -Each item follows the lifecycle defined in the -[`/implement-requirement` workflow](../../.agents/skills/implement-requirement/SKILL.md): -understand → design → plan → implement → merge design → cleanup. - -After the PR is merged, all obsolete working docs (design draft, plan doc) -must be deleted — see the workflow's Post-merge cleanup section. - ---- - - + configuration. **Area:** configuration / control plane. **Complexity:** Medium. + **Status:** Deferred until the file-backed service configuration contract. +- **[R144](R144-chunk-kv-partition-merge.md)** — adjacent partition merge. + **Area:** chunk-kv / KV / group 0. **Complexity:** High. **Status:** Deferred + until manifest reuse, transfer fencing, and transition recovery are stable. + +## Tree and memory + +- **[R4](R4-bounded-mempool.md)** — bounded memory pool for tree allocations. + **Area:** crowdb-tree engine. **Complexity:** Medium. +- **[R5](R5-rdma-alloc.md)** — RDMA-registered host-buffer allocation. + **Area:** crowdb-tree / RDMA. **Complexity:** High. **Status:** Design + proposal; blocked on an RDMA backend. +- **[R60](R60-tree-scan-sibling-leaf-readahead.md)** — sibling-leaf readahead on + cold scans. **Area:** crowdb-tree / scan / DiskIO. **Complexity:** Medium. + **Status:** Deferred pending cold file/block-backed measurements. +- **[R228](R228-tree-range-metrics.md)** — root/range page, logical-byte and live-KV + metrics for continuous split sizing and data-weight placement. **Area:** tree / + chunk-KV / console. **Complexity:** High. **Status:** Deferred implementation + pending statistics-format/performance review; count placement remains enabled. diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index 8f14a06c8..3b1affda0 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -1,123 +1,269 @@ -# CROWDB - Design: Access Dataset Service +# CROWDB - Design: Dataset -Dataset is CROWDB's AI-oriented access model for immutable generations, -samples, shards, tensors, batches, and streams. It offers both portable HTTP -access and a topology-aware native client with the same dataset semantics. +Dataset is CROWDB's sample-oriented data domain for AI, analytics, and data +preparation. It owns dataset metadata, immutable snapshots, sample selection, +field projection, bounded reads, and payload lifetime while using CROWDB KV and +chunks for durable state and bytes. -Depends on: [Access Server](../design-crowdb-access-server.md), -[Chunk I/O](../../chunkio/design-crowdb-chunkio.md), -[Chunk-KV](../../chunkds/design-crowdb-chunk-kv.md), and -[Chunk-KV client](../../chunkds/design-crowdb-chunk-kv-client.md). +The normative object and behavior contract is [CROWDB Dataset Specification](spec-crowdb-dataset.md). This design document describes the implementation architecture that realizes that contract; it does not redefine Dataset semantics. -Satisfies: dataset-oriented access whose routing, batching, and transfer model -is not constrained by S3 or Iceberg compatibility. +This design is independent of S3 and Iceberg. A Dataset may record an external +source reference, but that reference does not replace Dataset publication, +selection, or reclamation authority. ## Table of contents -1. [Intent and boundary](#1-intent-and-boundary) -2. [Dataset model](#2-dataset-model) -3. [Dual access surfaces](#3-dual-access-surfaces) -4. [Native and GPU-direct path](#4-native-and-gpu-direct-path) -5. [Publication and lifecycle](#5-publication-and-lifecycle) -6. [Relationship to other access models](#6-relationship-to-other-access-models) -7. [Correctness invariants](#7-correctness-invariants) - -## 1. Intent and boundary - -Dataset gives AI, analytics, and data-preparation applications concepts closer -to their work than buckets or tables alone. It owns generation publication, -sample and shard selection, batching, ordering, shuffle, streaming, prefetch, -and tensor-oriented delivery. - -Dataset does not own physical placement, replication, erasure coding, repair, -or disks. It is not an S3 extension, an Iceberg table convention, or a query -engine. - -## 2. Dataset model - -A dataset has stable identity and immutable generations. A generation selects a -manifest that maps logical concepts such as samples, shards, tensors, column -groups, or batches to validated byte ranges and encodings. CROWDB chunks own the -bytes; Dataset owns their logical organization and selection. - -One read selects one generation. New data and metadata are built out of view and -become visible through atomic generation publication. - -## 3. Dual access surfaces - -Dataset HTTP is an independent Access Server protocol. It provides portable -discovery, ingest, selection, and bounded streaming to clients that should not -embed CROWDB routing. - -The native Dataset client is a linked client library, not an HTTP wrapper. It -loads the same dataset authority, consumes a bounded versioned CROWDB routing -view, plans work locally, and connects directly to responsible metadata and -data services. This removes one routing and payload relay layer from the -performance path. - -The two surfaces may use different request shapes and transport mechanics. They -must select the same generation, enforce the same authorization, and return the -same logical data and integrity result. - -## 4. Native and GPU-direct path - -The native client owns routing, retry, prefetch, streaming, and application -buffer lifetime. A stale topology produces a bounded redirect, refresh, or -retry. Observing topology lets the client route work but never makes it a -placement authority. - -CPU delivery uses owner-backed CROWDB buffers. The accelerated direction is -direct delivery from DiskIO-owned buffers across RDMA into registered client GPU -memory. GPUDirect Storage, GPUDirect RDMA, cuObject-compatible transfer, or -native CROWDB RDMA may provide the mechanism. The goal is to avoid both an -Access Server payload bounce and Dataset-client CPU staging. - -Direct operations use short-lived capabilities bound to the principal, dataset -generation, operation, source and destination ranges, limits, expiry, and -topology epoch. Completion covers every delegated span and integrity check. -CPU fallback is chosen before direct submission and never continues a partial -direct transfer. - -## 5. Publication and lifecycle - -Ingest writes bounded candidates and publishes a complete immutable generation -only after its manifest, indexes, payloads, integrity state, and durability -conditions complete. Failed or losing candidates remain invisible. - -Readers retain their selected generation for the whole operation. Retention and -reclamation remove physical data only after no live dataset generation, lease, -or explicit cross-model reference can reach it. - -## 6. Relationship to other access models - -Dataset may explicitly reference immutable data associated with S3 or Iceberg. -Such a reference is generation-bound and preserves the source model's ownership -and lifetime rules. Dataset cannot overwrite an S3 object, commit an Iceberg -table, or reclaim source-owned bytes. - -Likewise, S3 compatibility and Iceberg table rules cannot restrict native -Dataset batching, shuffle, scatter/gather, or GPU delivery. - -## 7. Correctness invariants - -- **DATA-I1 — Generation consistency:** one operation uses one immutable - dataset generation. -- **DATA-I2 — Exact selection:** returned samples, shards, tensor ranges, and - destination offsets correspond exactly to the selected manifest. -- **DATA-I3 — Atomic publication:** readers observe absence or one complete - dataset generation, never a partially built manifest. -- **DATA-I4 — Native directness:** native access reaches responsible CROWDB - services without an Access Server request or payload hop. -- **DATA-I5 — Surface equivalence:** HTTP and native access return the same +1. [Scope and boundary](#1-scope-and-boundary) +2. [Authority and naming](#2-authority-and-naming) +3. [Dataset and snapshot model](#3-dataset-and-snapshot-model) +4. [Manifest, schema, samples, and fields](#4-manifest-schema-samples-and-fields) +5. [Selection and read plans](#5-selection-and-read-plans) +6. [Access surfaces and bounded delivery](#6-access-surfaces-and-bounded-delivery) +7. [Progress and lifecycle](#7-progress-and-lifecycle) +8. [Interactions](#8-interactions) +9. [Correctness invariants](#9-correctness-invariants) +10. [Implementation and I/O flows](#10-implementation-and-io-flows) +11. [Risks and open questions](#11-risks-and-open-questions) + +## 1. Scope and boundary + +Dataset provides a stable logical model for samples and heterogeneous fields. It +supports HTTP access and a separately maintained SDK with thin and fat client +modes, with equivalent selection and integrity semantics. It does not own disk placement, replication, +erasure coding, repair, or physical chunk layout. It does not become a query +engine or a training-framework-specific storage protocol. + +The core objects are Dataset, Snapshot, Manifest, Schema, Sample, Field, +Selection, Read Plan, and Cursor. Worker scheduling, tensor decoding, collation, +and model checkpoints remain in adapters or execution systems above Dataset. + +## 2. Authority and naming + +Dataset authority always supplies the fixed `dataset-ns` default/root namespace. Every Dataset +identity carries that namespace, even though the first implementation exposes no +namespace create, delete, or mutation operation. Dataset names are unique within +the supplied namespace. The representation keeps namespace in the key so a +future multi-namespace authority can be added without changing Dataset identity, +snapshot inheritance, chunk placement, or read ordering. + +## 3. Dataset and snapshot model + +A Dataset owns metadata, snapshots, manifests, samples, fields, and the payload +references needed to read them. A Snapshot is an immutable published view +identified by an opaque `snapshot_id`. `latest` and `stable` are mutable names +that resolve to real snapshot IDs; cursors and read plans store the real ID. + +A Snapshot has at most one `parent_snapshot_id`. A child applies additions, +deletions, and membership rules to its parent; the numeric order of IDs never +implies ancestry. Snapshot merge is explicit: a merge produces a new snapshot +with a declared parent and delta. Old snapshots remain readable until retention +releases them. + +Publication builds and validates the manifest, schema binding, indexes, payload +references, checksums, and durability state before atomically advancing a +mutable name. Readers observe either the old complete snapshot or the new +complete snapshot. + +## 4. Manifest, schema, samples, and fields + +The Manifest is the authority for a snapshot's schema binding, sample membership, +splits, selections, field locators, checksums, statistics, and parent delta. It +may be partitioned and indexed; a single hot key cannot be required for all +reads. Schema defines field type, structure, requiredness, missing-field +semantics, and evolution rules. + +`sample_id` is a stable opaque string supplied by the importer. Within one +snapshot, `(sample_id, field)` has one logical value. A field is either an inline +KV value for bounded metadata or an opaque chunk location string for larger +payloads. Dataset passes the location to the chunk system and never parses or +constructs its physical meaning. Copy-on-write shares unchanged values between +snapshots; field and sample deletion uses tombstones so historical snapshots +remain valid. + +A sample may contain image, video, audio, text, label, embedding, or metadata +fields. Fields are aligned by snapshot and sample identity, while decoding and +tensor composition belong to the consumer adapter. + +## 5. Selection and read plans + +A Split is a logical subset of a snapshot. A Selection in a Read Plan chooses +samples using prefix, range, explicit IDs, keyword, field match, and boolean +composition. Prefix, range, and explicit IDs can be checked for overlap directly; +metadata predicates require a scan against one fixed snapshot. The manifest +records any permitted overlap policy. + +A Read Plan binds one snapshot ID to selection, ordering, optional shuffle +parameters, and batch policy. The implementation supports bounded sample and +group shuffle. +Sample shuffle loads only sample IDs and lightweight metadata into bounded groups; +a selection that fits one group completes in one pass, while a larger selection +uses multiple groups with the same deterministic contract. Group shuffle remains +an explicit plan type for prefix, metadata, session, device, time-window, or +association-key grouping. Every shuffle has sample-count, retained-byte, batch, +prefetch, and in-flight limits and is rejected before payload reads when a limit +would be exceeded. Shuffle never moves or rewrites published payload. + +## 6. Access surfaces and bounded delivery + +HTTP is the portable Dataset surface. The native client loads the same authority +and a bounded versioned routing view, then connects directly to responsible +metadata and data services. Neither surface may change snapshot semantics. + +`GetSample`, `GetBatch`, and bounded Scan resolve metadata first, apply field +projection, and then read inline values or chunk ranges/streams. Batch size, +read window, prefetch, cache, and in-flight requests are bounded and propagate +backpressure. Direct GPU delivery may use short-lived capabilities and a CPU +fallback, but it must preserve the same selection and integrity result. + +## 7. Progress and lifecycle + +A cursor is valid only for its bound Read Plan and snapshot. Ordered reads store a +sample position; grouped shuffle stores group identity or index and an offset +within the group. The client advances a cursor after batch confirmation. A crash +may replay the last unconfirmed batch, but may not skip or change the snapshot. +Epoch and sampler state remain client state. + +An active-read lease protects payloads reachable from the selected snapshot while +the read is in progress. Normal close or cancel releases it immediately. If the +connection has no activity for 60 seconds, the lease is invalid and GC may treat +the read as disconnected; this TTL is the crash and network-failure fallback. + +Dataset owns the lifetime of its metadata and payload references. Reclamation +requires that no retained snapshot, tombstone needed by a retained history, or +active read lease can reach the value or chunk. External S3 or Iceberg references +are provenance unless an explicit cross-model retention contract says otherwise. + +## 8. Interactions + +Dataset metadata uses KV authority and can use chunk-backed range state for +large manifests or indexes. Chunk I/O owns location validation, range/stream +reads, checksums, and physical recovery. Access Server exposes HTTP and health +boundaries; the native client observes topology but never publishes placement. +Framework adapters map Read Plans to PyTorch `IterableDataset`, TensorFlow/JAX +iterators, Hugging Face streaming examples, or Ray blocks without adding storage +objects. + +S3 and Iceberg remain peer metadata domains. Dataset can consume immutable data +from either through an explicit source reference, but cannot commit their tables, +overwrite their objects, or reclaim their owned bytes. + +The Dataset library owns these semantics and storage clients. The existing Access +Server may expose Dataset HTTP on its own configured listener, alongside S3 and +Iceberg. A separate Dataset SDK project, maintained by this team, provides both a +thin HTTP client and a fat client that connects directly to internal Chunk-KV and +DiskIO. The same SDK owns the PyTorch `IterableDataset` adapter. These are two +client modes, not two Dataset authorities or a separate Dataset server product. + +## 9. Correctness invariants + +- **DATA-I1 — Namespace identity:** every Dataset carries the configured + default/root namespace, and a Dataset name is unique within it. +- **DATA-I2 — Snapshot consistency:** one operation resolves exactly one immutable + snapshot and never mixes fields from another snapshot. +- **DATA-I3 — Atomic publication:** readers observe no manifest or one complete + published snapshot. +- **DATA-I4 — Explicit ancestry:** only the selected snapshot's parent chain + explains inherited state; ID ordering is irrelevant. +- **DATA-I5 — Opaque locations:** Dataset does not interpret physical chunk + locations. +- **DATA-I6 — Logical shuffle:** ordering and shuffle do not move published + payload. +- **DATA-I7 — Bounded delivery:** batch, window, prefetch, cache, and in-flight + limits bound memory and registration. +- **DATA-I8 — Cursor binding:** a cursor and Read Plan cannot resume against a + different snapshot or selection identity. +- **DATA-I9 — Safe reclamation:** retained snapshots and active reads protect all + reachable metadata and payload. +- **DATA-I10 — Surface equivalence:** HTTP and native access return the same logical selection and integrity result. -- **DATA-I6 — Bounded streaming:** dataset size does not determine server or - client memory, registration, or prefetch depth. -- **DATA-I7 — Non-authoritative topology:** a client routing view cannot publish - or alter physical placement. -- **DATA-I8 — GPU final destination:** a direct GPU path does not relay payload - through Access Server or Dataset-client CPU memory. -- **DATA-I9 — Independent authority:** S3 and Iceberg semantics cannot redefine - Dataset publication, selection, batching, or transfer behavior. + +## 10. Implementation and I/O flows + +The implementation is split into a `crowdb-access-dataset` library and a thin +`crowdb-access-server` Dataset listener, following the S3 and Iceberg boundary. +`DatasetAuthority` owns keys, records, publication state, ancestry, retention, +and durable progress through the `DatasetStore` interface. `DatasetReadService` +owns admission, bounded delivery, plan/cursor execution, and lease lifetime. +`DatasetHttpService` owns HTTP framing, authentication, route/status mapping, +shutdown, and health wiring. `ChunkKvDatasetStore` adapts Dataset metadata and +payload access to the chunk-backed KV client without exposing physical layout +to the authority. + +### 10.1 Publication flow + +The HTTP or native client serializes a publication request and sends it to the +listener or authority. The authority validates the Dataset, parent, operation +token, and Manifest before any head mutation. It CAS-writes the operation +record, writes a `Prepared` Snapshot record, partitions and persists the +Manifest, then CAS-transitions the Snapshot to `Published` and advances the +head. A retry reads the operation record and resumes the same Snapshot. A +failure before the published transition leaves `latest` unchanged. Manifest +partitions and the head are never read through the HTTP listener directly; +all reads go through the authority's complete-snapshot checks. + +### 10.2 Read flow + +`DatasetReadService` validates the request and acquires a delivery-window +permit before admitting work. It then records the Dataset/Snapshot read lease, +loads the Manifest and inherited ancestry through `DatasetAuthority`, computes +bounded sample groups and projection, and resolves each field. Inline values +are verified locally; chunk locators are passed unchanged to the chunk store, +which performs range/stream I/O and integrity checks. The service releases the +lease and delivery permit after the batch result is formed, then persists the +next cursor when the operation is a Scan. Errors release lifecycle state before +being mapped to the native or HTTP error class. + +### 10.3 Retention and reclaim flow + +Retention mutations update Snapshot-scoped records through the authority. +Reclaim first walks the published head's explicit parent chain and checks +latest, stable, retained, and active-read protections. It then computes +Dataset-owned metadata and payload candidates, leaves shared or externally +owned locators untouched, deletes only eligible state, and persists completed +reclaim progress. Repeated reclaim and authority restart read the same durable +progress record and remain idempotent. + +### 10.4 Restart and parity flow + +The authority reconstructs its view from DatasetStore records on startup; no +in-memory head or lease registry is authoritative. HTTP and native entry +points call the same authority and read service, so parity tests compare +Snapshot IDs, manifests, sample ordering, checksums, cursors, leases, and +logical error classes. Restart tests reopen the same store and verify published +heads, Manifest partitions, cursors, retention state, and reclaim progress. + +Metadata mutations use the smallest possible authority operation. A single-key +CAS is reserved for a publication head, an operation phase, or an admission +fence whose conflict is the correctness decision. Do not CAS every sample, +field, payload block, or read cursor. Immutable payloads are written once and +referenced by the manifest; retry identity and persisted journals handle +recovery without a global CAS loop. + +Hot reads and planning use immutable snapshots, atomics, or `ArcSwap`-style +published views. Do not add a mutex or async lock to point reads, manifest +lookups, chunk reads, or stream delivery. Lifecycle-only coordination may use a +bounded lock when an existing owner requires it, with contention measured before +acceptance. + +Small values follow the existing small-object path and remain inline only when +the configured threshold admits them. Large fields use the large-write pipeline +with bounded blocks, backpressure, and chunk-native range/stream reads. Readers +fetch bounded frames and verify chunk/block integrity. The record contract may +require MD5 for compatibility or SHA-256 for canonical object identity; the +algorithm is selected by correctness requirements, never omitted to optimize a +benchmark. Digest cost, placement, and whether a full-object pass is needed are +measured in the relevant backlog item. Dataset does not add whole-object +buffering to the hot path. + +## 11. Risks and open questions + +Future namespaces add metadata fan-out and authorization scope. The first +implementation intentionally has no namespace operations; it only carries the +configured default/root namespace in every identity. + +Manifest partitioning, predicate indexes, cache invalidation, and direct GPU +transport require workload measurements. They are independent extensions and do +not alter the Dataset identity or snapshot contract. diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md new file mode 100644 index 000000000..6bc463386 --- /dev/null +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -0,0 +1,283 @@ + + + +# CROWDB Dataset Specification + +This document is the normative Dataset contract. It defines the objects, +identities, state transitions, observable behavior, and compatibility rules +shared by Dataset authorities, HTTP clients, native clients, and future SDKs. +The implementation design is described in +[Design: Dataset](design-crowdb-access-dataset.md); when the two documents +appear to disagree, this specification is authoritative. + +## Table of contents + +1. [Scope and terms](#1-scope-and-terms) +2. [Identity and names](#2-identity-and-names) +3. [Dataset records](#3-dataset-records) +4. [Operations](#4-operations) +5. [Manifest and field values](#5-manifest-and-field-values) +6. [Snapshot publication](#6-snapshot-publication) +7. [Selection, plans, and cursors](#7-selection-plans-and-cursors) +8. [Read behavior](#8-read-behavior) +9. [Retention and reclamation](#9-retention-and-reclamation) +10. [HTTP and native surfaces](#10-http-and-native-surfaces) +11. [Errors and compatibility](#11-errors-and-compatibility) +12. [Normative invariants](#12-normative-invariants) + +## 1. Scope and terms + +Dataset is a logical, sample-oriented collection. It stores metadata and +references to payload bytes; it does not define physical chunk layout, +replication, erasure coding, or framework tensor types. + +The following terms have one meaning throughout this specification: + +- **Dataset** is the named authority for immutable snapshots and their + metadata. +- **Snapshot** is an immutable view of a Dataset. It is identified by an + opaque `SnapshotId` and may name one parent Snapshot. +- **Manifest** is the complete, validated description of one Snapshot's + schema, sample membership, field values, partitions, and integrity metadata. +- **Schema** defines the fields, requiredness, and compatibility rules for a + Manifest. +- **Sample** is an opaque `sample_id` and its field map in one Snapshot. +- **Field** is a named value of one Sample. A missing field and a tombstone are + distinct: missing means no value is supplied by the visible ancestry; + tombstone explicitly hides an inherited value. +- **Locator** identifies payload bytes. Inline values are stored in metadata; + chunk locators are opaque strings interpreted only by the chunk subsystem. +- **Selection** identifies samples from one Snapshot. +- **Read Plan** binds a Selection, ordering, projection, batching, and optional + shuffle to one immutable Snapshot. +- **Cursor** records progress for one Read Plan and is acknowledged only after + the corresponding batch is accepted by the caller. +- **Active-read lease** protects data reachable from one Dataset/Snapshot while + a read is admitted or executing. + +Dataset does not infer semantics from a locator, a Snapshot ID's numeric +ordering, a sample ID's encoding, or an external source URI. + +## 2. Identity and names + +A Dataset identity is `(namespace, dataset_name)`. The initial authority uses +the configured `dataset-ns` root namespace for every Dataset. Names are unique +within that namespace and are compared as opaque UTF-8 names after validation. + +`SnapshotId` is immutable and opaque. `latest` and `stable` are mutable names +whose values are Snapshot IDs. A Read Plan, Cursor, Manifest, or lease stores +the resolved Snapshot ID and never stores a mutable name in its place. + +A Snapshot has zero or one declared parent. Parentage is explicit; ID ordering +never implies ancestry. A child may add, replace, or tombstone samples and +fields while inheriting all values not overridden by its visible parent chain. + +## 3. Dataset records + +A Dataset record establishes that the identity exists and owns its metadata. +Creating an existing identity is a conflict. Opening a missing identity is a +not-found error. Dataset deletion is outside this specification. + +A Snapshot record contains the Snapshot ID, optional parent, publication +operation identity, Manifest binding, and publication state. Publication states +are: + +- `Prepared`: the operation and candidate Snapshot are durable but the + Snapshot is not visible through `latest`. +- `Published`: the complete Manifest and binding are durable and the Snapshot + may become visible through `latest`. +- `Aborted`: the candidate is permanently unavailable for publication. + +A Manifest binding records the Snapshot it belongs to, its partition count, and +its integrity metadata. A binding for another Snapshot is invalid. Partitions +are an internal representation; readers observe one logical Manifest. + +## 4. Operations + +The Dataset authority provides these operations. Each operation is scoped by a +Dataset identity; operations that take a Snapshot ID MUST use the immutable ID, +not `latest` or `stable` as an implicit mutable reference. + +| Operation | Input | Successful result | +| --- | --- | --- | +| `CreateDataset` | Dataset identity | Creates the identity; duplicate creation conflicts. | +| `OpenDataset` | Dataset identity | Returns the Dataset record. | +| `PrepareSnapshot` | Parent, Manifest, operation token | Returns one durable candidate Snapshot ID that is not visible through `latest`. | +| `PublishSnapshot` | Parent, complete Manifest, operation token | Returns the published Snapshot ID and advances `latest` atomically. | +| `GetLatest` | Dataset identity | Returns the current Snapshot ID or no value for an empty Dataset. | +| `ListSnapshots` | Dataset identity | Returns published Snapshot records in declared ancestry order. | +| `GetSnapshot` | Snapshot ID | Returns the immutable Snapshot record. | +| `GetManifest` | Snapshot ID | Returns the complete validated logical Manifest. | +| `SetStable` | Published Snapshot ID | Sets the mutable `stable` name. | +| `RetainSnapshot` / `ReleaseSnapshot` | Snapshot ID | Adds or removes an explicit retention reference idempotently. | +| `ReclaimSnapshot` / `GetReclaimStatus` | Snapshot ID | Reclaims eligible state or returns durable reclaim progress. | +| `GetSample` / `GetBatch` | Snapshot ID, sample IDs, projection | Returns ordered samples and verified field values. | +| `Scan` | Read Plan, optional Cursor | Returns one bounded batch and the next Cursor. | +| `SaveProgress` / `Resume` | Read Plan, Cursor | Persists or loads plan-bound progress. | +| `Cancel` | Bound read or service | Stops admission and releases the active-read lease. | + +`PrepareSnapshot` and `PublishSnapshot` share the same operation token and +validation rules. A complete publication may be implemented as one call that +performs preparation and completion internally; it MUST have the same durable +state transitions. Read operations are side-effect free except for lease and +cursor state required by their documented lifecycle. + +## 5. Manifest and field values + +A Manifest MUST validate before publication. Validation rejects an empty +sample set, duplicate sample IDs, duplicate field names, missing required +fields, invalid checksums, invalid parent references, and incomplete or +inconsistent partition metadata. + +Within one Snapshot, `(sample_id, field_name)` has at most one value. A Schema +field can be required or optional. A required field removed by a child must be +represented by a tombstone or an explicitly compatible schema transition; +readers never silently combine incompatible schemas. + +A Field value is exactly one of: + +- **Inline**: bounded bytes plus the declared digest. +- **Chunk**: an opaque location, range or stream metadata as defined by the + chunk subsystem, and the declared digest. +- **Tombstone**: an explicit deletion marker with no payload. + +Readers verify the declared digest before returning bytes. External S3 or +Iceberg references are provenance and input metadata; they do not transfer +ownership of those systems' objects to Dataset. + +## 6. Snapshot publication + +Publication is an idempotent operation identified by an operation token. A +retry with the same token and equivalent parent and Manifest returns the same +Snapshot ID. A retry with divergent content is a conflict. + +The authority MUST perform publication in this order: + +1. Validate the Dataset, declared parent, operation token, and decoded Manifest. +2. Allocate or recover one candidate Snapshot ID for the operation. +3. Persist the prepared Snapshot record. +4. Persist every Manifest partition and the Snapshot binding. +5. Persist the published Snapshot record. +6. Advance `latest` atomically to the complete Snapshot. + +A failure before step 5 leaves the candidate invisible and recoverable by the +same operation token. A reader of `latest` MUST see either the previous +complete Snapshot or the new complete Snapshot, never a prepared or partial +Manifest. + +`stable` may be set only to an existing published Snapshot. Publishing a new +Snapshot never changes `stable` implicitly. A published Snapshot remains +addressable by ID until reclamation removes its metadata under Section 8. + +## 7. Selection, plans, and cursors + +A Selection is evaluated against exactly one Snapshot. Supported selection +forms are explicit sample IDs, prefix or range, and bounded metadata predicates; +compound forms preserve their declared order and duplicate policy. Invalid or +ambiguous selections are rejected before payload reads. + +A Read Plan contains the resolved Snapshot ID, Selection, ordering, projection, +batch size, and optional deterministic shuffle parameters. The plan identity +includes all of those values. A plan cannot be resumed with a different +Snapshot, Selection, projection, ordering, or shuffle configuration. + +A Cursor is either at the start or contains a plan-bound group and offset. The +caller acknowledges a batch by persisting the next Cursor. A failed or +unacknowledged batch may be replayed; a successful acknowledgement may not be +skipped or applied to another plan. + +Batch size, metadata bytes, prefetch, in-flight requests, and shuffle memory +are bounded. The authority rejects a plan that exceeds configured limits before +reading payload bytes. + +## 8. Read behavior + +`GetSample`, `GetBatch`, and bounded Scan first resolve Manifest metadata, +selection, projection, and ancestry, then fetch only the requested values. +Requested sample and projection order is preserved unless the plan explicitly +requests another ordering. Unknown samples, invalid projections, checksum +mismatches, and malformed cursors are errors. + +Each admitted read increments the active-read lease for its Dataset and +Snapshot. Normal completion, cancellation, and explicit close release the +lease. A bounded inactivity TTL expires a disconnected lease; the expiry is a +GC safety policy, not a publication or cursor transition. + +HTTP and native reads use the same authority semantics. Transport framing, +serialization, and topology discovery may differ, but the resolved Snapshot, +logical samples, field values, checksums, cursor result, and error class are +equivalent. + +## 9. Retention and reclamation + +`retain` and `release` apply to an immutable Snapshot ID. The latest Snapshot, +stable Snapshot, retained Snapshots, parent-reachable history, and Snapshots +with active-read leases are protected from reclamation. + +Reclamation is idempotent. It removes only unreachable Dataset-owned metadata +and payload references. A payload shared by two retained or readable +Snapshots remains available until its final Dataset owner is unreachable. +External S3, Iceberg, or other source references are never deleted by Dataset +reclamation. + +A reclaim status record reports the Snapshot, pending or completed state, and +reclaimed-count progress. Repeated requests and authority restarts preserve +that status and do not double-delete a payload. + +## 10. HTTP and native surfaces + +The HTTP surface is the portable wire contract. It exposes Dataset creation and +opening, latest and Snapshot inspection, publication and preparation, Manifest +retrieval, retention, stable selection, reclamation status, bounded reads, read +plans, progress, resume, and cancellation. + +The native surface exposes the same operations through typed calls. Every +operation identifies the Dataset explicitly or through a bound service and +carries a Snapshot ID where the operation is Snapshot-scoped. + +A configured HTTP bearer credential protects all Dataset routes. Deployments +that need separate management and data policies may enforce that distinction at +the service boundary without changing Dataset semantics. + +## 11. Errors and compatibility + +Errors preserve their logical class across surfaces: + +- invalid input, Manifest, selection, or cursor → client-invalid; +- missing Dataset, Snapshot, or Manifest → not-found; +- duplicate operation, parent race, protected Snapshot, or stale cursor → + conflict; +- checksum or corrupt durable state → integrity/unavailable; +- cancelled read or expired admission → cancelled; +- delivery-window or configured bound exceeded → backpressure/resource + exhausted; +- transient storage or service failure → unavailable and retryable only when + the operation is documented idempotent. + +Wire records are versioned. Unknown optional fields may be ignored, but a +reader MUST reject an unsupported version, missing required field, invalid +checksum, or incompatible state transition. New surfaces must preserve the +logical behavior in this specification before adding implementation-specific +extensions. + +## 12. Normative invariants + +- **DATA-S1 — Snapshot identity:** every operation resolves one immutable + Snapshot ID. +- **DATA-S2 — Complete visibility:** `latest` exposes only a published, + complete Manifest. +- **DATA-S3 — Explicit ancestry:** only declared parent links provide + inheritance. +- **DATA-S4 — Manifest uniqueness:** one Snapshot has one valid value per + `(sample_id, field_name)`. +- **DATA-S5 — Opaque location:** Dataset never parses or manufactures chunk + locations. +- **DATA-S6 — Plan binding:** a Cursor is valid only for its exact Read Plan. +- **DATA-S7 — Bounded delivery:** configured memory, batch, and in-flight + limits are enforced before payload admission. +- **DATA-S8 — Lease protection:** active reads protect all reachable metadata + and payloads until release or expiry. +- **DATA-S9 — Ownership-safe GC:** reclamation deletes only unowned Dataset + data and never external source objects. +- **DATA-S10 — Surface parity:** HTTP and native surfaces preserve the same + logical results and error classes. diff --git a/doc/design/access-server/design-crowdb-access-server.md b/doc/design/access-server/design-crowdb-access-server.md index af1fea5e1..14988f811 100644 --- a/doc/design/access-server/design-crowdb-access-server.md +++ b/doc/design/access-server/design-crowdb-access-server.md @@ -41,7 +41,7 @@ is an implementation choice and is not part of the architecture contract. | ------- | ------------------------------------------ | ---------------- | ---------------------------- | | S3 | Buckets, objects, multipart, S3 lifecycle | HTTP | Optional direct data plane | | Iceberg | Catalogs, namespaces, tables, snapshots | HTTP | Delegated immutable FileIO | -| Dataset | Generations, samples, shards, tensors | HTTP | Topology-aware native client | +| Dataset | Namespaces, snapshots, samples, fields | HTTP | Topology-aware native client | Related designs: @@ -98,8 +98,8 @@ on another model: - Iceberg owns catalog, table, snapshot, commit, immutable file, and reachability semantics. Its S3-shaped file locations do not make S3 the authority. -- Dataset owns generation publication, sample and shard selection, batching, - streaming, prefetch, and tensor-oriented delivery. +- Dataset owns namespace-scoped identity, snapshot publication, sample and field + selection, batching, streaming, prefetch, and bounded delivery. The models may share physical bytes only through an explicit reference contract. Sharing never transfers namespace, publication, authorization, or reclamation @@ -155,7 +155,7 @@ describes scheduling and stage measurements in detail. The Dataset native path embeds routing, retry, bounded planning, streaming, and buffer ownership in the application. It resolves one immutable dataset -generation and distributes work directly to responsible CROWDB services. A +snapshot and distributes work directly to responsible CROWDB services. A topology view routes work but never grants placement authority. The accelerated direction is an end-to-end path from storage to the final @@ -191,7 +191,7 @@ or retry. Protocol-local caches are discardable hints. Direct Dataset access also preserves the storage authority boundary. A client authenticates before receiving topology, metadata, or payload authority. Direct operations use short-lived capabilities bound to the principal, logical object, -immutable generation, operation, ranges, limits, expiry, and topology epoch. +immutable snapshot, operation, ranges, limits, expiry, and topology epoch. Topology and memory descriptors alone grant no access. All paths bound connections, metadata work, stream windows, retained buffers, @@ -199,7 +199,7 @@ registered memory, retries, and background work independently of total object, table, or dataset size. The process has a separate HTTP health listener configured by `HealthConfig` -(or `CROWDB_ACCESS_HEALTH_LISTEN`), defaulting to loopback port 9093. Its +(or `CROWDB_ACCESS_HEALTH_LISTEN`), defaulting to loopback port 9094. Its `/_crowdb/health/live` and `/_crowdb/health/ready` routes are independent of the S3 and Iceberg listeners. All three ports are distinct. Health serves no user-data routes; S3 requests cannot bypass authentication through a health @@ -216,7 +216,7 @@ address; a missing address requires deployment reconciliation before restart. - **AS-I3 — Native directness:** Dataset native access reaches responsible CROWDB services without an Access Server request or payload hop. - **AS-I4 — Semantic equivalence:** Dataset HTTP, native CPU, and accelerated - GPU paths select the same immutable generation and logical data. + GPU paths select the same immutable snapshot and logical data. - **AS-I5 — Storage authority:** no Access Server module or native client owns physical chunk placement, replicas, erasure-coding shards, or disks. - **AS-I6 — Stateless server:** an authoritative operation survives loss of the diff --git a/doc/design/chunkdb/design-crowdb-chunkdb.md b/doc/design/chunkdb/design-crowdb-chunkdb.md index 1632614a0..47da4a181 100644 --- a/doc/design/chunkdb/design-crowdb-chunkdb.md +++ b/doc/design/chunkdb/design-crowdb-chunkdb.md @@ -672,6 +672,15 @@ permission to reclaim data. sends one request per DiskDB data group and runs those requests concurrently. DiskDB persists each allocated busy block as Tentative before responding. +**Allocation routing failure**: Only a typed connection-establishment failure +before RPC submission permits rerouting the same allocation. ChunkDB refreshes +the service registry and makes at most one attempt at a different endpoint. +If discovery fails, the endpoint is unchanged, or no owner is advertised, it +returns the original failure. Any failure during RPC submission or while awaiting +the reply remains ambiguous and is never replayed by the allocator. Discovery +may refresh for the next independent request. Successful allocation adds no +registry read or lock beyond the existing periodic discovery policy. + **Success boundary**: The Active chunk and every referenced Tentative busy block are durable before ChunkDB returns success. DiskDB commit changes the matching busy records to Committed in one ordinary batch write. A diff --git a/doc/design/chunkds/design-crowdb-chunk-kv-server.md b/doc/design/chunkds/design-crowdb-chunk-kv-server.md index 546a25fd7..f287cb539 100644 --- a/doc/design/chunkds/design-crowdb-chunk-kv-server.md +++ b/doc/design/chunkds/design-crowdb-chunk-kv-server.md @@ -1,7 +1,7 @@ -# Chunk KV Server +# CROWDB - Design: Chunk KV Server `crowdb-chunk-kv-server` is the standalone ownership, routing, and network boundary for chunk-backed metadata partitions. It hosts zero or many R142 @@ -9,6 +9,19 @@ partition handles but never owns or bypasses a raw tree or stream. Group 0 is the durable catalog and monitor authority; monitor-issued serving grants are the only authority to admit data requests. +## Contents + +- [1. Catalog Publication](#1-catalog-publication) +- [2. Domain Monitors](#2-domain-monitors) +- [3. Serving Authority](#3-serving-authority) +- [4. Request Contract](#4-request-contract) +- [5. Ordered Reads](#5-ordered-reads) +- [6. Split Publication](#6-split-publication) +- [7. Child Balance State Machine](#7-child-balance-state-machine) +- [8. Placement Policy](#8-placement-policy) +- [9. Lifecycle and Observability](#9-lifecycle-and-observability) +- [Open Issues](#open-issues) + ## 1. Catalog Publication The catalog is one checksummed generation head over ordered immutable pages. @@ -139,9 +152,99 @@ routed client. ## 6. Split Publication -The split transition phases are `Planned`, `ParentPreparing`, -`ChildPrepared`, `CatalogCommitted`, and `Aborted`. The server and group-0 -monitor advance them in this order: +### 6.1. Publication boundary and durable steps + +The group-0 catalog head publication is the external range and assignment +cutover. A local writer handoff is a separate operation: the same process can +dispatch the old parent route to the retained parent and child before the new +catalog is visible. This compatibility must preserve every acknowledged WAL +record; it does not authorize a different owner to serve an unpublished range. + +The parent retains its existing tree and journal. Only the child receives new +storage. Handoff is parent-local range dispatch, not replacement of the parent +writer with another tree/WAL and not remote ownership transfer. The successful +durable handoff status update commits the obligation to resume this split; +in-memory dispatch changes afterward. Handoff status is distinct from final +`ChildPrepared` readiness and from external catalog publication. A candidate +epoch or a process-local dispatcher cannot substitute for this status. + +- **S1 — Plan:** persist the transition identity, expected parent assignment, + split key, both successor epochs and complete storage identities. Reserve the + affected partition and owner against another topology transition. +- **S2 — Prepare:** persist `ParentPreparing` before creating artifacts. Pin the + exact base, construct the bounded child view and establish the common WAL + frontier `C`. Persist the exact handoff status before changing local dispatch. + Parent journal appends and reads continue while the status is persisted; + no handoff waiting queue or mutation-worker wait for that update is allowed. + Bind the original parent storage, child storage, pinned base and replay + boundaries. Cover the additional parent suffix before actual dispatch in + child recovery; fixing `C` does not by itself end parent right-range writes. +- **S3 — Ready:** persist the exact retained-parent and child artifacts, + overlays, common cursor and readiness proof as `ChildPrepared`. A process-local + handle or heartbeat cannot substitute for this durable proof. +- **S4 — Publish:** validate that the current catalog still contains the planned + parent. Publish one successor head replacing it with two adjacent Serving + entries. This head is the external split commit point. On an ambiguous result, + reread the authoritative head and accept only the exact intended successors; + an unrelated newer generation is not proof that this split committed. +- **S5 — Reconcile:** persist `CatalogCommitted` after proving publication. + Install the matching catalog and grants; preserve old-route dispatch without + allowing a retired writer to append. Restart must use the published assignment + together with its exact durable split proof. +- **S6 — Materialize and retire:** checkpoint independently recoverable halves, + publish removal of their overlays and transition markers, then release pins. + Reclaim only artifacts with no catalog, replay, retry or pin references. + +Crash recovery must distinguish these boundaries: + +- Before handoff status commit, recover the published parent and retry or abort + preparation; no child mutation may have been acknowledged. +- After status commit but before in-memory dispatch, resume the same split. + Recover the complete original parent journal, including writes made while the + status update was in flight; an empty child WAL is valid. +- After dispatch but before catalog publication, resume the same split from the + original parent journal and the child journal. The old parent route remains + published; do not discard the child or choose authority by the maximum local + epoch. Neither final readiness nor catalog publication is needed to establish + that the committed handoff must be recovered. +- An uncertain handoff update requires exact durable transition reread. Stale + preparation or abort work must not overwrite a committed handoff. + While the process remains live, retain and retry the exact prepared child + base, WAL handle and pins. Do not reopen its WAL, release its pins or rebuild + a different candidate merely because publication returned an error. Writes + arriving on the parent meanwhile are covered by the later dispatch frontier. + Once dispatch is installed, its live writers own finalization; only a proven + pre-handoff abort may discard the pending candidate. +- After publication but before `CatalogCommitted` persistence, prove the exact + successor entries and finish the transition marker; do not roll back the split. +- After publication, recover each assigned half's complete own WAL. The child + inherits only the parent prefix through the final readiness frontier `C2` + recorded in its catalog artifact, then replays its own WAL. This bound applies + even when the child WAL is empty: later retained-parent records belong to the + parent's next epoch and must not extend child inheritance. Require matching + grants before admitting requests. + +The initial handoff status fixes `C`, before the eventual in-memory dispatch +frontier `C2`. While only that initial status exists, recovery derives `C2` from +the first child record's sequence minus one, or inherits the parent tail when +the child WAL is empty. Final readiness records the exact boundary explicitly; +published catalog recovery uses that boundary instead of rediscovering it from +the current parent tail. + +The durable handoff proof is stored in the split transition while its phase is +`ParentPreparing`. Revision CAS preserves an existing proof and reconciles an +unknown write result by exact reread. Initial and final replay frontiers remain +distinct; additional real-process crash windows are recorded under Open Issues. + +### 6.2. Local execution and storage retention + +The split transition phase symbols are `Planned`, `ParentPreparing`, +`ChildPrepared`, `CatalogCommitted`, and `Aborted`. A separately persisted +`handoff_proof` distinguishes preparation from committed local handoff before +final readiness. It binds the pinned tree base, original parent stream and +initial inherited frontier while the transition remains `ParentPreparing`. +Final readiness extends that frontier without replacing its base. The server and +group-0 monitor advance them in this order: 1. Group 0 persists the complete plan before local work begins. The existing parent keeps its identity, tree, stream, owner, and lower boundary; its next @@ -149,8 +252,11 @@ monitor advance them in this order: fixed by the plan. 2. The parent process persists `ParentPreparing`, renews its serving grant as an active owner, checkpoints the exact parent base, builds the child's - range-bounded base, performs the bounded writer handoff, and installs the - local child handle. + range-bounded base, fixes `C`, and persists the handoff status without stopping + parent writes or reads. It then installs parent-to-child dispatch. Left-range + writes retain the original parent journal; right-range writes move to the + child journal only when dispatch changes. Replay coverage includes parent + writes between the fixed frontier and that dispatch change. 3. Readiness binds the exact common cutover, the retained-parent next epoch, and the child's complete tail overlay. It is persisted before catalog publication. @@ -167,9 +273,11 @@ monitor advance them in this order: the actual writer or writers used by the operation. Before child materialization, heartbeat load reports the child as dependent. -After process restart, either split half may reopen with its historical tail -overlay in Prepared state. A matching serving grant alone cannot activate that -overlay. Startup loads the persisted split transition and requires its committed +After process restart, the retained parent reopens its original tree and WAL +through ordinary range-filtered recovery; it has no split tail overlay. The +child reopens its exact pinned base, replays the range-filtered parent suffix, +and then replays its own WAL. A matching serving grant alone cannot activate +the child overlay. Startup loads the persisted split transition and requires its committed phase, transition identity, range, owner, epoch and complete storage artifact to match the current Serving catalog entry. Transfer overlays retain their separate committed-transfer validation. Missing or conflicting evidence leaves the writer @@ -189,11 +297,77 @@ restart. Root reclamation treats the oldest durable pin as an upper bound. The owner releases the pin only after installing the newer catalog generation whose child has no overlay and whose parent and child have no split marker. Pin delete is idempotent and remains reconciliation work after an ambiguous response or a -crash. An abort before readiness removes unpublished child state and its pin; -after readiness, durable transition and catalog evidence decide cleanup. +crash. An abort before handoff commit may remove unpublished child state and its +pin. After handoff commit, absence of readiness or a child catalog entry is not +permission to abort or delete child storage; recover and complete the same split. + +The parent also retains its existing checkpoint generation while the child +depends on the inherited parent WAL prefix. Publishing that prefix into the +child's live tree does not release this dependency: the pinned recovery base +still needs those journal records after a crash. Parent checkpoint publication +remains suppressed by the existing transition pin until the catalog records an +independent child. Parent reads and journal appends continue throughout this +interval. The same completed catalog releases both generation pins. ## 7. Child Balance State Machine +### 7.1. Owner cutover and durable steps + +The first successful catalog head publication naming the target in +`TargetCatchingUp` is the owner cutover. The later `Serving` publication changes +readiness, not ownership. A candidate target epoch in a plan is not effective +tree or serving authority. Source release precedes the owner cutover and creates +a recovery obligation; it is not itself a catalog owner change. + +- **T1 — Plan:** persist the exact source assignment, target identity and higher + epoch, artifacts, lease bounds and transition identity before preparation. +- **T2 — Prepare source:** persist `SourcePreparing`, pin an immutable root, + initialize the independent target WAL and record replay start cursor `P`. + Source remains the published owner and accepts writes under its existing grant. +- **T3 — Prepare target:** persist `TargetPreparing`; open the pinned base and + replay the source suffix into a Prepared view. Persist readiness before + `TargetPrepared`. Preparation must not advance the shared tree's effective + owner epoch or disable source recovery. Target cannot admit data mutations. +- **T4 — Seal source:** after exact target readiness, close source admission, + drain admitted writes and persist release proof with final durable cursor `C`. + With an unreachable source, wait for lease exclusion and recover the complete + durable tail instead of treating `P` as final. No target data admission is + allowed while the catalog still names source. A durable release cannot be + undone merely because the catalog still has the old assignment. +- **T5 — Publish owner:** validate the source assignment and durable release, + then publish the target epoch, `TargetCatchingUp` and complete replay artifacts + in one catalog successor. Resolve ambiguous publication by exact head reread. + Effective target storage authority must follow this published assignment; + a crash before that authority update is completed must allow idempotent repair. +- **T6 — Catch up:** recover the complete source prefix through `C`, followed by + the target WAL. The published release proof permits durable unconditional + target appends; reads and conditional operations wait for the complete view. + Source cannot append, including through a stale client route or old grant. +- **T7 — Serve:** persist final readiness, publish the same target assignment + as Serving, persist `CatalogCommitted` and activate only with its exact grant. + A crash between head publication and phase persistence must finish the marker + from exact catalog evidence rather than strand or reverse the assignment. +- **T8 — Retire:** clear the completed overlay only after independent recovery + is proven. Release pins and reclaim old resources only after all recovery, + retry and forwarding references have cleared. + +Restart first reads the authoritative catalog and then the matching durable +transition. Before release it restores source; after release but before owner +publication it keeps source fenced and completes publication; after owner +publication it restores target. A local tree epoch, heartbeat or cached page +cannot independently choose the owner. Missing or conflicting proof fails closed. + +An abort is permitted only before durable release and owner publication. Persist +the exact abort before discarding the prepared target; stale workers must not +publish or activate that aborted transition. Candidate epochs are not reused, +and cleanup must retain any source-referenced base, WAL or pin. Restoring source +does not require synchronous deletion of unrelated target temporary objects. + +The effective-tree-authority ordering and publication/phase crash windows still +require implementation verification; known gaps are recorded under Open Issues. + +### 7.2. Execution, proofs and recovery + A balance transition persists source and target owners, increasing target epoch, source and target artifacts, readiness limits, old-grant deadline, phase, release proof, initial readiness proof, final catch-up proof, and @@ -319,7 +493,7 @@ The balance invariants are: Balancing targets at least `live_owner_count * target_partitions_per_owner`, defaulting to four partitions per owner. Split chooses the largest eligible -partition using approximate retained pack weights. A split boundary comes from +partition using approximate retained pack estimates. A split boundary comes from a resident index separator or a bounded small key window; exact byte or key medians are unnecessary. Live-key witnesses establish nonempty children when the observation is made. Concurrent mutations can change that observation. @@ -330,19 +504,89 @@ an unsplittable largest partition does not exclude the remaining candidates. Retained pack estimates use the opened manifest, including shared packs, rather than cumulative write counters. The estimate is deliberately coarse and can lag unsnapshotted mutations; observation does not scan remote metadata or data. -The retained parent and new split child stay local. Once independently -recoverable, count-correcting placement takes priority over further splitting; -weighted placement follows eligible splits. Placement minimizes partition-count -difference first, then durable-byte spread. A move must repair count imbalance -or improve weighted spread by at least 25%. Request rate and target headroom are -safety filters. A child with a parent-tail overlay is ineligible. The default -per-partition cooldown is one minute. Split pacing includes recent splits and -transfers; repeat-placement pacing includes only recent transfers, so a split -does not postpone a child's first placement. An owner participates in at most -one transition at a time. Transfer preparation, estimated catch-up and forwarding -retain independent ten-minute safety windows. Healthy, independently recoverable -partitions with capacity on an idle target must make actual balance progress -within forty seconds; splitting alone is not placement progress. +The retained parent and new split child stay local. Range splitting deliberately +uses an approximate boundary and does not guarantee equal child sizes. Split +sizing and minimum partition supply do not require exact placement equality. +A move uses assigned partition counts; byte estimates remain capacity filters. +An indivisible range may prevent further useful count balancing, in which case +placement remains unchanged. + +### 8.1 Count-Based Placement + +Placement currently uses assigned split count only: `w_i = p_i / P`, including +healthy owners with zero assignments. `byte_weight_percent` is reserved and must +be zero; nonzero settings are rejected until root/range statistics are available. +The data contribution is disabled. Physical shared-pack bytes, memtable bytes, +write counters and accumulated split ratios are not placement data weights. +Tolerance defaults to 20% relative to equal owner share; minimum global-loss +improvement defaults to 25%. Missing or epoch-mismatched assignment observations +still defer planning. Capacity/headroom remains a separate safety filter. + +Tree statistics do not yet expose current root/range page, logical-byte or live-KV +aggregates. Existing retained-pack estimates remain coarse inputs to automatic +split sizing/candidate ranking and capacity checks; they can include shared packs +and lag ordinary flushes. Count balancing does not make those estimates exact. +The split execution, boundary sampling, recovery and publication contracts remain +unchanged. Console does not display Weight or weighted balance explanations. + +### 8.2 Tolerance and Move Selection + +Equal owner weight is `1 / N`. The observation's relative deviation is +`D = max_i(abs(N * w_i - 1))`; `imbalance_tolerance_percent` bounds `100 * D`. +Within tolerance, no load-balancing move is planned. Outside tolerance, evaluate +candidate moves against the same global loss +`L = sum_i((N * w_i - 1)^2)` before and after the move. The partition count +contribution moves; the byte contribution is zero. The normalized denominators +stay fixed for each candidate evaluation. Only the two affected owner terms +change, so each candidate's loss delta requires constant arithmetic. + +A candidate must strictly reduce loss and meet +`minimum_weighted_improvement_percent` of the current loss. A count difference +does not authorize a move that worsens the global count score. Choose the greatest loss +reduction among safe eligible candidates; stable partition/target identities +break ties. Use validated bounded integer/fixed-point arithmetic, sufficient +intermediate width, and conservative threshold comparisons; UI rounding does +not decide eligibility. With unchanged observations, a reverse move increases +the same loss and cannot qualify. Observation changes remain subject to +thresholds and cooldown, rather than triggering actions for every small change. + +For example, two owners with weights 20% and 80% can move a partition whose +contribution is 40 percentage points to obtain 60% and 40%. A relative tolerance +of 20% includes that result because equal owner weight is 50%. This is an +illustration of configured thresholds, not a promise that split yields two +40-point children. When no safe indivisible candidate offers enough improvement, +report no useful move and keep the placement. + +Request rate, target headroom, fresh health, exact assignment and independent +recoverability are safety filters. A parent-tail overlay is ineligible. One +partition and each participating owner have at most one active transition. +Default per-partition cooldown is one minute; cooldown supplements scoring and +tolerance. Split pacing includes recent splits and transfers; repeat-placement +pacing includes only transfers, so split does not postpone first placement. +Transfer preparation, estimated catch-up and forwarding retain independent +ten-minute safety windows. A safe candidate outside tolerance that meets the +improvement threshold must make placement progress within forty seconds when +its cooldown has elapsed; splitting alone is not placement progress. + +### 8.3 Decision Observation + +The monitor's bounded decision observation identifies policy coefficients and +thresholds, catalog generation, assignment/owner identities, observation time, +source freshness, counts, estimated bytes, count/byte weight contributions, +owner weights, deviation and loss. It records the selected or best rejected +candidate's predicted weights and improvement, plus move/no-move reason: +within tolerance, insufficient benefit, unavailable observation, cooldown, +active transition, inherited overlay, unhealthy owner or capacity/rate limit. +Policy version identifies the scoring formula. Diagnostics retain at most 256 +owners, 4096 partition contributions and 2 MiB, independently of the planner's +complete metadata input. Omitted details display unavailable; they do not alter +planning. Unchanged diagnostics are renewed within half their freshness window +rather than written on every tick. Do not emit or persist all candidate pairs +on every tick. This diagnostic +observation is not catalog authority and does not advance catalog generation. +The observation remains backend diagnostic metadata; Console currently hides +Weight and balance explanations. Catalog generation/identity mismatch remains +explicit; UI refresh does not bypass stale cursor rejection. ## 9. Lifecycle and Observability @@ -380,13 +624,31 @@ generation. ## Open Issues +- **Prepared transfer crash coverage:** preparation preserves source tree + authority; mutable target authority requires the exact published owner + assignment. Fault injection must cover source and target restart around + that publication, including overlapping source checkpoint work. +- **Split handoff crash coverage:** the durable handoff status precedes local + dispatch and preserves the original parent tree and WAL. Recovery composes + the exact child base, range-filtered parent suffix and child WAL. Fault + injection must cover status persistence with continuing parent writes, + dispatch before final readiness and an unconfirmed status response. Status + persistence remains outside the mutation worker. +- **Head publication before phase persistence:** transfer and split publication + and their committed transition markers are separate writes. Recovery must + prove exact catalog results and finish markers after a crash in between; + fault injection must verify admission and eventual recovery at both transfer + publications and the split publication. +- **Abort racing publication:** an abort decision must be serialized with + publication under the monitor fence and transition revision. A previously + loaded readiness proof must not publish after an authoritative abort. Verify + this race before treating cleanup as safe. + - Real-process fault injection should continue expanding coverage of ambiguous catalog writes and process death at every balance phase. -- A source crash while a remote target is only prepared and a newer local split - is still materializing can advance stream writer authority ahead of the - catalog assignment. Immediate source recovery then fails closed on the stale - catalog epoch. Recovery should reconcile this exact interleaving without - weakening the single-writer or exact-transition proofs. +- Overlapping remote transfer preparation, local split materialization and + source restart need fault coverage that proves exact catalog and transition + reconciliation without weakening single-writer or replay epoch checks. - Completed transfers remove their catalog overlay, but physical reclamation of the departed source tree and stream, plus expiry of the stale-route forwarding grace period, still need a manifest-fenced background policy. diff --git a/doc/design/chunkds/design-crowdb-chunk-kv.md b/doc/design/chunkds/design-crowdb-chunk-kv.md index 1eaad720a..19dc025bf 100644 --- a/doc/design/chunkds/design-crowdb-chunk-kv.md +++ b/doc/design/chunkds/design-crowdb-chunk-kv.md @@ -33,7 +33,14 @@ adapters wrap crowdb-tree and R141 `ChunkStream`; injected implementations support deterministic tests. The raw stream handle is private, so a server cannot bypass partition ordering, trim, or epoch checks. -The tree is range-bounded at construction. Rust never implements the tree page +The tree enforces its logical request range at construction. A retained parent +keeps the original physical root, whose pages may still contain keys and +separators outside the narrowed range until background pruning completes. +Reopening that root validates frame checksums and structure without treating +those physical keys as corruption; point reads, scans and mutations still +enforce the current logical range. Importing a newly rebuilt range snapshot +continues to require every imported key to fit its destination range. +Rust never implements the tree page path through `crowdb-chunk-client`; the production tree selects R140's native page-store backend at runtime. A balance target opens the source's pinned tree manifest directly from shared page chunks and owns a distinct target WAL. It @@ -116,41 +123,48 @@ shrinks to `[old_start, split_key)`; the new child receives any identity, range, epoch, or stream fails closed. The new child initially remains on the parent owner. Placement is a later balance operation. +Split creates only the child's tree and WAL. The retained parent uses its +existing tree, WAL and mutation path; it is not rebuilt into a second +destination tree or moved to a replacement WAL. Local handoff means that +traffic still enters the parent, which dispatches the child range to the child. +It is distinct from the later catalog publication that exposes both ranges to +external routing. A higher candidate epoch alone proves neither boundary. + Split preparation follows these ordered steps: 1. Enter `SplitPreparing` and range-rebuild the child base from one exact pinned parent tree view. The parent remains the only mutation sequencer and continues its normal WAL and memtable path while the child tree, WAL, and live memtable are prepared. -2. Both destination stores and base trees are opened, and all fallible - preparation that does not depend on the final frontier is completed while - the parent continues serving. The parent mutation worker then processes one - ordered `SplitCutover` marker after all earlier mutations are durable and - applied. The marker records the right-range inheritance frontier `C`, - attaches the fixed parent view, seeds both writers from its bounded request - result cache instead of rescanning parent history, replays any previously - acknowledged destination-WAL tail, and installs key routing. Destination - tail replay is bounded preparation retry work; the marker performs no base - rebuild, parent-history replay, checkpoint, filtered publication, or - materialization. Mutations at or below `C` remain in parent order; later - right-range mutations enter the child WAL and live memtable. - Later left-range mutations continue through the retained parent. A request - that raced with route installation and reaches the old parent queue after - the marker is forwarded by key rather than assigned an old-parent sequence. -3. The route change does not seal, publish, checkpoint, or wait for a +2. Prepare the child storage and establish an ordered parent frontier `C`. + Fixing this frontier does not stop parent journal appends or reads. Persist + a handoff status in the split transition, binding the parent and child + identities, epochs, pinned base and WAL replay boundaries. Successful durable + status update is the local handoff commit point. Resolve an uncertain result + by reading that exact transition; an in-memory `SplitCutover` message is not + durable evidence. Do not await this update in the mutation worker or add a + split-owned request waiting queue. +3. After the handoff status is committed, install in-memory key dispatch to + the child. Until dispatch changes, writes continue to the original parent + WAL, including writes after `C`; afterward, left-range writes continue to + that same WAL and right-range writes enter the child WAL. The handoff proof + and replay rules must cover the parent suffix between fixing `C` and changing + dispatch exactly once. `C` alone cannot be treated as the final inherited + parent frontier if parent writes continued beyond it. The child view must + include those acknowledged writes for reads and conditional mutations. + Seed its bounded request result cache and recover any acknowledged child-WAL + tail without reevaluating conditions. A mutation racing dispatch must have + exactly one writer and preserve its logical request identity. +4. The route change does not seal, publish, checkpoint, or wait for a memtable. The child immediately reads its base, its live memtable, and the - right-range portion of the parent generation containing `C`. Conditional - mutations consult that same merged view. -4. Independently rotate the old parent memtable at its fixed `C` frontier. The - detached generation enters one split-owned shared queue. Post-route writes - are already isolated in the retained-parent or child writer memtable and - are not members of this shared generation. -5. In background, bulk-publish the shared generation once into each range: - only the retained range into the parent and only entries at or below `C` in - the child range into the child. Newer live-writer revisions win over an - older shared entry. Release the shared generation only after both filtered - publications and their durable frontiers are recorded in one immutable - `SplitArtifact`. + right-range inherited parent view, including the covered handoff interval. + Conditional mutations consult that same merged view. +5. Complete inherited-view publication in background, preserving the original + parent tree and its normal checkpoint path. The child materializes only its + range; newer child revisions win over older inherited entries. Retain the + shared view and replay dependencies until the complete inherited frontier + and durable storage proofs are recorded in one immutable `SplitArtifact`. + Handoff status is not final readiness and does not replace this artifact. 6. Publish one catalog generation that shrinks the retained parent (same ID, new epoch) and inserts exactly one new child. The parent owner already has both writers active, so refresh only publishes this catalog and serving @@ -163,21 +177,26 @@ child base is durable, preparation persists `root//pin/ = child_root_generation`. Readiness is not recorded until that operation succeeds. A retry with the same generation is idempotent; another generation under the same transition identity is corruption. -If local child construction fails before readiness, preparation removes the pin -while the parent is still the sole authority. +Preparation may remove the unpublished child's pin after an authoritative abort +only before handoff commit. After handoff commit, recovery continues the same +split and retains its base, WAL and pin even if final readiness is absent. A prepared or restarted child recovers in three layers: open the exact base -manifest, replay the retained parent suffix through `C` with range filtering, -then replay its own WAL from `C + 1`. Records at or below the base sequence -restore retained request outcomes without reapplying tree mutations. Failed +manifest, replay the retained parent suffix through the final inherited frontier +with range filtering, then replay its own WAL from the recorded child start. +The proof must include parent writes accepted while handoff status was being +persisted; the earlier fixed `C` is not automatically that final frontier. +Records at or below the base sequence restore retained request outcomes without +reapplying tree mutations. Failed conditions remain no-ops with their original result. Sequence gaps, a wrong source stream or epoch, a mismatched range, or a child WAL beginning anywhere -other than `C + 1` reject recovery. +other than the proof's exact child start reject recovery. The child recognizes two minimum-position namespaces. A retained parent-stream -position at or below `C` is satisfied by the inherited overlay frontier. A -child-stream position waits for the child's applied frontier. New mutations -return only child-stream positions. This preserves read-after-write across a +position covered by the inherited suffix is satisfied by the inherited overlay +frontier. A child-stream position waits for the child's applied frontier. New +child mutations return only child-stream positions; retained-parent mutations +continue returning parent-stream positions. This preserves read-after-write across a catalog split without treating two streams as one offset space. After activation, bounded background passes independently prune the retained @@ -197,11 +216,14 @@ The split invariants are: - **SPLIT-PARENT-STABLE:** split preserves the parent partition, tree, stream, owner, and lower range boundary and creates exactly one new child; -- **SPLIT-ONE-WRITER:** before `C` the old parent sequences the full range; - after `C` the retained parent and child sequence disjoint ranges; -- **SPLIT-ROUTE-FRONTIER:** `C` is the last parent-ordered right-range - mutation, not the later physical seal frontier; child mutations begin after - `C` and a raced old-parent admission is forwarded instead of resequenced; +- **SPLIT-ONE-WRITER:** before in-memory dispatch changes the parent sequences + the full range; afterward parent and child sequence disjoint ranges; +- **SPLIT-ROUTE-FRONTIER:** the fixed preparation frontier, durable handoff + commit and in-memory dispatch are distinct boundaries. Recovery includes all + parent writes through the final inherited frontier and the complete child + tail; no acknowledged mutation falls between them or executes twice; +- **SPLIT-HANDOFF-DURABLE:** persist handoff before child write admission; + restart after commit resumes the same split even before catalog publication; - **SPLIT-OVERLAY-DURABLE:** base plus parent suffix plus child WAL reconstructs values and request outcomes before activation; - **SPLIT-NO-FOREGROUND-WAIT:** routing does not wait for memtable seal, @@ -210,6 +232,12 @@ The split invariants are: - **SPLIT-PIN-BEFORE-RECLAIM:** physical deletion never precedes the last catalog, snapshot, forwarding, or retry reference. +Implementation verification remains required for the durable handoff status and +the exact replay/sequence representation covering continued parent appends +between fixing `C` and installing dispatch. It must not be resolved by pausing +parent writes, queueing requests behind status persistence, or retaining a +second destination tree/WAL for the parent. + ## 6. Remote Balance Storage Contract Balancing one independently recoverable child reuses the same base-plus-tail diff --git a/doc/design/console/design-crowdb-console-ui.md b/doc/design/console/design-crowdb-console-ui.md index 7da85e3f5..e12ec2615 100644 --- a/doc/design/console/design-crowdb-console-ui.md +++ b/doc/design/console/design-crowdb-console-ui.md @@ -334,9 +334,19 @@ provide observable capacity and bitmap fixtures. - **SVC-02:** Preserve desired configuration independently of PID. A restart retains endpoints, workspace, and data; refreshed properties show the new PID and separately observed readiness. Process identity is verified before signal. + Lifecycle acceptance first checks the submitted Start/Restart response, then + observes the new PID and process. An in-flight startup is not a completed + operation and must not trigger a duplicate cleanup restart. +- Page unload stops its deployment runner. Cancellation of an old page's + observation or progress request cannot publish a service failure. The next + page reloads the persisted plan and reconciles interrupted deployments; + restoring a cached history page also reloads before resuming its runner. - **SVC-03:** Startup observes dependencies: Group 0 before dependent metadata services; valid disks/ownership for storage; journal/data prerequisites before Chunk-KV bootstrap; Chunk-KV catalog before Access startup. +- Plans waiting for a registered DiskIO service to restart observe dependencies + every two seconds, including when Capacity data has not been loaded. The + ten-second missing-disk backoff applies only to other hardware waiting states. - Access provisioning durably records catalog initialization and activation request identities before issuing either mutation. Interrupted deployment reconciles committed state and resumes the same request. An existing catalog @@ -448,6 +458,8 @@ plan, never edited out of this specification to make a run pass. PID, and readiness. All instance labels follow §4. - **A04 / KV:** verify Group 0 and ordinary Group 1 with three replicas; select a group, inspect membership, and perform bounded exact/scan data operations. + Add members to the same group sequentially, refreshing membership after + each result; independent group preparation may run concurrently. - **A05 / Capacity:** create/register DiskGroup and disk, inspect owner/binding; select a disk with more than 32 zones, page, select Zone, and verify inline bitmap, decode offsets, muted colors, and error handling for unavailable owner. @@ -550,6 +562,16 @@ Unsupported features are explicit rather than represented as working controls. Changed observations require an explicit root refresh. History retains path, fences, offset, selected entry and byte format, then revalidates on return. +- **CKV-06:** Split properties, Overview and graph cards do not display Weight. + Full Range bounds remain visible and wrap independently. +- **CKV-07:** Weighted balance explanations and data/count contribution values + remain hidden until tree statistics can describe current range data reliably. + No tree statistics scan is triggered by opening or refreshing the console. +- **CKV-08:** Count-based placement remains a backend policy. Existing bounded + observations retain catalog/epoch freshness fences, but are not presented as + data weights. Browser assertions verify that Weight and weighted explanations + are absent while tree/page inspection and range navigation remain functional. + ## 21. Iceberg - **ICE-01:** The configured cluster Catalog loads automatically. Left shared diff --git a/doc/design/console/design-crowdb-console.md b/doc/design/console/design-crowdb-console.md index c025fdd61..bcf226358 100644 --- a/doc/design/console/design-crowdb-console.md +++ b/doc/design/console/design-crowdb-console.md @@ -285,9 +285,14 @@ the namespace. If a recorded port is held by another owner, restart reports the conflict; it does not silently select a different endpoint. Disposable deployment and E2E fixtures use ephemeral namespaces with the -same layout. All local CROWDB data, generated configuration, logs, benchmark -output, and coordination state stay below this root; no default path uses the -system temporary directory. Ordinary clean operations preserve persistent +same layout. A saved CLI cluster inside an ephemeral fixture inherits that +fixture's process and port ownership; its launcher may exit without releasing +ports still used by the fixture. Process records remain outside disposable +data so cleanup can terminate services even after data removal. Cleanup releases +these nested claims along with the fixture. All local CROWDB data, generated +configuration, logs, benchmark output, and coordination state stay below this +root; no default path uses the system temporary directory. Ordinary clean +operations preserve persistent namespaces. ## 5. Node Access Model diff --git a/doc/design/deploy/design-crowdb-deploy.md b/doc/design/deploy/design-crowdb-deploy.md new file mode 100644 index 000000000..cafc9a594 --- /dev/null +++ b/doc/design/deploy/design-crowdb-deploy.md @@ -0,0 +1,435 @@ + + + +# CROWDB - Design: Multi-node Deployment + +This document defines the target architecture for node discovery, Group-0 +bootstrap, shared console control and light-container deployment. It is a design +contract under review, not a claim that LAN discovery or containerd lifecycle +integration is implemented. Unresolved mechanisms are identified explicitly. + +The [console architecture](../console/design-crowdb-console.md) defines the +shared operation layer; the [Group-0 architecture](../kv/design-crowdb-kv-group0.md) +defines system metadata; the +[reconfiguration design](../kv/design-crowdb-kv-reconfiguration.md) governs +Paxos voting membership. This design connects those responsibilities without +making node discovery a membership protocol. + +## Table of Contents + +- [1. Goals and boundaries](#1-goals-and-boundaries) +- [2. Components and identities](#2-components-and-identities) +- [3. Invariants](#3-invariants) +- [4. Node discovery](#4-node-discovery) +- [5. Node and rack configuration](#5-node-and-rack-configuration) +- [6. Bootstrap and recovery](#6-bootstrap-and-recovery) +- [7. Shared UI and management operations](#7-shared-ui-and-management-operations) +- [8. Authorization and partitions](#8-authorization-and-partitions) +- [9. Image and runtime deployment](#9-image-and-runtime-deployment) +- [10. Resource and lifecycle management](#10-resource-and-lifecycle-management) +- [11. Configuration and observability](#11-configuration-and-observability) +- [12. Correctness and validation](#12-correctness-and-validation) +- [13. Decisions still required](#13-decisions-still-required) + +## 1. Goals and boundaries + +- Each node has a monitor and a Web UI. Operators can use any member's UI; + there is no permanently designated UI host. +- Monitors discover nodes on a management LAN without a hand-written peer list. + Discovery identifies nodes, not the individual servers hosted on them. +- Before Group 0 is initialized, UI operations are restricted to node/rack + preparation and Group-0 bootstrap. Other server deployment requires a ready + Group 0 and confirmed initial topology. +- Group 0 is the shared information platform and authority for cluster + membership, racks and deployment operations. +- Production deployment uses Linux light containers: OCI images package + binaries/dependencies, while network, disks and performance resources are + explicitly assigned from the host. +- Single-host tests simulate multiple nodes using isolated containers on a + dedicated Docker network and independent simulated disks. + +The deployment layer is not a general scheduler. Kubernetes and Nomad are not +the default deployment substrate. OCI image compatibility allows other runtimes +to consume an image; it does not establish complete Kubernetes deployment +support. Cross-subnet discovery has a seed fallback and does not require a +multicast relay. Independent initialized clusters are never automatically merged. + +## 2. Components and identities + +```text +operator browser + -> any node's Web UI / shared console operations + -> Group 0: committed topology and operation authority + -> target node monitor + -> runtime and managed server processes + +node monitor <-> management-network discovery <-> peer node monitors +``` + +- **Node:** a deployment and hardware-management identity. A production node + normally corresponds to a physical Linux host; a test node may be a container. + A node exists before any KV server is started. +- **Monitor:** the node management endpoint, discovery participant and local + process/container supervisor. Reuse `crowdb-monitor`; do not introduce a + second agent with overlapping responsibilities. +- **UI:** a presentation/API entry point into shared operations. It owns neither + confirmed topology nor a private cluster. Losing the initiating UI does not + delete a submitted operation. +- **Server:** a managed service/process deployment on a node. Server deployment + is distinct from node admission and Group-0 voting membership. +- **Cluster identity:** the durable identity associated with one initialized + Group 0. A node cannot infer or replace it from nearby multicast announcements. +- **Rack:** an editable logical grouping of nodes. A node-supplied rack value + is an initial hint; confirmed placement belongs to Group 0. +- **Physical failure domain:** actual shared host/rack infrastructure used by + placement policy. Virtual rack labels do not manufacture hardware independence. + +Persist node identity, cluster binding and accepted bootstrap information in +the node's host-mounted state root. Persist Group-0 replica state through its +existing WAL/storage contract. These identities are not image contents and are +not derived from IP addresses. Existing protocol identifier types remain the +canonical types; the exact allocation of pre-admission node identities must be +resolved before changing the public schema. + +## 3. Invariants + +- **I1 — Durable identity:** recreation, endpoint changes and rack reassignment + preserve node identity. Conflicting live endpoints claiming one identity + cause a visible conflict, not silent overwrite. +- **I2 — Candidate discovery:** discovery records are observations. They grant + no admission, voting membership, rack authority or execution permission. +- **I3 — Group-0 authority:** confirmed topology and management mutations use + Group 0. A local monitor/cache never substitutes for unavailable authority. +- **I4 — Bootstrap binding:** every selected monitor durably accepts one + compatible cluster/bootstrap manifest before starting its initial replica. + Conflicting manifests are rejected. No timeout permits replacement bootstrap. +- **I5 — Authorized execution:** a connected Group-0 replica is not proof of + authority. Management mutations require quorum-backed authorization, and + stale operations cannot execute after revocation or authority transition. +- **I6 — Explicit failure domains:** virtual rack assignment and physical + failure-domain identity are distinct placement inputs. +- **I7 — Runtime portability:** a supported-architecture OCI image carries + code; runtime inputs carry identity, network, devices and persistent state. +- **I8 — Bootstrap gate:** only Group-0 prerequisite processes and Group-0 + initialization run before the shared information platform is ready. Other + deployment requires committed initial topology. + +## 4. Node discovery + +### 4.1 mDNS and DNS-SD + +Monitors both announce and browse a CROWDB node discovery type. The proposed +type is `_crowdb-node._tcp.local.`; finalize the name before publication. + +- mDNS transports local-link DNS queries/responses using UDP port 5353 and + multicast groups `224.0.0.251` / `ff02::fb`. +- DNS-SD uses PTR records to enumerate instances, SRV to identify monitor host + and port, A/AAAA to resolve addresses, and TXT for bounded metadata. +- TXT carries only discovery essentials: identity, protocol compatibility and + optional cluster binding. Full node/hardware/rack information is obtained + through the monitor management handshake. + +These mechanisms come from [RFC 6762](https://www.rfc-editor.org/rfc/rfc6762.html) +and [RFC 6763](https://www.rfc-editor.org/rfc/rfc6763.html). The standards define +discovery and caching; CROWDB defines admission and authorization separately. +Implement standard announcements, refresh, cache expiry and clean departure, +rather than inventing a high-frequency broadcast heartbeat. + +Each monitor's discovered-peer cache feeds its local UI backend. Browser code +does not send mDNS traffic. Peers query/announce directly within the discovery +domain; there is no requirement for a separate gossip membership layer. + +### 4.2 Network scope and reachability + +Discovery runs only on configured management interfaces. Host-network nodes +use the management NIC/VLAN; Docker test nodes use their dedicated bridge +interface. The advertised monitor endpoint must be directly reachable by peers +in that scope. A wildcard bind address is not an advertised address. + +UI addresses exposed to browsers and monitor addresses used by peers are +separate endpoint purposes. In bridge tests, host UI ports are published while +peer monitor connections use container-network addresses. Production host +networking requires an explicit local port plan. + +mDNS does not normally cross routed subnet/VLAN boundaries. When unavailable, +configured peer/seed monitor endpoints provide the same handshake path. The +seed list is a discovery input, not an authoritative membership list. Monitor +connectivity verifies endpoint usability; discovery expiry alone does not +establish node failure or remove a confirmed member. + +### 4.3 Cluster separation and trust + +Classify a discovered peer as unbound, bound to this cluster, bound to another +cluster, incompatible or identity-conflicting. Only compatible candidates can +be selected for bootstrap/admission. A foreign cluster is displayed separately; +its topology is not imported automatically. + +Announcements are untrusted hints. The direct handshake and admission policy +must validate the management peer before privileged actions. Never advertise +credentials in TXT records. Trust provisioning remains a decision in section 13. + +## 5. Node and rack configuration + +Before bootstrap, an operator can inspect nodes, create/name virtual racks and +assign nodes. Node-provided rack hints can be overridden. The UI labels these +values as an uncommitted bootstrap draft, not shared authoritative metadata. +Drafts in different UIs can differ; consensus-backed UI consistency begins only +after Group 0 and topology publication are ready. + +The accepted bootstrap manifest captures the chosen initial node/rack mapping. +After bootstrap, all rack edits and node admissions are committed through +Group 0. Rediscovery cannot overwrite operator-confirmed rack placement. + +Physical host/failure-domain information remains independently inspectable. +Several test containers may occupy different virtual racks while sharing one +physical failure domain. Placement policy must not count those labels as +independent physical redundancy. + +## 6. Bootstrap and recovery + +### 6.1 State transitions + +The node's management lifecycle is: + +- **Unbound:** identity exists, discovery is active, no cluster is accepted. +- **Prepared:** a cluster identity and initial manifest are durably reserved; + the monitor accepts retries of that manifest and rejects conflicting ones. +- **Group-0 starting:** selected monitors provision/start their initial replicas + using the same fixed voting-member manifest. +- **Topology publishing:** Group 0 has usable quorum/leadership; initial hardware + and cluster metadata are being published idempotently. +- **Active:** shared topology is confirmed and general deployment is enabled. +- **Authority unavailable:** binding persists, diagnostics/discovery continue, + and new cluster-management mutations are disabled until authority recovers. + +Unavailability is not a transition back to Unbound. Clearing cluster binding +requires an explicit recovery/decommission procedure, not an automatic timeout. + +### 6.2 Initialization flow + +1. Any UI collects compatible candidates and the operator's rack assignments. +2. The operator explicitly selects initial Group-0 voting members. Candidate + count is not voting-member count; application nodes need not all vote. +3. Submit a bootstrap identity and immutable initial manifest to selected + monitors. Persist compatible preparation before starting replicas. +4. Start the minimum Group-0 services with the fixed manifest. Use existing + Paxos election and WAL durability, not discovery-cache membership. +5. Wait for quorum-backed readiness and publish initial metadata idempotently. +6. Persist/refresh cluster contact information for member UIs. Enable general + deployment only after initial topology is confirmed. + +Replica startup coordination and interrupted preparation recovery must be +specified before implementation. Durable per-node conflict rejection is +necessary but alone does not establish a complete distributed bootstrap +protocol. Two overlapping requests must never create incompatible groups; +arbitrary disjoint selections can form distinct clusters and must not be +advertised as one globally unique LAN cluster. + +### 6.3 Recovery and subsequent admission + +Restart resumes accepted identity and replica WAL state. Lost multicast does +not erase stored Group-0 contacts. An interrupted bootstrap reuses its operation +identity/manifest; partial metadata publication resumes without admitting +unselected nodes or repeating destructive provisioning. + +A newly discovered node enters the candidate view. A committed admission +operation binds it to the cluster and assigns its rack. Adding a deployment +node does not automatically alter Group-0 quorum. Voting-member changes follow +the existing reconfiguration contract. + +## 7. Shared UI and management operations + +All UIs invoke the same shared console operations and read confirmed state +from Group 0. Consistent reads support action confirmation; watch/refresh keeps +presentation current. A temporarily stale screen does not create a second +authority. The UI displays stale/unavailable state explicitly. + +Management operations have durable identity, target, requested action and +execution outcome. Group 0 commits the management intent/authorization; the +target monitor performs the side effect and records its result. This is not +an atomic transaction between a KV write and a runtime action. + +Retries therefore use operation identity and local execution recovery. Crash +after runtime success but before result publication must inspect runtime state +and complete/reconcile the same operation, rather than blindly repeat it. +Image digest and deployment-instance identity distinguish the intended instance +from a reused process ID or container name. + +Changing UI hosts does not require moving cluster state. Local secrets and +runtime observations are not replicated as public topology. Credentials needed +to authorize another UI must be provisioned through the chosen trust policy. + +## 8. Authorization and partitions + +For a three-voter Group 0 partitioned two-to-one, only the two-voter side can +commit management mutations. The minority cannot use its local replica or +monitor cache as replacement authority and cannot bootstrap a new Group 0. + +The deployment policy is that a node may execute controlled actions only while +holding valid authorization from the current Group 0. Define bounded grants +and renewal for operations that outlive a single consensus commit. Fencing +generations distinguish old ownership/authorization from current grants; +execution boundaries must reject expired or superseded commands. + +A bare increasing token is insufficient unless the receiving execution path +knows which generations are current and validates them. Likewise, a lease is +insufficient unless timing, renewal, expiry and restart behavior are defined. +No design claim of stale-node exclusion is valid until those enforcement +details are resolved and tested. + +Monitors and UIs remain available for diagnostics during authority loss. +The exact meaning of stopping an unauthorized node's existing data services +is unresolved: management authorization does not automatically revoke every +data group's independent Paxos authority. Section 13 makes that choice explicit. +Group-0 replica processes must be allowed to recover quorum even while general +deployment is gated; stopping all replicas on authority loss would prevent +recovery. + +## 9. Image and runtime deployment + +### 9.1 Common OCI image + +Publish OCI images to Docker Hub or another OCI-compatible registry. The same +image for a supported CPU architecture contains monitor/UI/server binaries and +their dependencies. Node-local state and cluster identity are external mounts. +Tags identify releases for humans; production plans pin immutable image digests. +Architecture-specific builds may share a multi-platform image index only when +their dependencies have been validated. + +Docker and containerd consume these images. `nerdctl` is an operator CLI for +containerd, not an additional daemon. Production monitor runtime control uses +containerd APIs without depending on dockerd. See the +[nerdctl reference](https://github.com/containerd/nerdctl) for CLI/runtime +terminology; its CLI compatibility does not define CROWDB lifecycle semantics. + +### 9.2 Production profile + +- Linux host, containerd runtime and host networking. +- Explicit management/data-plane network selection; discovery uses management. +- Stable disk/device identity and explicitly permitted block-device access. +- Host-persistent config, node state, server state, logs and crash artifacts. +- Explicit CPU/memory and device assignments; resource claims are validated + before runtime side effects. + +Host networking removes a container bridge/NAT layer but does not itself +guarantee high performance. Disk passthrough does not itself bypass filesystem +cache: the storage engine's raw-block/direct-I/O path defines that behavior. + +### 9.3 Single-host test profile + +- One container per simulated node on a dedicated user-defined Docker bridge. +- Unique persistent roots and identities; independent simulated disk files or + loop devices, never overlapping writes to one backing disk. +- Identical internal listener ports are allowed because network namespaces + differ; browser UI access uses distinct published host ports. +- Actual multicast discovery is exercised within the bridge. Separate networks + isolate independent discovery tests. + +Docker bridge behavior is described in the +[Docker reference](https://docs.docker.com/engine/network/drivers/bridge/). +The test profile measures functional correctness, not raw-disk production +performance. Multi-host host-network tests separately validate LAN discovery +and real resource access. macOS-hosted Linux containers require a VM/network +setup that is tested explicitly; image portability does not make the VM's LAN +multicast behavior equivalent to Linux host networking. + +## 10. Resource and lifecycle management + +Containers are packaging, resource/isolation and upgrade units. Deployment +plans explicitly assign host resources; monitors validate and execute, rather +than autonomously choose global placement. + +- **CPU/memory:** cpuset, memory limits and optional NUMA assignments define + container boundaries. Application thread placement stays within those limits. + IRQ/kernel affinity tuning is host policy, not image content. +- **Disk:** stable device references, exclusive intended ownership and supported + engine access modes. Important data never lives only in the writable layer. +- **Network:** host ports and management/data interfaces are explicit. + RDMA device access and TCP fallback remain the responsibility of their + transport contracts, not the discovery mechanism. +- **Lifecycle:** explicit start, stop, drain, upgrade and rollback operations. + Image upgrades pin digests and preserve state. Stateful service rollback is + allowed only within its storage/protocol compatibility contract. +- **Monitoring:** container and process health are separate observations. + Existing monitor supervision can recover eligible process exits; it must not + silently override a deliberate stop, expired authorization or upgrade drain. +- **Runtime access:** monitor receives the minimum runtime/device privileges + needed. Application/UI containers do not all receive the containerd socket. + +No automatic global placement/config reconciliation is required. Monitor +detects drift against confirmed deployment inputs and reports it; local +liveness recovery follows explicit policy. Repair/upgrade progress and failure +remain visible to every authorized UI through shared operation records. + +## 11. Configuration and observability + +Local startup inputs are limited to bootstrap prerequisites and process/runtime +settings: persistent root, management interfaces, listen/advertise addresses, +optional discovery seeds, trust inputs and runtime endpoint. Group 0 owns +confirmed cluster topology and management intent. This does not move every +process tuning parameter into Group 0; respect the existing +[configuration architecture](../config/design-crowdb-config.md). + +Expose: + +- Discovery peer state, endpoint reachability, last observation/expiry, + incompatible identities and multicast-interface status. +- Bootstrap manifest identity, preparation/start/publication progress and the + reason general deployment is gated. +- Group-0 authority/renewal status, rejected stale operations and recovery needs. +- Desired deployment inputs, actual image digest/resources, process/container + health, drift and operation outcomes. + +Keep local logs/crash artifacts persistent and collect asynchronously. Do not +replicate every log line into Group 0 or expose credentials in discovery/UI +diagnostics. + +## 12. Correctness and validation + +- Restart and endpoint changes preserve identity; duplicated roots cause a + visible conflict (I1). +- Three monitors discover each other on a real test bridge and LAN; expiry + affects candidates, not confirmed topology (I2). +- Multiple UIs can edit local drafts, but only the accepted manifest becomes + shared topology. General deployment stays gated through incomplete publication + (I3, I8). +- Concurrent overlapping bootstraps, partial preparation, initiator crashes + and interrupted publication recover without conflicting replicas or identity + replacement (I4). +- Partition tests distinguish reachable replicas from quorum authority and + deliver delayed/retried commands to verify stale-action rejection (I5). +- Multiple virtual racks on one host preserve one physical failure-domain + identity (I6). +- Docker bridge and Linux containerd host-network profiles use the same image + digest and exercise restart, persistent disk/state and resource boundaries + (I7). +- Existing Group-0 membership changes remain governed by reconfiguration; + deployment-node discovery never changes a voting set. + +Before adoption, measure discovery latency/traffic and stale-view bounds at the +intended LAN size. Broad ecosystem use of mDNS is not proof of unlimited +single-domain scalability. + +## 13. Decisions still required + +The architectural flow is settled; the following mechanisms need explicit +agreement before their implementation: + +- **Bootstrap coordination:** one selected coordinator with durable monitor + preparation and resumable delivery, or a quorum/prepare protocol over fixed + selected membership. Define conflicts, cancellation and recovery; avoid + assuming local reservations alone establish global agreement. +- **Strict node activity policy:** decide whether Group-0 authorization expiry + stops existing data-serving processes, including reads, or only gates + management/new deployment while data groups retain their own authority. + Define expiry bounds and fencing enforcement. Group-0 recovery and diagnostic + processes remain exempt from general deployment gating. +- **Admission trust:** explicit UI approval with authenticated management peers, + or preprovisioned deployment credentials for automatic admission. Specify how + every authorized UI obtains access without public secret replication. +- **Pre-admission identity allocation:** retain canonical numeric node IDs with + a defined collision/allocation scheme, or use a durable discovery identity + mapped to a confirmed node ID at admission. IP-derived identities are excluded. +- **Monitor placement:** a host service controlling containers, or a dedicated + management container with narrowly scoped runtime access. Choose the production + recovery/security boundary independently of application image compatibility. diff --git a/doc/design/diskio/design-crowdb-diskio.md b/doc/design/diskio/design-crowdb-diskio.md index f57df351f..d03d1eed4 100644 --- a/doc/design/diskio/design-crowdb-diskio.md +++ b/doc/design/diskio/design-crowdb-diskio.md @@ -474,13 +474,14 @@ Two registration modes: A `PollingMode` config per pipeline: -- **Classic** — `io_uring_wait_cqe_timeout` every iteration, 50ms - timeout. Simplest mode; used by crowdb-tree's single-pipeline - topology. +- **Classic** — event-wait on the poll thread's private eventfd with a + bounded timeout. Both submissions and kernel completions wake it; + no pipeline waits on an empty CQ while another pipeline has ready work. + Used by crowdb-tree's single-pipeline topology. - **Hybrid** `{ busy_poll_budget }` — busy-poll the CQ ring's shared memory via `io_uring_peek_cqe` with no syscall while I/O is active. After `busy_poll_budget` consecutive empty peeks, transition to - `epoll_wait` on the pipeline's eventfd. Any CQE resets the counter, + `epoll_wait` on the poll thread's private eventfd. Any CQE resets the counter, returning to busy-poll. Gives sub-µs CQE dispatch during I/O bursts, sleeps when idle, no core burned at idle. Default for `UringEngine`. - **Sqpoll** `{ sq_thread_idle }` — opt-in for sustained high-IOPS: @@ -492,6 +493,12 @@ When multiple Hybrid-mode pipelines share a poll thread, the thread uses the **minimum** `busy_poll_budget` across its assigned Hybrid pipelines — the most conservative budget wins. +Each pipeline registers its owning poll thread's private eventfd with +`io_uring`. The poll thread drains ready CQEs from every assigned pipeline +and then signals the separate pipeline eventfds consumed by external +reactors. External notifications therefore follow callback completion; +external consumers cannot steal the poll thread's wakeup. + ### 7.4 Batched SQE Submission Client threads fill SQE slots (lock-free CAS on shadow tail + per-slot @@ -501,12 +508,12 @@ thread submits all pending SQEs in one `io_uring_enter` per iteration `io_uring_enter`; only the polling thread does. In busy-poll mode (I/O active), SQEs are submitted within <1µs of -being filled. In event-wait mode (idle), the first client to fill an -SQE after the thread sleeps writes the pipeline's eventfd (waking the -thread); subsequent clients in the same burst skip the eventfd write -via a `thread_sleeping` CAS — at most one eventfd write per sleep -cycle. For N=32 SQEs in a burst: batching = 2 syscalls, unbatched = -32. **16× reduction.** +being filled. The first client changing `pending_submit` from false to +true writes the owning poll thread's private eventfd. Subsequent clients +in the same pending batch skip that write. The eventfd remains readable +if the thread has not entered event-wait yet, so a submission cannot lose +its wakeup. The thread publishes newly pending SQEs after waking and +before draining completions. ### 7.5 Pipeline Topology diff --git a/doc/design/kv/design-crowdb-kv-server.md b/doc/design/kv/design-crowdb-kv-server.md index 556eaa905..cf0699df4 100644 --- a/doc/design/kv/design-crowdb-kv-server.md +++ b/doc/design/kv/design-crowdb-kv-server.md @@ -45,8 +45,20 @@ library. - **`memory`** — in-memory, non-durable. Explicit low-durability choice for tests and dev. -`--kv-backend` (`text` default, or `block` for `O_DIRECT`) only applies -when `--kv-engine crowdb-tree` is selected. +`--kv-backend` defaults to `block`: the tree uses an aligned block page +store with direct I/O on Linux. `file` explicitly selects buffered file +storage, and `mem-block` explicitly selects an in-memory block device. + +`--wal-backend` defaults to `block-device`: aligned real-file block storage +with direct I/O on Linux and durable synchronization before acknowledgement. +`file`, `uring` and `mem-block` remain explicit alternatives. File-backed +device simulation still traverses the filesystem and can incur variable VFS +and filesystem-log synchronization latency. Backend selection does not disable +durable synchronization. + +Backend selection must match an existing data directory. Defaults select the +backend for a new deployment; they do not convert stored tree formats. Existing +file-backend deployments continue to specify `file` explicitly on restart. ### 2.2 Startup ordering diff --git a/doc/design/kv/design-crowdb-kv-wal.md b/doc/design/kv/design-crowdb-kv-wal.md index 7de8dbdbf..cfc7212bd 100644 --- a/doc/design/kv/design-crowdb-kv-wal.md +++ b/doc/design/kv/design-crowdb-kv-wal.md @@ -348,6 +348,13 @@ The split matters: replay/restore are purely *local* (this node's WAL), but `Acc ### 6.1 Replay — rebuild acceptor state (`replay_group`) 1. Discover all segments under `/group/seg-*.ck`, order by `segment_id`. + A zero-byte final segment on each WAL disk can result from a crash between + file creation and header writing. Remove that empty tail; it contains no + acknowledged records. A nonempty unreadable segment, invalid header, or + empty intermediate segment prevents local recovery; never skip it and serve + a partial acceptor history. The direct block backend + aligns physical reads and copies the requested byte range, including short + header reads and record ranges spanning block boundaries. 2. Walk each segment's records in order; verify `magic`, `version`, `crc32c`. On the first failure, **truncate that segment at the offset and stop** (a torn tail from a crash mid-write); later segments are still processed. diff --git a/doc/design/tree/design-crowdb-tree-engine.md b/doc/design/tree/design-crowdb-tree-engine.md index 6cf4c78b2..5c9ad8203 100644 --- a/doc/design/tree/design-crowdb-tree-engine.md +++ b/doc/design/tree/design-crowdb-tree-engine.md @@ -505,7 +505,7 @@ guard). Design rules: - **Allocator seam.** `alloc()` routes owned allocations larger than `kInlineCap` through a single internal allocator hook (today: glibc `malloc`); a size-classed pool or RDMA-pinned allocator could slot in here - later with no call-site changes. See [`todo_code.md`](../../todo_code.md) for + later with no call-site changes. See [`todo_code.md`](../../working/todo_code.md) for why that hasn't been done speculatively. - **MemTable = `absl::btree_map`.** The KEY stays `std::string`, the VALUE is a `cell_entry{slot, flags, cell}`. The diff --git a/doc/design/tree/design-crowdb-tree.md b/doc/design/tree/design-crowdb-tree.md index dd5ed61fd..781a470b4 100644 --- a/doc/design/tree/design-crowdb-tree.md +++ b/doc/design/tree/design-crowdb-tree.md @@ -13,7 +13,7 @@ consumed from the Rust `crowdb-kv` crate over a C ABI. It records the decisions behind that design and maps the sub-design documents. `libcrowdb-tree` is fully implemented, wired into `crowdb-kv` (`CrowdbTreeEngine`), and shipped; this document set is the durable record of *why* it looks the way it does, not a build plan. -See [`todo_code.md`](../todo_code.md) for anything still open. +See [`todo_code.md`](../../working/todo_code.md) for anything still open. ## Table of Contents diff --git a/doc/doc_index.md b/doc/doc_index.md index 02affbead..bc01fa918 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -20,6 +20,7 @@ listed document or section needed by the task. | `doc/design/rpc/design-crowdb-rpc.md` | RPC engine, wire format, FFI, and transport. | | `doc/design/console/design-crowdb-console.md` | Console architecture and service lifecycle. | | `doc/design/config/design-crowdb-config.md` | Configuration ownership, precedence, validation, reload. | +| `doc/design/deploy/design-crowdb-deploy.md` | Node discovery, Group-0 bootstrap, multi-node deployment. | | `doc/design/access-server/design-crowdb-access-server.md` | S3, Iceberg, native Dataset access, and GPU delivery. | ## Backlog (`doc/backlog/`) @@ -121,6 +122,7 @@ Temporary plans live under `doc/working/`; flow analyses live under | `doc/design/access-server/iceberge/iceberg-table-spec-1.11.0.md` | Backed-up normative Iceberg table format specification. | | `doc/design/access-server/iceberge/iceberg-rest-catalog-open-api-1.11.0.yaml` | Backed-up normative Iceberg REST Catalog OpenAPI. | | `doc/design/access-server/dataset/design-crowdb-access-dataset.md` | Dataset HTTP, native topology access, and GPU delivery. | +| `doc/design/access-server/dataset/spec-crowdb-dataset.md` | Normative Dataset concepts, behavior, invariants, and compatibility. | Prefer the most specific match. Open a root design only for cross-topic work; update its row when a permanent document is renamed or materially rescoped. diff --git a/doc/draft.md b/doc/draft.md deleted file mode 100644 index 5195820cd..000000000 --- a/doc/draft.md +++ /dev/null @@ -1,289 +0,0 @@ - - - -# Docker Hub repository drafts - -Editing notes (do not paste this section into Docker Hub): - -- Copy each description into **Description**, select the suggested categories, - and copy the content under **Repository overview** into the overview editor. -- These are English drafts for two different user audiences. -- The S3 image publication is being added to the release workflow. Publish its - tags before posting the S3 quick start as available to users. -- Examples follow the current container configuration and retained client tests. - They have not been rerun against the public registry images for this draft. -- Category names follow [Docker Hub repository information](https://docs.docker.com/docker-hub/repos/manage/information/). - -## crowdb/crowdb-s3 - -### Description - -S3-compatible object storage with native CROWDB storage, in one container for development and testing. - -### Category - -- Databases & storage -- Developer tools - -### Repository overview - -# CROWDB S3 - -Run S3-compatible object storage locally with one Docker command. Use boto3 to -create buckets, upload objects, and read them back without setting up a -separate storage cluster. - -CROWDB is an open-source distributed storage platform. This image packages its -storage services and S3 endpoint into a single-node development environment. - -## Supported operations - -- Bucket creation, listing, existence checks, and deletion. -- Object upload, overwrite, download, metadata retrieval, and deletion. -- Prefix and delimiter listing with ListObjectsV2 and continuation tokens. -- Range and conditional reads. -- Multipart upload, part replacement, part listing, completion, and abort. -- AWS Signature Version 4 authentication and presigned request verification. -- Content-MD5 validation and S3-style ETags. - -The repository includes boto3 end-to-end tests for core operations, multipart -uploads, interrupted requests, and recovery after service restart. - -## Quick start - -Requires Docker on Linux amd64, or a Docker environment capable of running -Linux amd64 containers. - -```sh -docker run -d --name crowdb-s3 \ - --mount type=volume,source=crowdb-s3-data,target=/opt/crowdb/data \ - -p 127.0.0.1:9091:9091 \ - crowdb/crowdb-s3:latest -``` - -Check startup and wait until the health status is `healthy`: - -```sh -docker inspect --format '{{.State.Health.Status}}' crowdb-s3 -docker logs crowdb-s3 -``` - -Display the generated client credentials: - -```sh -docker exec crowdb-s3 crowdb-monitor credentials show --format env -``` - -Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_DEFAULT_REGION` in -your client environment using the displayed values. Keep credentials private. -The S3 endpoint is `http://127.0.0.1:9091` and uses path-style addressing. - -## Upload and download with boto3 - -Install the client in your Python environment: - -```sh -python -m pip install boto3 -``` - -Run after setting the AWS environment variables above: - -```python -import boto3 -from botocore.config import Config - -s3 = boto3.client( - "s3", - endpoint_url="http://127.0.0.1:9091", - config=Config( - s3={"addressing_style": "path"}, - request_checksum_calculation="when_required", - response_checksum_validation="when_required", - ), -) - -s3.create_bucket(Bucket="example") -s3.put_object(Bucket="example", Key="hello.txt", Body=b"Hello from CROWDB!") -print(s3.get_object(Bucket="example", Key="hello.txt")["Body"].read()) -print(s3.list_objects_v2(Bucket="example")["Contents"]) -``` - -The checksum configuration matches the retained container boto3 example. -Compatibility with every SDK default or S3 tool is not implied. - -## Data and tags - -- Mount `/opt/crowdb/data` to retain data and generated credentials across - container recreation. Reuse the same volume when recreating the container. -- `latest` is the moving release tag; select a version tag for an explicit - release. Version tags can be replaced when that release is republished; - pin an image digest for exact image identity. -- An optional console is available by also mapping container port `9090`. - -## Development preview scope - -This image runs a single-node profile with one voting replica and one copy of -stored data. It provides no redundancy. Use disposable data for development, -testing, and public evaluation. Production use and on-disk upgrade compatibility -are not supported. - -The supported S3 API is a subset of Amazon S3. Server-side copy, batch deletion, -versioning, lifecycle policies, bucket policies/ACLs, server-side encryption, -object tags, replication, and event notifications are outside the current -advertised surface. See the project documentation for updates. - -## Related image - -[crowdb/crowdb-iceberg](https://hub.docker.com/r/crowdb/crowdb-iceberg) provides -an Iceberg-focused quick start. Both repositories contain the same runtime: -one Access Server listens on S3 port `9091` and Iceberg port `9092`. Map both ports -if you need both interfaces in one container. Their namespaces and authorization -remain separate; general S3 buckets do not expose Iceberg table files. - -## Links and license - -- [Project website](https://crowdb.dev/) -- [Source and issues](https://github.com/buzzcrow/crowdb) -- [Container documentation](https://github.com/buzzcrow/crowdb/blob/main/container/single-node-container/README.md) -- [License: Apache-2.0](https://github.com/buzzcrow/crowdb/blob/main/LICENSE) - -## crowdb/crowdb-iceberg - -### Description - -Apache Iceberg REST catalog with native CROWDB storage, in one container for development and testing. - -### Category - -- Databases & storage -- Data science -- Developer tools - -### Repository overview - -# CROWDB Iceberg - -Create Apache Iceberg tables, append data, and read it back with PyIceberg—all -from a single container providing a REST catalog and native table-file storage. -No separate object store is needed for this local workflow. - -CROWDB is an open-source distributed storage platform. Iceberg catalog, table, -snapshot, and immutable file semantics are implemented directly over CROWDB's -storage core. - -## What is included - -- An Apache Iceberg REST catalog endpoint. -- Namespace and table operations, snapshot metadata, and atomic table commits. -- Native immutable table-file storage and delegated FileIO credentials. -- A single-node storage environment with automatic initialization and generated - client credentials. -- A web console, available on container port `9090`. - -The container acceptance suite exercises PyIceberg table creation, append, -Arrow/pandas reads, and retrieval after service recovery and container restart. - -## Quick start - -Requires Docker on Linux amd64, or a Docker environment capable of running -Linux amd64 containers. The example uses host port `9092`, matching the tested -Iceberg catalog and file endpoint setup. - -```sh -docker run -d --name crowdb-iceberg \ - --mount type=volume,source=crowdb-iceberg-data,target=/opt/crowdb/data \ - -p 127.0.0.1:9092:9092 \ - crowdb/crowdb-iceberg:latest -``` - -Check startup and wait until the health status is `healthy`: - -```sh -docker inspect --format '{{.State.Health.Status}}' crowdb-iceberg -docker logs crowdb-iceberg -``` - -Display the generated client credentials: - -```sh -docker exec crowdb-iceberg crowdb-monitor credentials show --format env -``` - -Set `ICEBERG_TOKEN` in your client environment using the displayed value. Keep -the token private. The REST catalog URI is `http://127.0.0.1:9092`. - -## Create and read a table with PyIceberg - -Install the clients in your Python environment: - -```sh -python -m pip install 'pyiceberg[pyarrow]' pandas -``` - -Run after setting `ICEBERG_TOKEN`: - -```python -import os - -import pandas as pd -import pyarrow as pa -from pyiceberg.catalog import load_catalog - -catalog = load_catalog( - "crowdb", - type="rest", - uri="http://127.0.0.1:9092", - token=os.environ["ICEBERG_TOKEN"], -) - -catalog.create_namespace("demo") -rows = pa.Table.from_pandas( - pd.DataFrame({"order_id": [1, 2], "amount": [120, 80]}), - preserve_index=False, -) -table = catalog.create_table("demo.orders", schema=rows.schema) -table.append(rows) -print(catalog.load_table("demo.orders").scan().to_pandas()) -``` - -Use a fresh namespace/table for subsequent runs, or load the existing table -instead of creating it again. The catalog provides the table's FileIO -configuration; this example needs no separately configured S3 service. - -## Data and tags - -- Mount `/opt/crowdb/data` to retain tables and generated credentials across - container recreation. Reuse the same volume when recreating the container. -- `latest` is the moving release tag; select a version tag for an explicit - release. Version tags can be replaced when that release is republished; - pin an image digest for exact image identity. -- To open the console, also publish `-p 127.0.0.1:9090:9090` and visit - `http://127.0.0.1:9090`. - -## Development preview scope - -This image runs a single-node profile with one voting replica and one copy of -stored data. It provides no redundancy. Use disposable data for development, -testing, and public evaluation. Production use and on-disk upgrade compatibility -are not supported. Dataset access and direct GPU delivery are not available in -this image. - -The example covers a local PyIceberg workflow. Compatibility with every Iceberg -engine, file format, or deployment configuration is not implied. Start with the -published quick start and tested client recipes. - -## Related image - -[crowdb/crowdb-s3](https://hub.docker.com/r/crowdb/crowdb-s3) provides an -object-storage-focused quick start. Both repositories contain the same runtime: -one Access Server listens on Iceberg port `9092` and general S3 port `9091`. -Map both ports if you need both interfaces in one container. Iceberg table -storage and general S3 buckets have separate namespaces and authorization. - -## Links and license - -- [Project website](https://crowdb.dev/) -- [Iceberg quick start](https://crowdb.dev/docs/quickstart/) -- [Source and issues](https://github.com/buzzcrow/crowdb) -- [Container documentation](https://github.com/buzzcrow/crowdb/blob/main/container/single-node-container/README.md) -- [License: Apache-2.0](https://github.com/buzzcrow/crowdb/blob/main/LICENSE) diff --git a/doc/working/dataset-todo.md b/doc/working/dataset-todo.md new file mode 100644 index 000000000..68157d322 --- /dev/null +++ b/doc/working/dataset-todo.md @@ -0,0 +1,80 @@ + + + +# CrowDB Dataset TODO + +Upstream design and contract: + +- `/cjdata/cpp/workshop/crowdb/proposal-storage/dataset-crowdb-scope.md` +- `/cjdata/cpp/workshop/crowdb/proposal-storage/dataset-crowdb-design.md` +- `/cjdata/cpp/workshop/crowdb/proposal-storage/dataset-use-cases.md` +- `/cjdata/cpp/workshop/crowdb/proposal-storage/dataset-crowdb-plan.md` + +Goal: track the remaining design and implementation gaps required for a working +Dataset core. R213–R226 now cover the completed authority, Manifest, +publication, read-plan, retention, and API parity foundations. The Dataset SDK +is intentionally deferred until the core Dataset path is usable. + +Current implementation status: R213–R226 are complete. Durable per-snapshot +leases, ownership-aware reclamation, complete HTTP publication, and +parity/recovery coverage are in place; only the deferred R220 SDK remains. + +The backlog items are ordered by dependency: publication recovery (R222), +retention and GC (R225), then complete API parity (R226). Small wording or +scope corrections belong in this TODO; implementation work belongs in the +linked backlog item. + +## P0: publication and read correctness + +- [x] **Atomic snapshot publication** (R222): Manifest partitions, binding, publication state, and head advancement are now ordered so `latest` exposes only a complete Snapshot. Files: `lib/crowdb-access-dataset/src/authority.rs`, `record.rs`, `publication.rs`. +- [x] **Publication retry and recovery** (R222): Prepared operations recover through their durable operation token; retries preserve one Snapshot ID and reject divergent requests. +- [x] **Safe retention and GC** (R225): Make snapshot reclamation require retention/release authority and protect latest/stable snapshots, retained parent chains, and active reads. `reclaim_snapshot_metadata` must not unconditionally delete a live snapshot. Files: `authority.rs`, `retention.rs`, `lease.rs`. +- [x] **Per-read GC protection** (R225): Track active Read Plans by Dataset and Snapshot, rather than using one global lease or one boolean for all reads. Protect the referenced payload until the read completes, is released, or expires under an explicit policy. + +## P1: Snapshot and Manifest model + +- [x] **Opaque locator contract** (R214): Store and pass the chunk location as an opaque location string according to the Dataset contract. Dataset code must not parse or infer chunk identity from the locator. +- [x] **Manifest completeness validation** (R222): Persisted Manifest bindings now carry partition count and digest, reject zero/incomplete partitions, validate parent availability, and reject divergent re-publication. + +## P1: Read Plan, streaming, and resume + + +## P1: Public control and data surfaces + +- [x] **Complete control API** (R226): Add the Dataset operations required by the contract: ListSnapshots, PrepareSnapshot, PublishSnapshot, OpenSnapshot, GetManifest, RetainSnapshot, and ReleaseSnapshot. +- [x] **Complete read API** (R226): Add stable operations for GetSample, GetBatch, bounded Scan, StartReadPlan, NextBatch, SaveProgress, and Resume. HTTP and native surfaces must use the same authority and return equivalent logical results. +- [x] **End-to-end HTTP publication** (R226): Make the HTTP client able to publish a complete validated Dataset snapshot, including Manifest persistence and publication completion, rather than publishing only an opaque reference. +- [x] **Status and error mapping** (R226): Map not-found, conflict, invalid Manifest, checksum, transient, cancellation, backpressure, and service-unavailable states to stable native and HTTP errors. + +## P2: lifecycle and operational behavior + +- [x] **Mutable names** (R225): Implement and validate `latest` and `stable` as mutable names that never replace the immutable Snapshot ID stored in Read Plans or cursors. +- [x] **Snapshot listing and inspection** (R226): Expose snapshot ancestry, publication state, Manifest completeness, retention state, and provenance for operators. +- [x] **Dataset-owned payload lifecycle** (R225): Track Dataset-owned chunk references separately from external source references. External source refs must not control Dataset payload GC. +- [x] **Shared payload reclamation** (R225): Reclaim a chunk only after no retained Snapshot and no active Read Plan references it, including inherited fields and shared locators. +- [x] **Authentication and authorization scope** (R226): The HTTP service applies its configured bearer token to every Dataset route, covering management, snapshot access, publication, retention, reclamation, and cancellation; deployments can place finer-grained policy at the service boundary. + +## Verification gaps + +- [x] **Atomic publication tests** (R222): Cover prepared visibility, snapshot-write crash recovery, duplicate operation tokens, head advancement, and declared parent ancestry. +- [x] **GC tests** (R225): Cover retained parent chains, latest/stable protection, active reads by Snapshot, lease expiry, shared chunks, external source refs, and idempotent deletion. +- [x] **HTTP/native parity tests** (R226): Complete publication and bounded reads run through HTTP and native surfaces, with stable status mapping, durable lease cleanup, and restart recovery coverage. + +## Deferred + +- [ ] **Dataset SDK** ([R220](../backlog/R220-dataset-fat-client-plan.md)): Start only after the core Dataset publication, payload read, Snapshot membership, bounded streaming, retention, and resume semantics are working and covered by the tests above. The SDK must adapt the stable core contract rather than define new storage semantics. + +## Files to revisit + +- `lib/crowdb-access-dataset/src/authority.rs` +- `lib/crowdb-access-dataset/src/manifest.rs` +- `lib/crowdb-access-dataset/src/planner.rs` +- `lib/crowdb-access-dataset/src/shuffle.rs` +- `lib/crowdb-access-dataset/src/transport.rs` +- `lib/crowdb-access-dataset/src/chunk_store.rs` +- `lib/crowdb-access-dataset/src/retention.rs` +- `lib/crowdb-access-dataset/src/cursor.rs` +- `lib/crowdb-access-dataset/src/wire.rs` +- `app/crowdb-access-server/src/dataset.rs` +- `lib/crowdb-access-dataset/tests/` +- `app/crowdb-access-server/tests/` diff --git a/doc/working/plan-chunk-kv-cutover.md b/doc/working/plan-chunk-kv-cutover.md new file mode 100644 index 000000000..05af12aa0 --- /dev/null +++ b/doc/working/plan-chunk-kv-cutover.md @@ -0,0 +1,258 @@ + + + +# Chunk KV Cutover Plan + +Upstream: [Chunk KV Server design](../design/chunkds/design-crowdb-chunk-kv-server.md), +sections 6–9. Results: [intel7960 test matrix](test.md). +Goal: recover split/transfer at durable boundaries without lost writes, and +make automatic placement stable, inexpensive and explainable. + +Temporary plan: completed implementation detail is condensed below; historical +source and evidence remain in local commits and the referenced artifacts. +Delete this plan after the remaining audits and acceptance are complete. + +## Current placement decision — 2026-10-09 + +- R228 and R229 are deferred by the user. Resume Console/native acceptance + and the cutover audits; do not continue membership implementation or metrics. +- Data Weight is deferred to [tree range metrics](../backlog/R228-tree-range-metrics.md). + Placement uses split counts with existing tolerance/cooldown; byte coefficient + must be zero. Console Weight and weighted explanation are hidden. +- Tree metrics implementation is not part of this cutover work. No tree writer, + split/move execution or recovery changes are authorized by this deferral. +- Size-driven weighted acceptance is superseded by the deferred requirement, + not marked passed or ignored. Retain count convergence/data integrity coverage. +- Existing automatic split has count-shortfall and coarse-pack size triggers; + the latter and size ranking require replacement after tree metrics acceptance. + +## Status — 2026-10-09 + +- Local commits: `1a471f43`, `407b5cab`, `a664e357`, `38389b0c`; no push performed. +- Split retains the original parent tree/WAL and creates one child. Durable + handoff precedes memory routing; catalog publication is the external cutover. + Recovery opens the exact base, range-filters the inherited parent WAL, then + replays the child's WAL. Initial unpublished handoff derives the dispatch + frontier from child WAL evidence; final readiness/catalog fixes the published + frontier, even when the child's WAL is empty. +- Verified fixes include prepared-target immutable root access, uncertain + handoff candidate reuse, current-writer routing, unpublished dispatcher reuse, + narrowed-range lazy recovery, aligned block-WAL reads/fail-closed replay, + narrowly scoped empty-final-tail cleanup, block tree/WAL defaults, I/O completion + wakeup and format-compatible bulk WAL byte encoding. Focused regressions pass. +- Before the balance implementation, eight of nine Linux matrix tasks pass; + extra Rust Iceberg SDK passes 5/5. + Baseline full UI passes 225/225, C++ 942/942, Core 987/987, Storage 839/839. + Separately scheduled SDK cases ran and passed; no functional Linux skip is + accepted. This historical baseline is superseded by the complete current + Console acceptance below. Exact counts/times are in `test.md`. +- No new crash observed in the latest acceptance attempts. Previous crash + investigation is retained under `/tmp/crowdb-core-investigation`. +- File-backed simulated device sync delays are confirmed and accepted by the + user; sync stays enabled. Native API preparation may use 10 seconds and native + fixture cases 180 seconds; ordinary UI assertions/actions remain 3 seconds. +- Transfer/publication race and native overlay lifetime items below are audits, + not confirmed defects or authorization for speculative production changes. + +## Constraints + +- Run builds, tests, lint and executables through `pixi run`; runtime suites + run sequentially. Commit only when asked; never push without authorization. +- Preserve exact identities, epochs, root lineage, revision CAS and recovery + pins. No epoch guessing, caller retries, weakened assertions or deadline inflation. +- Keep hot paths lock-free. Review any unavoidable lock, queue, blocking work or + material performance cost with the user before implementation. +- The user authorizes balance algorithm/configuration and UI implementation. + Split/move execution, handoff and recovery changes still require prior review. + No new hot-path locks or speculative cutover changes are authorized. + +## 1. Placement policy and deferred tree metrics + +- [x] **Verify count-only placement and hidden Weight**: byte coefficient defaults + to zero and nonzero configurations are rejected. Both selectors retain shared + count scoring, 20% tolerance, 25% improvement, cooldown and safety filters. + Equal split counts must not move merely because shared-pack estimates differ. + Hide graph/property Weight and balance explanations; verify native Page flow. + Files: protocol/server config, balance tests, console components/catalog spec. + Verified: 24 focused Rust cases, 7 UI unit cases, Rust fmt/clippy, UI lint/build + pass. Actual native Page flow passes: browser 2.3s, owned cluster 345.98s; + zero ignored. Weight/property/explanation absence and Page inspection asserted. +- [x] **Finish Console acceptance**: complete gate passes 350 Rust test + executions, zero failed/ignored, and all 23 native browser cases across four + fixture phases. Approved allocation fix reroutes only typed pre-send connection + failures to a changed owner endpoint once; ambiguous sent requests are never + replayed. Both regressions and all eight S3 cluster cases pass. Native Start + asserts the original response before process state and avoids duplicate cleanup + submissions. Measured ChunkKV Start is 789/3800ms; response budget is 5s, + process/DOM stays 3s. No core startup change. Full UI passes 229/229. + Evidence and matrix: `test.md`, balance-policy artifact directory. +- Tree metrics/data-weight work is deferred to + [R228](../backlog/R228-tree-range-metrics.md), requiring core review before + implementation. Size-triggered split currently uses coarse pack estimates; + count-shortfall split remains available. No writer/checkpoint change here. +- The previous size-weight acceptance failed at 968.43s because opened manifest + estimates did not update after ordinary flush. It is superseded by the deferred + metrics contract, not counted passed or ignored. Evidence remains under + `.crowdb-runtime/artifacts/balance-policy-20261009/`. + +## 2. Remaining cutover boundary audits + +Only change behavior when a concrete regression proves a gap; any core fix +requires the user's review. Existing durable handoff/replay contracts stay intact. + +- [x] **Review publication boundaries**: compare S1–S6/T1–T8 with monitor and + local publication, revision CAS, grants and startup. Distinguish initial + unpublished handoff recovery from final published replay bounds. + Files: monitor `chunk_kv/catalog.rs`, server `catalog/transition.rs`, design. + Both publish immutable pages before expected-head revision CAS and recognize + exact successor entries idempotently; phase persistence is a subsequent CAS. + The remaining startup/abort races below are not proven by that comparison. +- [ ] **Recover release before publication**: prove old-source admission stays + fenced by exact release/lease evidence and reconciliation can still start. + Files: server `main.rs`, `catalog/recovery.rs`, `serving/transition_runtime.rs`. + Review finding: old Serving catalog plus old valid grant reaches generic + `activate_recovered_partition`; the source branch does not read transfer + release proof. `suspend_for_transfer` fences the live lifecycle in memory. + Need a controlled restart regression before changing activation; do not claim + this window safe from ordinary transfer/state-machine tests. + Proposed reviewed scope: load durable transfer evidence once on the grant/ + recovery control path, match exact source instance/partition/epoch, and keep + recovered source non-serving after release even if old catalog/grant remains. + Retain transition processing so publication can finish. No per-request read, + lock, WAL rerouting or epoch change. Controlled restart regression must prove + old-source mutation rejection and eventual target recovery. Await approval. +- [ ] **Repair publication before phase marker**: cover owner and Serving + publication independently; exact catalog entries and proofs allow idempotent + repair and cannot reactivate the source. Files: server `server/activation.rs`, + monitor `chunk_kv/catalog.rs`, `catalog_transition_test.rs`, transfer recovery tests. +- [x] **Verify split abort**: reject abort after durable handoff or with stale + proof; preserve child WAL omitted by the old catalog and resume the same split. + Files: server `serving/split.rs`, local split owner, split abort/recovery tests. + `control_store_test`, `split_prepare_retry_test` and handoff recovery tests pass; + committed proof cannot be cleared/replaced, unknown uncommitted status permits + exact abort, and committed dispatch resumes the same base. +- [ ] **Reconcile published split before marker**: recover both exact successor + assignments and matching authority; historical routing cannot select an old + writer over the current assignment. Files: server activation/reconcile and + `split_recovery_test.rs`. +- [ ] **Verify grant-generation refresh**: immediate catalog refresh is already + implemented; prove new grants do not leave routes fenced until periodic refresh. + Files: server `main.rs`, native S3/Iceberg catalog-change acceptance. +- [ ] **Order abort against publication**: controlled interleavings must prove + monitor fencing, transition revision and expected-head CAS exclude stale + publication. Files: monitor catalog, local transition and catalog transition tests. +- [ ] **Retire exact candidate resources**: durable abort precedes idempotent + cleanup; preserve source references, base pins, retry floors and grace. Verify + monotonic candidate epochs across abort/restart. Files: transition worker, + monitor, transfer recovery tests. +- [ ] **Audit native overlay lifetime**: prove raw source-tree pointers remain + valid when external/session handles disappear, finalization fails or overlays + are removed concurrently. Establish a failing regression before ownership changes. + Files: `lib/crowdb-chunk-kv/src/partition/tree.rs`, + `lib/crowdb-tree/ffi/src/tree.rs`, `lib/crowdb-tree/src/btree/memtable_sources.cpp`. + C++ stores a raw source-tree pointer; Rust's safe install method takes a + temporary borrow and does not retain source ownership. Normal successful + finalization clears the overlay, but failed finalization/source-handle loss + and readers concurrent with clear still need a lifetime regression. No new + crash has been observed and no ownership fix is applied. + +- Deferred **membership CAS contract (R229)**: exact UI topology reproduction + also fails (2/3 ready groups). User requires sequential UI additions and shared + KV-client/server protection with explicit rejection across UI servers. + Approved: one complete members record, epoch CAS, Installing/Ready completion + fencing, exact-epoch recovery, no separate operation ID or legacy compatibility. + Sequential topology fixture passes 5/5 (33.1s), with the original leader + assertions/budgets unchanged. Approved contract and recovery boundaries are in + [R229](../backlog/R229-kv-membership-conflict.md), executed through + [membership plan](plan-kv-membership-cas.md); no membership core change + is applied. + +Native lifecycle follow-up: recorded restart opens its first tree root at +09:21:27.697, then three more roots at 09:21:30.298–30.485; the RPC listener +starts at 09:21:30.532 (about 3.15s after process startup). Recovery is sequential +and includes journal replay. These timestamps do not attribute the delay to +fsync. The user authorized a measured, modest Start-response budget after +reviewing timing distribution; 10s is not the default target. The focused run passes with a 5s response budget: ChunkDB 215ms, DiskIO +366ms, ChunkKV 789ms, Access 178ms (browser 9.2s; owned fixture 375.59s). +Full Console provides a second Start sample: ChunkDB 255ms, DiskIO 466ms, +ChunkKV 3800ms, Access 148ms. Keep the 5s native response budget: 3s fails +legitimate recovery, while these samples do not justify 10s. These samples +and the prior 3.15s listener sample establish variation, not percentiles. Keep per-service millisecond output for full-suite verification. Process/DOM assertions stay at 3s. Recovery and +readiness behavior are unchanged. Complete UI passes 229/229; the native +full Console rerun passes (350 Rust executions, zero failed/ignored; all +23 native browser cases, including the three dedicated phases). + +## 3. Final verification and documentation + +- [ ] **Controlled crash/race coverage**: consolidate before/after durable-step + tests, ambiguous replies, stale revisions/grants, concurrent checkpoints and + both restarts. Assert exclusive ownership, WAL order, acknowledged values, + exact-root lineage and eventual recovery. Capture authoritative metadata before + teardown for any newly failing transition; retain actual crash/core evidence. +- [x] **Complete Console matrix**: rerun native inspection, journal interruption + and dedicated production split/count-placement phases after confirmed changes. Eight + independent tasks already pass; rerun them only where new diffs affect them. + Complete task passes: 350 Rust executions, zero failed/ignored; all 23 native + browser cases run across four fixture phases. Updated `test.md` with the + approximate 1588.13s complete task interval and separate phase timings. +- [ ] **Performance and quality gates**: inspect scans/serialization/locking and + bound planning/diagnostic work. Run affected tests, fmt, clippy, relevant C++ + gates and UI lint/specs. Make performance claims only from measured workloads. +- [ ] **Reconcile permanent docs**: formula, thresholds and UI contract are + updated. Track deferred tree metrics separately and resolve remaining cutover + Open Issues only after review and passing regressions; remove this plan after + final acceptance. + +## Latest focused audit verification + +- Independent native phases now pass without ignored cases: prerequisite-plan + resumption 19.32s, journal owner interruption 54.26s, and production split/ + count convergence with complete data checks 350.85s (4/4/4). + The plan fixture pauses after ChunkDB rather than before its three instances + exist. UI observes registered DiskIO restart every 2s when Capacity is not + loaded; 20 focused hook tests pass. Serial deployment responses are checked + separately before durable progress. Core storage/transition behavior is unchanged. + Logs: `native-plan-response-order.out`, `native-journal-final.out`, and + `native-count-final.out` under the balance-policy artifact directory. +- Full workspace Rust clippy, fmt, CI task mapping and UI lint/build pass. + Complete UI passes 166 unit plus 63 browser cases (229/229), 307.40s. + An earlier complete rerun passed automatic service + deployment but failed bucket clicking because creation consumed 2.92s of the + same action window. The test now asserts the bucket creation response before + selecting its UI item; each original action/response budget remains 3s. +- Reload failure is diagnosed separately: the old page's canceled server-list + fetch persisted `failed` over a saved `waiting` plan. Page unload now stops + the runner and guards cancellation paths before failure publication; cached + history restoration reloads persisted authority. The 21 hook tests and entire + eight-case rack/node spec pass (32.5s browser). Full UI acceptance passes + in `ui-reload-acceptance-final.out`; no timeout or assertion is weakened. +- 21 server cases pass across catalog transition, control store, split recovery, + current assignment, transfer state machine and transfer storage. +- 10 library cases pass across native split handoff, split handoff recovery, + ingress and prepared-candidate retry. Zero ignored. These are focused evidence, + not complete real-process coverage of every listed crash/race window. +- Current count-only Core passes 994/994, zero ignored; UI passes 166 unit and + 63/63 browser cases. Earlier Group 1990 replica 19902 remained unknown/term 1 + while its leader was term 2 in a failed observation window. Preserved trace: + `.crowdb-runtime/artifacts/balance-policy-20261009/ui-leader-election-failure/`. + Log: `.crowdb-runtime/artifacts/balance-policy-20261009/count-only-core.out`. + Tree metrics remain deferred to R228. + +## Evidence and file inventory + +- Successful final independent matrix: + `.crowdb-runtime/artifacts/measure-tests/20261009T022131.804646Z/`. + Core/Storage/Access: `20261009T002906.349723Z/` under the same directory. +- Latest UI/Core/Storage pass: `20261009T053116.399322Z/` under the same + measurement directory. Latest incomplete Console attempt: + `20261009T052410.043897Z/`. Balance-specific focused results and final + clippy output: `.crowdb-runtime/artifacts/balance-policy-20261009/`. + Withdrawn readiness/order experiments: + `.crowdb-runtime/artifacts/native-catalog-readiness-experiments/`. + Earlier speculative cutover changes: + `.crowdb-runtime/artifacts/cutover-review-withdrawn/`. +- Production: protocol policy/status; Group-0 monitor/catalog; ChunkKV server + storage, serving, routing, activation/reconcile; owning ChunkKV/tree libraries. +- Unit/integration: server planning, control-store, split/transfer recovery, + catalog transition and native overlay tests. E2E: web native provisioning/ + balance and existing catalog, journal, Iceberg/S3 browser specs. diff --git a/doc/working/plan-kv-membership-cas.md b/doc/working/plan-kv-membership-cas.md new file mode 100644 index 000000000..1531f9d25 --- /dev/null +++ b/doc/working/plan-kv-membership-cas.md @@ -0,0 +1,77 @@ + + + +# KV Membership CAS Plan + +Upstream: [R229](../backlog/R229-kv-membership-conflict.md). +Goal: publish complete members through epoch CAS and reject concurrent/stale +configuration submissions before they can change durable or running groups. + +Execution is deferred by the user on 2026-10-09. The first protocol/client task +is committed as `339ef482`; do not resume the remaining tasks until requested. + +## Protocol and shared authority + +- [x] **Implement record and KV client CAS**: add complete member/epoch/state + types, canonical validation, a typed group-members key and linearizable + snapshots. Fresh creation starts Installing at epoch 1; changes require the + expected Ready epoch. Preserve previous members during installation for + removal/restart scope. Complete only the exact Installing snapshot by CAS. + Files: protocol `kv_membership.rs`, `key/kv_cluster.rs`; + KV client `membership.rs`; protocol/client integration tests. +- [ ] **Wire new group authority**: creation, bootstrap and membership readers + use complete records; remove unconditional/member-per-key mutation authority + without legacy fallback. Files: client `transport/cluster.rs`, + `hardware/sysmd.rs`; Console `ops/kv_logical/publication.rs`. + +## Server submission and propagation + +- [ ] **Guard before durable write**: validate epoch/full configuration and + atomically protect the persistence-to-publication interval. Include remote + add/remove/batch, replacement, restore and reconcile. Inspect lifecycle + cancellation and persisted config identity before selecting the guard. + Files: KV server `mgmt/replica_ops.rs`, `recovery/group_rebuild.rs`; + KV `cluster/px_kv_store.rs`, `group_config.rs`, `node_config.rs`. +- [ ] **Propagate one authorized epoch**: KV client management submission sends + complete configuration/epoch; Console maps conflicts to 409. Confirm exact + installation/fencing before Ready. Capture no-effects vs unknown failures; + never clear Installing on timeout. Files: client `transport/cluster.rs`; + Console `ops/kv_logical.rs`; Web `mgmt/replica_ops.rs`. +- [ ] **Resume after interruption**: restore durable successor, reconcile the + exact authoritative epoch and reject delayed prior configuration/completion. + Required removal/fencing scope must survive coordinator restart. + +## Acceptance and completion + +- [ ] **Control-plane races**: real Group-0 CAS concurrent clients, Installing + rejection, stale expected epoch, exact completion, stale completion and + independent groups. Then controlled local persistence/publication overlaps. +- [ ] **Crash and unknown outcome**: lost CAS/reply, partial installation, + restart after durable write and before memory publication; delayed old request. +- [ ] **UI and docs**: sequential submissions and explicit conflict across two + UI servers. Update permanent reconfiguration/console specifications once + implemented. Keep topology setup separate from concurrent conflict coverage. +- [ ] **Gates and cleanup**: affected protocol/client/server/Console tests, + Rust fmt/clippy and affected UI spec; requirement commits include only this + requirement's coherent work. Preserve other uncommitted changes. Never push. + Delete R229, its index entry and this plan only after all acceptance passes. + +## Verification + +- Protocol 2/2 and real Group-0 CAS 1/1 passed, zero ignored. Coverage includes + independent clients, Installing rejection, stale completion, exact Ready + completion confirmation and independent groups. Workspace fmt and affected + protocol/client all-target clippy passed. Production submission is not wired + yet; this does not establish server persistence/publication safety. +- Protocol: `pixi run cargo test -p crowdb-protocol --test kv_membership_test`. +- Shared CAS: `pixi run cargo test -p crowdb-kv-client --test membership_cas_test`. +- Server/Console: commands from R229; runtime suites sequential and owned. +- UI: affected replica dialog tests and topology/conflict browser scenarios. +- Gates: `pixi run rs-fmt-check` and affected-crate clippy with `-D warnings`. + +## Adjacent unfinished work + +Native lifecycle response still exceeds its unchanged 3s budget; investigate +actual recovery cost rather than claiming fsync or weakening tests. Complete +Console/native matrix remains separate from this requirement. Chunk-KV cutover +core proposals retain their own review boundary. diff --git a/doc/working/plan-test-task-completeness.md b/doc/working/plan-test-task-completeness.md new file mode 100644 index 000000000..4fba8c22f --- /dev/null +++ b/doc/working/plan-test-task-completeness.md @@ -0,0 +1,52 @@ + + + +# Access and Iceberg Test Task Completeness Plan + +Upstream: [test matrix](test.md), existing Pixi SDK preparation tasks. +Goal: each named task runs all its functional acceptance cases in one invocation, +including environment-specific preparation, without manual follow-up or retries. + +## Tasks + +- [x] **Close task coverage**: `test-access` currently leaves + `official_boto3_recognizes_a_copy_error_after_http_200_and_keepalives` to another + task. Dispatch that exact test in the pinned Boto3 environment from the same + parent task. Reuse its SDK environment setup; avoid running the full Boto3 + suite unnecessarily. `test-iceberg-e2e` must invoke the existing Java FileIO + preparation task for all three `official_java_` cases after its Python/native + phases. Keep sequential owned cluster execution and fail-fast behavior. + Files: `tools/pixi-tasks/test-access.sh`, `test-iceberg-e2e.sh`, existing SDK + setup scripts and `pixi.toml` only if a narrowly scoped task is needed. +- [ ] **Verify exact case inventory**: distinguish helper subprocess entry points + from acceptance cases. The native fault-listener child is executed by its + parent crash matrix, not a standalone acceptance case. Keep manual stress + profiles explicit; list any remaining exclusions and reasons instead of + counting another task's results as this task's completion. +- [~] **Verify one-call acceptance in CI**: run `pixi run test-access` and + `pixi run -e iceberg-e2e test-iceberg-e2e` sequentially, capture complete logs + and independent wall times. Confirm the Boto3 case and all three Java cases + execute and pass in their parent task invocation. No retry loop, weakened + assertion or core code change. Missing required dependencies fail the task. +- [ ] **Reconcile counts and matrix**: deduplicate cases by test binary/name + across ordinary and explicit ignored-test dispatches. Do not add ignored + lines to totals after that exact case passed in a later phase of the same + invocation. Mark complete only after the entire task exits successfully; + update counts, actual timings and retained logs in `test.md`. + +## Verification + +- Existing Rust assertions remain unchanged; this is task orchestration work. +- Check modified shell syntax through `pixi run bash -n