From eb1d1ec214517e32e203f3cfbae4607d7c6df502 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Thu, 27 Aug 2026 14:57:21 -0500 Subject: [PATCH] refactor(mem-wal): finish the SSTable/frozen naming pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #7943 and #7957, which renamed the persisted unit to SSTable and the compaction progress to `compacted_sstables` but left two terms behind in the internals. **`L0` -> SSTable.** 26 doc-comment and test-name references survived the earlier passes. The format spec has no L0 concept — a flushed MemTable is an SSTable, and SSTables are compacted into the base table — so the term only appeared in prose. **Memtable `seal` -> `freeze`.** The state was already `frozen` in 267 places (`MemTable::freeze`, `is_frozen`, `frozen_memtable_grace`, `frozen_count`, `FrozenMemTable`), but the action was `seal`. That left one memtable lifecycle described by two verbs, and collided with `ShardStatus::Sealed`, which is the unrelated drop-table 2PC marker. Renamed: `force_seal_active` -> `force_freeze_active`, `SealFence` -> `FreezeFence`, `SealFence::sealed_generation` -> `FreezeFence::frozen_generation`. `ShardStatus::Sealed` and the `SEALED` proto enum are unchanged — sealing is now exclusively a shard-lifecycle term. Also fixes two `region_id` locals in `wal.rs` tests, left over from the region -> shard rename in #6367. ## Not renamed - `bucket_sharding` / `num_buckets`: the sharding transform, not a shard. - `mem_wal_kv_point_lookup.rs` "L0 SSTs": that is RocksDB's L0 in the comparison benchmark, where the term is correct. - Histogram buckets, and the Harley-Seal popcount in `hamming.rs`. ## Compatibility MemWAL is experimental. The renamed API is Rust-only — no Python, Java, or protobuf surface — and there is no on-disk or wire change. Renamed directly without deprecation shims, matching #7943. ## Validation - `cargo check -p lance --tests --benches` - `cargo clippy -p lance --tests --benches` (no new warnings) - `cargo test -p lance --lib mem_wal` — 669 passed - `cargo fmt --all` Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgVf5C2yRbwK6pe1aMYkmg --- .../mem_wal/fts/mem_wal_fts_read_bench.rs | 2 +- .../mem_wal/kv/mem_wal_kv_point_lookup.rs | 6 +- .../mem_wal_point_lookup_bench.rs | 2 +- .../vector/hnsw/mem_wal_recall_hnsw.rs | 8 +- rust/lance/src/dataset/mem_wal.rs | 2 +- rust/lance/src/dataset/mem_wal/index/fts.rs | 26 +- rust/lance/src/dataset/mem_wal/memtable.rs | 8 +- .../src/dataset/mem_wal/memtable/flush.rs | 2 +- rust/lance/src/dataset/mem_wal/observer.rs | 2 +- .../src/dataset/mem_wal/scanner/builder.rs | 2 +- .../src/dataset/mem_wal/scanner/collector.rs | 6 +- .../src/dataset/mem_wal/scanner/planner.rs | 4 +- .../dataset/mem_wal/scanner/vector_search.rs | 2 +- rust/lance/src/dataset/mem_wal/wal.rs | 4 +- rust/lance/src/dataset/mem_wal/write.rs | 332 +++++++++--------- rust/lance/tests/mem_wal/mod.rs | 2 +- 16 files changed, 205 insertions(+), 205 deletions(-) diff --git a/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs b/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs index 462acb5b717..11197408b0b 100644 --- a/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs +++ b/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs @@ -732,7 +732,7 @@ async fn run_search(args: &Args) -> Result { tokio::time::sleep(flush_wait).await; } } - // Wait for any triggered (sealed) memtable flushes to commit to the + // Wait for any triggered (frozen) memtable flushes to commit to the // manifest before we snapshot it — otherwise the SSTables // race the read and may not all be visible yet. writer.wait_for_flush_drain().await?; diff --git a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs index 335a41d3bbd..9b6c70cb0f6 100644 --- a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs +++ b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs @@ -774,7 +774,7 @@ async fn run_lance( // No-flush config: every *memtable*-flush threshold is set above the // dataset so the single active MemTable holds all rows (no generation is - // sealed to disk). Read visibility is gated on the WAL durability + // frozen to disk). Read visibility is gated on the WAL durability // watermark (`max_visible_batch_position`), which only advances on a WAL // flush — so we use `durable_write=true`: each put flushes its batch to // the WAL and awaits, which both populates the maintained BTree and @@ -801,7 +801,7 @@ async fn run_lance( let writer = dataset.mem_wal_writer(shard_id, config).await?; // --- write phase --- - // LSM: split rows into `generations+1` parts; seal+flush after each of the + // LSM: split rows into `generations+1` parts; freeze+flush after each of the // first `generations` parts (each becomes an on-disk generation) and leave // the last part in the active MemTable. let gens = args.generations; @@ -827,7 +827,7 @@ async fn run_lance( lo = hi; } if g < gens { - writer.force_seal_active().await?; + writer.force_freeze_active().await?; for _ in 0..600 { let n = writer .manifest() diff --git a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs index 0b92d1d9c73..b72f85acf8f 100644 --- a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs +++ b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs @@ -5,7 +5,7 @@ //! //! Measures lookup latency against three tiers of the LSM tree: //! - Base table (on-disk, compacted data) -//! - SSTables (on-disk L0) +//! - SSTables (on-disk, uncompacted) //! - Active MemTable (in-memory write buffer) //! //! Two phases, selected with `--phase`: diff --git a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs index 625900cf834..837bf3d6e5d 100644 --- a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs +++ b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs @@ -440,12 +440,12 @@ async fn run_checkpoint( use std::io::Write; std::io::stdout().flush().ok(); - // Flush mode: seal the active MemTable to a persistent on-disk generation + // Flush mode: freeze the active MemTable to a persistent on-disk generation // (single-partition IVF_HNSW_SQ at the base dataset's storage version) and // print its dataset path. Validates we can flush cp=100k/500k/1M. if let Some(dir) = &local_dir { - let seal_start = Instant::now(); - writer.force_seal_active().await?; + let freeze_start = Instant::now(); + writer.force_freeze_active().await?; let mut waited = 0u64; let gen_path = loop { if let Some(m) = writer.manifest().await? @@ -474,7 +474,7 @@ async fn run_checkpoint( "SSTABLE_OK cp={} id_offset={} flush_s={:.2} path={}", cp, id_offset, - seal_start.elapsed().as_secs_f64(), + freeze_start.elapsed().as_secs_f64(), gen_path ); std::io::stdout().flush().ok(); diff --git a/rust/lance/src/dataset/mem_wal.rs b/rust/lance/src/dataset/mem_wal.rs index 962a01c8111..f45e7dbfe61 100644 --- a/rust/lance/src/dataset/mem_wal.rs +++ b/rust/lance/src/dataset/mem_wal.rs @@ -141,7 +141,7 @@ pub use sharding::{ evaluate_sharding_spec_with_source_columns, }; pub use wal::{BatchDurableWatcher, WalAppendResult, WalAppender, WalReadEntry, WalTailer}; -pub use write::SealFence; +pub use write::FreezeFence; pub use write::ShardWriter; pub use write::ShardWriterConfig; pub use write::WriteResult; diff --git a/rust/lance/src/dataset/mem_wal/index/fts.rs b/rust/lance/src/dataset/mem_wal/index/fts.rs index fb34b9a2f79..60d3c2afe05 100644 --- a/rust/lance/src/dataset/mem_wal/index/fts.rs +++ b/rust/lance/src/dataset/mem_wal/index/fts.rs @@ -640,12 +640,12 @@ impl BatchMeta { /// the tail's term map. An estimate — the node layout is crossbeam-internal. const SKIPMAP_ENTRY_OVERHEAD: usize = 32; -/// Size of a sealed batch block. Small enough that copying the partial tail -/// block on append stays cheap; large enough that sealing (which clones the +/// Size of a frozen batch block. Small enough that copying the partial tail +/// block on append stays cheap; large enough that freezing (which clones the /// block-pointer vec) is rare. const BATCH_BLOCK: usize = 64; -/// Append-only, structurally-shared log of visible batch metadata. Sealed full +/// Append-only, structurally-shared log of visible batch metadata. Frozen full /// blocks are immutable and shared across snapshots; only the current partial /// block is copied on append, so publishing a batch is amortized O(1) (instead /// of copying every batch pointer per publish — O(batches²) per generation) @@ -653,7 +653,7 @@ const BATCH_BLOCK: usize = 64; #[derive(Debug, Clone)] struct BatchLog { /// Immutable full blocks (each `BATCH_BLOCK` long), shared across snapshots. - sealed: Arc]>>>, + frozen: Arc]>>>, /// The current partial block (`< BATCH_BLOCK` entries). tail: Arc<[Arc]>, len: usize, @@ -662,28 +662,28 @@ struct BatchLog { impl BatchLog { fn empty() -> Self { Self { - sealed: Arc::new(Vec::new()), + frozen: Arc::new(Vec::new()), tail: Arc::from(Vec::>::new().into_boxed_slice()), len: 0, } } - /// A new log with `meta` appended; shares every sealed block with `self`, + /// A new log with `meta` appended; shares every frozen block with `self`, /// copying only the partial tail block. fn pushed(&self, meta: Arc) -> Self { let mut tail: Vec> = self.tail.to_vec(); tail.push(meta); if tail.len() == BATCH_BLOCK { - let mut sealed = (*self.sealed).clone(); - sealed.push(Arc::from(tail.into_boxed_slice())); + let mut frozen = (*self.frozen).clone(); + frozen.push(Arc::from(tail.into_boxed_slice())); Self { - sealed: Arc::new(sealed), + frozen: Arc::new(frozen), tail: Arc::from(Vec::>::new().into_boxed_slice()), len: self.len + 1, } } else { Self { - sealed: Arc::clone(&self.sealed), + frozen: Arc::clone(&self.frozen), tail: Arc::from(tail.into_boxed_slice()), len: self.len + 1, } @@ -693,15 +693,15 @@ impl BatchLog { fn get(&self, i: usize) -> Option<&Arc> { let block = i / BATCH_BLOCK; let off = i % BATCH_BLOCK; - match self.sealed.get(block) { + match self.frozen.get(block) { Some(b) => b.get(off), - None if block == self.sealed.len() => self.tail.get(off), + None if block == self.frozen.len() => self.tail.get(off), None => None, } } fn iter(&self) -> impl Iterator> { - self.sealed + self.frozen .iter() .flat_map(|b| b.iter()) .chain(self.tail.iter()) diff --git a/rust/lance/src/dataset/mem_wal/memtable.rs b/rust/lance/src/dataset/mem_wal/memtable.rs index 151294c035e..ab76141ef2d 100644 --- a/rust/lance/src/dataset/mem_wal/memtable.rs +++ b/rust/lance/src/dataset/mem_wal/memtable.rs @@ -130,8 +130,8 @@ const PK_BLOOM_FILTER_FPP: f64 = 0.00057; /// measurement, and a memory view need not carry it per memtable. /// /// Counted as index memory rather than row data: it is an auxiliary lookup -/// structure, and a fixed term in the `max_memtable_size` seal trigger would -/// make every memtable seal a constant early. +/// structure, and a fixed term in the `max_memtable_size` freeze trigger would +/// make every memtable freeze a constant early. pub fn pk_bloom_filter_bytes() -> usize { static BYTES: std::sync::OnceLock = std::sync::OnceLock::new(); *BYTES.get_or_init(|| { @@ -737,7 +737,7 @@ impl MemTable { } /// Whether every committed batch in this memtable is WAL-durable, given the - /// writer-global durability cursor. The L0 flush's precondition. + /// writer-global durability cursor. The SSTable flush's precondition. pub fn all_flushed_to_wal(&self, durable: usize) -> bool { self.batch_store.pending_wal_flush_count(durable) == 0 } @@ -903,7 +903,7 @@ mod tests { assert_eq!(total_rows, 15); } - /// `all_flushed_to_wal(durable)` is the L0 flush's precondition (`flush.rs:171`): + /// `all_flushed_to_wal(durable)` is the SSTable flush's precondition (`flush.rs:171`): /// false while any committed batch is still un-appended, true once the /// durability watermark covers every one of them. #[tokio::test] diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs index c9c4c0e4dd4..dc868dffb09 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs @@ -1254,7 +1254,7 @@ mod tests { .await .unwrap(); - // Nothing is durable yet, so the L0 flush must refuse. + // Nothing is durable yet, so the SSTable flush must refuse. let durable = 0; assert!(!memtable.all_flushed_to_wal(durable)); diff --git a/rust/lance/src/dataset/mem_wal/observer.rs b/rust/lance/src/dataset/mem_wal/observer.rs index a567e953e8e..0e3a61627cf 100644 --- a/rust/lance/src/dataset/mem_wal/observer.rs +++ b/rust/lance/src/dataset/mem_wal/observer.rs @@ -24,7 +24,7 @@ pub trait WalObserver: Send + Sync + Debug { /// `durable_write` put waits on. fn on_wal_flush(&self, _duration: Duration, _bytes: usize) {} - /// A frozen memtable became an L0 SSTable. Orders of magnitude longer + /// A frozen memtable became an SSTable. Orders of magnitude longer /// than a WAL flush. fn on_memtable_flush(&self, _duration: Duration, _rows: usize) {} } diff --git a/rust/lance/src/dataset/mem_wal/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/scanner/builder.rs index 9da723886f1..e7a0696cb42 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/builder.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/builder.rs @@ -284,7 +284,7 @@ impl LsmScanner { /// SSTables) without including a base Lance table. /// /// This is useful when the caller owns the base read path separately and - /// only needs the WAL's contribution: active memtable ∪ L0 SSTables. + /// only needs the WAL's contribution: active memtable ∪ SSTables. /// Deduplication semantics are unchanged — newer generations /// still win on PK conflicts. /// diff --git a/rust/lance/src/dataset/mem_wal/scanner/collector.rs b/rust/lance/src/dataset/mem_wal/scanner/collector.rs index a5d0ecd0b65..b44d11faa22 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/collector.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/collector.rs @@ -33,7 +33,7 @@ impl InMemoryMemTableRef { /// Row-data bytes: the buffered batches. /// /// This is the **flush unit**, not the memtable's footprint — it drives the - /// `max_memtable_size` seal trigger, so it stays a function of the rows in + /// `max_memtable_size` freeze trigger, so it stays a function of the rows in /// it. Use [`Self::resident_bytes`] to budget memory. pub fn row_bytes(&self) -> usize { self.batch_store.row_bytes() @@ -91,7 +91,7 @@ pub struct InMemoryMemTables { /// When the base table is omitted (see [`Self::without_base_table`]), `collect` /// returns only SSTable and active-memtable sources. This is used /// by callers that own the base read path elsewhere and only need the WAL's -/// fresh tier (active memtable ∪ L0 SSTables). +/// fresh tier (active memtable ∪ SSTables). pub struct LsmDataSourceCollector { /// Base Lance table (None when scanning only the fresh tier). base_table: Option>, @@ -241,7 +241,7 @@ impl LsmDataSourceCollector { /// scan sources, in **ascending generation order**. The planner relies /// on this: it reverses sources to generation-DESC so the newest row /// wins the dedup tiebreaker (see `LsmScanPlanner::plan_scan`). Active - /// is the newest generation; frozen are older sealed ones — so without + /// is the newest generation; the frozen ones are older — so without /// this sort a stale frozen row could outrank a re-write in the active /// memtable for the same pk. fn in_memory_sources(shard_id: Uuid, mems: &InMemoryMemTables) -> Vec { diff --git a/rust/lance/src/dataset/mem_wal/scanner/planner.rs b/rust/lance/src/dataset/mem_wal/scanner/planner.rs index 2a83447fb17..768c917852b 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/planner.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/planner.rs @@ -818,7 +818,7 @@ mod integration_tests { assert_eq!(results.get(&6), Some(&"active_6".to_string())); } - /// Regression for the concurrent-read-vs-flush hole: a sealed + /// Regression for the concurrent-read-vs-flush hole: a frozen /// (frozen-awaiting-flush) memtable is not yet recorded as an /// SSTable, but its rows must still be in the scan's read union and /// dedup correctly by generation across the active/frozen seam. @@ -855,7 +855,7 @@ mod integration_tests { .with_sstable(1, "gen_1".to_string()) .with_sstable(2, "gen_2".to_string()); - // Frozen gen3 (sealed, NOT in the manifest) and active gen4. + // Frozen gen3 (NOT in the manifest) and active gen4. let (frozen_store, frozen_index) = pk_indexed(&[create_test_batch(&schema, &[6, 7], "frozen")]); let frozen = InMemoryMemTableRef { diff --git a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs index 3c314b36a75..c78a699a729 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs @@ -3350,7 +3350,7 @@ mod tests { assert_eq!( id1.len(), 1, - "newest-wins: id=1 must appear exactly once after a same-L0 override, got {:?}", + "newest-wins: id=1 must appear exactly once after a same-generation override, got {:?}", rows ); assert!( diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs index 6f31824ee7b..2f7803a0bc3 100644 --- a/rust/lance/src/dataset/mem_wal/wal.rs +++ b/rust/lance/src/dataset/mem_wal/wal.rs @@ -1788,8 +1788,8 @@ mod tests { #[tokio::test] async fn test_track_batch_watcher_blocks_until_flush() { let (store, base_path, _temp_dir) = create_local_store().await; - let region_id = Uuid::new_v4(); - let flusher = build_test_flusher(store, &base_path, region_id, 1); + let shard_id = Uuid::new_v4(); + let flusher = build_test_flusher(store, &base_path, shard_id, 1); let schema = create_test_schema(); let batch_store = Arc::new(BatchStore::with_capacity(10)); diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs index 6be5588f74c..7d3539c6c64 100644 --- a/rust/lance/src/dataset/mem_wal/write.rs +++ b/rust/lance/src/dataset/mem_wal/write.rs @@ -792,7 +792,7 @@ impl ShardMemory { /// Deliberately *not* `active_bytes() - index_bytes()`. That difference is /// what the batches pin — whole parent buffers, unbounded above this figure /// once any batch is a zero-copy slice — and is what the ceiling is built - /// on. Use this to reason about when a memtable seals, not about what it + /// on. Use this to reason about when a memtable freezes, not about what it /// costs. pub fn row_bytes(&self) -> usize { match &self.0 { @@ -827,7 +827,7 @@ impl ShardMemory { } } - /// Resident bytes of sealed memtables whose flush has not committed. + /// Resident bytes of frozen memtables whose flush has not committed. /// Always `0` in WAL-only mode. pub fn frozen_bytes(&self) -> usize { match &self.0 { @@ -843,7 +843,7 @@ impl ShardMemory { } } - /// Resident bytes of sealed memtables that have flushed and are lingering + /// Resident bytes of frozen memtables that have flushed and are lingering /// out `frozen_memtable_grace` so in-flight as-of reads stay batch-resolved. /// /// Real memory, but no flush reclaims it — the sweeper does, on a timer. A @@ -946,7 +946,7 @@ impl ShardMemory { /// Returned by [`ShardMemory::drain`] so a blocking controller can tell a wait /// that ends from one that cannot. The distinction is not academic: a flush /// that fails leaves its generation resident and charged with nothing queued to -/// retry it, and index memory is charged to the ceiling while the seal trigger +/// retry it, and index memory is charged to the ceiling while the freeze trigger /// measures row bytes — either can put a shard over budget with no flush in /// flight, and only a new write would start one. #[derive(Debug)] @@ -1075,7 +1075,7 @@ impl BackpressureController for LocalBackpressureController { /// would have no event to end on. The /// error names the breakdown, because the two ways to get there — a flush /// that failed and left its generation charged, or index memory carrying a - /// memtable past the ceiling while the seal trigger still sees small row + /// memtable past the ceiling while the freeze trigger still sees small row /// bytes — are both configuration or operational conditions an operator has /// to act on rather than wait out. async fn maybe_apply_backpressure(&self, shard: ShardMemory) -> Result<()> { @@ -1142,7 +1142,7 @@ impl BackpressureController for LocalBackpressureController { return Err(Error::backpressure(format!( "shard is at its memtable ceiling with no flush outstanding, so waiting \ cannot drain it: unflushed_bytes={}, max={}, active_bytes={} (of which \ - index_bytes={}), frozen_bytes={}. The active memtable seals on resident \ + index_bytes={}), frozen_bytes={}. The active memtable freezes on resident \ bytes, so reaching here means a flush failed and left its generation \ charged with nothing queued to retry it", unflushed_memtable_bytes, @@ -1165,7 +1165,7 @@ pub struct WriteResult { pub batch_positions: std::ops::Range, } -/// A sealed memtable kept queryable in memory. `flushed_at_ms` is `None` while +/// A frozen memtable kept queryable in memory. `flushed_at_ms` is `None` while /// the generation is still awaiting (or retrying) its flush, and `Some(t)` once /// the flush commits — after which it lingers for `frozen_memtable_grace` so /// in-flight as-of reads keep batch-resolved membership, then is swept. @@ -1199,10 +1199,10 @@ struct FrozenMemTable { struct ResidentMemTables { /// The active memtable, or `None` before the first publish. active: Option, - /// Sealed memtables whose flush has not committed — including any left + /// Frozen memtables whose flush has not committed — including any left /// resident by a *failed* flush, which are the ones most worth metering. frozen: Vec, - /// Sealed memtables whose flush *did* commit, lingering out + /// Frozen memtables whose flush *did* commit, lingering out /// `frozen_memtable_grace` before `SweepExpired` drops them. /// /// Held apart from `frozen` rather than dropped from the view: no flush can @@ -1215,7 +1215,7 @@ struct ResidentMemTables { /// The oldest flush still outstanding, or `None` when none is. /// /// Deliberately not backfilled with the active memtable's watcher. That - /// watcher only fires when the active memtable is sealed, which only a put + /// watcher only fires when the active memtable is frozen, which only a put /// does — so offering it to a waiter that is itself holding puts out names /// an event that cannot arrive. `None` is the honest answer, and /// [`ShardMemory::drain`] turns it into one. @@ -1258,7 +1258,7 @@ struct WriterState { /// count: sizes are read live off `frozen_memtables` (see /// [`ResidentMemTables`]), so there is nothing here to keep paired. frozen_flush_watchers: VecDeque, - /// Sealed memtables, kept queryable so a concurrent reader sees no hole + /// Frozen memtables, kept queryable so a concurrent reader sees no hole /// between `freeze_memtable` and the flush task's manifest commit, and for /// `frozen_memtable_grace` beyond it so as-of reads stay batch-resolved. /// Pushed in `freeze_memtable`; stamped `flushed_at_ms` by `flush_memtable` @@ -1317,29 +1317,29 @@ fn now_millis() -> u64 { /// shard between our `claim_epoch` and this replay, fencing us. /// Outcome of replaying a shard's WAL into memory. struct ReplayResult { - /// The active memtable — the final, partial one replay left unsealed. A fresh - /// shard yields an empty one; every sealed memtable was flushed to a Lance + /// The active memtable — the final, partial one replay left unfrozen. A fresh + /// shard yields an empty one; every frozen memtable was flushed to a Lance /// generation during replay and is not returned. active: MemTable, /// One past the highest WAL entry position observed — the next write position. next_wal_position: u64, } -/// Replay a shard's WAL into memory, flushing sealed memtables as the batch store +/// Replay a shard's WAL into memory, flushing frozen memtables as the batch store /// fills. /// /// A single memtable holds at most `max_memtable_batches` batches, but a WAL is /// unbounded — so replay must rotate exactly as the live write path does. It -/// seals a full memtable and, because the data is already durable, flushes it to +/// freezes a full memtable and, because the data is already durable, flushes it to /// a Lance generation right here (the same `MemTableFlusher::flush` the live path -/// uses), rather than holding every sealed memtable in memory until open +/// uses), rather than holding every frozen memtable in memory until open /// finishes. That bounds resident memory to ~two memtables and truncates the WAL /// as it goes, so a later reopen replays only the unflushed tail. Only the final /// partial memtable is returned, as the active one. /// /// `make_memtable(generation, global_offset)` builds a fresh, cursor-bound /// memtable. Rotation happens at WAL-entry boundaries, never mid-entry, so each -/// sealed memtable covers a clean range of complete entries and stamps the last +/// frozen memtable covers a clean range of complete entries and stamps the last /// one as its SSTable's `replay_after_wal_entry_position`. #[allow(clippy::too_many_arguments)] async fn replay_memtable_from_wal( @@ -1395,7 +1395,7 @@ async fn replay_memtable_from_wal( .map(|b| ensure_tombstone_column(b, &storage_schema)) .collect::>>()?; - // Seal + flush at the entry boundary on the *same* criteria the + // Freeze + flush at the entry boundary on the *same* criteria the // live path uses (`maybe_trigger_memtable_flush`): the memtable // is at or over `max_memtable_size` bytes, or this whole entry // won't fit the batch store. The byte trigger is the one that @@ -1405,7 +1405,7 @@ async fn replay_memtable_from_wal( // capacity when the final active memtable is indexed. // // Rotate at the entry boundary so no entry is split across two - // memtables and each sealed one covers a clean range of complete + // memtables and each frozen one covers a clean range of complete // entries. Never rotate an empty memtable — if a single entry // has more batches than a memtable can hold, a fresh one would // overflow too, the same hard limit the live put path has, left @@ -1425,7 +1425,7 @@ async fn replay_memtable_from_wal( let generation = active.generation() + 1; let global_end = store.global_end(); - // The sealed data is already durable in the WAL — mark it + // The frozen data is already durable in the WAL — mark it // so the flush's `all_flushed_to_wal` precondition holds and // no WAL re-append is attempted. wal_flusher.advance_durable(global_end); @@ -1456,7 +1456,7 @@ async fn replay_memtable_from_wal( // Rebuild the active memtable's in-memory indexes from the batches just // replayed, so readers see them through the index path — matching what the - // pre-crash writer's flush would have done. Sealed memtables needed no + // pre-crash writer's flush would have done. Frozen memtables needed no // in-memory index build: they were flushed straight to disk and are gone. if let Some(indexes) = active.indexes_arc() { let batch_count = active.batch_count(); @@ -1479,7 +1479,7 @@ async fn replay_memtable_from_wal( }) } -/// Whether a memtable has reached the threshold at which it should be sealed and +/// Whether a memtable has reached the threshold at which it should be frozen and /// flushed. /// /// The single source of truth for the flush trigger, shared by the live put path @@ -1499,7 +1499,7 @@ async fn replay_memtable_from_wal( /// - **Resident total** against `max_resident_bytes`, the backpressure ceiling. /// The row window bounds neither what the batches *pin* (a one-row slice holds /// its whole parent) nor what the indexes hold, so without this arm a memtable -/// can carry a shard past its ceiling with no seal reachable — and the valve, +/// can carry a shard past its ceiling with no freeze reachable — and the valve, /// finding nothing outstanding to wait on, would refuse writes that can never /// succeed. This is the drain path that makes the ceiling live rather than a /// trap. @@ -1518,7 +1518,7 @@ fn memtable_reached_flush_threshold( /// What this memtable holds in memory: the heap its batches pin plus its /// in-memory indexes. The same quantity [`ShardMemory`] reports for the active -/// memtable, so the seal trigger and the ceiling that gates writes measure the +/// memtable, so the freeze trigger and the ceiling that gates writes measure the /// same thing. fn memtable_resident_bytes(memtable: &MemTable) -> usize { memtable.batch_store().retained_bytes() @@ -1526,7 +1526,7 @@ fn memtable_resident_bytes(memtable: &MemTable) -> usize { + super::memtable::pk_bloom_filter_bytes() } -/// Flush a sealed replay memtable to a Lance generation, choosing the indexed +/// Flush a frozen replay memtable to a Lance generation, choosing the indexed /// path when secondary indexes are configured (mirroring the live memtable-flush /// handler). Commits the manifest, stamping `covered` as the generation's /// `replay_after_wal_entry_position` so a later reopen skips these entries. @@ -1748,7 +1748,7 @@ impl SharedWriterState { // The outgoing memtable may still owe an index apply — the puts that // filled it triggered one, but the task need not have drained yet, and - // this is the last chance to name that store. Its L0 flush is gated on + // this is the last chance to name that store. Its SSTable flush is gated on // the WAL append (below), not on indexing, so without this its tail could // stay unindexed and invisible for the rest of its life. let pending_index_apply = match old_memtable.indexes_arc() { @@ -2071,9 +2071,9 @@ impl ShardWriter { )?; // An HNSW graph reserves its whole capacity before the first insert, - // but the seal trigger only measures row bytes. A reservation with no + // but the freeze trigger only measures row bytes. A reservation with no // room left under the ceiling puts the shard over budget at zero rows - // — nothing to seal, so nothing to flush, so every put stalls and then + // — nothing to freeze, so nothing to flush, so every put stalls and then // fails as `Error::Backpressure`, which is supposed to mean "retry // later". Reject the config instead; only the built-in valve reads // this ceiling. @@ -2081,13 +2081,13 @@ impl ShardWriter { // The headroom the check below reserves for rows. At zero it // reserves nothing, so a fresh memtable's index reservation may // *equal* the ceiling — over budget before its first row, with an - // empty memtable there is nothing to seal, and the writer refuses + // empty memtable there is nothing to freeze, and the writer refuses // its first write forever. A zero threshold is degenerate anyway: - // it seals every memtable at every insert. + // it freezes every memtable at every insert. if config.max_memtable_size == 0 { return Err(Error::invalid_input( "max_memtable_size must be greater than zero: it is both the \ - seal threshold for row data and the headroom reserved for rows \ + freeze threshold for row data and the headroom reserved for rows \ under max_unflushed_memtable_bytes, and at zero a writer with \ in-memory indexes can be at its ceiling before its first row", )); @@ -2105,7 +2105,7 @@ impl ShardWriter { } let reserved = indexes.resident_bytes() + super::memtable::pk_bloom_filter_bytes(); // Room for a full memtable of rows on top, or the ceiling is - // crossed before `max_memtable_size` can seal. + // crossed before `max_memtable_size` can freeze. let needed = reserved.saturating_add(config.max_memtable_size); if needed > config.max_unflushed_memtable_bytes { return Err(Error::invalid_input(format!( @@ -2113,7 +2113,7 @@ impl ShardWriter { max_memtable_rows={}, and max_memtable_size={} must fit alongside them, \ needing {needed} bytes; max_unflushed_memtable_bytes={} is below that, \ so the active memtable would cross the backpressure ceiling before \ - accruing enough row bytes to seal, stalling every write. Raise \ + accruing enough row bytes to freeze, stalling every write. Raise \ max_unflushed_memtable_bytes to at least {needed}, or lower \ max_memtable_rows / max_memtable_size", config.max_memtable_rows, @@ -2283,8 +2283,8 @@ impl ShardWriter { Ok(memtable) }; - // The flusher writes sealed memtables to Lance generations — both the - // ones replay seals below and the ones the live path freezes later. + // The flusher writes frozen memtables to Lance generations — both the + // ones replay freezes below and the ones the live path freezes later. let flusher = Arc::new( MemTableFlusher::new( object_store.clone(), @@ -2298,7 +2298,7 @@ impl ShardWriter { ); // Replay any WAL entries written after the last successfully-flushed - // SSTable, flushing sealed memtables to Lance SSTables as the batch + // SSTable, flushing frozen memtables to Lance SSTables as the batch // store fills. Each entry's writer_epoch is checked against ours; an entry // with a strictly greater epoch means a successor claimed the shard // between our `claim_epoch` and replay, so we abort with a fence error. @@ -2350,7 +2350,7 @@ impl ShardWriter { // Mark the active memtable's replayed batches durable. They came *from* // the WAL, and replay has already re-derived its indexes over them. // - // Without this the durability cursor stays at the last sealed generation, + // Without this the durability cursor stays at the last frozen generation, // so the next WAL flush re-covers the active tail: it re-appends the // already-durable rows *and* re-inserts every replayed row into the // indexes. None of the three in-memory indexes is idempotent (HNSW mints @@ -2360,7 +2360,7 @@ impl ShardWriter { // — and it compounds, because the WAL now holds those rows twice. // // `global_end()` is the writer-global batch count through this memtable, - // since its coordinate continues where the last sealed generation ended. + // since its coordinate continues where the last frozen generation ended. wal_flusher.advance_durable(memtable.batch_store().global_end()); wal_flusher @@ -2392,7 +2392,7 @@ impl ShardWriter { last_wal_flush_trigger_time: 0, }; // Seed before the first freeze: replay above may already have filled the - // memtable, and nothing else publishes until it seals. + // memtable, and nothing else publishes until it freezes. publish_memory(&memory, &state); let state = Arc::new(RwLock::new(state)); @@ -2786,11 +2786,11 @@ impl ShardWriter { // poisoned writer can't drift further from the durable WAL. self.wal_flusher.check_poisoned()?; - // The seal check runs inside the lock immediately after an insert, but the + // The freeze check runs inside the lock immediately after an insert, but the // index apply that follows it runs *outside* — and replay hands back a // memtable whose indexes were built after its last check too. Either way - // the ceiling can be reached with nothing sealed, and the valve below - // would then refuse a write that a seal would have admitted. Re-run the + // the ceiling can be reached with nothing frozen, and the valve below + // would then refuse a write that a freeze would have admitted. Re-run the // check here so the wait has a flush to end on. // // The read is two relaxed loads; the lock is taken only when the shard is @@ -2799,7 +2799,7 @@ impl ShardWriter { >= self.config.max_unflushed_memtable_bytes { let mut state = state_lock.write().await; - // Nothing to seal in an empty memtable, and freezing one would spin: + // Nothing to freeze in an empty memtable, and freezing one would spin: // an injected controller skips the open-time reservation check, so a // fresh memtable can sit above this ceiling on its indexes alone. if state.memtable.batch_count() > 0 { @@ -3198,13 +3198,13 @@ impl ShardWriter { } } - /// Seal the active memtable so it's queued for L0 flush. Errors in + /// Freeze the active memtable so it's queued for SSTable flush. Errors in /// WAL-only mode or if this writer has been fenced by a successor. /// - /// The returned [`SealFence`] is what makes a *bounded* wait possible: - /// a caller that needs "everything written before my seal is in L0" - /// awaits [`SealFence::wait`], which covers the flushes outstanding at - /// seal time and nothing else. [`Self::wait_for_flush_drain`] instead + /// The returned [`FreezeFence`] is what makes a *bounded* wait possible: + /// a caller that needs "everything written before my freeze is an SSTable" + /// awaits [`FreezeFence::wait`], which covers the flushes outstanding at + /// freeze time and nothing else. [`Self::wait_for_flush_drain`] instead /// loops until the frozen set is *empty*, re-collecting it each round, /// so it also waits on every memtable frozen *while it waits*. Under /// sustained writes that set may never empty. @@ -3216,8 +3216,8 @@ impl ShardWriter { /// writer process, or to drain the WAL ahead of a format change so /// the next epoch starts with no replayable entries from the old /// layout. - #[instrument(name = "sw_force_seal_active", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))] - pub async fn force_seal_active(&self) -> Result { + #[instrument(name = "sw_force_freeze_active", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))] + pub async fn force_freeze_active(&self) -> Result { match &self.mode { WriterMode::MemTable { state, @@ -3227,7 +3227,7 @@ impl ShardWriter { self.check_fenced().await?; self.wal_flusher.check_poisoned()?; let mut state = state.write().await; - let sealed_generation = if state.memtable.batch_count() == 0 { + let frozen_generation = if state.memtable.batch_count() == 0 { None } else { let generation = state.memtable.generation(); @@ -3236,30 +3236,30 @@ impl ShardWriter { }; // Capture the outstanding set under the same lock that froze, // so no freeze can slip between the two and widen the fence. - // It covers more than the generation just sealed: a + // It covers more than the generation just frozen: a // size/interval trigger freezes generations asynchronously, so // an empty active memtable does *not* mean every pre-call write - // already reached L0. Watchers are popped as flushes settle, so + // already became an SSTable. Watchers are popped as flushes settle, so // whatever remains here is exactly what is still owed. - Ok(SealFence { - sealed_generation, + Ok(FreezeFence { + frozen_generation, watchers: state.frozen_flush_watchers.iter().cloned().collect(), }) } WriterMode::WalOnly { .. } => Err(Error::invalid_input( - "force_seal_active not available in WAL-only mode (no MemTable)", + "force_freeze_active not available in WAL-only mode (no MemTable)", )), } } - /// Block until every frozen memtable in the L0 flush queue has + /// Block until every frozen memtable in the SSTable flush queue has /// landed and been recorded in the manifest. Does not wait on the - /// active memtable — call [`Self::force_seal_active`] first if you + /// active memtable — call [`Self::force_freeze_active`] first if you /// want everything-on-disk. Errors in WAL-only mode, or if any /// awaited flush reports `DurabilityResult::Failed`. /// /// Useful in tests for deterministic post-flush assertions. In - /// production prefer [`SealFence::wait`] from the seal itself: this + /// production prefer [`FreezeFence::wait`] from the freeze itself: this /// drains the queue to empty, including memtables frozen after the /// call, so under sustained writes it may not return. #[instrument(name = "sw_wait_for_flush_drain", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))] @@ -3393,7 +3393,7 @@ impl ShardWriter { } => { // Drain *both* tasks against the active memtable. The index apply // and the WAL append are independent now, so closing has to - // settle both: the L0 flush below turns this memtable into a + // settle both: the SSTable flush below turns this memtable into a // Lance generation, and a generation whose indexes never saw the // tail is a generation with a hole in it. let st = state.read().await; @@ -3539,7 +3539,7 @@ pub struct MemTableStats { pub pending_wal_batch_count: usize, pub pending_wal_row_count: usize, pub pending_wal_estimated_bytes: usize, - /// Frozen memtables in the read view: sealed-awaiting-flush, plus flushed + /// Frozen memtables in the read view: awaiting flush, plus flushed /// ones still inside `frozen_memtable_grace`. pub frozen_count: usize, } @@ -3551,34 +3551,34 @@ pub struct WalStats { pub next_wal_entry_position: u64, } -/// The L0 flushes outstanding when [`ShardWriter::force_seal_active`] +/// The SSTable flushes outstanding when [`ShardWriter::force_freeze_active`] /// returned — the exact predicate for "everything written before that call -/// is in L0". +/// is an SSTable". /// -/// The set is fixed at seal time, so [`Self::wait`] is bounded no matter how +/// The set is fixed at freeze time, so [`Self::wait`] is bounded no matter how /// many memtables freeze while it waits, and each entry reports the outcome of /// its own flush. That is why the fence is a captured watcher set and not a /// generation number compared against `ShardManifest::current_generation`: the /// manifest advances to `generation + 1` on every committed flush without /// checking for a gap, so a later generation's success moves it past a -/// generation that failed and never reached L0 — the watermark would report +/// generation that failed and never became an SSTable — the watermark would report /// durability that does not exist. #[derive(Debug)] -pub struct SealFence { - sealed_generation: Option, +pub struct FreezeFence { + frozen_generation: Option, watchers: Vec, } -impl SealFence { - /// The generation the seal froze, or `None` when the active memtable - /// was empty (a no-op seal). Independent of what the fence covers: an +impl FreezeFence { + /// The generation this call froze, or `None` when the active memtable + /// was empty (a no-op freeze). Independent of what the fence covers: an /// empty active memtable says nothing about generations frozen earlier /// and still awaiting flush, which the fence still waits on. - pub fn sealed_generation(&self) -> Option { - self.sealed_generation + pub fn frozen_generation(&self) -> Option { + self.frozen_generation } - /// Block until every flush this fence covers has landed in L0. + /// Block until every flush this fence covers has landed as an SSTable. /// /// Errors if any of them reports `DurabilityResult::Failed`, or if the /// flush handler exited without reporting. @@ -3788,7 +3788,7 @@ impl WalFlushHandler { /// So the target is a function of `durable`, not of when the timer fired. /// /// Safe to walk the frozen list because a store that still owes an append - /// cannot be swept: its L0 flush is blocked on the completion cell that only + /// cannot be swept: its SSTable flush is blocked on the completion cell that only /// that append fires. /// /// In WAL-only mode there is a single FIFO pending queue and no memtable @@ -3903,7 +3903,7 @@ struct MemTableFlushHandler { /// flush commits and the memtable set changes. memory: Arc>, flusher: Arc, - /// Source of the writer-global durability cursor, which the L0 flush asserts + /// Source of the writer-global durability cursor, which the SSTable flush asserts /// covers the whole frozen memtable before it writes a generation. wal_flusher: Arc, epoch: u64, @@ -5092,7 +5092,7 @@ mod tests { .await .unwrap(); writer.delete(vec![id_only_keys(&[2])]).await.unwrap(); - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); assert_eq!( @@ -5137,12 +5137,12 @@ mod tests { .put(vec![create_test_batch(&schema, 0, 5)]) .await .unwrap(); - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); // gen 2: tombstone for id=0 (a later generation than the live row). writer.delete(vec![id_only_keys(&[0])]).await.unwrap(); - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); assert_eq!( @@ -5191,10 +5191,10 @@ mod tests { .put(vec![create_test_batch(&schema, 0, 5)]) .await .unwrap(); - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); writer.delete(vec![id_only_keys(&[0])]).await.unwrap(); - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); assert_eq!( @@ -5623,7 +5623,7 @@ mod tests { .unwrap(); // Freeze the active memtable and wait until it lands on disk. - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); // Resolve the SSTable recorded in the manifest. @@ -5725,7 +5725,7 @@ mod tests { durable_write: false, max_wal_buffer_size: 1024 * 1024, max_wal_flush_interval: Some(Duration::from_millis(10)), - max_memtable_size: 1024, // small enough to seal within the loop below + max_memtable_size: 1024, // small enough to freeze within the loop below frozen_memtable_grace: Duration::from_secs(600), manifest_scan_batch_size: 2, ..Default::default() @@ -5754,7 +5754,7 @@ mod tests { assert_eq!( memory.frozen_bytes(), 0, - "every seal flushed, so nothing is owed to flush" + "every freeze flushed, so nothing is owed to flush" ); // Owing nothing to flush is not the same as holding nothing. Those @@ -6303,7 +6303,7 @@ mod tests { /// This is what replaced a pair of incremented counters. A counter has no /// way back: one missed decrement is permanent, and the pod eventually /// refuses every write with a memtable that reads empty. So this walks the - /// writer through open, puts, a seal, a flush commit, and puts into the + /// writer through open, puts, a freeze, a flush commit, and puts into the /// fresh memtable, checking the invariant after each. #[rstest] #[case::grace_keeps_handles(Duration::from_secs(600))] @@ -6316,7 +6316,7 @@ mod tests { shard_id: Uuid::new_v4(), durable_write: false, max_wal_flush_interval: Some(Duration::from_millis(10)), - // Small enough that the loop below seals several times. + // Small enough that the loop below freezes several times. max_memtable_size: 2048, frozen_memtable_grace: grace, ..Default::default() @@ -6336,12 +6336,12 @@ mod tests { assert_no_drift(&writer, &format!("put {i}")).await; } - // Seals happened above; make sure at least one did, or this test proves + // Freezes happened above; make sure at least one did, or this test proves // nothing about the freeze path. assert!( writer.memory().frozen_bytes() > 0 || writer.memtable_stats().await.unwrap().frozen_count > 0, - "the loop must have sealed at least once for this to cover freeze" + "the loop must have frozen at least once for this to cover freeze" ); writer.wait_for_flush_drain().await.unwrap(); @@ -6420,16 +6420,16 @@ mod tests { writer.close().await.unwrap(); } - /// Two seal predicates exist — `MemTable::should_flush` and + /// Two freeze predicates exist — `MemTable::should_flush` and /// `memtable_reached_flush_threshold` — and their *row-window* arms must trip /// at the same byte. They did not: one counted the PK bloom filter and the /// other did not, so they disagreed by a fixed offset on every memtable. /// - /// Only that arm is shared. `memtable_reached_flush_threshold` also seals on + /// Only that arm is shared. `memtable_reached_flush_threshold` also freezes on /// resident bytes, which `should_flush` knows nothing about, so the ceiling /// below is held out of range to compare like with like. #[tokio::test] - async fn test_both_seal_predicates_share_one_byte_arm() { + async fn test_both_freeze_predicates_share_one_byte_arm() { let schema = create_test_schema(); let mut memtable = MemTable::with_capacity(schema.clone(), 1, vec![], CacheConfig::default(), 64).unwrap(); @@ -6457,8 +6457,8 @@ mod tests { assert_eq!( memtable_reached_flush_threshold(&memtable, bytes, usize::MAX, 1), expected, - "the two seal predicates disagree at {bytes}; a bloom-sized offset \ - between them makes every memtable seal early on one path" + "the two freeze predicates disagree at {bytes}; a bloom-sized offset \ + between them makes every memtable freeze early on one path" ); } } @@ -7379,9 +7379,9 @@ mod tests { writer.abort().await.unwrap(); let err = writer - .force_seal_active() + .force_freeze_active() .await - .expect_err("force_seal_active must surface the failed dispatch"); + .expect_err("force_freeze_active must surface the failed dispatch"); assert!( err.to_string().contains("channel closed"), "unexpected error: {err}" @@ -7399,7 +7399,7 @@ mod tests { /// A WAL holding more batches than one memtable's capacity must reopen. /// /// One memtable holds at most `max_memtable_batches` batches, but a WAL is - /// unbounded, so replay has to rotate — seal the full memtable, start a fresh + /// unbounded, so replay has to rotate — freeze the full memtable, start a fresh /// one — exactly as the live write path does. Before, replay stuffed /// everything into a single memtable and `open()` failed outright with /// "MemTable batch store is full", leaving the shard permanently unopenable. @@ -7473,12 +7473,12 @@ mod tests { .await .expect("a WAL larger than one memtable must still reopen"); - // Rotation produced sealed memtables, and replay flushed each to a Lance + // Rotation produced frozen memtables, and replay flushed each to a Lance // generation rather than holding it in memory or leaving it in the WAL. let manifest = writer_b.manifest().await.unwrap().unwrap(); assert!( !manifest.sstables.is_empty(), - "replay must have sealed and flushed at least one full memtable" + "replay must have frozen and flushed at least one full memtable" ); // Every row survived, split between the SSTables and the @@ -7490,7 +7490,7 @@ mod tests { ); writer_b.close().await.unwrap(); - // Because the sealed memtables were flushed, the manifest's replay cursor + // Because the frozen memtables were flushed, the manifest's replay cursor // advanced past their WAL entries — so a second reopen replays only the // tail and still accounts for every row. The WAL truncates across reopens // rather than growing without bound. @@ -8489,11 +8489,11 @@ mod tests { } #[tokio::test] - async fn test_force_seal_active_and_wait_for_flush_drain() { + async fn test_force_freeze_active_and_wait_for_flush_drain() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = create_test_schema(); - // Thresholds high enough that auto-flush won't fire; the seal is + // Thresholds high enough that auto-flush won't fire; the freeze is // the only thing that should rotate the memtable. let config = ShardWriterConfig { shard_id: Uuid::new_v4(), @@ -8522,8 +8522,8 @@ mod tests { .put(vec![create_test_batch(&schema, 0, 10)]) .await .unwrap(); - let fence = writer.force_seal_active().await.unwrap(); - assert_eq!(fence.sealed_generation(), Some(initial_gen)); + let fence = writer.force_freeze_active().await.unwrap(); + assert_eq!(fence.frozen_generation(), Some(initial_gen)); fence.wait().await.unwrap(); let stats = writer.memtable_stats().await.unwrap(); @@ -8540,8 +8540,8 @@ mod tests { writer.close().await.unwrap(); } - /// A durable put returns only once its WAL flush landed, and the seal - /// fence resolves only once the sealed memtable reached L0 — so both + /// A durable put returns only once its WAL flush landed, and the freeze + /// fence resolves only once the frozen memtable became an SSTable — so both /// callbacks have fired by the time this asserts, without sleeping. #[tokio::test] async fn test_observer_sees_both_flush_kinds() { @@ -8572,7 +8572,7 @@ mod tests { let sink: Arc = observer.clone(); let config = ShardWriterConfig { observer: Some(sink), - ..seal_fence_test_config(Uuid::new_v4()) + ..freeze_fence_test_config(Uuid::new_v4()) }; let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) @@ -8584,7 +8584,7 @@ mod tests { .await .unwrap(); writer - .force_seal_active() + .force_freeze_active() .await .unwrap() .wait() @@ -8603,7 +8603,7 @@ mod tests { /// WAL-durable. Both fence tests tear the background tasks down before /// freezing, and a freeze still owing an index apply or a WAL append /// would dispatch onto a closed channel and poison the writer. - fn seal_fence_test_config(shard_id: Uuid) -> ShardWriterConfig { + fn freeze_fence_test_config(shard_id: Uuid) -> ShardWriterConfig { ShardWriterConfig { shard_id, shard_spec_id: 0, @@ -8615,20 +8615,20 @@ mod tests { } } - /// An empty active memtable does not mean every pre-call write is in - /// L0: a size/interval trigger swaps generation N for an empty N+1 - /// while N's flush is still in flight. The seal must fence that - /// generation anyway — reporting "nothing sealed, nothing to wait for" + /// An empty active memtable does not mean every pre-call write is an + /// SSTable: a size/interval trigger swaps generation N for an empty N+1 + /// while N's flush is still in flight. The freeze must fence that + /// generation anyway — reporting "nothing frozen, nothing to wait for" /// lets a caller acknowledge a flush that has not happened. #[tokio::test] - async fn test_force_seal_active_fences_pending_generation_when_active_is_empty() { + async fn test_force_freeze_active_fences_pending_generation_when_active_is_empty() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = create_test_schema(); let writer = ShardWriter::open( store, base_path, base_uri, - seal_fence_test_config(Uuid::new_v4()), + freeze_fence_test_config(Uuid::new_v4()), schema.clone(), vec![], ) @@ -8655,11 +8655,11 @@ mod tests { WriterMode::WalOnly { .. } => unreachable!("opened in memtable mode"), }; - let fence = writer.force_seal_active().await.unwrap(); + let fence = writer.force_freeze_active().await.unwrap(); assert_eq!( - fence.sealed_generation(), + fence.frozen_generation(), None, - "the active memtable was empty, so this seal froze nothing" + "the active memtable was empty, so this freeze froze nothing" ); assert!( tokio::time::timeout(Duration::from_millis(200), fence.wait()) @@ -8675,14 +8675,14 @@ mod tests { /// so a later generation's success moves it past one that failed — a /// watermark comparison would report durability that does not exist. #[tokio::test] - async fn test_force_seal_active_fence_ignores_manifest_generation_advance() { + async fn test_force_freeze_active_fence_ignores_manifest_generation_advance() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = create_test_schema(); let writer = ShardWriter::open( store, base_path, base_uri, - seal_fence_test_config(Uuid::new_v4()), + freeze_fence_test_config(Uuid::new_v4()), schema.clone(), vec![], ) @@ -8693,20 +8693,20 @@ mod tests { .await .unwrap(); - // No flush task, so the sealed generation never reaches L0. + // No flush task, so the frozen generation never becomes an SSTable. writer.task_executor.shutdown_all().await.unwrap(); - let fence = writer.force_seal_active().await.unwrap(); - let sealed = fence - .sealed_generation() + let fence = writer.force_freeze_active().await.unwrap(); + let frozen = fence + .frozen_generation() .expect("the active memtable held rows"); - // Advance the manifest past the sealed generation, as a later + // Advance the manifest past the frozen generation, as a later // generation's successful flush would. writer .manifest_store .commit_update(writer.epoch(), |current| ShardManifest { version: current.next_version(), - current_generation: sealed + 2, + current_generation: frozen + 2, ..current.clone() }) .await @@ -8716,12 +8716,12 @@ mod tests { tokio::time::timeout(Duration::from_millis(200), fence.wait()) .await .is_err(), - "generation {sealed} never reached L0; a manifest advance past it must not satisfy its fence" + "generation {frozen} never became an SSTable; a manifest advance past it must not satisfy its fence" ); } /// `abort` tears down the background flush tasks WITHOUT flushing — - /// buffered memtable rows are discarded, not sealed into an L0 + /// buffered memtable rows are discarded, not frozen into an SSTable /// generation the way `close` would. Idempotent on a second call. #[tokio::test] async fn test_abort_discards_without_flushing_and_is_idempotent() { @@ -8758,8 +8758,8 @@ mod tests { writer.abort().await.unwrap(); - // No generation was sealed — contrast with `close`, which flushes - // the 10 buffered rows into a new L0 generation. + // No generation was frozen — contrast with `close`, which flushes + // the 10 buffered rows into a new SSTable. let flushed_after = writer .manifest() .await @@ -8768,7 +8768,7 @@ mod tests { .unwrap_or(0); assert_eq!( flushed_after, flushed_before, - "abort must not flush a new L0 generation" + "abort must not flush a new SSTable" ); // Idempotent: re-cancels the already-cancelled token, joins an @@ -8776,7 +8776,7 @@ mod tests { writer.abort().await.unwrap(); } - /// On a successful flush commit the sealed generation's rows land in the + /// On a successful flush commit the frozen generation's rows land in the /// manifest immediately, but the in-memory handle is NOT dropped — it /// lingers for `frozen_memtable_grace` (so in-flight as-of reads keep /// batch-resolved membership), then is swept by the `SweepExpired` ticker. @@ -8805,7 +8805,7 @@ mod tests { .put(vec![create_test_batch(&schema, 0, 10)]) .await .unwrap(); - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); // Recorded in the manifest at commit time. @@ -8863,7 +8863,7 @@ mod tests { .put(vec![create_test_batch(&schema, 0, 10)]) .await .unwrap(); - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); // Rows are durably in the manifest... @@ -8935,10 +8935,10 @@ mod tests { } /// Regression: a transient flush failure must NOT reopen the - /// concurrent-read-vs-flush hole. The sealed generation stays in the + /// concurrent-read-vs-flush hole. The frozen generation stays in the /// queryable set (rows intact) until a later flush or WAL replay. /// Failure is induced deterministically by fencing the writer with a - /// successor before the seal, so the flush's `check_fenced` rejects it. + /// successor before the freeze, so the flush's `check_fenced` rejects it. #[tokio::test] async fn test_frozen_retained_after_failed_flush() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; @@ -8975,7 +8975,7 @@ mod tests { .unwrap(); assert!(writer_b.epoch() > writer_a.epoch()); - // `force_seal_active` would reject up-front on a fenced writer; + // `force_freeze_active` would reject up-front on a fenced writer; // freeze directly so the failure surfaces at flush-commit time — // exactly the freeze/flush race the fix guards. match &writer_a.mode { @@ -8996,14 +8996,14 @@ mod tests { "fenced flush should fail the drain" ); - // The hole did not reopen: the sealed generation is still queryable + // The hole did not reopen: the frozen generation is still queryable // with its rows, alongside the new (empty) active generation. let refs = writer_a.in_memory_memtable_refs().await.unwrap(); - assert_eq!(refs.frozen.len(), 1, "sealed generation must be retained"); + assert_eq!(refs.frozen.len(), 1, "frozen generation must be retained"); assert_eq!(refs.frozen[0].generation, initial_gen); assert!( !refs.frozen[0].batch_store.is_empty(), - "retained sealed memtable must still hold its rows" + "retained frozen memtable must still hold its rows" ); assert_eq!(refs.active.generation, initial_gen + 1); @@ -9022,7 +9022,7 @@ mod tests { // Charging those bytes must not become a trap. Nothing retries the // failed generation, and its watcher came off the queue when the flush // reported, so there is no event left that would drain the pool — while - // the valve itself is what keeps the puts that could seal a new + // the valve itself is what keeps the puts that could freeze a new // generation from arriving. Waiting here would never end, so the // controller refuses instead. assert!( @@ -9049,11 +9049,11 @@ mod tests { /// A one-row slice pins its whole parent, so a memtable can hold megabytes /// while its row window reads a few dozen bytes. With only the row arm the - /// shard crossed its ceiling with nothing sealed and no seal reachable — the + /// shard crossed its ceiling with nothing frozen and no freeze reachable — the /// valve then found `Drain::Stalled` and refused every put, permanently. /// The resident arm is what makes that shard drain instead. #[tokio::test] - async fn test_pinned_parents_seal_before_the_ceiling_traps_the_writer() { + async fn test_pinned_parents_freeze_before_the_ceiling_traps_the_writer() { let (store, base_path, base_uri, _temp) = create_local_store().await; let schema = Arc::new(ArrowSchema::new(vec![Field::new( "id", @@ -9065,7 +9065,7 @@ mod tests { let config = ShardWriterConfig { shard_id: Uuid::new_v4(), // Far above anything the one-row windows will ever sum to, so the row - // arm cannot be what seals here. + // arm cannot be what freezes here. max_memtable_size: 1024 * 1024, // Never filled, so the capacity arm cannot be it either. max_memtable_batches: 1024, @@ -9086,7 +9086,7 @@ mod tests { .unwrap(); tokio::time::timeout(STALL_GRACE * 10, writer.put(vec![parent.slice(0, 1)])) .await - .expect("a shard the resident arm can seal must not park forever") + .expect("a shard the resident arm can freeze must not park forever") .unwrap_or_else(|e| { panic!("put {i} was refused, so the ceiling is still a trap: {e}") }); @@ -9095,13 +9095,13 @@ mod tests { let stats = writer.memtable_stats().await.unwrap(); assert!( stats.generation > 0, - "the pinned parents must have sealed a generation; row bytes never \ + "the pinned parents must have frozen a generation; row bytes never \ came close to max_memtable_size" ); assert!( writer.memory().row_bytes() < 1024, "the row window must still be tiny — otherwise the row arm did the \ - sealing and this proves nothing" + freezing and this proves nothing" ); writer.close().await.unwrap(); @@ -9111,7 +9111,7 @@ mod tests { /// arm that has to notice a memtable whose indexes rather than its rows are /// filling the ceiling. #[tokio::test] - async fn test_resident_arm_seals_on_index_memory() { + async fn test_resident_arm_freezes_on_index_memory() { let schema = create_test_schema(); let mut memtable = MemTable::with_capacity(schema.clone(), 1, vec![], CacheConfig::default(), 64).unwrap(); @@ -9131,22 +9131,22 @@ mod tests { // Row arm way out of range; only the resident arm can fire. assert!( memtable_reached_flush_threshold(&memtable, usize::MAX, resident, 1), - "resident bytes at the ceiling must seal" + "resident bytes at the ceiling must freeze" ); assert!( !memtable_reached_flush_threshold(&memtable, usize::MAX, resident + 1, 1), - "and must not seal below it" + "and must not freeze below it" ); } - /// The post-insert seal check runs inside the writer lock; the index apply + /// The post-insert freeze check runs inside the writer lock; the index apply /// that follows it runs outside. So a put's index growth is invisible to the /// only check that put makes, and the *next* put is gated by the valve before /// it can insert and check again — leaving a shard over its ceiling with - /// nothing sealed and every write refused. Replay reaches the same state by + /// nothing frozen and every write refused. Replay reaches the same state by /// building its final memtable's indexes after its last check. #[tokio::test] - async fn test_index_growth_after_the_seal_check_still_drains() { + async fn test_index_growth_after_the_freeze_check_still_drains() { let (store, base_path, base_uri, _t) = create_local_store().await; let schema = Arc::new(ArrowSchema::new(vec![ Field::new("id", DataType::Int32, false), @@ -9168,7 +9168,7 @@ mod tests { let config = ShardWriterConfig { shard_id: Uuid::new_v4(), // As high as the open-time check allows, keeping the row arm out of - // reach of the rows below so only the resident arm can seal. + // reach of the rows below so only the resident arm can freeze. max_memtable_size: ceiling - 64 * 1024, max_memtable_batches: 1024, // Above the rows alone, below rows plus the index heap they build. @@ -9199,9 +9199,9 @@ mod tests { tokio::time::timeout(STALL_GRACE * 10, writer.put(vec![batch])) .await - .expect("a shard with a sealable memtable must not park forever") + .expect("a shard with a freezable memtable must not park forever") .unwrap_or_else(|e| { - panic!("put {i} was refused; index growth outran the seal check: {e}") + panic!("put {i} was refused; index growth outran the freeze check: {e}") }); } @@ -9216,7 +9216,7 @@ mod tests { /// `max_memtable_size` is the headroom the reservation check reserves for /// rows. At zero it reserves none, so a fresh memtable's index reservation /// could equal the ceiling exactly — admissible by the check, yet over budget - /// before its first row, with an empty memtable offering nothing to seal. The + /// before its first row, with an empty memtable offering nothing to freeze. The /// writer refused its first write and never recovered. #[tokio::test] async fn test_open_rejects_a_zero_row_headroom() { @@ -9289,8 +9289,8 @@ mod tests { } /// An HNSW graph is charged from `max_memtable_rows` before the first - /// insert, while only row bytes seal a memtable. Sized past the ceiling it - /// would put the shard over budget at zero rows with nothing to seal and so + /// insert, while only row bytes freeze a memtable. Sized past the ceiling it + /// would put the shard over budget at zero rows with nothing to freeze and so /// nothing to flush — every put stalling, then failing as `Backpressure`, /// which means "retry later" and never comes true. That is a config error, /// so it has to land at `open`, not on put #1. @@ -9341,7 +9341,7 @@ mod tests { /// The other side of the gate: a ceiling with room for the reservation *and* /// a full memtable of rows on top opens, and keeps taking writes across a - /// seal — the frozen generation gives the valve a flush to park on, so the + /// freeze — the frozen generation gives the valve a flush to park on, so the /// wait ends instead of refusing. #[tokio::test] async fn test_writer_with_indexes_under_the_ceiling_keeps_accepting_writes() { @@ -9358,7 +9358,7 @@ mod tests { shard_id: Uuid::new_v4(), max_memtable_rows: sizing.max_memtable_rows, max_memtable_size, - // Two generations' worth, so a seal does not immediately re-park the + // Two generations' worth, so a freeze does not immediately re-park the // writer on a ceiling it cannot clear. max_unflushed_memtable_bytes: 2 * (reserved + max_memtable_size), ..Default::default() @@ -9377,7 +9377,7 @@ mod tests { .expect("a reservation that leaves room under the ceiling must open"); // Enough rows to carry row bytes past `max_memtable_size` several times - // over, so the run spans seals rather than sitting in one memtable. + // over, so the run spans freezes rather than sitting in one memtable. for round in 0..8i32 { let rows = 64; let ids: Vec = (0..rows).map(|i| round * rows + i).collect(); @@ -9401,11 +9401,11 @@ mod tests { .unwrap_or_else(|e| panic!("put in round {round} was refused: {e}")); } - // Without a seal this would only prove one memtable fits, which is not + // Without a freeze this would only prove one memtable fits, which is not // the case that stalls. assert!( writer.memtable_stats().await.unwrap().generation > 0, - "the run must cross a seal for the drain path to have been exercised" + "the run must cross a freeze for the drain path to have been exercised" ); writer.close().await.unwrap(); @@ -9718,7 +9718,7 @@ mod shard_writer_tests { writer.delete(vec![keys]).await.unwrap(); // The drain must not error on the empty HNSW. - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); // The tombstone-only generation still flushed (data without an HNSW index). @@ -10589,7 +10589,7 @@ mod shard_writer_tests { .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)]) .await .expect("Failed to write"); - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer .wait_for_flush_drain() .await @@ -10685,7 +10685,7 @@ mod shard_writer_tests { .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)]) .await .expect("Failed to write"); - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.expect("flush failed"); let manifest = writer.manifest().await.unwrap().expect("manifest exists"); @@ -10777,7 +10777,7 @@ mod shard_writer_tests { .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)]) .await .expect("Failed to write"); - writer.force_seal_active().await.unwrap(); + writer.force_freeze_active().await.unwrap(); writer.wait_for_flush_drain().await.expect("flush failed"); let manifest = writer.manifest().await.unwrap().expect("manifest exists"); diff --git a/rust/lance/tests/mem_wal/mod.rs b/rust/lance/tests/mem_wal/mod.rs index c617c2a1589..b3bfecbe4d2 100644 --- a/rust/lance/tests/mem_wal/mod.rs +++ b/rust/lance/tests/mem_wal/mod.rs @@ -88,7 +88,7 @@ async fn durable_put_does_not_alias_across_memtable_generations() { // Write and fully flush generation N, then rotate to generation N+1. writer_a.put(vec![first_batch]).await.unwrap(); let first_generation = writer_a.memtable_stats().await.unwrap().generation; - writer_a.force_seal_active().await.unwrap(); + writer_a.force_freeze_active().await.unwrap(); writer_a.wait_for_flush_drain().await.unwrap(); let current_generation = writer_a.memtable_stats().await.unwrap().generation; assert_eq!(