Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -732,7 +732,7 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
tokio::time::sleep(flush_wait).await;
}
}
// Wait for any triggered (sealed) memtable flushes to commit to the
// Wait for any triggered (frozen) memtable flushes to commit to the
// manifest before we snapshot it — otherwise the SSTables
// race the read and may not all be visible yet.
writer.wait_for_flush_drain().await?;
Expand Down
6 changes: 3 additions & 3 deletions rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -774,7 +774,7 @@ async fn run_lance(

// No-flush config: every *memtable*-flush threshold is set above the
// dataset so the single active MemTable holds all rows (no generation is
// sealed to disk). Read visibility is gated on the WAL durability
// frozen to disk). Read visibility is gated on the WAL durability
// watermark (`max_visible_batch_position`), which only advances on a WAL
// flush — so we use `durable_write=true`: each put flushes its batch to
// the WAL and awaits, which both populates the maintained BTree and
Expand All @@ -801,7 +801,7 @@ async fn run_lance(
let writer = dataset.mem_wal_writer(shard_id, config).await?;

// --- write phase ---
// LSM: split rows into `generations+1` parts; seal+flush after each of the
// LSM: split rows into `generations+1` parts; freeze+flush after each of the
// first `generations` parts (each becomes an on-disk generation) and leave
// the last part in the active MemTable.
let gens = args.generations;
Expand All @@ -827,7 +827,7 @@ async fn run_lance(
lo = hi;
}
if g < gens {
writer.force_seal_active().await?;
writer.force_freeze_active().await?;
for _ in 0..600 {
let n = writer
.manifest()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
//!
//! Measures lookup latency against three tiers of the LSM tree:
//! - Base table (on-disk, compacted data)
//! - SSTables (on-disk L0)
//! - SSTables (on-disk, uncompacted)
//! - Active MemTable (in-memory write buffer)
//!
//! Two phases, selected with `--phase`:
Expand Down
8 changes: 4 additions & 4 deletions rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
Original file line number Diff line number Diff line change
Expand Up @@ -440,12 +440,12 @@ async fn run_checkpoint(
use std::io::Write;
std::io::stdout().flush().ok();

// Flush mode: seal the active MemTable to a persistent on-disk generation
// Flush mode: freeze the active MemTable to a persistent on-disk generation
// (single-partition IVF_HNSW_SQ at the base dataset's storage version) and
// print its dataset path. Validates we can flush cp=100k/500k/1M.
if let Some(dir) = &local_dir {
let seal_start = Instant::now();
writer.force_seal_active().await?;
let freeze_start = Instant::now();
writer.force_freeze_active().await?;
let mut waited = 0u64;
let gen_path = loop {
if let Some(m) = writer.manifest().await?
Expand Down Expand Up @@ -474,7 +474,7 @@ async fn run_checkpoint(
"SSTABLE_OK cp={} id_offset={} flush_s={:.2} path={}",
cp,
id_offset,
seal_start.elapsed().as_secs_f64(),
freeze_start.elapsed().as_secs_f64(),
gen_path
);
std::io::stdout().flush().ok();
Expand Down
2 changes: 1 addition & 1 deletion rust/lance/src/dataset/mem_wal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ pub use sharding::{
evaluate_sharding_spec_with_source_columns,
};
pub use wal::{BatchDurableWatcher, WalAppendResult, WalAppender, WalReadEntry, WalTailer};
pub use write::SealFence;
pub use write::FreezeFence;
pub use write::ShardWriter;
pub use write::ShardWriterConfig;
pub use write::WriteResult;
Expand Down
26 changes: 13 additions & 13 deletions rust/lance/src/dataset/mem_wal/index/fts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -640,20 +640,20 @@ impl BatchMeta {
/// the tail's term map. An estimate — the node layout is crossbeam-internal.
const SKIPMAP_ENTRY_OVERHEAD: usize = 32;

/// Size of a sealed batch block. Small enough that copying the partial tail
/// block on append stays cheap; large enough that sealing (which clones the
/// Size of a frozen batch block. Small enough that copying the partial tail
/// block on append stays cheap; large enough that freezing (which clones the
/// block-pointer vec) is rare.
const BATCH_BLOCK: usize = 64;

/// Append-only, structurally-shared log of visible batch metadata. Sealed full
/// Append-only, structurally-shared log of visible batch metadata. Frozen full
/// blocks are immutable and shared across snapshots; only the current partial
/// block is copied on append, so publishing a batch is amortized O(1) (instead
/// of copying every batch pointer per publish — O(batches²) per generation)
/// while keeping O(1) index for `batch_for`.
#[derive(Debug, Clone)]
struct BatchLog {
/// Immutable full blocks (each `BATCH_BLOCK` long), shared across snapshots.
sealed: Arc<Vec<Arc<[Arc<BatchMeta>]>>>,
frozen: Arc<Vec<Arc<[Arc<BatchMeta>]>>>,
/// The current partial block (`< BATCH_BLOCK` entries).
tail: Arc<[Arc<BatchMeta>]>,
len: usize,
Expand All @@ -662,28 +662,28 @@ struct BatchLog {
impl BatchLog {
fn empty() -> Self {
Self {
sealed: Arc::new(Vec::new()),
frozen: Arc::new(Vec::new()),
tail: Arc::from(Vec::<Arc<BatchMeta>>::new().into_boxed_slice()),
len: 0,
}
}

/// A new log with `meta` appended; shares every sealed block with `self`,
/// A new log with `meta` appended; shares every frozen block with `self`,
/// copying only the partial tail block.
fn pushed(&self, meta: Arc<BatchMeta>) -> Self {
let mut tail: Vec<Arc<BatchMeta>> = self.tail.to_vec();
tail.push(meta);
if tail.len() == BATCH_BLOCK {
let mut sealed = (*self.sealed).clone();
sealed.push(Arc::from(tail.into_boxed_slice()));
let mut frozen = (*self.frozen).clone();
frozen.push(Arc::from(tail.into_boxed_slice()));
Self {
sealed: Arc::new(sealed),
frozen: Arc::new(frozen),
tail: Arc::from(Vec::<Arc<BatchMeta>>::new().into_boxed_slice()),
len: self.len + 1,
}
} else {
Self {
sealed: Arc::clone(&self.sealed),
frozen: Arc::clone(&self.frozen),
tail: Arc::from(tail.into_boxed_slice()),
len: self.len + 1,
}
Expand All @@ -693,15 +693,15 @@ impl BatchLog {
fn get(&self, i: usize) -> Option<&Arc<BatchMeta>> {
let block = i / BATCH_BLOCK;
let off = i % BATCH_BLOCK;
match self.sealed.get(block) {
match self.frozen.get(block) {
Some(b) => b.get(off),
None if block == self.sealed.len() => self.tail.get(off),
None if block == self.frozen.len() => self.tail.get(off),
None => None,
}
}

fn iter(&self) -> impl Iterator<Item = &Arc<BatchMeta>> {
self.sealed
self.frozen
.iter()
.flat_map(|b| b.iter())
.chain(self.tail.iter())
Expand Down
8 changes: 4 additions & 4 deletions rust/lance/src/dataset/mem_wal/memtable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,8 @@ const PK_BLOOM_FILTER_FPP: f64 = 0.00057;
/// measurement, and a memory view need not carry it per memtable.
///
/// Counted as index memory rather than row data: it is an auxiliary lookup
/// structure, and a fixed term in the `max_memtable_size` seal trigger would
/// make every memtable seal a constant early.
/// structure, and a fixed term in the `max_memtable_size` freeze trigger would
/// make every memtable freeze a constant early.
pub fn pk_bloom_filter_bytes() -> usize {
static BYTES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
*BYTES.get_or_init(|| {
Expand Down Expand Up @@ -737,7 +737,7 @@ impl MemTable {
}

/// Whether every committed batch in this memtable is WAL-durable, given the
/// writer-global durability cursor. The L0 flush's precondition.
/// writer-global durability cursor. The SSTable flush's precondition.
pub fn all_flushed_to_wal(&self, durable: usize) -> bool {
self.batch_store.pending_wal_flush_count(durable) == 0
}
Expand Down Expand Up @@ -903,7 +903,7 @@ mod tests {
assert_eq!(total_rows, 15);
}

/// `all_flushed_to_wal(durable)` is the L0 flush's precondition (`flush.rs:171`):
/// `all_flushed_to_wal(durable)` is the SSTable flush's precondition (`flush.rs:171`):
/// false while any committed batch is still un-appended, true once the
/// durability watermark covers every one of them.
#[tokio::test]
Expand Down
2 changes: 1 addition & 1 deletion rust/lance/src/dataset/mem_wal/memtable/flush.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1254,7 +1254,7 @@ mod tests {
.await
.unwrap();

// Nothing is durable yet, so the L0 flush must refuse.
// Nothing is durable yet, so the SSTable flush must refuse.
let durable = 0;
assert!(!memtable.all_flushed_to_wal(durable));

Expand Down
2 changes: 1 addition & 1 deletion rust/lance/src/dataset/mem_wal/observer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ pub trait WalObserver: Send + Sync + Debug {
/// `durable_write` put waits on.
fn on_wal_flush(&self, _duration: Duration, _bytes: usize) {}

/// A frozen memtable became an L0 SSTable. Orders of magnitude longer
/// A frozen memtable became an SSTable. Orders of magnitude longer
/// than a WAL flush.
fn on_memtable_flush(&self, _duration: Duration, _rows: usize) {}
}
2 changes: 1 addition & 1 deletion rust/lance/src/dataset/mem_wal/scanner/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ impl LsmScanner {
/// SSTables) without including a base Lance table.
///
/// This is useful when the caller owns the base read path separately and
/// only needs the WAL's contribution: active memtable ∪ L0 SSTables.
/// only needs the WAL's contribution: active memtable ∪ SSTables.
/// Deduplication semantics are unchanged — newer generations
/// still win on PK conflicts.
///
Expand Down
6 changes: 3 additions & 3 deletions rust/lance/src/dataset/mem_wal/scanner/collector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ impl InMemoryMemTableRef {
/// Row-data bytes: the buffered batches.
///
/// This is the **flush unit**, not the memtable's footprint — it drives the
/// `max_memtable_size` seal trigger, so it stays a function of the rows in
/// `max_memtable_size` freeze trigger, so it stays a function of the rows in
/// it. Use [`Self::resident_bytes`] to budget memory.
pub fn row_bytes(&self) -> usize {
self.batch_store.row_bytes()
Expand Down Expand Up @@ -91,7 +91,7 @@ pub struct InMemoryMemTables {
/// When the base table is omitted (see [`Self::without_base_table`]), `collect`
/// returns only SSTable and active-memtable sources. This is used
/// by callers that own the base read path elsewhere and only need the WAL's
/// fresh tier (active memtable ∪ L0 SSTables).
/// fresh tier (active memtable ∪ SSTables).
pub struct LsmDataSourceCollector {
/// Base Lance table (None when scanning only the fresh tier).
base_table: Option<Arc<Dataset>>,
Expand Down Expand Up @@ -241,7 +241,7 @@ impl LsmDataSourceCollector {
/// scan sources, in **ascending generation order**. The planner relies
/// on this: it reverses sources to generation-DESC so the newest row
/// wins the dedup tiebreaker (see `LsmScanPlanner::plan_scan`). Active
/// is the newest generation; frozen are older sealed ones — so without
/// is the newest generation; the frozen ones are older — so without
/// this sort a stale frozen row could outrank a re-write in the active
/// memtable for the same pk.
fn in_memory_sources(shard_id: Uuid, mems: &InMemoryMemTables) -> Vec<LsmDataSource> {
Expand Down
4 changes: 2 additions & 2 deletions rust/lance/src/dataset/mem_wal/scanner/planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -818,7 +818,7 @@ mod integration_tests {
assert_eq!(results.get(&6), Some(&"active_6".to_string()));
}

/// Regression for the concurrent-read-vs-flush hole: a sealed
/// Regression for the concurrent-read-vs-flush hole: a frozen
/// (frozen-awaiting-flush) memtable is not yet recorded as an
/// SSTable, but its rows must still be in the scan's read union and
/// dedup correctly by generation across the active/frozen seam.
Expand Down Expand Up @@ -855,7 +855,7 @@ mod integration_tests {
.with_sstable(1, "gen_1".to_string())
.with_sstable(2, "gen_2".to_string());

// Frozen gen3 (sealed, NOT in the manifest) and active gen4.
// Frozen gen3 (NOT in the manifest) and active gen4.
let (frozen_store, frozen_index) =
pk_indexed(&[create_test_batch(&schema, &[6, 7], "frozen")]);
let frozen = InMemoryMemTableRef {
Expand Down
2 changes: 1 addition & 1 deletion rust/lance/src/dataset/mem_wal/scanner/vector_search.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3350,7 +3350,7 @@ mod tests {
assert_eq!(
id1.len(),
1,
"newest-wins: id=1 must appear exactly once after a same-L0 override, got {:?}",
"newest-wins: id=1 must appear exactly once after a same-generation override, got {:?}",
rows
);
assert!(
Expand Down
4 changes: 2 additions & 2 deletions rust/lance/src/dataset/mem_wal/wal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1788,8 +1788,8 @@ mod tests {
#[tokio::test]
async fn test_track_batch_watcher_blocks_until_flush() {
let (store, base_path, _temp_dir) = create_local_store().await;
let region_id = Uuid::new_v4();
let flusher = build_test_flusher(store, &base_path, region_id, 1);
let shard_id = Uuid::new_v4();
let flusher = build_test_flusher(store, &base_path, shard_id, 1);

let schema = create_test_schema();
let batch_store = Arc::new(BatchStore::with_capacity(10));
Expand Down
Loading
Loading