diff --git a/Cargo.toml b/Cargo.toml index 93bf735..8cd6a41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ categories = ["database-implementations", "data-structures", "caching"] [features] perf_measurements = ["dep:performance_measurement", "dep:performance_measurement_codegen"] s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktable_codegen/s3-support"] +versioned-row-publication = ["worktable_codegen/versioned-row-publication"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/README.md b/README.md index d2d09f3..f4a487c 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,30 @@ S3 support layers *on top of* the disk engine rather than replacing it. worktable = { version = "0.9", features = ["s3-support"] } # S3 sync, optional ``` +## Concurrent read/write publication + +The default build preserves the existing lowest-latency page path and requires +applications to exclude reads that overlap page-byte mutation. Applications +that need generated reads to overlap updates, inserts, deletes, and vacuum can +opt into immutable row-version publication: + +```toml +[dependencies] +worktable = { version = "0.9", features = ["versioned-row-publication"] } +``` + +In this mode, generated reads acquire an immutable owned row version instead +of borrowing the mutable archived page image. Writers replace a per-row version +only after a complete page mutation, insert visibility is an atomic lifecycle +transition after every index is installed, and deleted or relocated links are +not reused until readers that could have captured them have drained. Page bytes +remain the persistence image and are internally serialized; range queries are +still non-snapshot reads. The mode intentionally trades memory, an atomic +read-side grace-period counter, and publication bookkeeping for this stronger +concurrent-read contract. See +[`docs/versioned-row-publication.md`](docs/versioned-row-publication.md) for the +protocol and its scope. + ## Relationship to `data_bucket` WorkTable is built on [`data_bucket`](https://crates.io/crates/data_bucket), which @@ -396,4 +420,3 @@ enum WorkTableError Check out - [Examples](./examples) - diff --git a/benches/cases/unique_index.rs b/benches/cases/unique_index.rs index 7eb69e7..acf75f5 100644 --- a/benches/cases/unique_index.rs +++ b/benches/cases/unique_index.rs @@ -1,6 +1,7 @@ use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, black_box, criterion_group}; use std::sync::Arc; use tokio::runtime::Runtime; +use worktable::prelude::SelectQueryExecutor; use crate::common::*; @@ -64,6 +65,26 @@ fn select_by_unique_index(c: &mut Criterion) { }); } +fn select_by_unique_index_range(c: &mut Criterion) { + let table = UniqueIndexWorkTable::default(); + + for i in 1..=1000i64 { + let row = UniqueIndexRow { + id: table.get_next_pk().into(), + test: i, + another: i as u64, + }; + table.insert(row).unwrap(); + } + + c.bench_function("unique_index_select_by_test_range", |b| { + b.iter(|| { + let test = fastrand::i64(1..=1000); + black_box(table.select_by_test_range(test..=test).execute()) + }) + }); +} + fn update(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let table = Arc::new(UniqueIndexWorkTable::default()); @@ -224,6 +245,7 @@ criterion_group! { insert, select_by_pk, select_by_unique_index, + select_by_unique_index_range, update, delete, upsert_insert, diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 05ace59..e6c5f78 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -8,6 +8,7 @@ repository = "https://github.com/pathscale/WorkTable" [features] s3-support = [] +versioned-row-publication = [] [lib] name = "worktable_codegen" diff --git a/codegen/src/generators/in_memory/queries/select.rs b/codegen/src/generators/in_memory/queries/select.rs index a86ddea..7ba14ec 100644 --- a/codegen/src/generators/in_memory/queries/select.rs +++ b/codegen/src/generators/in_memory/queries/select.rs @@ -29,9 +29,13 @@ impl InMemoryGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let iter = self.0.primary_index.pk_map .iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/in_memory/table/impls.rs b/codegen/src/generators/in_memory/table/impls.rs index b647c68..4f42537 100644 --- a/codegen/src/generators/in_memory/table/impls.rs +++ b/codegen/src/generators/in_memory/table/impls.rs @@ -110,9 +110,13 @@ impl InMemoryGenerator { range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); + let read_guard = self.0.data.read_guard(); let rows = self.0.primary_index.pk_map .range(converted_range) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); #pk_sorted_by } @@ -262,6 +266,7 @@ impl InMemoryGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { + let _read_guard = self.0.data.read_guard(); let first = self.0.primary_index.pk_map.iter().next().map(|(k, v)| (k.clone(), v.0)); let Some((mut k, link)) = first else { return Ok(()) diff --git a/codegen/src/generators/in_memory/table/index_fns.rs b/codegen/src/generators/in_memory/table/index_fns.rs index 0a73cce..e82d75f 100644 --- a/codegen/src/generators/in_memory/table/index_fns.rs +++ b/codegen/src/generators/in_memory/table/index_fns.rs @@ -83,13 +83,39 @@ impl InMemoryGenerator { row.#row_field_ident.eq(&by) } }; - - Ok(quote! { - pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { + let select = if cfg!(feature = "versioned-row-publication") { + quote! { + loop { + let link: Link = self.0.indexes.#field_ident + .get(#by) + .map(|kv| kv.get().value.into())?; + if let Ok(row) = self.0.data.select_non_ghosted(link) { + if #predicate_matches { + return Some(row); + } + } + + let current_link: Option = self.0.indexes.#field_ident + .get(#by) + .map(|kv| kv.get().value.into()); + if current_link == Some(link) { + return None; + } + } + } + } else { + quote! { let link: Link = self.0.indexes.#field_ident.get(#by).map(|kv| kv.get().value.into())?; let row = self.0.data.select_non_ghosted(link).ok()?; #predicate_matches.then_some(row) } + }; + + Ok(quote! { + pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { + let _read_guard = self.0.data.read_guard(); + #select + } }) } @@ -121,10 +147,14 @@ impl InMemoryGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let rows = self.0.indexes.#field_ident .get(#by) .into_iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()) + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) .filter(move |r| &r.#row_field_ident == &by); SelectQueryBuilder::new(rows) @@ -143,21 +173,52 @@ impl InMemoryGenerator { let type_ = columns_map.get(i).ok_or(syn::Error::new(i.span(), "Row not found"))?; let fn_name = Ident::new(format!("select_by_{i}_range").as_str(), Span::mixed_site()); let field_ident = &idx.name; + let row_field_ident = &idx.field; let column_pascal = Ident::new(&i.to_string().to_case(Case::Pascal), Span::mixed_site()); + let revalidate = cfg!(feature = "versioned-row-publication"); let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( quote! { std::ops::RangeBounds<#type_> }, - quote! { + if revalidate { + quote! { + ( + predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), + predicate_range.1.as_ref().map(|v| OrderedFloat(*v)), + ) + } + } else { + quote! { ( range.start_bound().map(|v| OrderedFloat(*v)), range.end_bound().map(|v| OrderedFloat(*v)), ) + } }, ) + } else if revalidate { + ( + quote! { std::ops::RangeBounds<#type_> }, + quote! { predicate_range.clone() }, + ) } else { (quote! { std::ops::RangeBounds<#type_> }, quote! { range }) }; + let predicate_setup = revalidate.then(|| { + quote! { + let predicate_range = ( + range.start_bound().cloned(), + range.end_bound().cloned(), + ); + } + }); + let predicate_filter = revalidate.then(|| { + quote! { + .filter(move |row| { + std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + }) + } + }); Ok(quote! { pub fn #fn_name(&self, range: R) -> SelectQueryBuilder<#row_ident, @@ -167,9 +228,15 @@ impl InMemoryGenerator { where R: #range_bounds { + #predicate_setup + let read_guard = self.0.data.read_guard(); let rows = self.0.indexes.#field_ident .range(#range_arg) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + #predicate_filter; SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/codegen/src/generators/persist/queries/select.rs b/codegen/src/generators/persist/queries/select.rs index 5db8e71..84501b2 100644 --- a/codegen/src/generators/persist/queries/select.rs +++ b/codegen/src/generators/persist/queries/select.rs @@ -29,9 +29,13 @@ impl PersistGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let iter = self.0.primary_index.pk_map .iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index eafdc3a..7481669 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -196,9 +196,13 @@ impl PersistGenerator { range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); + let read_guard = self.0.data.read_guard(); let rows = self.0.primary_index.pk_map .range(converted_range) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); #pk_sorted_by } @@ -369,6 +373,7 @@ impl PersistGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { + let _read_guard = self.0.data.read_guard(); let first = self.0.primary_index.pk_map.iter().next().map(|(k, v)| (k.clone(), v.0)); let Some((mut k, link)) = first else { return Ok(()) diff --git a/codegen/src/generators/persist/table/index_fns.rs b/codegen/src/generators/persist/table/index_fns.rs index eb1ac14..8768de4 100644 --- a/codegen/src/generators/persist/table/index_fns.rs +++ b/codegen/src/generators/persist/table/index_fns.rs @@ -83,13 +83,39 @@ impl PersistGenerator { row.#row_field_ident.eq(&by) } }; - - Ok(quote! { - pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { + let select = if cfg!(feature = "versioned-row-publication") { + quote! { + loop { + let link: Link = self.0.indexes.#field_ident + .get(#by) + .map(|kv| kv.get().value.into())?; + if let Ok(row) = self.0.data.select_non_ghosted(link) { + if #predicate_matches { + return Some(row); + } + } + + let current_link: Option = self.0.indexes.#field_ident + .get(#by) + .map(|kv| kv.get().value.into()); + if current_link == Some(link) { + return None; + } + } + } + } else { + quote! { let link: Link = self.0.indexes.#field_ident.get(#by).map(|kv| kv.get().value.into())?; let row = self.0.data.select_non_ghosted(link).ok()?; #predicate_matches.then_some(row) } + }; + + Ok(quote! { + pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { + let _read_guard = self.0.data.read_guard(); + #select + } }) } @@ -121,10 +147,14 @@ impl PersistGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let rows = self.0.indexes.#field_ident .get(#by) .into_iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()) + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) .filter(move |r| &r.#row_field_ident == &by); SelectQueryBuilder::new(rows) @@ -143,21 +173,52 @@ impl PersistGenerator { let type_ = columns_map.get(i).ok_or(syn::Error::new(i.span(), "Row not found"))?; let fn_name = Ident::new(format!("select_by_{i}_range").as_str(), Span::mixed_site()); let field_ident = &idx.name; + let row_field_ident = &idx.field; let column_pascal = Ident::new(&i.to_string().to_case(Case::Pascal), Span::mixed_site()); + let revalidate = cfg!(feature = "versioned-row-publication"); let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( quote! { std::ops::RangeBounds<#type_> }, - quote! { + if revalidate { + quote! { + ( + predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), + predicate_range.1.as_ref().map(|v| OrderedFloat(*v)), + ) + } + } else { + quote! { ( range.start_bound().map(|v| OrderedFloat(*v)), range.end_bound().map(|v| OrderedFloat(*v)), ) + } }, ) + } else if revalidate { + ( + quote! { std::ops::RangeBounds<#type_> }, + quote! { predicate_range.clone() }, + ) } else { (quote! { std::ops::RangeBounds<#type_> }, quote! { range }) }; + let predicate_setup = revalidate.then(|| { + quote! { + let predicate_range = ( + range.start_bound().cloned(), + range.end_bound().cloned(), + ); + } + }); + let predicate_filter = revalidate.then(|| { + quote! { + .filter(move |row| { + std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + }) + } + }); Ok(quote! { pub fn #fn_name(&self, range: R) -> SelectQueryBuilder<#row_ident, @@ -167,9 +228,15 @@ impl PersistGenerator { where R: #range_bounds { + #predicate_setup + let read_guard = self.0.data.read_guard(); let rows = self.0.indexes.#field_ident .range(#range_arg) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + #predicate_filter; SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/codegen/src/generators/read_only/queries/select.rs b/codegen/src/generators/read_only/queries/select.rs index 9e700ad..51dda41 100644 --- a/codegen/src/generators/read_only/queries/select.rs +++ b/codegen/src/generators/read_only/queries/select.rs @@ -29,9 +29,13 @@ impl ReadOnlyGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let iter = self.0.primary_index.pk_map .iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/read_only/table/impls.rs b/codegen/src/generators/read_only/table/impls.rs index c1ffd3a..0d0cb5a 100644 --- a/codegen/src/generators/read_only/table/impls.rs +++ b/codegen/src/generators/read_only/table/impls.rs @@ -192,9 +192,13 @@ impl ReadOnlyGenerator { range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); + let read_guard = self.0.data.read_guard(); let rows = self.0.primary_index.pk_map .range(converted_range) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); #pk_sorted_by } @@ -277,6 +281,7 @@ impl ReadOnlyGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { + let _read_guard = self.0.data.read_guard(); let first = self.0.primary_index.pk_map.iter().next().map(|(k, v)| (k.clone(), v.0)); let Some((mut k, link)) = first else { return Ok(()) diff --git a/codegen/src/generators/read_only/table/index_fns.rs b/codegen/src/generators/read_only/table/index_fns.rs index f626ad7..82738c7 100644 --- a/codegen/src/generators/read_only/table/index_fns.rs +++ b/codegen/src/generators/read_only/table/index_fns.rs @@ -83,13 +83,39 @@ impl ReadOnlyGenerator { row.#row_field_ident.eq(&by) } }; - - Ok(quote! { - pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { + let select = if cfg!(feature = "versioned-row-publication") { + quote! { + loop { + let link: Link = self.0.indexes.#field_ident + .get(#by) + .map(|kv| kv.get().value.into())?; + if let Ok(row) = self.0.data.select_non_ghosted(link) { + if #predicate_matches { + return Some(row); + } + } + + let current_link: Option = self.0.indexes.#field_ident + .get(#by) + .map(|kv| kv.get().value.into()); + if current_link == Some(link) { + return None; + } + } + } + } else { + quote! { let link: Link = self.0.indexes.#field_ident.get(#by).map(|kv| kv.get().value.into())?; let row = self.0.data.select_non_ghosted(link).ok()?; #predicate_matches.then_some(row) } + }; + + Ok(quote! { + pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { + let _read_guard = self.0.data.read_guard(); + #select + } }) } @@ -121,10 +147,14 @@ impl ReadOnlyGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let rows = self.0.indexes.#field_ident .get(#by) .into_iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()) + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) .filter(move |r| &r.#row_field_ident == &by); SelectQueryBuilder::new(rows) @@ -143,21 +173,52 @@ impl ReadOnlyGenerator { let type_ = columns_map.get(i).ok_or(syn::Error::new(i.span(), "Row not found"))?; let fn_name = Ident::new(format!("select_by_{i}_range").as_str(), Span::mixed_site()); let field_ident = &idx.name; + let row_field_ident = &idx.field; let column_pascal = Ident::new(&i.to_string().to_case(Case::Pascal), Span::mixed_site()); + let revalidate = cfg!(feature = "versioned-row-publication"); let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( quote! { std::ops::RangeBounds<#type_> }, - quote! { + if revalidate { + quote! { + ( + predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), + predicate_range.1.as_ref().map(|v| OrderedFloat(*v)), + ) + } + } else { + quote! { ( range.start_bound().map(|v| OrderedFloat(*v)), range.end_bound().map(|v| OrderedFloat(*v)), ) + } }, ) + } else if revalidate { + ( + quote! { std::ops::RangeBounds<#type_> }, + quote! { predicate_range.clone() }, + ) } else { (quote! { std::ops::RangeBounds<#type_> }, quote! { range }) }; + let predicate_setup = revalidate.then(|| { + quote! { + let predicate_range = ( + range.start_bound().cloned(), + range.end_bound().cloned(), + ); + } + }); + let predicate_filter = revalidate.then(|| { + quote! { + .filter(move |row| { + std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + }) + } + }); Ok(quote! { pub fn #fn_name(&self, range: R) -> SelectQueryBuilder<#row_ident, @@ -167,9 +228,15 @@ impl ReadOnlyGenerator { where R: #range_bounds { + #predicate_setup + let read_guard = self.0.data.read_guard(); let rows = self.0.indexes.#field_ident .range(#range_arg) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + #predicate_filter; SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/docs/versioned-row-publication.md b/docs/versioned-row-publication.md new file mode 100644 index 0000000..fbcc289 --- /dev/null +++ b/docs/versioned-row-publication.md @@ -0,0 +1,86 @@ +# Versioned row publication + +Status: feature-gated prototype behind `versioned-row-publication`. + +## Problem + +The original fast path deserializes directly from archived bytes in an +`UnsafeCell` page while same-size updates mutate those bytes in place. Returning +an owned row prevents references from escaping, but it does not make a read +that overlaps a write data-race-free: the deserializer can still access bytes +while a writer changes them. A sequence counter cannot repair this in Rust, +because detecting a race after it happened does not make the racy byte access +defined. + +Physical links introduce a second issue. A reader can obtain a link from an +index, pause, and resume after delete or vacuum has reused that address for a +different row. Predicate revalidation helps, but reclamation must also cover +the interval from index lookup through acquisition of a stable row version. + +## Protocol + +With the feature enabled, `DataPages` maintains two representations: + +- Archived page bytes are the compact persistence and mutation image. All + accesses that can overlap a mutation are serialized by an internal page + barrier. +- A concurrent link map holds an immutable application-visible row version. + Each slot contains an `Arc` behind a short per-slot pointer lock and + atomic ghost, deleted, and vacuum lifecycle bits. + +The generated API follows these publication rules: + +1. **Read.** Acquire a read-grace guard before consulting an index. Resolve the + link, acquire an `Arc` to its complete published version, check lifecycle and + index predicates, clone the owned row, and release the guard. Unique and + primary-key point reads retry when the mapping swings to a replacement link + while it is being resolved. A reader never accesses mutable archived bytes + after a slot has been hydrated. +2. **Insert.** Serialize the complete row and stage a ghosted version. Install + the primary and secondary indexes. Only after every checked index insert + succeeds does the lifecycle transition publish the version with release + ordering. Failed inserts retire an unpublished version. +3. **Update.** Hold the generated row/field lock, mutate the archived image + under the page barrier, deserialize the completed wrapper, then replace the + immutable version. A concurrent reader can return the complete old version + or the complete new version, never a partially updated row. +4. **Delete.** Remove index reachability, mark the version deleted, and retire + its physical link. The empty-link allocator cannot reuse it while a reader + that could have captured the old index entry remains active. +5. **Vacuum.** Copy a complete row to a staged destination version, swing its + indexes, and retire the source publication and page. Retired links, slots, + and pages become reusable only after a read-side grace period. +6. **Reload.** Persisted tables hydrate immutable slots lazily under the page + barrier. Subsequent generated reads use the published version map. + +The grace period is quiescent-state reclamation: a feature-only atomic counter +tracks generated reads, and retirement queues are drained when that counter is +zero. `Arc` ownership independently keeps a version alive after a reader has +acquired it. + +## Guarantees and non-guarantees + +For generated table APIs in this mode: + +- reads do not race with mutation of archived page bytes; +- a read returns a complete row version; +- ghosted or deleted versions are not returned; +- a retired physical link is not reused while a pre-existing generated read + can still resolve it; and +- unique, non-unique point, and secondary-index range lookups revalidate each + resolved row predicate. + +This is not MVCC and does not add multi-operation transactions or snapshot +range scans. A scan may include or omit a concurrently inserted or updated row. +Point-read retry may starve under perpetual replacement churn. +The guarantee also does not cover callers that bypass generated table methods +and directly invoke low-level `Data` page mutation APIs. + +## Cost model and rollout + +The feature is off by default. It adds one owned row copy plus slot/map +metadata per live physical link, an atomic increment/decrement per generated +read, a concurrent publication-map lookup, and writer-side page serialization. +Those costs are inappropriate to impose silently on latency-sensitive users. +The default path remains unchanged; benchmark results for both modes must be +reported before this feature is proposed for default enablement. diff --git a/src/in_memory/mod.rs b/src/in_memory/mod.rs index 2b02cf0..0da4038 100644 --- a/src/in_memory/mod.rs +++ b/src/in_memory/mod.rs @@ -1,9 +1,11 @@ mod data; mod empty_link_registry; mod pages; +#[cfg(feature = "versioned-row-publication")] +mod publication; mod row; pub use data::{DATA_INNER_LENGTH, Data, ExecutionError as DataExecutionError}; pub use empty_link_registry::EmptyLinkRegistry; -pub use pages::{DataPages, ExecutionError as PagesExecutionError}; -pub use row::{ArchivedRowWrapper, Query, RowWrapper, StorableRow}; +pub use pages::{DataPages, ExecutionError as PagesExecutionError, ReadGuard as DataPagesReadGuard}; +pub use row::{ArchivedRowWrapper, PublicationSafe, Query, RowWrapper, StorableRow}; diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index 08a41fb..475a3ba 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -1,5 +1,7 @@ use data_bucket::page::PageId; use derive_more::{Display, Error, From}; +#[cfg(feature = "versioned-row-publication")] +use parking_lot::Mutex; use parking_lot::RwLock; #[cfg(feature = "perf_measurements")] use performance_measurement_codegen::performance_measurement; @@ -10,7 +12,12 @@ use rkyv::{ ser::{Serializer, allocator::ArenaHandle, sharing::Share}, util::AlignedVec, }; +#[cfg(feature = "versioned-row-publication")] +use std::collections::HashMap; use std::collections::VecDeque; +#[cfg(feature = "versioned-row-publication")] +use std::hash::{BuildHasherDefault, Hasher}; +use std::marker::PhantomData; use std::{ fmt::Debug, sync::Arc, @@ -18,7 +25,11 @@ use std::{ }; use crate::in_memory::empty_link_registry::EmptyLinkRegistry; +#[cfg(feature = "versioned-row-publication")] +use crate::in_memory::publication::{DELETED, GHOSTED, PublishedRow, VACUUMED}; use crate::prelude::ArchivedRowWrapper; +#[cfg(feature = "versioned-row-publication")] +use crate::util::OffsetEqLink; use crate::{ in_memory::{ DATA_INNER_LENGTH, Data, DataExecutionError, @@ -31,11 +42,78 @@ fn page_id_mapper(page_id: usize) -> usize { page_id - 1usize } +/// `OffsetEqLink` already reduces publication keys to a trusted internal u64 +/// storage offset. Avoid hashing that offset again on every versioned read. +#[cfg(feature = "versioned-row-publication")] +#[derive(Default)] +struct PublicationHasher(u64); + +#[cfg(feature = "versioned-row-publication")] +impl Hasher for PublicationHasher { + fn finish(&self) -> u64 { + self.0 + } + + fn write(&mut self, bytes: &[u8]) { + let mut hash = 0xcbf29ce484222325u64; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + self.0 = hash; + } + + fn write_u64(&mut self, value: u64) { + self.0 = value; + } +} + +#[cfg(feature = "versioned-row-publication")] +type PublicationMap = + HashMap, Arc>, BuildHasherDefault>; + +pub struct ReadGuard<'a> { + #[cfg(feature = "versioned-row-publication")] + active_readers: &'a AtomicU64, + marker: PhantomData<&'a ()>, +} + +impl Drop for ReadGuard<'_> { + fn drop(&mut self) { + #[cfg(feature = "versioned-row-publication")] + self.active_readers.fetch_sub(1, Ordering::SeqCst); + } +} + #[derive(Debug)] pub struct DataPages where Row: StorableRow, { + /// Immutable application-visible row versions. Published readers never + /// borrow the mutable archived page image. + #[cfg(feature = "versioned-row-publication")] + published_rows: RwLock>, + + /// Protects the mutable page image used by writers, vacuum, and + /// persistence. Application reads use `published_rows` after hydration. + #[cfg(feature = "versioned-row-publication")] + page_access: RwLock<()>, + + /// Read-side grace period protecting the interval from index lookup until + /// an immutable row version has been acquired. + #[cfg(feature = "versioned-row-publication")] + active_readers: AtomicU64, + + #[cfg(feature = "versioned-row-publication")] + retired_links: Mutex>, + + #[cfg(feature = "versioned-row-publication")] + retired_pages: Mutex>, + + #[cfg(feature = "versioned-row-publication")] + retired_publications: Mutex>>, + /// Pages vector. Currently, not lock free. pages: RwLock::WrappedRow, DATA_LENGTH>>>>, @@ -66,8 +144,126 @@ where Row: StorableRow, ::WrappedRow: RowWrapper, { + #[cfg(feature = "versioned-row-publication")] + fn publication_flags(row: &::WrappedRow) -> u8 { + let mut flags = 0; + if row.is_ghosted() { + flags |= GHOSTED; + } + if row.is_deleted() { + flags |= DELETED; + } + if row.is_vacuumed() { + flags |= VACUUMED; + } + flags + } + + #[cfg(feature = "versioned-row-publication")] + fn publish_wrapped_row(&self, link: Link, wrapped: ::WrappedRow) { + let flags = Self::publication_flags(&wrapped); + let row = wrapped.get_inner(); + let key = OffsetEqLink(link); + + let mut published_rows = self.published_rows.write(); + if let Some(slot) = published_rows.get(&key).cloned() { + drop(published_rows); + slot.replace(row, flags); + } else { + published_rows.insert(key, Arc::new(PublishedRow::new(row, flags))); + } + } + + #[cfg(feature = "versioned-row-publication")] + fn stage_published_row(&self, link: Link, row: Row) { + let wrapped = ::WrappedRow::from_inner(row); + self.publish_wrapped_row(link, wrapped); + } + + #[cfg(feature = "versioned-row-publication")] + fn published_slot(&self, link: Link) -> Option>> { + self.published_rows.read().get(&OffsetEqLink(link)).cloned() + } + + #[cfg(feature = "versioned-row-publication")] + fn published_slot_or_hydrate(&self, link: Link) -> Result>, ExecutionError> + where + <::WrappedRow as Archive>::Archived: + Deserialize<::WrappedRow, HighDeserializer>, + { + if let Some(slot) = self.published_slot(link) { + return Ok(slot); + } + + let _page_access = self.page_access.read(); + if let Some(slot) = self.published_slot(link) { + return Ok(slot); + } + + let pages = self.pages.read(); + let page = pages + .get(page_id_mapper(link.page_id.into())) + .ok_or(ExecutionError::PageNotFound(link.page_id))?; + let wrapped = page.get_row(link).map_err(ExecutionError::DataPageError)?; + let flags = Self::publication_flags(&wrapped); + let slot = Arc::new(PublishedRow::new(wrapped.get_inner(), flags)); + let mut published_rows = self.published_rows.write(); + Ok(published_rows.entry(OffsetEqLink(link)).or_insert(slot).clone()) + } + + pub fn read_guard(&self) -> ReadGuard<'_> { + #[cfg(feature = "versioned-row-publication")] + self.active_readers.fetch_add(1, Ordering::SeqCst); + + ReadGuard { + #[cfg(feature = "versioned-row-publication")] + active_readers: &self.active_readers, + marker: PhantomData, + } + } + + #[cfg(feature = "versioned-row-publication")] + fn reclaim_retired(&self) { + if self.active_readers.load(Ordering::SeqCst) != 0 { + return; + } + + let mut retired_links = self.retired_links.lock(); + let mut retired_pages = self.retired_pages.lock(); + let mut retired_publications = self.retired_publications.lock(); + if self.active_readers.load(Ordering::SeqCst) != 0 { + return; + } + + let mut published_rows = self.published_rows.write(); + for link in retired_links.drain(..) { + published_rows.remove(&OffsetEqLink(link)); + self.empty_links.push(link); + } + for link in retired_publications.drain(..) { + published_rows.remove(&link); + } + drop(published_rows); + if !retired_pages.is_empty() { + let mut empty_pages = self.empty_pages.write(); + empty_pages.extend(retired_pages.drain(..)); + } + } + pub fn new() -> Self { Self { + #[cfg(feature = "versioned-row-publication")] + published_rows: RwLock::new(PublicationMap::default()), + #[cfg(feature = "versioned-row-publication")] + page_access: RwLock::new(()), + #[cfg(feature = "versioned-row-publication")] + active_readers: AtomicU64::new(0), + #[cfg(feature = "versioned-row-publication")] + retired_links: Mutex::new(Vec::new()), + #[cfg(feature = "versioned-row-publication")] + retired_pages: Mutex::new(Vec::new()), + #[cfg(feature = "versioned-row-publication")] + retired_publications: Mutex::new(Vec::new()), // We are starting ID's from `1` because `0`'s page in file is info page. pages: RwLock::new(vec![Arc::new(Data::new(1.into()))]), empty_links: EmptyLinkRegistry::::default(), @@ -85,6 +281,18 @@ where } else { let last_page_id = vec.len(); Self { + #[cfg(feature = "versioned-row-publication")] + published_rows: RwLock::new(PublicationMap::default()), + #[cfg(feature = "versioned-row-publication")] + page_access: RwLock::new(()), + #[cfg(feature = "versioned-row-publication")] + active_readers: AtomicU64::new(0), + #[cfg(feature = "versioned-row-publication")] + retired_links: Mutex::new(Vec::new()), + #[cfg(feature = "versioned-row-publication")] + retired_pages: Mutex::new(Vec::new()), + #[cfg(feature = "versioned-row-publication")] + retired_publications: Mutex::new(Vec::new()), pages: RwLock::new(vec), empty_links: EmptyLinkRegistry::default(), empty_pages: Default::default(), @@ -97,13 +305,20 @@ where pub fn insert(&self, row: Row) -> Result where - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + Row: Archive + + Clone + + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, { - let general_row = ::WrappedRow::from_inner(row); + let general_row = ::WrappedRow::from_inner(row.clone()); + + #[cfg(feature = "versioned-row-publication")] + self.reclaim_retired(); if let Some(link) = self.empty_links.pop_max() { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); let pages = self.pages.read(); let current_page: usize = page_id_mapper(link.page_id.into()); let page = &pages[current_page]; @@ -113,6 +328,8 @@ where if let Some(l) = left_link { self.empty_links.push(l); } + #[cfg(feature = "versioned-row-publication")] + self.stage_published_row(link, row.clone()); return Ok(link); } Err(e) => match e { @@ -129,11 +346,19 @@ where loop { let (link, tried_page) = { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); let pages = self.pages.read(); let current_page = page_id_mapper(self.current_page_id.load(Ordering::Acquire) as usize); let page = &pages[current_page]; - (page.save_row(&general_row), current_page) + let link = page.save_row(&general_row); + #[cfg(feature = "versioned-row-publication")] + if let Ok(saved_link) = &link { + self.stage_published_row(*saved_link, row.clone()); + } + + (link, current_page) }; match link { Ok(link) => { @@ -191,12 +416,17 @@ where /// Allocates a new page or reuses a free page from `empty_pages`. /// Does **NOT** set the page as `current`. pub fn allocate_new_or_pop_free(&self) -> Arc::WrappedRow, DATA_LENGTH>> { + #[cfg(feature = "versioned-row-publication")] + self.reclaim_retired(); + let page_id = { let mut empty_pages = self.empty_pages.write(); empty_pages.pop_front() }; if let Some(page_id) = page_id { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); let pages = self.pages.read(); let index = page_id_mapper(page_id.into()); let page = pages[index].clone(); @@ -216,54 +446,104 @@ where #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "DataPages"))] pub fn select>(&self, link: L) -> Result where - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + Row: Archive + + Clone + + for<'a> Serialize, Share>, rkyv::rancor::Error>>, <::WrappedRow as Archive>::Archived: Portable + Deserialize<::WrappedRow, HighDeserializer>, { let link = link.into(); - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; - Ok(gen_row.get_inner()) + #[cfg(feature = "versioned-row-publication")] + { + let slot = self.published_slot_or_hydrate(link)?; + Ok(slot.snapshot().as_ref().clone()) + } + + #[cfg(not(feature = "versioned-row-publication"))] + { + let pages = self.pages.read(); + let page = pages + .get(page_id_mapper(link.page_id.into())) + .ok_or(ExecutionError::PageNotFound(link.page_id))?; + let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; + Ok(gen_row.get_inner()) + } } pub fn select_non_ghosted(&self, link: Link) -> Result where - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + Row: Archive + + Clone + + for<'a> Serialize, Share>, rkyv::rancor::Error>>, <::WrappedRow as Archive>::Archived: Portable + Deserialize<::WrappedRow, HighDeserializer>, { - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; - if gen_row.is_ghosted() { - return Err(ExecutionError::Ghosted); + #[cfg(feature = "versioned-row-publication")] + { + let slot = self.published_slot_or_hydrate(link)?; + let (row, flags) = slot.load(); + if flags & GHOSTED != 0 { + return Err(ExecutionError::Ghosted); + } + if flags & DELETED != 0 { + return Err(ExecutionError::Deleted); + } + Ok(row.as_ref().clone()) + } + + #[cfg(not(feature = "versioned-row-publication"))] + { + let pages = self.pages.read(); + let page = pages + .get(page_id_mapper(link.page_id.into())) + .ok_or(ExecutionError::PageNotFound(link.page_id))?; + let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; + if gen_row.is_ghosted() { + return Err(ExecutionError::Ghosted); + } + Ok(gen_row.get_inner()) } - Ok(gen_row.get_inner()) } pub fn select_non_vacuumed(&self, link: Link) -> Result where - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + Row: Archive + + Clone + + for<'a> Serialize, Share>, rkyv::rancor::Error>>, <::WrappedRow as Archive>::Archived: Portable + Deserialize<::WrappedRow, HighDeserializer>, { - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; - if gen_row.is_ghosted() { - return Err(ExecutionError::Ghosted); + #[cfg(feature = "versioned-row-publication")] + { + let slot = self.published_slot_or_hydrate(link)?; + let (row, flags) = slot.load(); + if flags & GHOSTED != 0 { + return Err(ExecutionError::Ghosted); + } + if flags & VACUUMED != 0 { + return Err(ExecutionError::Vacuumed); + } + if flags & DELETED != 0 { + return Err(ExecutionError::Deleted); + } + Ok(row.as_ref().clone()) } - if gen_row.is_vacuumed() { - return Err(ExecutionError::Vacuumed); + + #[cfg(not(feature = "versioned-row-publication"))] + { + let pages = self.pages.read(); + let page = pages + .get(page_id_mapper(link.page_id.into())) + .ok_or(ExecutionError::PageNotFound(link.page_id))?; + let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; + if gen_row.is_ghosted() { + return Err(ExecutionError::Ghosted); + } + if gen_row.is_vacuumed() { + return Err(ExecutionError::Vacuumed); + } + Ok(gen_row.get_inner()) } - Ok(gen_row.get_inner()) } #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "DataPages"))] @@ -272,6 +552,8 @@ where Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, Op: Fn(&<::WrappedRow as Archive>::Archived) -> Res, { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.read(); let pages = self.pages.read(); let page = pages .get::(page_id_mapper(link.page_id.into())) @@ -287,18 +569,31 @@ where where Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, <::WrappedRow as Archive>::Archived: Portable, + <::WrappedRow as Archive>::Archived: + Deserialize<::WrappedRow, HighDeserializer>, Op: FnMut(&mut <::WrappedRow as Archive>::Archived) -> Res, { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); let pages = self.pages.read(); let page = pages .get(page_id_mapper(link.page_id.into())) .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let gen_row = unsafe { - page.get_mut_row_ref(link) - .map_err(ExecutionError::DataPageError)? - .unseal_unchecked() + let res = { + let gen_row = unsafe { + page.get_mut_row_ref(link) + .map_err(ExecutionError::DataPageError)? + .unseal_unchecked() + }; + op(gen_row) }; - let res = op(gen_row); + + #[cfg(feature = "versioned-row-publication")] + { + let wrapped = page.get_row(link).map_err(ExecutionError::DataPageError)?; + self.publish_wrapped_row(link, wrapped); + } + Ok(res) } @@ -310,19 +605,24 @@ where /// - The operation does not cause data races or memory corruption. pub unsafe fn update(&self, row: Row, link: Link) -> Result where - Row: Archive, + Row: Archive + Clone, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); let pages = self.pages.read(); let page = pages .get(page_id_mapper(link.page_id.into())) .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let gen_row = ::WrappedRow::from_inner(row); - unsafe { + let gen_row = ::WrappedRow::from_inner(row.clone()); + let result = unsafe { page.save_row_by_link(&gen_row, link) .map_err(ExecutionError::DataPageError) - } + }?; + #[cfg(feature = "versioned-row-publication")] + self.stage_published_row(link, row); + Ok(result) } pub fn delete(&self, link: Link) -> Result<(), ExecutionError> @@ -330,15 +630,26 @@ where Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, { unsafe { self.with_mut_ref(link, |r| r.delete())? } + #[cfg(feature = "versioned-row-publication")] + { + self.retired_links.lock().push(link); + self.reclaim_retired(); + } + + #[cfg(not(feature = "versioned-row-publication"))] self.empty_links.push(link); Ok(()) } pub fn select_raw(&self, link: Link) -> Result, ExecutionError> { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.read(); let pages = self.pages.read(); let page = pages .get(page_id_mapper(link.page_id.into())) @@ -348,7 +659,15 @@ where pub fn mark_page_empty(&self, page_id: PageId) { if u32::from(page_id) != self.current_page_id.load(Ordering::Acquire) { + #[cfg(feature = "versioned-row-publication")] + { + self.retired_pages.lock().push(page_id); + self.reclaim_retired(); + } + + #[cfg(not(feature = "versioned-row-publication"))] let mut g = self.empty_pages.write(); + #[cfg(not(feature = "versioned-row-publication"))] g.push_back(page_id); } } @@ -395,6 +714,8 @@ where } pub fn get_bytes(&self) -> Vec<([u8; DATA_LENGTH], u32)> { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.read(); let pages = self.pages.read(); pages .iter() @@ -402,6 +723,75 @@ where .collect() } + pub(crate) fn reset_page(&self, page_id: PageId) -> Result<(), ExecutionError> { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); + let pages = self.pages.read(); + let page = pages + .get(page_id_mapper(page_id.into())) + .ok_or(ExecutionError::PageNotFound(page_id))?; + page.reset(); + + Ok(()) + } + + /// Copies a row to another page without exposing either mutable byte + /// image to application readers. + pub(crate) unsafe fn move_row_for_vacuum( + &self, + from_link: Link, + to_page_id: PageId, + ) -> Result<(Vec, Link), ExecutionError> + where + Row: Clone, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, + { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); + let pages = self.pages.read(); + let from_page = pages + .get(page_id_mapper(from_link.page_id.into())) + .ok_or(ExecutionError::PageNotFound(from_link.page_id))?; + let to_page = pages + .get(page_id_mapper(to_page_id.into())) + .ok_or(ExecutionError::PageNotFound(to_page_id))?; + + let raw_data = from_page + .get_raw_row(from_link) + .map_err(ExecutionError::DataPageError)?; + let archived = unsafe { + from_page + .get_mut_row_ref(from_link) + .map_err(ExecutionError::DataPageError)? + .unseal_unchecked() + }; + archived.set_in_vacuum_process(); + let new_link = to_page.save_raw_row(&raw_data).map_err(ExecutionError::DataPageError)?; + + #[cfg(feature = "versioned-row-publication")] + { + let old_wrapped = from_page.get_row(from_link).map_err(ExecutionError::DataPageError)?; + self.publish_wrapped_row(from_link, old_wrapped); + let new_wrapped = to_page.get_row(new_link).map_err(ExecutionError::DataPageError)?; + self.publish_wrapped_row(new_link, new_wrapped); + } + + Ok((raw_data, new_link)) + } + + pub(crate) fn retire_published_link(&self, link: Link) { + #[cfg(feature = "versioned-row-publication")] + { + self.retired_publications.lock().push(OffsetEqLink(link)); + self.reclaim_retired(); + } + + #[cfg(not(feature = "versioned-row-publication"))] + let _ = link; + } + pub fn get_page_count(&self) -> usize { self.pages.read().len() } @@ -466,7 +856,11 @@ mod tests { use std::collections::HashSet; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; + #[cfg(feature = "versioned-row-publication")] + use std::sync::mpsc; use std::thread; + #[cfg(feature = "versioned-row-publication")] + use std::time::Duration; use std::time::Instant; use parking_lot::RwLock; @@ -603,6 +997,104 @@ mod tests { assert_eq!(res.err(), Some(PagesExecutionError::Ghosted)) } + #[cfg(feature = "versioned-row-publication")] + #[test] + fn versioned_insert_stays_hidden_until_unghost() { + let pages = DataPages::::new(); + let row = TestRow { a: 7, b: 9 }; + let link = pages.insert(row).unwrap(); + + assert_eq!(pages.select_non_ghosted(link), Err(ExecutionError::Ghosted)); + unsafe { + pages.with_mut_ref(link, |archived| archived.unghost()).unwrap(); + } + assert_eq!(pages.select_non_ghosted(link), Ok(row)); + } + + #[cfg(feature = "versioned-row-publication")] + #[test] + fn versioned_reader_observes_old_row_while_page_update_is_incomplete() { + let pages = Arc::new(DataPages::::new()); + let link = pages.insert(TestRow { a: 0, b: 0 }).unwrap(); + unsafe { + pages.with_mut_ref(link, |row| row.unghost()).unwrap(); + } + + let (first_field_written_tx, first_field_written_rx) = mpsc::channel(); + let (finish_update_tx, finish_update_rx) = mpsc::channel(); + let writer_pages = pages.clone(); + let writer = thread::spawn(move || unsafe { + writer_pages + .with_mut_ref(link, |archived| { + archived.inner.a = 1.into(); + first_field_written_tx.send(()).unwrap(); + finish_update_rx.recv().unwrap(); + archived.inner.b = 1.into(); + }) + .unwrap(); + }); + + first_field_written_rx.recv().unwrap(); + let (read_tx, read_rx) = mpsc::channel(); + let reader_pages = pages.clone(); + let reader = thread::spawn(move || { + read_tx.send(reader_pages.select_non_ghosted(link)).unwrap(); + }); + + assert_eq!( + read_rx.recv_timeout(Duration::from_secs(1)).unwrap(), + Ok(TestRow { a: 0, b: 0 }), + "reader must use the old immutable version instead of page bytes" + ); + + finish_update_tx.send(()).unwrap(); + writer.join().unwrap(); + reader.join().unwrap(); + assert_eq!(pages.select_non_ghosted(link), Ok(TestRow { a: 1, b: 1 })); + } + + #[cfg(feature = "versioned-row-publication")] + #[test] + fn retired_version_survives_link_reuse_for_in_flight_reader() { + let pages = DataPages::::new(); + let link = pages.insert(TestRow { a: 1, b: 1 }).unwrap(); + unsafe { + pages.with_mut_ref(link, |row| row.unghost()).unwrap(); + } + let old_slot = pages.published_slot(link).unwrap(); + let old_version = old_slot.snapshot(); + + pages.delete(link).unwrap(); + let reused_link = pages.insert(TestRow { a: 2, b: 2 }).unwrap(); + assert_eq!(reused_link, link); + assert_eq!(pages.select_non_ghosted(reused_link), Err(ExecutionError::Ghosted)); + unsafe { + pages.with_mut_ref(reused_link, |row| row.unghost()).unwrap(); + } + + assert_eq!(old_version.as_ref(), &TestRow { a: 1, b: 1 }); + assert_eq!(pages.select_non_ghosted(reused_link), Ok(TestRow { a: 2, b: 2 })); + } + + #[cfg(feature = "versioned-row-publication")] + #[test] + fn read_grace_period_prevents_link_aba() { + let pages = DataPages::::new(); + let old_link = pages.insert(TestRow { a: 1, b: 1 }).unwrap(); + unsafe { + pages.with_mut_ref(old_link, |row| row.unghost()).unwrap(); + } + + let read_guard = pages.read_guard(); + pages.delete(old_link).unwrap(); + let new_link = pages.insert(TestRow { a: 2, b: 2 }).unwrap(); + assert_ne!(new_link, old_link, "retired link was reused by an active reader"); + + drop(read_guard); + pages.reclaim_retired(); + assert!(pages.get_empty_links().contains(&old_link)); + } + #[test] fn select_non_vacuumed_returns_row_when_valid() { let pages = DataPages::::new(); diff --git a/src/in_memory/publication.rs b/src/in_memory/publication.rs new file mode 100644 index 0000000..a0f0907 --- /dev/null +++ b/src/in_memory/publication.rs @@ -0,0 +1,51 @@ +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, Ordering}; + +use parking_lot::RwLock; + +pub(super) const GHOSTED: u8 = 1 << 0; +pub(super) const DELETED: u8 = 1 << 1; +pub(super) const VACUUMED: u8 = 1 << 2; + +/// One immutable application-visible row version plus atomic lifecycle bits. +/// +/// Readers hold an `Arc` to a complete version, so replacing or retiring a +/// version cannot invalidate an in-flight read. The short per-row lock only +/// protects the `Arc` pointer; readers never access mutable archived bytes. +pub(super) struct PublishedRow { + row: RwLock>, + flags: AtomicU8, +} + +impl PublishedRow { + pub(super) fn new(row: Row, flags: u8) -> Self { + Self { + row: RwLock::new(Arc::new(row)), + flags: AtomicU8::new(flags), + } + } + + pub(super) fn replace(&self, row: Row, flags: u8) { + *self.row.write() = Arc::new(row); + self.flags.store(flags, Ordering::Release); + } + + pub(super) fn load(&self) -> (Arc, u8) { + let flags = self.flags.load(Ordering::Acquire); + let row = self.row.read().clone(); + (row, flags) + } + + pub(super) fn snapshot(&self) -> Arc { + self.row.read().clone() + } +} + +impl Debug for PublishedRow { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PublishedRow") + .field("flags", &self.flags.load(Ordering::Relaxed)) + .finish_non_exhaustive() + } +} diff --git a/src/in_memory/row.rs b/src/in_memory/row.rs index 2d75a8b..115b9b7 100644 --- a/src/in_memory/row.rs +++ b/src/in_memory/row.rs @@ -2,10 +2,22 @@ use std::fmt::Debug; use rkyv::Archive; +#[cfg(feature = "versioned-row-publication")] +pub trait PublicationSafe: Send + Sync + 'static {} + +#[cfg(feature = "versioned-row-publication")] +impl PublicationSafe for T {} + +#[cfg(not(feature = "versioned-row-publication"))] +pub trait PublicationSafe {} + +#[cfg(not(feature = "versioned-row-publication"))] +impl PublicationSafe for T {} + /// Common trait for the `Row`s that can be stored on the [`Data`] page. /// /// [`Data`]: crate::in_memory::data::Data -pub trait StorableRow { +pub trait StorableRow: PublicationSafe { type WrappedRow: Archive + Debug; } diff --git a/src/table/mod.rs b/src/table/mod.rs index cb54aaa..298b403 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -23,7 +23,7 @@ use rkyv::ser::Serializer; use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; -use rkyv::{Archive, Deserialize, Serialize}; +use rkyv::{Archive, Deserialize, Portable, Serialize}; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; @@ -152,11 +152,30 @@ where <::WrappedRow as Archive>::Archived: Deserialize<::WrappedRow, HighDeserializer>, { - let link: Option = self.primary_index.pk_map.get(&pk).map(|v| v.get().value.into()); - if let Some(link) = link { - self.data.select_non_ghosted(link).ok() - } else { - None + let _read_guard = self.data.read_guard(); + #[cfg(feature = "versioned-row-publication")] + { + loop { + let link: Link = self.primary_index.pk_map.get(&pk).map(|v| v.get().value.into())?; + if let Ok(row) = self.data.select_non_ghosted(link) { + return Some(row); + } + + let current_link: Option = self.primary_index.pk_map.get(&pk).map(|v| v.get().value.into()); + if current_link == Some(link) { + return None; + } + } + } + + #[cfg(not(feature = "versioned-row-publication"))] + { + let link: Option = self.primary_index.pk_map.get(&pk).map(|v| v.get().value.into()); + if let Some(link) = link { + self.data.select_non_ghosted(link).ok() + } else { + None + } } } @@ -168,7 +187,9 @@ where + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, PrimaryKey: Clone, AvailableTypes: 'static, AvailableIndexes: AvailableIndex, @@ -184,9 +205,9 @@ where if let Err(e) = self.indexes.save_row(row.clone(), link) { return match e { IndexError::AlreadyExists { at, inserted_already } => { - self.data.delete(link).map_err(WorkTableError::PagesError)?; self.primary_index.remove(&pk, link); self.indexes.delete_from_indexes(row, link, inserted_already)?; + self.data.delete(link).map_err(WorkTableError::PagesError)?; Err(WorkTableError::AlreadyExists(at.to_string_value())) } @@ -216,7 +237,9 @@ where + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, PrimaryKey: Clone, SecondaryEvents: Debug + Default + Clone + TableSecondaryIndexEventsOps, SecondaryIndexes: TableSecondaryIndex @@ -334,7 +357,9 @@ where + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, PrimaryKey: Clone, AvailableTypes: 'static, AvailableIndexes: Debug + AvailableIndex, @@ -391,7 +416,9 @@ where + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, PrimaryKey: Clone, SecondaryEvents: Debug + Default + Clone + TableSecondaryIndexEventsOps, SecondaryIndexes: TableSecondaryIndex diff --git a/src/table/vacuum/vacuum.rs b/src/table/vacuum/vacuum.rs index 1cb3d8b..6fec037 100644 --- a/src/table/vacuum/vacuum.rs +++ b/src/table/vacuum/vacuum.rs @@ -207,13 +207,11 @@ where } fn free_page(&self, page_id: PageId) { - let p = self.data_pages.get_page(page_id).expect("should exist as called"); - p.reset() + self.data_pages.reset_page(page_id).expect("should exist as called") } async fn move_data_from(&self, from: PageId, to: PageId) -> (bool, bool) { let to_page = self.data_pages.get_page(to).expect("should exist as link exists"); - let from_page = self.data_pages.get_page(from).expect("should exist as link exists"); let to_free_space = to_page.free_space(); let page_start = OffsetEqLink::<_>(Link { @@ -268,18 +266,13 @@ where self.lock_manager.remove_with_lock_check(&pk); continue; } - let raw_data = from_page - .get_raw_row(from_link.0) - .expect("link is not bigger than free offset"); - unsafe { + let (raw_data, new_link) = unsafe { self.data_pages - .with_mut_ref(from_link.0, |r| r.set_in_vacuum_process()) - .expect("link should be valid") - } - let new_link = to_page - .save_raw_row(&raw_data) - .expect("page is not full as checked on links collection"); + .move_row_for_vacuum(from_link.0, to) + .expect("links and destination capacity were checked") + }; self.update_index_after_move(pk.clone(), from_link.0, new_link, raw_data); + self.data_pages.retire_published_link(from_link.0); lock.unlock(); self.lock_manager.remove_with_lock_check(&pk); diff --git a/tests/worktable/float.rs b/tests/worktable/float.rs index 29747ee..576c707 100644 --- a/tests/worktable/float.rs +++ b/tests/worktable/float.rs @@ -54,6 +54,38 @@ fn unique_float_point_read_revalidates_the_returned_row() { assert_eq!(table.select_by_value(second.value), Some(second)); } +#[cfg(feature = "versioned-row-publication")] +#[test] +fn float_range_read_revalidates_each_resolved_row() { + let table = TestFloatWorkTable::default(); + let inside = TestFloatRow { + id: table.get_next_pk().into(), + test: 1, + another: 10.0, + exchange: "inside".to_string(), + }; + let outside = TestFloatRow { + id: table.get_next_pk().into(), + test: 2, + another: 100.0, + exchange: "outside".to_string(), + }; + table.insert(inside.clone()).unwrap(); + table.insert(outside.clone()).unwrap(); + + let outside_link = table + .0 + .primary_index + .pk_map + .get(&TestFloatPrimaryKey(outside.id)) + .map(|entry| entry.get().value.0) + .unwrap(); + TableIndex::insert(&table.0.indexes.another_idx, OrderedFloat(15.0), outside_link); + + let rows = table.select_by_another_range(0.0..20.0).execute().unwrap(); + assert_eq!(rows, vec![inside]); +} + #[test] fn select_all_range_float_test() { let table = TestFloatWorkTable::default(); diff --git a/tests/worktable/index/range.rs b/tests/worktable/index/range.rs index 99591ce..39f98dd 100644 --- a/tests/worktable/index/range.rs +++ b/tests/worktable/index/range.rs @@ -32,6 +32,39 @@ worktable!( } ); +#[cfg(feature = "versioned-row-publication")] +#[test] +fn range_read_revalidates_each_resolved_row() { + let table = RangeTestWorkTable::default(); + let inside = RangeTestRow { + id: table.get_next_pk().into(), + value: 10, + name: "inside".to_string(), + }; + let outside = RangeTestRow { + id: table.get_next_pk().into(), + value: 100, + name: "outside".to_string(), + }; + table.insert(inside.clone()).unwrap(); + table.insert(outside.clone()).unwrap(); + + let outside_link = table + .0 + .primary_index + .pk_map + .get(&RangeTestPrimaryKey(outside.id)) + .map(|entry| entry.get().value.0) + .unwrap(); + + // Model the transient state where an index entry still falls inside the + // requested range but its completed row version has already moved out. + TableIndex::insert(&table.0.indexes.val_idx, 15, outside_link); + + let rows = table.select_by_value_range(0..20).execute().unwrap(); + assert_eq!(rows, vec![inside]); +} + #[test] fn test_range_select_basic() { let table = RangeTestWorkTable::default();