diff --git a/changelog.d/rm-185-malformed-locks.md b/changelog.d/rm-185-malformed-locks.md new file mode 100644 index 000000000..7cb3efb1b --- /dev/null +++ b/changelog.d/rm-185-malformed-locks.md @@ -0,0 +1,17 @@ +### Rust MongoDB server: a malformed `dropIndexes`, `dbStats` or `listDatabases` takes no lock + +mongod parses a command before it takes the command's locks, so a request +it refuses as malformed takes none. `dropIndexes` without `index`, and +`dbStats` and `listDatabases` with a wrongly typed or unknown field, now +parse before their locks here too, and their `serverStatus.locks` deltas +match mongod 8.2.11's (none). The other malformed commands of RM-185 were +closed by RM-386. + +#### Fixed + +- `dropIndexes`, `dbStats` and `listDatabases` parse in + `parse_before_locks`, before authorization, admission and locks. +- The lock comparison's index-build check no longer requires a build poll of + a `createIndexes` that mongod refuses before any build. + +Fixes RM-185 (tests/backlog/test_rm_185_locks.py::test_lock_counts_of_other_malformed_commands) diff --git a/changelog.d/rm-350-nonlogged-crash-retry.md b/changelog.d/rm-350-nonlogged-crash-retry.md new file mode 100644 index 000000000..608ef83b8 --- /dev/null +++ b/changelog.d/rm-350-nonlogged-crash-retry.md @@ -0,0 +1,15 @@ +### A retryable write retried after a crash under `--data-nonlogged` is not applied again + +A data-nonlogged store writes a session's `config.transactions` row and a +retryable `findAndModify`'s image without an oplog entry, and recovers its +data tables from the oplog after a crash, so a SIGKILL lost both: a retried +insert failed on a duplicate key and a retried `findAndModify` changed its +document a second time. Crash recovery now rebuilds the row and the image as +it replays each retryable write's entry, as mongod 8.2.11's +`SessionUpdateTracker` does, and both retries are answered as after a clean +restart. The change is in `secantus-storage`, which the Rust PostgreSQL +server links too. + +#### Fixed + +- Fixes RM-350 (tests/backlog/test_rm_350_retry_after_restart.py::test_a_retry_after_a_crash_under_data_nonlogged_applies_nothing_twice) diff --git a/changelog.d/rm-423-meta-sort.md b/changelog.d/rm-423-meta-sort.md new file mode 100644 index 000000000..e1380e39a --- /dev/null +++ b/changelog.d/rm-423-meta-sort.md @@ -0,0 +1,11 @@ +### Rust MongoDB server: an aggregate `$sort` on a `$meta` key runs + +A `$sort` stage keyed by `{$meta: "randVal"}` or `{$meta: "searchScore"}` +was refused as an unsupported stage; it now sorts as mongod 8.2 does. + +#### Fixed + +- `{$sort: {s: {$meta: "randVal"}}}` orders the documents by a random value + drawn for each, and a `searchScore` key, which has no value without + `$search`, ties every document, so the other keys decide the order. + Fixes RM-423 (tests/backlog/test_rm_found_2026_11_08_rm332.py::test_a_sort_on_a_meta_key_runs) diff --git a/changelog.d/rm-425-cmd-line-opts.md b/changelog.d/rm-425-cmd-line-opts.md new file mode 100644 index 000000000..278e7da56 --- /dev/null +++ b/changelog.d/rm-425-cmd-line-opts.md @@ -0,0 +1,13 @@ +### Rust MongoDB server: `getCmdLineOpts` reports the command line + +`secantusd-rs` now answers `getCmdLineOpts` as mongod 8.2 does: `argv` is the +command line the process was started with, and `parsed` holds each option it +was given under its mongod configuration-file path (`--port 27031 +--storage-path D` is `{net: {port: 27031}, storage: {dbPath: D}}`). The same +`parsed` is the `Options set by command line` startup log line and the +`cmdLine` of each `local.startup_log` record. Flags with no mongod option +(`--oplog-async`, `--cache-size`, ...) do not appear. + +#### Fixed + +- Fixes RM-425 (tests/backlog/test_rm_cmd_line_opts.py::test_get_cmd_line_opts_reports_the_command_line) diff --git a/crates/secantus-commands/src/admin.rs b/crates/secantus-commands/src/admin.rs index 4095b31ae..b2c8b0a2e 100644 --- a/crates/secantus-commands/src/admin.rs +++ b/crates/secantus-commands/src/admin.rs @@ -4075,23 +4075,30 @@ fn index_conflict( } /// `dropIndexes` — drop a named index, or all of them with `"*"`. -pub fn drop_indexes(doc: &Document, ctx: &mut CommandContext) -> HandlerResult { +/// `dropIndexes`'s `index`, parsed as mongod's IDL parses it: before the +/// command's locks, so a request without it takes none (RM-185, measured +/// 8.2.11). Returns the list of names when `index` is one. +pub fn drop_indexes_parse(doc: &Document) -> Result>, CommandError> { // A list of names is a third form `index` takes, beside a name and a key. let names: Option> = match doc.get("index") { Some(Bson::Array(a)) => a.iter().map(|v| v.as_str().map(String::from)).collect(), _ => None, }; if doc.get("index").is_none() { - return Ok(CommandError::new( + return Err(CommandError::new( 40414, "IDLFailedToParse", "BSON field 'dropIndexes.index' is missing but a required field", - ) - .into_reply()); + )); } if names.is_none() { argtypes::require_index_name_or_key(doc, "index", "dropIndexes.index")?; } + Ok(names) +} + +pub fn drop_indexes(doc: &Document, ctx: &mut CommandContext) -> HandlerResult { + let names = drop_indexes_parse(doc)?; let coll = coll_arg(doc, "dropIndexes")?; let storage = ctx.storage()?; let exists = storage diff --git a/crates/secantus-commands/src/diagnostics.rs b/crates/secantus-commands/src/diagnostics.rs index 672b8edfa..3a0b7ace1 100644 --- a/crates/secantus-commands/src/diagnostics.rs +++ b/crates/secantus-commands/src/diagnostics.rs @@ -77,19 +77,43 @@ pub fn fsync(doc: &Document, ctx: &mut CommandContext) -> HandlerResult { /// `getCmdLineOpts` — parsed command line; drivers read /// `parsed.security.authorization` to detect `--auth`. pub fn get_cmd_line_opts(_doc: &Document, ctx: &mut CommandContext) -> HandlerResult { - let mut argv = vec![Bson::String("secantus".to_string())]; - if ctx.require_auth { - argv.push(Bson::String("--auth".to_string())); - } + let argv: Vec = match COMMAND_LINE.get() { + Some((argv, _)) => argv.iter().cloned().map(Bson::String).collect(), + None => { + let mut argv = vec![Bson::String("secantus".to_string())]; + if ctx.require_auth { + argv.push(Bson::String("--auth".to_string())); + } + argv + } + }; Ok(doc! { "argv": argv, "parsed": parsed_options(ctx.require_auth), "ok": 1.0 }) } +/// The process's command line, recorded once by the `secantusd-rs` binary +/// before it serves: `argv` as the process was started, and the options it +/// gave under their mongod configuration-file paths (RM-425). An embedded +/// server has no command line and leaves it unset. +static COMMAND_LINE: std::sync::OnceLock<(Vec, Document)> = std::sync::OnceLock::new(); + +/// Record the process's command line for `getCmdLineOpts`, the `Options set +/// by command line` log line and `local.startup_log`'s `cmdLine`. The first +/// call wins. +pub fn set_command_line(argv: Vec, parsed: Document) { + let _ = COMMAND_LINE.set((argv, parsed)); +} + /// The options the server was started with, as `getCmdLineOpts` reports /// them under `parsed`; also the `cmdLine` of each `local.startup_log` -/// record, which mongod takes from the same place. +/// record, which mongod takes from the same place. Without a recorded +/// command line (an embedded server) it is `{net: {}, storage: {}}`, and +/// authorization from a configuration file shows as mongod shows it. pub fn parsed_options(require_auth: bool) -> Document { - let mut parsed = doc! { "net": {}, "storage": {} }; - if require_auth { + let mut parsed = match COMMAND_LINE.get() { + Some((_, parsed)) => parsed.clone(), + None => doc! { "net": {}, "storage": {} }, + }; + if require_auth && !parsed.contains_key("security") { parsed.insert("security", doc! { "authorization": "enabled" }); } parsed diff --git a/crates/secantus-commands/src/lib.rs b/crates/secantus-commands/src/lib.rs index 955bf467c..ff78e8499 100644 --- a/crates/secantus-commands/src/lib.rs +++ b/crates/secantus-commands/src/lib.rs @@ -1790,6 +1790,9 @@ fn parse_before_locks( "count" => crud::count_parse(doc), "findAndModify" | "findandmodify" => findandmodify::parse(doc), "createIndexes" => argtypes::require_index_specs(doc), + "dropIndexes" => admin::drop_indexes_parse(doc).map(drop), + "dbStats" | "dbstats" => stats_admin::parse_db_stats(doc).map(drop), + "listDatabases" => stats_admin::parse_list_databases(doc).map(drop), "drop" => admin::drop_parse(doc), "listCollections" => admin::list_collections_parse(doc), "listIndexes" => admin::list_indexes_parse(doc), diff --git a/crates/secantus-commands/src/stats_admin.rs b/crates/secantus-commands/src/stats_admin.rs index f17d1519c..f433904a0 100644 --- a/crates/secantus-commands/src/stats_admin.rs +++ b/crates/secantus-commands/src/stats_admin.rs @@ -142,8 +142,9 @@ fn is_timeseries_definition(definition: &Document) -> bool { .is_some_and(|stage| stage.contains_key("$_internalUnpackBucket")) } -/// `dbStats`. -pub fn db_stats(doc: &Document, ctx: &mut CommandContext) -> HandlerResult { +/// `dbStats`'s fields, parsed as mongod's IDL parses them: before the +/// command's locks, so a malformed one takes none (RM-185, measured 8.2.11). +pub fn parse_db_stats(doc: &Document) -> Result<(i64, bool), CommandError> { let name = doc.keys().next().map_or("dbStats", String::as_str); unknown_field(doc, "dbStats", &[name, "scale", "freeStorage"])?; let scale = scale_of(doc, "dbStats")?.unwrap_or(1); @@ -172,6 +173,12 @@ pub fn db_stats(doc: &Document, ctx: &mut CommandContext) -> HandlerResult { )) } }; + Ok((scale, free)) +} + +/// `dbStats`. +pub fn db_stats(doc: &Document, ctx: &mut CommandContext) -> HandlerResult { + let (scale, free) = parse_db_stats(doc)?; let t = totals(ctx, &ctx.db_name)?; let scaled = |n: i64| n as f64 / scale as f64; let mut out = doc! { @@ -599,8 +606,9 @@ fn bounded_size( Ok((size, count)) } -/// `listDatabases`. -pub fn list_databases(doc: &Document, ctx: &mut CommandContext) -> HandlerResult { +/// `listDatabases`'s fields, parsed as mongod's IDL parses them: before +/// the command's locks (RM-185, measured 8.2.11). Returns `nameOnly`. +pub fn parse_list_databases(doc: &Document) -> Result { unknown_field( doc, "listDatabases", @@ -639,6 +647,12 @@ pub fn list_databases(doc: &Document, ctx: &mut CommandContext) -> HandlerResult )) } } + Ok(name_only) +} + +/// `listDatabases`. +pub fn list_databases(doc: &Document, ctx: &mut CommandContext) -> HandlerResult { + let name_only = parse_list_databases(doc)?; if ctx.db_name != "admin" { return Err(err( 13, diff --git a/crates/secantus-core/src/sorter.rs b/crates/secantus-core/src/sorter.rs index b853b4a38..8690f3d18 100644 --- a/crates/secantus-core/src/sorter.rs +++ b/crates/secantus-core/src/sorter.rs @@ -1163,13 +1163,26 @@ pub fn sort_documents_placed( sizing: &Sizing, limit: Option, ) -> R<(Vec, Vec)> { - let sort = crate::aggregate::sort_fields(spec)?; + let (sort, metas) = meta_sort_fields(spec)?; let mut keyed: Vec<(Vec, Document)> = Vec::with_capacity(docs.len()); for d in docs { - keyed.push((crate::aggregate::sort_key_values(&d, &sort)?, d)); + let mut key = if sort.is_empty() { + Vec::new() + } else { + crate::aggregate::sort_key_values(&d, &sort)? + }; + for (at, meta) in &metas { + key.insert(*at, meta.value()); + } + keyed.push((key, d)); + } + let mut dirs: Vec = sort.iter().map(|(_, rev)| *rev).collect(); + for (at, _) in &metas { + // A `$meta` key sorts descending, as mongod's does. + dirs.insert(*at, true); } let cmp = |ka: &[Bson], kb: &[Bson]| { - for (i, (_, rev)) in sort.iter().enumerate() { + for (i, rev) in dirs.iter().enumerate() { let c = crate::order::cmp(&ka[i], &kb[i]); let c = if *rev { c.reverse() } else { c }; if c != Ordering::Equal { @@ -1185,10 +1198,84 @@ pub fn sort_documents_placed( .unzip()) } +/// A `$meta` key of a `$sort` stage. +enum MetaKey { + /// `{$meta: "randVal"}`: a fresh random double for each document. + RandVal, + /// `{$meta: "searchScore"}`: no value without `$search`, so every + /// document ties on it. + SearchScore, +} + +impl MetaKey { + fn value(&self) -> Bson { + match self { + MetaKey::RandVal => Bson::Double(rand::random::()), + MetaKey::SearchScore => Bson::Null, + } + } +} + +/// The field keys of `spec`, and its `$meta` keys with the place each takes +/// in the sort key. mongod 8.2.1 sorts a `$sort` stage by `{$meta: +/// "randVal"}` and by `{$meta: "searchScore"}` (RM-423); any other `$meta`, +/// and a spec with no key at all, is left to [`crate::aggregate::sort_fields`]. +#[allow(clippy::type_complexity)] +fn meta_sort_fields(spec: &Bson) -> R<(Vec<(String, bool)>, Vec<(usize, MetaKey)>)> { + let Bson::Document(d) = spec else { + return Ok((crate::aggregate::sort_fields(spec)?, Vec::new())); + }; + let meta = |v: &Bson| match v { + Bson::Document(m) if m.len() == 1 => match m.get("$meta") { + Some(Bson::String(k)) if k == "randVal" => Some(MetaKey::RandVal), + Some(Bson::String(k)) if k == "searchScore" => Some(MetaKey::SearchScore), + _ => None, + }, + _ => None, + }; + let mut fields = Document::new(); + let mut metas = Vec::new(); + for (i, (k, v)) in d.iter().enumerate() { + match meta(v) { + Some(m) => metas.push((i, m)), + None => { + fields.insert(k.clone(), v.clone()); + } + } + } + if metas.is_empty() || !fields.is_empty() { + let sort = crate::aggregate::sort_fields(&Bson::Document(fields))?; + return Ok((sort, metas)); + } + Ok((Vec::new(), metas)) +} + #[cfg(test)] mod tests { use super::*; + /// RM-423: a `searchScore` key ties every document, leaving the order + /// to the other keys and then to arrival; a `randVal` key alone sorts. + #[test] + fn a_meta_sort_key_ties_or_draws() { + let docs: Vec = (0..6).map(|i| doc! { "_id": i, "a": i % 2 }).collect(); + let sizing = Sizing { + sizes: None, + source: SortSource::QueryLayer, + fields: None, + added: None, + budget: memlimit::MAX_BYTES, + }; + let spec = Bson::Document(doc! { "a": -1, "s": { "$meta": "searchScore" } }); + let out = sort_documents(docs.clone(), &spec, &sizing, None).unwrap(); + let ids: Vec = out.iter().map(|d| d.get_i32("_id").unwrap()).collect(); + assert_eq!(ids, vec![1, 3, 5, 0, 2, 4]); + let spec = Bson::Document(doc! { "s": { "$meta": "randVal" } }); + assert_eq!(sort_documents(docs, &spec, &sizing, None).unwrap().len(), 6); + let spec = Bson::Document(doc! { "s": { "$meta": "textScore" } }); + assert!(sort_documents(Vec::new(), &spec, &sizing, None).is_err()); + } + fn items(n: i32, len: usize) -> Vec<(Vec, Document)> { (0..n) .map(|i| { diff --git a/crates/secantus-server/src/args.rs b/crates/secantus-server/src/args.rs index aa096154c..724b9641b 100644 --- a/crates/secantus-server/src/args.rs +++ b/crates/secantus-server/src/args.rs @@ -33,6 +33,39 @@ pub struct ParsedRun { /// `--repair`: repair the store and exit instead of serving (mongod's /// `--repair`). Command line only, as mongod's is. pub repair: bool, + /// The options given on the command line, as mongod's `getCmdLineOpts` + /// reports them under `parsed`: each under its configuration-file path + /// (`--port 27031 --dbpath D` is `{net: {port: 27031}, storage: {dbPath: + /// D}}`), keys in order (mongod builds it from an ordered map). Only the + /// flags with a mongod option appear (RM-425). + pub cmd_line: bson::Document, +} + +/// Put `value` at the dotted configuration path `path` in `doc`, creating +/// the sub-documents on the way. +fn set_option(doc: &mut bson::Document, path: &[&str], value: bson::Bson) { + let (last, parents) = path.split_last().expect("non-empty path"); + let mut cur = doc; + for key in parents { + if !matches!(cur.get(*key), Some(bson::Bson::Document(_))) { + cur.insert(*key, bson::Document::new()); + } + cur = cur.get_document_mut(*key).expect("just inserted"); + } + cur.insert(*last, value); +} + +/// Sort every level of `doc` by key, as mongod's option map is ordered. +fn sort_options(doc: bson::Document) -> bson::Document { + let mut pairs: Vec<(String, bson::Bson)> = doc.into_iter().collect(); + pairs.sort_by(|a, b| a.0.cmp(&b.0)); + pairs + .into_iter() + .map(|(k, v)| match v { + bson::Bson::Document(d) => (k, bson::Bson::Document(sort_options(d))), + other => (k, other), + }) + .collect() } /// A fully-resolved run configuration: where to bind, the WiredTiger home, and @@ -269,6 +302,7 @@ pub fn parse_args(args: &[String]) -> Result { let mut tls_ca_file: Option = None; let mut tls_require_client_cert = false; let mut repair = false; + let mut cmd_line = bson::Document::new(); let mut iter = args.iter(); while let Some(arg) = iter.next() { @@ -291,16 +325,33 @@ pub fn parse_args(args: &[String]) -> Result { match flag { "--help" | "-h" => return Ok(Parsed::Help(usage())), "--version" => return Ok(Parsed::Version(version_text("secantusd-rs"))), - "--config" => config_path = Some(PathBuf::from(take_value("--config")?)), - "--host" => o.host = Some(take_value("--host")?), + "--config" => { + let v = take_value("--config")?; + set_option(&mut cmd_line, &["config"], v.clone().into()); + config_path = Some(PathBuf::from(v)); + } + "--host" => { + let v = take_value("--host")?; + set_option(&mut cmd_line, &["net", "bindIp"], v.clone().into()); + o.host = Some(v); + } "--port" => { let raw = take_value("--port")?; o.port = Some(raw.parse::().map_err(|_| { format!("--port expects an integer in 0..=65535, got {raw:?}") })?); + set_option( + &mut cmd_line, + &["net", "port"], + bson::Bson::Int32(i32::from(o.port.unwrap_or_default())), + ); + } + "--storage-path" => { + let v = take_value("--storage-path")?; + set_option(&mut cmd_line, &["storage", "dbPath"], v.clone().into()); + o.storage_path = Some(v); } - "--storage-path" => o.storage_path = Some(take_value("--storage-path")?), "--log-level" => { let lvl = take_value("--log-level")?; validate_log_level(&lvl, "secantusd-rs").map_err(|_| { @@ -356,14 +407,27 @@ pub fn parse_args(args: &[String]) -> Result { format!("--oplog-max-entries expects a non-negative integer, got {raw:?}") })?); } - "--auth" => o.auth = Some(true), + "--auth" => { + set_option( + &mut cmd_line, + &["security", "authorization"], + "enabled".into(), + ); + o.auth = Some(true); + } "--standalone" => o.standalone = Some(true), - "--repair" => repair = true, + "--repair" => { + set_option(&mut cmd_line, &["repair"], true.into()); + repair = true; + } "--enable-test-commands" => o.enable_test_commands = Some(true), - "--setParameter" => o - .set_parameters - .get_or_insert_with(Vec::new) - .push(take_value("--setParameter")?), + "--setParameter" => { + let v = take_value("--setParameter")?; + if let Some((name, value)) = v.split_once('=') { + set_option(&mut cmd_line, &["setParameter", name], value.into()); + } + o.set_parameters.get_or_insert_with(Vec::new).push(v); + } "--tls-cert-file" => tls_cert_file = Some(take_value("--tls-cert-file")?), "--tls-key-file" => tls_key_file = Some(take_value("--tls-key-file")?), "--tls-ca-file" => tls_ca_file = Some(take_value("--tls-ca-file")?), @@ -415,6 +479,7 @@ pub fn parse_args(args: &[String]) -> Result { config_path, overrides: o, repair, + cmd_line: sort_options(cmd_line), }))) } @@ -550,6 +615,30 @@ mod tests { parse_args(&owned) } + #[test] + fn cmd_line_names_each_option_under_its_mongod_path() { + let pr = match parse(&[ + "--storage-path", + "D", + "--port=27031", + "--auth", + "--oplog-async", + ]) + .unwrap() + { + Parsed::Run(pr) => pr, + other => panic!("expected Run, got {other:?}"), + }; + assert_eq!( + pr.cmd_line, + bson::doc! { + "net": {"port": 27031}, + "security": {"authorization": "enabled"}, + "storage": {"dbPath": "D"}, + } + ); + } + /// Parse into a [`ParsedRun`], then resolve with no TOML file so the result /// reflects CLI-over-defaults (the common test path). fn run(words: &[&str]) -> CliArgs { diff --git a/crates/secantus-storage/src/lib.rs b/crates/secantus-storage/src/lib.rs index 437b6e51c..b3472deaf 100644 --- a/crates/secantus-storage/src/lib.rs +++ b/crates/secantus-storage/src/lib.rs @@ -9962,10 +9962,12 @@ impl Storage { Ok(None) }; let outcome = before.and_then(|before| { + let retry_before = self.replayed_retry_pre_image(&entry)?; let applied = self.apply_replay_entry_idempotent(&entry)?; if let (true, Some(record)) = (applied, before) { self.write_pre_image_records_alone(&[record])?; } + self.replay_session_state(&entry, applied, retry_before)?; Ok(applied) }); REPLAY_TS.with(|c| c.set(0)); @@ -9994,6 +9996,112 @@ impl Storage { self.stable_checkpoint() } + /// The document a retryable `findAndModify`'s replayed entry changes, as + /// it stands before the replay, when the image a retry of it returns is + /// that pre-image (`rw.img: "preImage"`). + fn replayed_retry_pre_image(&self, entry: &Document) -> Result> { + let Ok(rw) = entry.get_document("rw") else { + return Ok(None); + }; + if rw.get_str("img") != Ok("preImage") { + return Ok(None); + } + self.replayed_target(entry) + } + + /// The stored document an update or delete oplog entry names by `_id`. + fn replayed_target(&self, entry: &Document) -> Result> { + let id = match entry.get_str("op") { + Ok("u") => entry.get_document("o2").ok().and_then(|o2| o2.get("_id")), + Ok("d") => entry.get_document("o").ok().and_then(|o| o.get("_id")), + _ => None, + }; + let Some(id) = id else { return Ok(None) }; + let Some((db, coll)) = entry.get_str("ns").ok().and_then(|ns| ns.split_once('.')) else { + return Ok(None); + }; + match self.find_by_id(db, coll, id)? { + Some(bytes) => Ok(Some(decode_doc(&bytes)?)), + None => Ok(None), + } + } + + /// What mongod's `SessionUpdateTracker` rebuilds as recovery applies a + /// retryable write's oplog entry: the session's `config.transactions` + /// row (`{_id: lsid, txnNum, lastWriteOpTime, lastWriteDate}`) and, for + /// a `findAndModify` that changed a document, its retry image in + /// `config.image_collection`. A data-nonlogged store writes both without + /// an oplog entry into tables a crash rolls back to the checkpoint, so + /// replay is the only place they come back from (RM-350). An entry the + /// checkpoint already held (`applied` false) rewrites the row, which the + /// checkpoint may still lack, but no pre-image: the document it would be + /// read from is already changed. + fn replay_session_state( + &self, + entry: &Document, + applied: bool, + retry_before: Option, + ) -> Result<()> { + let Ok(rw) = entry.get_document("rw") else { + return Ok(()); + }; + let (Ok(lsid), Ok(txn)) = (rw.get_document("lsid"), rw.get_i64("txnNumber")) else { + return Ok(()); + }; + let Some(Bson::Timestamp(ts)) = entry.get("ts").cloned() else { + return Ok(()); + }; + let wall = entry + .get_datetime("wall") + .copied() + .unwrap_or_else(|_| bson::DateTime::now()); + let no_vars = Document::new(); + let upsert = |coll: &str, row: Document| -> Result<()> { + self.update_matching( + "config", + coll, + &bson::doc! {"_id": lsid.clone()}, + &row, + false, + true, + &[], + &no_vars, + None, + None, + false, + ) + .map(|_| ()) + }; + upsert( + "transactions", + bson::doc! { + "_id": lsid.clone(), + "txnNum": txn, + "lastWriteOpTime": {"ts": ts, "t": 1_i64}, + "lastWriteDate": wall, + }, + )?; + let image = match rw.get_str("img") { + Ok("postImage") => self.replayed_target(entry)?, + Ok("preImage") if applied => retry_before, + _ => None, + }; + if let (Some(image), Ok(kind)) = (image, rw.get_str("img")) { + upsert( + "image_collection", + bson::doc! { + "_id": lsid.clone(), + "txnNum": txn, + "ts": ts, + "imageKind": kind, + "image": image, + "invalidated": false, + }, + )?; + } + Ok(()) + } + /// The `config.system.preimages` document crash recovery records for /// the update or delete stored at `seq`, when its collection records /// pre-images: the document it changes, as it is before the replay. diff --git a/crates/secantusdb/src/main.rs b/crates/secantusdb/src/main.rs index b9c3814c3..8ae301ab3 100644 --- a/crates/secantusdb/src/main.rs +++ b/crates/secantusdb/src/main.rs @@ -101,6 +101,13 @@ fn main() -> ExitCode { } }; + // What `getCmdLineOpts` reports: the command line as the process was + // started, and the options it gave (RM-425). + secantus_commands::diagnostics::set_command_line( + std::env::args().collect(), + parsed.cmd_line.clone(), + ); + // Layer the TOML file (explicit --config or auto-discovered) underneath the // CLI flags, then derive the resolved run args (TLS pairing + standalone → // replica_set_name). A config error (missing file, unknown key/table, diff --git a/tasks/backlog-rust-mongodb.md b/tasks/backlog-rust-mongodb.md index 842861bb8..42f496532 100644 --- a/tasks/backlog-rust-mongodb.md +++ b/tasks/backlog-rust-mongodb.md @@ -1296,7 +1296,7 @@ These are explicit non-goals. Don't add them without a reason. the log is mongod's structured log, and since 2026-11-13 this server starts it as mongod 8.2.11 does on `initandlisten`: `MongoDB starting`, `Build Info` (23403), `Operating System` (51765), `Options set by command - line` (21951, which carries RM-425's `parsed` until that is fixed), the + line` (21951), the lock file's `Detected unclean shutdown` (22271), `Opening WiredTiger` (22315), `WiredTiger opened` (4795906) and, last, `mongod startup complete` (8423403). mongod writes about sixty more there that this @@ -2121,29 +2121,15 @@ live. Not filed here: `$rankFusion` / `$score` / `$scoreFusion` (RM-198), (RM-418). Verify: tests/backlog/test_rm_found_2026_11_10_rm418.py::test_sbe_find_shapes_are_explained_as_sbe Backlog-ID: RM-543 -- [ ] **An aggregate `$sort` on a `$meta` key is refused.** mongod 8.2.1 - sorts `{$sort: {s: {$meta: "randVal"}}}` (and a `searchScore` key, - which has no value without `$search` and so ties); this server - refuses any `$meta` key in the stage with 2 "aggregation pipeline - uses a stage or operator not supported by the Rust server". A `find` - sort by `{$meta: "randVal"}` answers on both. Found 2026-11-08 while - working RM-332. - Verify: tests/backlog/test_rm_found_2026_11_08_rm332.py::test_a_sort_on_a_meta_key_runs - Backlog-ID: RM-423 -- [ ] **A retryable write retried after a crash under `--data-nonlogged` is - applied again.** A restarted server reads a session's - `config.transactions` row, its oplog entries and its - `config.image_collection` image back (since 2026-11-08), but that mode - writes the row and the image without an oplog entry and recovers its - data tables by replaying the oplog, so a SIGKILL loses both: a retried - insert fails on a duplicate key, a retried `findAndModify` changes the - document a second time. mongod 8.2.11 rebuilds the row and the image - from the oplog in recovery (its `SessionUpdateTracker`; the image from - the document the replayed write changed) and answers both retries as - after a clean restart. The fix is in `recover_from_oplog` - (`secantus-storage`), which the PG server links too. - Backlog-ID: RM-350 - Verify: tests/backlog/test_rm_350_retry_after_restart.py::test_a_retry_after_a_crash_under_data_nonlogged_applies_nothing_twice +- [ ] **A `$sort` on a field and `{$meta: "randVal"}` before a `$project` + is not refused.** mongod 8.2.11 refuses `[{$sort: {a: 1, s: {$meta: + "randVal"}}}, {$project: {a: 1, _id: 0}}]` with 16410 ("FieldPath + field names may not start with '$', given '$computed1'", from its + pipeline rewrite); the same sort alone, or `{$meta: "randVal"}` as the + only key, answers on both, and this server sorts the pipeline. Found + 2026-11-14 while working RM-423. + Verify: tests/backlog/test_rm_found_2026_11_08_rm332.py::test_a_rand_sort_beside_a_field_under_a_project_is_refused + Backlog-ID: RM-691 ### 7.085 The shared Rust storage, audited -- 2026-10-20 @@ -2193,16 +2179,6 @@ inject (a panic, a power loss, a failing checkpoint). collection, readable by a scan and not by `_id`. Backlog-ID: RM-318 Verify: none -- needs a salvage that brings back a deleted document beside its replacement, which no quick test can stage -- [ ] **`getCmdLineOpts` does not report the command line.** mongod 8.2.1 - answers with `argv` as the process was started and `parsed` holding - each option given, under its configuration-file path (`--port 27031 - --dbpath D` is `{net: {port: 27031}, storage: {dbPath: D}}`); this - server answers `argv: ["secantus"]` (and `--auth`) and `parsed: {net: - {}, storage: {}}` whatever it was started with. The same `parsed` is - the `cmdLine` of each `local.startup_log` record, which shares the - gap. Found while fixing RM-319 (2026-11-08). - Backlog-ID: RM-425 - Verify: tests/backlog/test_rm_cmd_line_opts.py::test_get_cmd_line_opts_reports_the_command_line ### 7.084 What an authenticated user may run -- 2026-10-14 @@ -2289,65 +2265,6 @@ Still different: in memory answers 115 with this server's own text, because mongod's in-memory engine is Enterprise-only and its message could not be measured. -- [ ] **`serverStatus.locks`: locks taken before a command parses, the - transaction commands, and the commands not measured.** The lock - manager (`lock_manager.rs`, RM-034) takes, outside a transaction, the - rounds mongod 8.2.11 takes (`statement_rounds`, measured per command): - reads, the write commands, `findAndModify`, `create` / `drop` / - `dropDatabase`, index builds, `dropIndexes`, `$out`, `dbStats`, and - `$merge`, which takes a write round per document as it writes - (`write_rounds`, two more when it creates its target, and with - `whenMatched: "fail"` mongod's retries after a duplicate). Since - 2026-10-24 also the commands that read no data (none), `rolesInfo` / - `usersInfo`, `dataSize` / `filemd5` / the plan-cache commands, - `explain`, `dbHash` and `listDatabases` (one more global IS per - database whose size it reads), and the global IS of a slow - operation's log line (RM-299). All count as mongod's (the regression - tests in `tests/backlog/test_rm_185_locks.py`). An index build's count - of the global IS is not fixed on mongod: its build runs on a thread of - its own and the command takes the IS each time it looks at it, 1 to 4 - times in one shape measured. This server takes mongod's usual count (2, - and 4 with every operation logged as slow: macOS 8.2.11 gave exactly - that in 20 of 20 runs; the Linux build sometimes polls once less), and - the tests accept any count mongod gave in its five runs for that one - lock (Joe, 2026-10-27). mongod takes a command's locks inside it, - after it parses, so a malformed one takes none: since 2026-11-08 - `explain`, `dataSize`, the plan cache commands and - `validateDBMetadata` parse before their locks here too - (`parse_before_locks`), and `validateDBMetadata` takes a round on each - database it reads and on each of its collections. Still different: - the other commands find a malformed field in their handler, after - their locks -- `find` with a wrongly typed `filter` or an unknown - field, `aggregate` without `pipeline` or on `1` with no stage to read - from, `getMore` of a name, `listCollections` and `listDatabases` with - a wrongly typed `filter`, `insert`, `update`, `createIndexes` and - `dropIndexes` without their required field, and `dbStats` with a - wrongly typed `scale` take their chain or rounds where mongod takes - none; the transaction commands, which need a replica set, were measured on a - one-member mongod 8.2.11 set (2026-10-17) cannot be pinned the same - way: its replication threads take the global IS about twice in any - window, and its writes take rounds for the oplog and the session table - too (a plain insert three where a standalone takes one; a one-insert - transaction with its commit 3 collection IX, 2 global IX and 8 on the - state transition lock), machinery this server does not run; and the - commands neither measured on 2026-10-24 (a sweep of the 115 both - servers list, in valid forms) nor named above still take the generic - chain -- those that write, wait, change the server or need a replica - set were left out of the sweep. mongod's internal resources - (`Metadata`, `oplog`, `Mutex`, `MultiDocumentTransactionsBarrier`) are - locked by its background threads only and are absent here. - **Time-based plan yields** (measured 2026-10-21): mongod's executor - yields after `internalQueryExecYieldPeriodMS` (10 ms), and each re-take - of a lock counts. `$out` reads 7 global IS on Linux x86-64 8.2.12 (CI, - the exemplar build: Joe, 2026-10-21), 8 on macOS arm64 idle and 9 on a - loaded Mac; with the period off, macOS reads 7 too. So this server - takes Linux's 7, and the regression tests' reference mongod runs with - the period off, which gives Linux's counts on any machine and stops - them moving with load. The counts that still moved under load were - mongod's slow-operation log (RM-274, closed 2026-10-24): the tests set - `slowms` out of reach on both servers. - Backlog-ID: RM-185 - Verify: tests/backlog/test_rm_185_locks.py::test_lock_counts_of_other_malformed_commands - **Scope decision — `replSetReconfig` takes one member, this server** (2026-10-20, under "Explicitly out of scope: real replica sets"). mongod 8.2.11 accepts a configuration that adds a member it cannot reach and diff --git a/tests/backlog/test_rm_185_locks.py b/tests/backlog/test_rm_185_locks.py index 2a476b703..3a9997546 100644 --- a/tests/backlog/test_rm_185_locks.py +++ b/tests/backlog/test_rm_185_locks.py @@ -175,7 +175,9 @@ def _same_counts(got: list, want: list) -> None: if label.startswith("createIndex"): polls = [run.get(_BUILD_POLLS, 0) for run in _RUNS[label]] mine = ours.get(_BUILD_POLLS, 0) - assert mine >= 1, label + # A build that ran polls at least once; a malformed request, + # which mongod refuses before any build, polls none. + assert mine >= min(1, min(polls)), label assert min(polls) <= mine <= max(polls), ( f"{label!r}: this server took {_BUILD_POLLS} {mine} times; " f"mongod's five runs took {polls}" @@ -798,10 +800,10 @@ def _parsed_after_locking(client: Any) -> list: ) -@pytest.mark.backlog("RM-185") +@pytest.mark.backlog_fixed("RM-185") def test_lock_counts_of_other_malformed_commands(rust_mongo: Any, mongod: Any) -> None: - """The same, for commands whose handler finds the error after the + """The same, for commands whose handler found the error after the statement's locks: mongod refuses each having taken no lock, and this - server takes the command's chain or rounds first (measured 8.2.11, - 2026-11-08).""" + server took the command's chain or rounds first (measured 8.2.11, + 2026-11-08). Each parses before its locks now (RM-386, RM-185).""" _same_counts(_parsed_after_locking(rust_mongo), _reference(_parsed_after_locking, mongod)) diff --git a/tests/backlog/test_rm_350_retry_after_restart.py b/tests/backlog/test_rm_350_retry_after_restart.py index ac437f6ff..635693579 100644 --- a/tests/backlog/test_rm_350_retry_after_restart.py +++ b/tests/backlog/test_rm_350_retry_after_restart.py @@ -198,15 +198,16 @@ def test_each_kind_of_retry_after_a_restart(tmp_path: Any) -> None: ] -@pytest.mark.backlog("RM-350") +@pytest.mark.backlog_fixed("RM-350") def test_a_retry_after_a_crash_under_data_nonlogged_applies_nothing_twice(tmp_path: Any) -> None: """mongod 8.2.11 killed with SIGKILL after two journaled retryable writes answers their retries as after a clean restart: recovery rebuilds the session's `config.transactions` row and the `findAndModify`'s image - from the oplog. This server under `--data-nonlogged` writes both without - an oplog entry and recovers its data tables by replaying the oplog, so a - crash loses them: the insert fails on a duplicate key and the - `findAndModify` runs a second time.""" + from the oplog. This server under `--data-nonlogged` wrote both without + an oplog entry and recovered its data tables by replaying the oplog, so a + crash lost them: the insert failed on a duplicate key and the + `findAndModify` ran a second time. Replay now rebuilds both, as mongod's + `SessionUpdateTracker` does.""" store = str(tmp_path / "store") a, b = _session(), _session() journaled = {"writeConcern": {"j": True}} diff --git a/tests/backlog/test_rm_cmd_line_opts.py b/tests/backlog/test_rm_cmd_line_opts.py index 7bab5f100..470a61689 100644 --- a/tests/backlog/test_rm_cmd_line_opts.py +++ b/tests/backlog/test_rm_cmd_line_opts.py @@ -17,9 +17,9 @@ from tests.backlog.conftest import _RS_BANNER, RS_BINARY, drain_output -@pytest.mark.backlog("RM-425") +@pytest.mark.backlog_fixed("RM-425") def test_get_cmd_line_opts_reports_the_command_line(tmp_path: pathlib.Path) -> None: - """mongod's `parsed` names each option it was given, and `argv` is the + """`parsed` named each option it was given, and `argv` was the command line itself.""" store = tmp_path / "store" argv = [str(RS_BINARY), "--port", "0", "--storage-path", str(store)] diff --git a/tests/backlog/test_rm_found_2026_11_08_rm332.py b/tests/backlog/test_rm_found_2026_11_08_rm332.py index ce1a18842..ab6dbff83 100644 --- a/tests/backlog/test_rm_found_2026_11_08_rm332.py +++ b/tests/backlog/test_rm_found_2026_11_08_rm332.py @@ -19,8 +19,54 @@ def _meta_sort(client: Any) -> tuple: ) -@pytest.mark.backlog("RM-423") +@pytest.mark.backlog_fixed("RM-423") def test_a_sort_on_a_meta_key_runs(rust_mongo: Any, mongod: Any) -> None: """mongod 8.2.1 sorts by `{$meta: "randVal"}` in a `$sort` stage; the - Rust server refuses the stage as unsupported.""" + Rust server refused the stage as unsupported.""" assert _meta_sort(rust_mongo) == _meta_sort(mongod) + + +def _search_score_sort(client: Any) -> tuple: + coll = client.rm332found.c + coll.drop() + coll.insert_many([{"_id": i, "a": i % 2} for i in range(6)]) + return mongo_answer( + lambda: [ + d["_id"] for d in coll.aggregate([{"$sort": {"a": -1, "s": {"$meta": "searchScore"}}}]) + ] + ) + + +@pytest.mark.backlog_fixed("RM-423") +def test_a_search_score_sort_key_ties(rust_mongo: Any, mongod: Any) -> None: + """mongod 8.2.11 sorts `{a: -1, s: {$meta: "searchScore"}}` by `a` alone, + the documents tying on the score they have none of; the Rust server + refused the stage.""" + assert _search_score_sort(rust_mongo) == _search_score_sort(mongod) + + +def _rand_sort_then_project(client: Any) -> tuple: + coll = client.rm332found.c + coll.drop() + coll.insert_many([{"_id": i, "a": i % 2} for i in range(6)]) + return mongo_answer( + lambda: [ + d["a"] + for d in coll.aggregate( + [ + {"$sort": {"a": 1, "s": {"$meta": "randVal"}}}, + {"$project": {"a": 1, "_id": 0}}, + ] + ) + ] + ) + + +@pytest.mark.backlog("RM-691") +def test_a_rand_sort_beside_a_field_under_a_project_is_refused( + rust_mongo: Any, mongod: Any +) -> None: + """mongod 8.2.11 refuses `{$sort: {a: 1, s: {$meta: "randVal"}}}` + followed by a `$project` with 16410 (a `$computed1` field path its + pipeline rewrite makes); the Rust server sorts it.""" + assert _rand_sort_then_project(rust_mongo) == _rand_sort_then_project(mongod)