From 77892887f028c85f4199d2361bda056b208bb446 Mon Sep 17 00:00:00 2001 From: Joe Drumgoole Date: Sat, 14 Nov 2026 17:09:16 +0000 Subject: [PATCH 1/5] Rust MongoDB server: a rooted $or's branch in front of SBE stages is cached under the subplanner's o key (RM-429) Co-Authored-By: Claude Opus 5.5 --- .claude/skills/rust-mongodb-explain/SKILL.md | 2 +- changelog.d/rm-429-or-branch-sbe.md | 14 ++++++++ .../src/explain/cache_key.rs | 35 ++++++++++++++++++- .../src/explain/change_stream.rs | 2 ++ crates/secantus-commands/src/explain/mod.rs | 1 + crates/secantus-commands/src/explain/order.rs | 11 +++--- tasks/backlog-rust-mongodb.md | 10 ------ tests/backlog/test_plan_cache_or_branches.py | 9 ++--- 8 files changed, 61 insertions(+), 23 deletions(-) create mode 100644 changelog.d/rm-429-or-branch-sbe.md diff --git a/.claude/skills/rust-mongodb-explain/SKILL.md b/.claude/skills/rust-mongodb-explain/SKILL.md index 8f9c28c85..73b3ee9a8 100644 --- a/.claude/skills/rust-mongodb-explain/SKILL.md +++ b/.claude/skills/rust-mongodb-explain/SKILL.md @@ -3,4 +3,4 @@ name: rust-mongodb-explain description: 'How the Rust MongoDB server reproduces mongod explain, the classic plan cache and document return order (crates/secantus-commands/src/explain/). Fires when touching explain, the planner, the plan cache, the storage router, or result order on the Rust MongoDB server.' --- -**The Rust MongoDB server reproduces mongod's `explain` (2026-10-12)** -- `indexBounds`, the `IDHACK` / `EXPRESS_IXSCAN` / `COUNT_SCAN` / `DISTINCT_SCAN` / `PROJECTION_COVERED` executors and every command's stages, because they turned out to follow fixed rules whenever only one index can serve the query: its `explain` (`crates/secantus-commands/src/explain/`) derives the plan from mongod's planning rules — bounds, residual filters, sort and covering, and the `count` / `distinct` / write stages — and derives `executionStats` by walking that plan over the documents. Since 2026-10-27 it also keeps mongod 8.2's classic plan cache (`explain/plan_cache.rs`, `explain/cache_key.rs`, RM-154): the query-shape key and its hashes byte for byte, inactive and active entries, replanning, the 5,000-entry eviction, what clears it, and `$planCacheStats`; the trial ranks plans with mongod's tie-breaking bonuses. A pipeline front SBE runs (`plan::Query::sbe`, since 2026-11-08, RM-340) is planned without index intersections and cached under its `s` key by the keys and documents its winner read (`worksType: "reads"`), replanned when its reads reach ten times that (`trial::within_reads`). Not yet: a rooted `$or`'s branches in such a front (RM-429). Past 2,000 documents a trial no longer reads the collection whole: it runs over what its candidates' scans read from the store in the works it takes (`order::Lazy::reached`, `stream::scan_reach`), read again for eight times as many works when it ran out of them (`plan::over_reach`; RM-342). It reports the plan mongod runs, which is not always how the Rust storage router reads the documents (`tasks/backlog-rust-mongodb.md` 7.075). Sweep: `tools/probes/explain_commands.py`, 786 replies. **Since 2026-10-13 the same plan sets the ORDER the Rust server returns documents in** (`explain/order.rs`, `tasks/backlog-rust-mongodb.md` 7.080): the router says only which documents match, and the plan says in what order mongod reads them, which is also which document a `limit: 1` delete or an update without `multi` changes. A sort is still the store's own; the plan decides how documents that compare equal are left. For a sort the predicate's index does not return, the planner counts mongod's trial of the two candidate plans (`plan::counted`, else the trial) on a collection of any size. Sweep: `tools/probes/find_order.py`, 6,531 replies. When a change touches the router or the planner, run it: a router change can no longer move the order, and a planner change now can. +**The Rust MongoDB server reproduces mongod's `explain` (2026-10-12)** -- `indexBounds`, the `IDHACK` / `EXPRESS_IXSCAN` / `COUNT_SCAN` / `DISTINCT_SCAN` / `PROJECTION_COVERED` executors and every command's stages, because they turned out to follow fixed rules whenever only one index can serve the query: its `explain` (`crates/secantus-commands/src/explain/`) derives the plan from mongod's planning rules — bounds, residual filters, sort and covering, and the `count` / `distinct` / write stages — and derives `executionStats` by walking that plan over the documents. Since 2026-10-27 it also keeps mongod 8.2's classic plan cache (`explain/plan_cache.rs`, `explain/cache_key.rs`, RM-154): the query-shape key and its hashes byte for byte, inactive and active entries, replanning, the 5,000-entry eviction, what clears it, and `$planCacheStats`; the trial ranks plans with mongod's tie-breaking bonuses. A pipeline front SBE runs (`plan::Query::sbe`, since 2026-11-08, RM-340) is planned without index intersections and cached under its `s` key by the keys and documents its winner read (`worksType: "reads"`), replanned when its reads reach ten times that (`trial::within_reads`). A rooted `$or`'s branches in such a front are planned in the classic engine and cached with `worksType: "works"` under the subplanner's `o` key (RM-429). Past 2,000 documents a trial no longer reads the collection whole: it runs over what its candidates' scans read from the store in the works it takes (`order::Lazy::reached`, `stream::scan_reach`), read again for eight times as many works when it ran out of them (`plan::over_reach`; RM-342). It reports the plan mongod runs, which is not always how the Rust storage router reads the documents (`tasks/backlog-rust-mongodb.md` 7.075). Sweep: `tools/probes/explain_commands.py`, 786 replies. **Since 2026-10-13 the same plan sets the ORDER the Rust server returns documents in** (`explain/order.rs`, `tasks/backlog-rust-mongodb.md` 7.080): the router says only which documents match, and the plan says in what order mongod reads them, which is also which document a `limit: 1` delete or an update without `multi` changes. A sort is still the store's own; the plan decides how documents that compare equal are left. For a sort the predicate's index does not return, the planner counts mongod's trial of the two candidate plans (`plan::counted`, else the trial) on a collection of any size. Sweep: `tools/probes/find_order.py`, 6,531 replies. When a change touches the router or the planner, run it: a router change can no longer move the order, and a planner change now can. diff --git a/changelog.d/rm-429-or-branch-sbe.md b/changelog.d/rm-429-or-branch-sbe.md new file mode 100644 index 000000000..a7d90d8b5 --- /dev/null +++ b/changelog.d/rm-429-or-branch-sbe.md @@ -0,0 +1,14 @@ +### Rust MongoDB server: a rooted `$or`'s branches are cached in front of SBE stages + +A rooted `$or` in front of stages mongod runs in SBE (a `$group`, say) now +caches each branch the way mongod 8.2.11 does: a classic entry +(`worksType: "works"`) under a key whose shape ends in `o`, the flag mongod's +`encodeClassic` gives a subplanner's branch of an SBE-compatible query, so it +never shares an entry with a classic query of the same clause. + +#### Fixed + +- `$planCacheStats` lists the branch entries of such a pipeline, with + mongod's `planCacheShapeHash` and `planCacheKey`. + +Fixes RM-429 (tests/backlog/test_plan_cache_or_branches.py::test_an_or_branch_before_sbe_stages_is_cached) diff --git a/crates/secantus-commands/src/explain/cache_key.rs b/crates/secantus-commands/src/explain/cache_key.rs index 9a22ead4c..643544787 100644 --- a/crates/secantus-commands/src/explain/cache_key.rs +++ b/crates/secantus-commands/src/explain/cache_key.rs @@ -763,6 +763,9 @@ pub(crate) struct Parts<'a> { pub distinct: Option<&'a str>, /// Planned by the classic planner and run in SBE. pub sbe: bool, + /// A rooted `$or`'s branch, which the classic subplanner plans and + /// caches (`CanonicalQuery::forSubPlanner`). + pub subplanner: bool, } /// `encodeClassic`: the query shape. @@ -780,7 +783,15 @@ pub(crate) fn shape(e: &Expr, p: &Parts) -> String { out.push('0'); } out.push('f'); - out.push(if p.sbe { 's' } else { 'c' }); + // `c` for a plan the classic engine runs, `s` for one planned by the + // classic planner and run in SBE, and `o` for a subplanner's branch of + // a query SBE runs: its entry stores `works`, and the flag keeps a + // classic query from sharing it (mongod 8.2.11, RM-429). + out.push(match (p.sbe, p.subplanner) { + (false, _) => 'c', + (true, false) => 's', + (true, true) => 'o', + }); out } @@ -1037,6 +1048,27 @@ mod tests { assert_eq!(hex(compute_hash("res/im/||||fc")), "4F486929"); } + /// A subplanner's branch of a query SBE runs ends its shape in `o` + /// (mongod 8.2.11, 2026-11-14: the branch `{a: {$lt: 3}, b: {$lt: 50}}` + /// of a rooted `$or` in front of a `$group`, over `a_1` and `b_1`, RM-429). + #[test] + fn a_subplanner_branch_of_an_sbe_query_is_keyed_o() { + let e = parse(&doc! { "a": {"$lt": 3}, "b": {"$lt": 50} }).unwrap(); + let projection = doc! { "b": 1, "_id": 0 }; + let parts = Parts { + sort: None, + projection: Some(&projection), + collation: None, + distinct: None, + sbe: true, + subplanner: true, + }; + let s = shape(&e, &parts); + assert_eq!(s, "an[lta,ltb]||b||fo"); + assert_eq!(hex(compute_hash(&s)), "EDB869B0"); + assert_eq!(hex(key_hash(&format!("{s}<1><1>"))), "CA4248A6"); + } + /// The read's query settings move the key, not the shape hash (mongod /// 8.2.11, 2026-11-10: `find({x: 1})`, no index, RM-418). #[test] @@ -1095,6 +1127,7 @@ mod tests { collation: None, distinct: None, sbe: false, + subplanner: false, }; let shape_of = |q: Document| shape(&parse(&q).unwrap(), &none); assert_eq!( diff --git a/crates/secantus-commands/src/explain/change_stream.rs b/crates/secantus-commands/src/explain/change_stream.rs index 31241d532..67cffef89 100644 --- a/crates/secantus-commands/src/explain/change_stream.rs +++ b/crates/secantus-commands/src/explain/change_stream.rs @@ -748,6 +748,7 @@ pub(super) fn reply( collation: None, distinct: None, sbe: false, + subplanner: false, }, &cache_key::Indexability { catalog: &[], @@ -874,6 +875,7 @@ mod tests { collation: None, distinct: None, sbe: false, + subplanner: false, }, &cache_key::Indexability { catalog: &[], diff --git a/crates/secantus-commands/src/explain/mod.rs b/crates/secantus-commands/src/explain/mod.rs index 3b7328941..946b778cd 100644 --- a/crates/secantus-commands/src/explain/mod.rs +++ b/crates/secantus-commands/src/explain/mod.rs @@ -817,6 +817,7 @@ fn cache_key_of( collation, distinct, sbe, + subplanner: false, }; let ix = cache_key::Indexability { catalog: &full, diff --git a/crates/secantus-commands/src/explain/order.rs b/crates/secantus-commands/src/explain/order.rs index d11eaad3e..2ebefc126 100644 --- a/crates/secantus-commands/src/explain/order.rs +++ b/crates/secantus-commands/src/explain/order.rs @@ -806,12 +806,6 @@ impl plan::CacheHook for ReadCache<'_> { if !(top || branch) || !cacheable(q) { return None; } - // A branch of a rooted `$or` in front of stages SBE runs is cached - // by mongod under a key this does not yet build (RM-429): it is - // left uncached rather than share the entry of a classic query. - if branch && self.ask.asked.sbe { - return None; - } let parts = super::cache_key::Parts { sort: if top { q.sort } else { None }, // A branch is keyed with the query's projection (measured @@ -819,7 +813,10 @@ impl plan::CacheHook for ReadCache<'_> { projection: self.ask.asked.projection, collation: q.collation, distinct: if top { self.ask.asked.distinct } else { None }, - sbe: q.sbe, + // The subplanner plans a branch in the classic engine (so `q.sbe` + // is false for it), but keys it by whether the query runs in SBE. + sbe: if branch { self.ask.asked.sbe } else { q.sbe }, + subplanner: branch, }; let filter = if top { self.ask.filter } else { q.canonical }; let implies = |partial: &Document| self.storage.query_implies_partial(filter, partial); diff --git a/tasks/backlog-rust-mongodb.md b/tasks/backlog-rust-mongodb.md index 42f496532..79bd70065 100644 --- a/tasks/backlog-rust-mongodb.md +++ b/tasks/backlog-rust-mongodb.md @@ -2458,16 +2458,6 @@ Still different, all 77: itself. Backlog-ID: RM-056 Verify: none -- performance budget (timings against an earlier build) -- [ ] **A rooted `$or` in front of stages SBE runs caches no branch.** - mongod 8.2.11 caches each branch as a classic entry (`worksType: - "works"`) under a key that is neither the classic branch's nor the `s` - key of the same clause with the pipeline's projection (`CA4248A6` - for the test's clause, where those two are `0B947F35` and - `6F377EC6`): its encoding is still to be found. Until then such a - branch is not cached here, so that it cannot share a classic - query's entry. - Backlog-ID: RM-429 - Verify: tests/backlog/test_plan_cache_or_branches.py::test_an_or_branch_before_sbe_stages_is_cached - [ ] **`$planCacheStats` lists no `estimatedSizeBytes` on Linux.** The field is mongod's estimate of the entry's C++ objects, so its figure depends on how mongod was built (RM-341): this server reproduces diff --git a/tests/backlog/test_plan_cache_or_branches.py b/tests/backlog/test_plan_cache_or_branches.py index 28cbdc9f5..8596280ce 100644 --- a/tests/backlog/test_plan_cache_or_branches.py +++ b/tests/backlog/test_plan_cache_or_branches.py @@ -73,11 +73,12 @@ def _behind_sbe(client: Any) -> Any: return _entries(col) -@pytest.mark.backlog("RM-429") +@pytest.mark.backlog_fixed("RM-429") def test_an_or_branch_before_sbe_stages_is_cached(rust_mongo: Any, mongod: Any) -> None: """mongod 8.2.11 caches the branch of a rooted `$or` in front of a `$group` it runs in SBE as a classic entry (`worksType: "works"`) under - a key (`CA4248A6` here) that is neither the classic branch's - (`0B947F35`) nor the `s` key of the same clause; this server caches no - branch for such a pipeline.""" + a key (`CA4248A6` here) whose shape ends in `o`, the flag + `encodeClassic` gives a subplanner's branch of an SBE-compatible query, + rather than `c` or `s`. This server cached no branch for such a + pipeline.""" assert _behind_sbe(rust_mongo) == _behind_sbe(mongod) From f7d56a54e0c58432f2f6e16c54f859f01a9fe949 Mon Sep 17 00:00:00 2001 From: Joe Drumgoole Date: Sat, 14 Nov 2026 17:52:11 +0000 Subject: [PATCH 2/5] Rust MongoDB server: a compound index scan over several intervals examines mongod's keys (RM-433) Co-Authored-By: Claude Opus 5.5 --- changelog.d/rm-433-compound-scan-keys.md | 18 +++++ crates/secantus-commands/src/explain/exec.rs | 66 ++++++++++++++----- tasks/backlog-rust-mongodb.md | 20 ------ .../backlog/test_rm_found_2026_11_08_rm341.py | 7 +- .../backlog/test_rm_found_2026_11_08_rm342.py | 18 +++-- 5 files changed, 82 insertions(+), 47 deletions(-) create mode 100644 changelog.d/rm-433-compound-scan-keys.md diff --git a/changelog.d/rm-433-compound-scan-keys.md b/changelog.d/rm-433-compound-scan-keys.md new file mode 100644 index 000000000..ca6c0ef6d --- /dev/null +++ b/changelog.d/rm-433-compound-scan-keys.md @@ -0,0 +1,18 @@ +### Rust MongoDB server: a compound index scan over several intervals examines mongod's keys + +A scan of `a_1_c_1` for `{a: {$in: [3, 60]}, c: {$in: [1, 5]}}` now +examines 16 keys with 5 seeks, as mongod 8.2.11's does, in `explain` and in +the plan-cache trial whose works score the candidates. + +#### Fixed + +- Past the last interval of a later field, the scan ends when every earlier + field is on the point that is its last interval, as mongod's + `IndexBoundsChecker` does, instead of examining one more key. +- A seek to a field's next interval lands at the start of the later fields' + first intervals, so the keys before them are not examined. A losing + `$nin` candidate over three intervals of one field now seeks and returns + as mongod's does in the trial, and scores the same. + +Fixes RM-433 (tests/backlog/test_rm_found_2026_11_08_rm342.py::test_a_scan_of_two_in_lists_examines_mongods_keys) +Fixes RM-438 (tests/backlog/test_rm_found_2026_11_08_rm341.py::test_a_nin_candidate_reads_mongods_keys) diff --git a/crates/secantus-commands/src/explain/exec.rs b/crates/secantus-commands/src/explain/exec.rs index 6e0df4cb0..8e7d5b6a9 100644 --- a/crates/secantus-commands/src/explain/exec.rs +++ b/crates/secantus-commands/src/explain/exec.rs @@ -536,20 +536,23 @@ impl Scan { // `IndexBoundsChecker::getStartSeekPoint`: each field's first // interval's start. Where that start is exclusive, a key equal to it // there is still before the seek point. - let start: Vec<(&[u8], bool)> = self + let start: Vec<(Vec, bool)> = self .ranges .iter() - .map(|list| { - list.first() - .map_or((&[][..], true), |r| (r.0.as_slice(), r.1)) - }) + .map(|list| list.first().map_or((vec![], true), |r| (r.0.clone(), r.1))) .collect(); + self.skip_before(&start); + } + + /// Move past every entry before the seek point `start` (each field's + /// value, and whether a key equal to it there is at the point or + /// before it). + fn skip_before(&mut self, start: &[(Vec, bool)]) { while self.pos < self.entries.len() { let key = &self.entries[self.pos].0; let before = key .iter() - .map(Vec::as_slice) - .zip(start.iter().copied()) + .zip(start) .find_map(|(k, (s, inclusive))| { if k != s { Some(k < s) @@ -567,6 +570,15 @@ impl Scan { } } + /// Whether every field before `field` holds its last interval, and that + /// interval is a point: no later key can fall inside the bounds again. + fn on_last_points(&self, key: &[Vec], field: usize) -> bool { + key[..field].iter().zip(&self.ranges).all(|(part, list)| { + list.last() + .is_some_and(|r| r.0 == r.2 && r.1 && r.3 && inside(part, r)) + }) + } + /// One unit of mongod's work: an entry inside the bounds, a key that /// sends the scan seeking, or the end. The key that ends a scan of /// several intervals is examined in the work that ends it; a single run @@ -597,22 +609,40 @@ impl Scan { return Step::End; } self.keys += 1; - if !ahead && field == 0 { + // `IndexBoundsChecker::checkKey`: past the last interval of a + // field, the scan is over when no earlier field can move on to + // another key -- the first field, or earlier fields each on the + // point that is their last interval. + if !ahead && self.on_last_points(&key, field) { self.done = true; return Step::End; } - // Seek: past every key that fails the same way. - let prefix = key[..field].to_vec(); self.pos += 1; - while self.pos < self.entries.len() { - let other = &self.entries[self.pos].0; - if other[..field] != prefix[..] { - break; - } - if ahead && !matches!(self.outside(other), Some((f, true)) if f == field) { - break; + if ahead { + // Seek to the key's prefix, the field's next interval's + // start, and the first interval's start of each later field. + let part = &key[field]; + let next = self.ranges[field] + .iter() + .find(|r| part.as_slice() < r.0.as_slice() || (!r.1 && part == &r.0)) + .map(|r| (r.0.clone(), r.1)); + let mut start: Vec<(Vec, bool)> = + key[..field].iter().map(|k| (k.clone(), true)).collect(); + start.extend(next); + start.extend( + self.ranges[field + 1..] + .iter() + .map(|list| list.first().map_or((vec![], true), |r| (r.0.clone(), r.1))), + ); + self.skip_before(&start); + } else { + // Seek past every key with the same prefix. + let prefix = key[..field].to_vec(); + while self.pos < self.entries.len() + && self.entries[self.pos].0[..field] == prefix[..] + { + self.pos += 1; } - self.pos += 1; } self.seek_pending = true; return Step::Seek; diff --git a/tasks/backlog-rust-mongodb.md b/tasks/backlog-rust-mongodb.md index 79bd70065..cbce7a018 100644 --- a/tasks/backlog-rust-mongodb.md +++ b/tasks/backlog-rust-mongodb.md @@ -2474,17 +2474,6 @@ Still different, all 77: built for, each fitted by probing that platform's mongod 8.2.11.** Backlog-ID: RM-442 Verify: tests/backlog/test_rm_340_342_plan_cache_gaps.py::test_an_entry_reports_its_estimated_size_on_linux -- [ ] **A compound index scan whose second field has several intervals - examines a key more than mongod's for each value of the first.** - Over `a_1_c_1` (2,000 documents, `a: i % 100`, `c: i % 7`), mongod - 8.2.11 scans `{a: {$in: [3, 60]}, c: {$in: [1, 5]}}` examining 16 keys - with 5 seeks and `{a: 3, c: {$in: [1, 5]}}` 7 with 2; this server's - scan model (`exec::Scan`, which the trial steps) counts 18 with 7 and - 8 with 3, in `explain` and in a plan cache entry's - `creationExecStats`, where the extra works also move the scores. - Found 2026-11-08 (RM-342). - Verify: tests/backlog/test_rm_found_2026_11_08_rm342.py::test_a_scan_of_two_in_lists_examines_mongods_keys - Backlog-ID: RM-433 - [ ] **`explain`'s winning plan has no work accounting.** mongod 8.2.11's `executionStats` gives each stage its `works`, `advanced`, `needTime`, `needYield`, `isEOF` and timings, a FETCH its `alreadyHasObj`, an @@ -2533,15 +2522,6 @@ Still different, all 77: reports 83 works and neither field. Found 2026-11-08 (RM-341). Verify: tests/backlog/test_rm_found_2026_11_08_rm341.py::test_a_sort_merge_candidate_counts_as_mongods Backlog-ID: RM-437 -- [ ] **A scan over several intervals of one field seeks and returns - differently in the trial.** mongod 8.2.11's losing candidate for - `{a: {$lt: 5}, b: {$nin: [3, 4]}}` reads `b_1` over three intervals, - examining 101 keys in 3 seeks and returning 99; this server's scan - model seeks twice and returns 100, in `creationExecStats` (and so in - its score). Perhaps RM-433's cause on one field. Found 2026-11-08 - (RM-341). - Verify: tests/backlog/test_rm_found_2026_11_08_rm341.py::test_a_nin_candidate_reads_mongods_keys - Backlog-ID: RM-438 - [ ] **A candidate's FETCH filter puts an inexact `$exists` after the other predicates.** mongod 8.2.11's losing candidate for `{a: {$lt: 5}, b: {$exists: true}}` over `b_1` rechecks `{$and: [{b: {$exists: diff --git a/tests/backlog/test_rm_found_2026_11_08_rm341.py b/tests/backlog/test_rm_found_2026_11_08_rm341.py index 2e4952c33..75d7642ae 100644 --- a/tests/backlog/test_rm_found_2026_11_08_rm341.py +++ b/tests/backlog/test_rm_found_2026_11_08_rm341.py @@ -95,12 +95,13 @@ def _losers(client: Any, q: dict) -> list: return out -@pytest.mark.backlog("RM-438") +@pytest.mark.backlog_fixed("RM-438") def test_a_nin_candidate_reads_mongods_keys(rust_mongo: Any, mongod: Any) -> None: """mongod 8.2.11's losing candidate for `{a: {$lt: 5}, b: {$nin: [3, 4]}}` -- `b_1` read over three intervals -- examines 101 keys in the - trial, seeking 3 times and returning 99; this server's seeks twice and - returns 100.""" + trial, seeking 3 times and returning 99; this server's sought twice and + returned 100, until a seek to the next interval landed at its start + (RM-433's fix).""" q = {"a": {"$lt": 5}, "b": {"$nin": [3, 4]}} assert _losers(rust_mongo, q) == _losers(mongod, q) diff --git a/tests/backlog/test_rm_found_2026_11_08_rm342.py b/tests/backlog/test_rm_found_2026_11_08_rm342.py index 59ffbe9a3..230094f3a 100644 --- a/tests/backlog/test_rm_found_2026_11_08_rm342.py +++ b/tests/backlog/test_rm_found_2026_11_08_rm342.py @@ -25,18 +25,24 @@ def _stages(stage: Any) -> list: return out -@pytest.mark.backlog("RM-433") +@pytest.mark.backlog_fixed("RM-433") def test_a_scan_of_two_in_lists_examines_mongods_keys(rust_mongo: Any, mongod: Any) -> None: """mongod 8.2.11 scans `{a: {$in: [3, 60]}, c: {$in: [1, 5]}}` over `a_1_c_1` (2,000 documents, `a: i % 100`, `c: i % 7`) examining 16 - keys; this server's scan counts 18.""" + keys with 5 seeks, and `{a: 3, c: {$in: [1, 5]}}` 7 with 2; this + server's scan counted 18 with 7 and 8 with 3: it went on past the last + interval of `c` when `a` was on its last point, and a seek to the next + `a` landed before the first interval of `c`.""" - def keys(client: Any) -> list: - f = {"a": {"$in": [3, 60]}, "c": {"$in": [1, 5]}} + def keys(client: Any, f: dict) -> list: return [s.get("keysExamined") for s in _stages(_explained(client, f))] - assert keys(mongod) == [None, 16] - assert keys(rust_mongo) == keys(mongod) + two = {"a": {"$in": [3, 60]}, "c": {"$in": [1, 5]}} + one = {"a": 3, "c": {"$in": [1, 5]}} + assert keys(mongod, two) == [None, 16] + assert keys(mongod, one) == [None, 7] + for f in (two, one): + assert keys(rust_mongo, f) == keys(mongod, f) @pytest.mark.backlog("RM-434") From 48ff604ee58a15ef523b509f2f1ba9b9c285dfd7 Mon Sep 17 00:00:00 2001 From: Joe Drumgoole Date: Sat, 14 Nov 2026 18:04:26 +0000 Subject: [PATCH 3/5] Rust MongoDB server: explain's winning plan reports its work accounting (RM-434; RM-692 filed) Co-Authored-By: Claude Opus 5.5 --- changelog.d/rm-434-explain-work-accounting.md | 16 +++++++ crates/secantus-commands/src/explain/exec.rs | 16 ++++++- crates/secantus-commands/src/explain/mod.rs | 30 +++++++++++++ .../secantus-commands/src/explain/sbe/mod.rs | 2 +- .../secantus-commands/src/explain/stream.rs | 2 +- crates/secantus-commands/src/explain/trial.rs | 10 +++++ tasks/backlog-rust-mongodb.md | 28 +++++------- .../backlog/test_rm_found_2026_11_08_rm310.py | 4 +- .../backlog/test_rm_found_2026_11_08_rm342.py | 4 +- .../backlog/test_rm_found_2026_11_14_rm434.py | 43 +++++++++++++++++++ 10 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 changelog.d/rm-434-explain-work-accounting.md create mode 100644 tests/backlog/test_rm_found_2026_11_14_rm434.py diff --git a/changelog.d/rm-434-explain-work-accounting.md b/changelog.d/rm-434-explain-work-accounting.md new file mode 100644 index 000000000..1d1f650cb --- /dev/null +++ b/changelog.d/rm-434-explain-work-accounting.md @@ -0,0 +1,16 @@ +### Rust MongoDB server: `explain`'s winning plan reports its work accounting + +`executionStats` now gives every stage of the winning plan the counters +mongod 8.2.11 gives: `works`, `advanced`, `needTime`, `needYield`, +`saveState`, `restoreState` and `isEOF`, and an index scan's `seeks`, +`dupsTested` and `dupsDropped`. Before, a stage had only its `nReturned` and +examined counts. + +#### Fixed + +- A `find`, `distinct` or aggregate plan's `executionStages` carry each + stage's work accounting, counted by working the plan to its end as mongod + works the winner. Under an aggregate's `$cursor`, `saveState` and + `restoreState` count the batches the cursor loaded. +- Fixes RM-434 (tests/backlog/test_rm_found_2026_11_08_rm342.py::test_explain_gives_each_stage_its_work_accounting, tests/backlog/test_rm_found_2026_11_08_rm310.py::test_a_distinct_scan_run_reports_its_work_counters) +- Filed RM-692: a `$cursor` that the pipeline stops reading still reports a plan run to its end. diff --git a/crates/secantus-commands/src/explain/exec.rs b/crates/secantus-commands/src/explain/exec.rs index 8e7d5b6a9..d2160562d 100644 --- a/crates/secantus-commands/src/explain/exec.rs +++ b/crates/secantus-commands/src/explain/exec.rs @@ -1406,5 +1406,19 @@ pub(crate) fn run(plan: &Node, env: &Env) -> Stats { root.extra.insert("nSkipped", num(skipped)); } while root.pull(env).is_some() {} - root.stats() + let mut stats = root.stats(); + if let Some(worked) = super::trial::worked_through(plan, env) { + graft_work(&mut stats, &worked); + } + stats +} + +/// Each stage's work accounting from `worked`, a tree of the same plan. +fn graft_work(stats: &mut Stats, worked: &Stats) { + stats.work = worked.work; + if stats.children.len() == worked.children.len() { + for (s, w) in stats.children.iter_mut().zip(&worked.children) { + graft_work(s, w); + } + } } diff --git a/crates/secantus-commands/src/explain/mod.rs b/crates/secantus-commands/src/explain/mod.rs index 946b778cd..5c28d9e95 100644 --- a/crates/secantus-commands/src/explain/mod.rs +++ b/crates/secantus-commands/src/explain/mod.rs @@ -1391,6 +1391,33 @@ fn set_aside(st: &mut Stats) { } /// Raise each counter of `run` to what `trial` counted in the same stage. +/// A `$cursor`'s plan, saved when the cursor is prepared and after each +/// batch it loads, restored before each (`DocumentSourceCursor::loadBatch`). +/// The plan here ran to its end; where the stages after the cursor stopped +/// pulling first, mongod's stopped with them, and the run's work counts are +/// not its: they are left out (RM-692). +fn cursor_yields(run: &mut Stats, pulled: Option) { + if pulled.is_some_and(|p| p < run.n_returned) { + strip_work(run); + return; + } + let (_, loads, _) = sbe::batches(run.n_returned, None); + stream::yielded( + run, + stream::Yields { + saved: loads + 1, + restored: loads, + }, + ); +} + +fn strip_work(stats: &mut Stats) { + stats.work = None; + for c in &mut stats.children { + strip_work(c); + } +} + fn at_least(run: &mut Stats, trial: &Stats) { run.n_returned = run.n_returned.max(trial.n_returned); run.keys = run.keys.max(trial.keys); @@ -2069,6 +2096,9 @@ fn reply_for(r: &Request, verbosity: Verbosity, ctx: &mut CommandContext) -> Han if let Some((_, _, trial)) = tried.first() { at_least(&mut run, trial); } + if !s.rest.is_empty() { + cursor_yields(&mut run, cursor_pulled); + } run }); let mut reply = doc! { "explainVersion": "1" }; diff --git a/crates/secantus-commands/src/explain/sbe/mod.rs b/crates/secantus-commands/src/explain/sbe/mod.rs index 3b2026181..ea36b2f41 100644 --- a/crates/secantus-commands/src/explain/sbe/mod.rs +++ b/crates/secantus-commands/src/explain/sbe/mod.rs @@ -119,7 +119,7 @@ pub(crate) enum Pull { /// that comes back full is followed by another unless the stages after the /// cursor have their `demand`, so a plan that returns exactly a batch's /// worth is asked once more and answers nothing. -fn batches(n: i64, demand: Option) -> (i64, i64, bool) { +pub(crate) fn batches(n: i64, demand: Option) -> (i64, i64, bool) { let (mut got, mut loads, mut size) = (0_i64, 0_i64, INITIAL_BATCH); loop { let this = size.min(n - got); diff --git a/crates/secantus-commands/src/explain/stream.rs b/crates/secantus-commands/src/explain/stream.rs index 05c44e254..0b6e90d8b 100644 --- a/crates/secantus-commands/src/explain/stream.rs +++ b/crates/secantus-commands/src/explain/stream.rs @@ -3411,7 +3411,7 @@ impl WriteRun { } /// Every stage of `stats` went through `y`. -fn yielded(stats: &mut Stats, y: Yields) { +pub(crate) fn yielded(stats: &mut Stats, y: Yields) { if let Some(w) = stats.work.as_mut() { w.saved = y.saved; w.restored = y.restored; diff --git a/crates/secantus-commands/src/explain/trial.rs b/crates/secantus-commands/src/explain/trial.rs index e7f8ec3e9..61240d575 100644 --- a/crates/secantus-commands/src/explain/trial.rs +++ b/crates/secantus-commands/src/explain/trial.rs @@ -813,3 +813,13 @@ pub(crate) fn within_reads(plan: &Node, env: &Env, need: i64, budget: i64) -> Op } } } + +/// The plan worked to its end, as mongod's executor works the winning plan +/// (the trial's works and the run's after it are one stage tree's): what +/// each stage counted. `None`: a stage this does not step. +pub(crate) fn worked_through(plan: &Node, env: &Env) -> Option { + let mut cache = Entries::new(); + let mut stage = Stage::new(plan, env, &mut cache)?; + while !matches!(stage.work(env), Work::Eof) {} + Some(stage.stats()) +} diff --git a/tasks/backlog-rust-mongodb.md b/tasks/backlog-rust-mongodb.md index cbce7a018..bd6ab1246 100644 --- a/tasks/backlog-rust-mongodb.md +++ b/tasks/backlog-rust-mongodb.md @@ -2474,24 +2474,16 @@ Still different, all 77: built for, each fitted by probing that platform's mongod 8.2.11.** Backlog-ID: RM-442 Verify: tests/backlog/test_rm_340_342_plan_cache_gaps.py::test_an_entry_reports_its_estimated_size_on_linux -- [ ] **`explain`'s winning plan has no work accounting.** mongod 8.2.11's - `executionStats` gives each stage its `works`, `advanced`, `needTime`, - `needYield`, `isEOF` and timings, a FETCH its `alreadyHasObj`, an - IXSCAN its `seeks`, `dupsTested` and `dupsDropped`. This server's - `executionStages` give a stage only its `nReturned` and examined - counts (`allPlansExecution` and `$planCacheStats` do carry the - accounting the trial counted). `tools/probes/explain_commands.py` - leaves these fields out of its comparison. Found 2026-11-08 (RM-342). - (This was RM-446: A DISTINCT_SCAN plan's `executionStats` gives no work - counters.) mongod 8.2.11's `executionStages` for a DISTINCT_SCAN plan -- - of `distinct`, and of a `$group` it answers from one -- gives every - stage's `works`, `advanced`, `needTime`, `needYield`, `saveState`, - `restoreState` and `isEOF` (`distinct("x", {x: {$gt: 0}})` over `x_1`: - `works: 4`). This server's gives `nReturned` and the scan's own fields, - none of them. The trial's counters, in `allPlansExecution`, are given. - Found 2026-11-08 (RM-310). - Verify: tests/backlog/test_rm_found_2026_11_08_rm342.py::test_explain_gives_each_stage_its_work_accounting, tests/backlog/test_rm_found_2026_11_08_rm310.py::test_a_distinct_scan_run_reports_its_work_counters - Backlog-ID: RM-434 +- [ ] **A `$cursor` the pipeline stops reading reports a plan run to its + end.** Under `[{$match: {a: {$gt: 2}}}, {$unwind: "$b"}, {$limit: 3}]` + (100 documents, `a_1`), mongod 8.2.11's `DocumentSourceCursor` loads one + batch of 32 documents and stops: its FETCH and IXSCAN report + `nReturned` 32, 32 works, `saveState` 2, `restoreState` 1, `isEOF` 0. + This server's classic run goes to the end (70 documents), and leaves the + work accounting out where its stages' counts are not mongod's + (`cursor_yields`). Found 2026-11-14 (RM-434). + Verify: tests/backlog/test_rm_found_2026_11_14_rm434.py::test_a_cursor_the_pipeline_stops_reading_reports_one_batch + Backlog-ID: RM-692 - [ ] **A `$sort` after a `$group` mongod pushes into SBE counts its documents smaller than mongod's.** `[{$group: {_id: "$a"}}, {$sort: {_id: 1}}]` over ten documents: mongod 8.2.11's `$sort` reports diff --git a/tests/backlog/test_rm_found_2026_11_08_rm310.py b/tests/backlog/test_rm_found_2026_11_08_rm310.py index 062d020d0..8d5f5cdbc 100644 --- a/tests/backlog/test_rm_found_2026_11_08_rm310.py +++ b/tests/backlog/test_rm_found_2026_11_08_rm310.py @@ -50,12 +50,12 @@ def _distinct_scan_runs(client: Any) -> dict: } -@pytest.mark.backlog("RM-434") +@pytest.mark.backlog_fixed("RM-434") def test_a_distinct_scan_run_reports_its_work_counters(rust_mongo: Any, mongod: Any) -> None: """mongod 8.2.11's `executionStats` of a DISTINCT_SCAN plan, for `distinct` and for a `$group` it answers from one, gives every stage's `works`, `advanced`, `needTime`, `needYield`, `saveState`, - `restoreState` and `isEOF`. This server's gives none of them.""" + `restoreState` and `isEOF`. This server's gave none of them.""" got, want = _distinct_scan_runs(rust_mongo), _distinct_scan_runs(mongod) assert want["distinct"][1]["works"] == 4 assert got == want diff --git a/tests/backlog/test_rm_found_2026_11_08_rm342.py b/tests/backlog/test_rm_found_2026_11_08_rm342.py index 230094f3a..0a9a2e4a6 100644 --- a/tests/backlog/test_rm_found_2026_11_08_rm342.py +++ b/tests/backlog/test_rm_found_2026_11_08_rm342.py @@ -45,13 +45,13 @@ def keys(client: Any, f: dict) -> list: assert keys(rust_mongo, f) == keys(mongod, f) -@pytest.mark.backlog("RM-434") +@pytest.mark.backlog_fixed("RM-434") def test_explain_gives_each_stage_its_work_accounting(rust_mongo: Any, mongod: Any) -> None: """mongod 8.2.11's `executionStats` gives each stage of the winning plan its work accounting -- `works`, `advanced`, `needTime`, `needYield`, `isEOF`, and a FETCH's `alreadyHasObj`, an IXSCAN's `seeks`, `dupsTested` and `dupsDropped` -- for `{a: 3}` over `a_1_c_1`. - This server's stages carry only their `nReturned` and examined counts.""" + This server's stages carried only their `nReturned` and examined counts.""" def fields(client: Any) -> list: return [ diff --git a/tests/backlog/test_rm_found_2026_11_14_rm434.py b/tests/backlog/test_rm_found_2026_11_14_rm434.py new file mode 100644 index 000000000..08dcda163 --- /dev/null +++ b/tests/backlog/test_rm_found_2026_11_14_rm434.py @@ -0,0 +1,43 @@ +"""Divergences found while giving the winning plan its work accounting +(RM-434, 2026-11-14), each measured against mongod 8.2.11.""" + +from __future__ import annotations + +from typing import Any + +import pytest + + +def _cursor_stages(client: Any) -> list: + client.drop_database("rm434_found") + col = client.rm434_found.c + col.insert_many([{"_id": i, "a": i % 10, "b": [1, 2]} for i in range(100)]) + col.create_index("a") + e = col.database.command( + "explain", + { + "aggregate": "c", + "pipeline": [{"$match": {"a": {"$gt": 2}}}, {"$unwind": "$b"}, {"$limit": 3}], + "cursor": {}, + }, + verbosity="executionStats", + ) + stage = e["stages"][0]["$cursor"]["executionStats"]["executionStages"] + out = [] + while stage: + out.append({k: stage.get(k) for k in ("stage", "nReturned", "works", "saveState", "isEOF")}) + stage = stage.get("inputStage") + return out + + +@pytest.mark.backlog("RM-692") +def test_a_cursor_the_pipeline_stops_reading_reports_one_batch( + rust_mongo: Any, mongod: Any +) -> None: + """mongod 8.2.11's `$cursor` under `[$match, $unwind, $limit: 3]` loads + one batch of 32 documents and stops: its FETCH and IXSCAN report + `nReturned` 32, 32 works, `saveState` 2 and `isEOF` 0. This server's + plan runs to the end (70 documents) and reports no work accounting.""" + want = _cursor_stages(mongod) + assert want[0] == {"stage": "FETCH", "nReturned": 32, "works": 32, "saveState": 2, "isEOF": 0} + assert _cursor_stages(rust_mongo) == want From 69bfa81c2f6a65b829b004213f4396292edc3dba Mon Sep 17 00:00:00 2001 From: Joe Drumgoole Date: Sat, 14 Nov 2026 18:42:27 +0000 Subject: [PATCH 4/5] Rust MongoDB server: a $sort after a $group run in SBE counts the groups as SBE builds them (RM-435; RM-693 filed) Co-Authored-By: Claude Opus 5.5 --- changelog.d/rm-435-sbe-group-sort-size.md | 18 +++++++ crates/secantus-core/src/aggregate.rs | 48 +++++++++++++++++++ crates/secantus-core/src/docstore.rs | 20 ++++++++ tasks/backlog-rust-mongodb.md | 20 ++++---- .../backlog/test_rm_found_2026_11_08_rm071.py | 34 +++++++++++-- tests/backlog/test_rm_found_rm435.py | 37 ++++++++++++++ 6 files changed, 164 insertions(+), 13 deletions(-) create mode 100644 changelog.d/rm-435-sbe-group-sort-size.md create mode 100644 tests/backlog/test_rm_found_rm435.py diff --git a/changelog.d/rm-435-sbe-group-sort-size.md b/changelog.d/rm-435-sbe-group-sort-size.md new file mode 100644 index 000000000..13031a100 --- /dev/null +++ b/changelog.d/rm-435-sbe-group-sort-size.md @@ -0,0 +1,18 @@ +### Rust MongoDB server: a `$sort` after a `$group` run in SBE counts mongod's bytes + +A `$sort` behind a `$group` that mongod 8.2 runs in SBE now reports the +`totalDataSizeSortedBytesEstimate` mongod does: SBE hands each group to the +classic `$sort` as a document built field by field, each sub-document its +own built storage, which costs more than the document the classic engine +builds. + +#### Fixed + +- `[{$group: {_id: "$a"}}, {$sort: {_id: 1}}]` over ten documents counted + 351 bytes where mongod 8.2.11 counts 735. + Fixes RM-435 (tests/backlog/test_rm_found_2026_11_08_rm071.py::test_a_sort_after_a_pushed_down_group_sizes_what_sbe_hands_it) + +#### Found + +- RM-693: a `$group` with `$mod` in its `_id`, or an expression object in an + accumulator, is not pushed into SBE. diff --git a/crates/secantus-core/src/aggregate.rs b/crates/secantus-core/src/aggregate.rs index 98a5a0505..ffa54559a 100644 --- a/crates/secantus-core/src/aggregate.rs +++ b/crates/secantus-core/src/aggregate.rs @@ -300,6 +300,35 @@ pub fn apply_pipeline( .as_ref() .filter(|_| pipeline_sort) .map(|st| st.iter().map(docstore::Stored::for_sorter).collect()); + // A `$group` mongod ran in SBE hands the classic `$sort` after it + // each group as a document built field by field from its slots + // (`MutableDocument`), over an empty BSON object (measured + // 8.2.11: `[{$group: {_id: "$a"}}, {$sort: {_id: 1}}]` counts + // 245 bytes a group, its 16-byte key included, RM-435). + let sizes = match sizes { + None if source == Some(crate::sorter::SortSource::Stored) + && after_sbe_group(before) => + { + // A key that is a sub-document is the group's built one, + // not a document read from BSON as the sorter sizes it. + let fields = sort_fields(spec)?; + docs.iter() + .map(|d| { + let key: usize = sort_key_values(d, &fields)? + .iter() + .filter_map(Bson::as_document) + .map(|k| { + docstore::sbe_built(k).approximate() + - memlimit::document_size(k) + }) + .sum(); + Ok(docstore::sbe_built(d).for_sorter() + key) + }) + .collect::>>() + .map(Some)? + } + sizes => sizes, + }; // Where each document goes, to keep its storage beside it: the // sorter's own order where it sorts them, since its ties come // in the C++ library's order (`sorter::sort_held`). @@ -2140,6 +2169,25 @@ pub fn limit_for_pushdown(after: &[Bson]) -> Option { limit } +/// Whether `before` ends with the SBE section [`crate::sbe::mark`] closes +/// after a `$group`: the documents it hands on are SBE's output. +fn after_sbe_group(before: &[Bson]) -> bool { + let engine = |st: &Bson, on: bool| { + st.as_document() + .and_then(|d| d.get(crate::sbe::ENGINE_STAGE)) + .and_then(Bson::as_bool) + == Some(on) + }; + match before { + [.., group, last] => { + engine(last, false) + && stage_name_of(group) == "$group" + && before.iter().any(|st| engine(st, true)) + } + _ => false, + } +} + /// Which of mongod's sorts a `$sort` after `before` is, for what each /// document counts (`sorter`): the stages that change a document leave it /// counted for its cache. diff --git a/crates/secantus-core/src/docstore.rs b/crates/secantus-core/src/docstore.rs index 79e1c21ed..9035714b1 100644 --- a/crates/secantus-core/src/docstore.rs +++ b/crates/secantus-core/src/docstore.rs @@ -461,6 +461,26 @@ fn produced(expr: &Bson, v: &Bson) -> Held { } } +/// A document SBE hands the classic engine (`sbe::value` to `Document`, +/// as a `$group` pushed into SBE returns its groups): built field by field +/// over an empty BSON object, each sub-document a storage of its own built +/// the same way (measured 8.2.11, RM-435). +pub fn sbe_built(doc: &Document) -> Stored { + let mut s = Stored::built(); + for (k, v) in doc { + s.append(k, sbe_value(v)); + } + s +} + +fn sbe_value(v: &Bson) -> Held { + match v { + Bson::Document(d) => Held::Document(Rc::new(RefCell::new(sbe_built(d)))), + Bson::Array(a) => Held::Array(a.iter().map(sbe_value).collect()), + _ => Held::Bytes(held(v)), + } +} + /// `Value::getApproximateSize` of `v`, the value `expr` evaluated to. pub fn produced_size(expr: &Bson, v: &Bson) -> usize { memlimit::VALUE + produced_held(expr, v) diff --git a/tasks/backlog-rust-mongodb.md b/tasks/backlog-rust-mongodb.md index bd6ab1246..09ce07481 100644 --- a/tasks/backlog-rust-mongodb.md +++ b/tasks/backlog-rust-mongodb.md @@ -1736,7 +1736,7 @@ live. Not filed here: `$rankFusion` / `$score` / `$scoreFusion` (RM-198), document (a `MutableDocument` with room for its two fields), and the `$sort` counts 2450 bytes over ten groups; this server counts the BSON each encodes, 1170, as it keeps no storage behind an SBE - `$group` (see also RM-435). Found 2026-11-13 (RM-519). + `$group`. Found 2026-11-13 (RM-519). Verify: tests/backlog/test_rm_found_rm519.py::test_a_project_after_a_pushed_down_group_builds_its_documents Backlog-ID: RM-638 - [ ] **A `$sort` behind a `$setWindowFields` that sorted by the same key @@ -2484,15 +2484,15 @@ Still different, all 77: (`cursor_yields`). Found 2026-11-14 (RM-434). Verify: tests/backlog/test_rm_found_2026_11_14_rm434.py::test_a_cursor_the_pipeline_stops_reading_reports_one_batch Backlog-ID: RM-692 -- [ ] **A `$sort` after a `$group` mongod pushes into SBE counts its - documents smaller than mongod's.** `[{$group: {_id: "$a"}}, {$sort: - {_id: 1}}]` over ten documents: mongod 8.2.11's `$sort` reports - `totalDataSizeSortedBytesEstimate` 735 for the three groups the SBE - plan hands it, this server's 351, its estimate of the same three - documents as the classic engine builds them. Found 2026-11-08 - (RM-071). - Verify: tests/backlog/test_rm_found_2026_11_08_rm071.py::test_a_sort_after_a_pushed_down_group_sizes_what_sbe_hands_it - Backlog-ID: RM-435 +- [ ] **A `$group` whose `_id` takes `$mod`, or whose accumulator takes an + expression object, is not pushed into SBE.** `[{$group: {_id: {$mod: + ["$b", 2]}}}, {$sort: {_id: 1}}]`, and `{_id: "$a", q: {$push: {s: + "$s"}}}` (`$addToSet` too): mongod 8.2.11 runs the `$group` in SBE, + `explainVersion` "2" and no `$group` in `stages`; this server explains + version "1" with a classic `$group`. The `$sort`'s byte count after it + already matches. Found 2026-11-14 (RM-435). + Verify: tests/backlog/test_rm_found_rm435.py::test_a_group_mongod_pushes_into_sbe_is_pushed + Backlog-ID: RM-693 - [ ] **`estimatedSizeBytes` is left out of entries whose index tree this server does not build.** It sizes an entry's `PlanCacheIndexTree` from the canonical query and the clauses the winning plan's one index diff --git a/tests/backlog/test_rm_found_2026_11_08_rm071.py b/tests/backlog/test_rm_found_2026_11_08_rm071.py index ecc38f0fc..1e31e4517 100644 --- a/tests/backlog/test_rm_found_2026_11_08_rm071.py +++ b/tests/backlog/test_rm_found_2026_11_08_rm071.py @@ -18,12 +18,40 @@ def _sort_bytes(client: Any) -> Any: return e["stages"][1].get("totalDataSizeSortedBytesEstimate") -@pytest.mark.backlog("RM-435") +@pytest.mark.backlog_fixed("RM-435") def test_a_sort_after_a_pushed_down_group_sizes_what_sbe_hands_it( rust_mongo: Any, mongod: Any ) -> None: """mongod 8.2.11 sorts the three groups `[{$group: {_id: "$a"}}, {$sort: - {_id: 1}}]` makes in SBE as 735 bytes; this server's `$sort` counts the - same three documents as 351.""" + {_id: 1}}]` makes in SBE as 735 bytes; this server's `$sort` counted the + same three documents as 351, as the classic engine builds them.""" assert _sort_bytes(mongod) == 735 assert _sort_bytes(rust_mongo) == _sort_bytes(mongod) + + +def _sort_bytes_of(client: Any, group: dict) -> Any: + client.drop_database("rm071_found") + col = client.rm071_found.c + col.insert_many([{"a": i % 3, "b": i, "s": "x" * (i + 20)} for i in range(10)]) + pipeline = [{"$group": group}, {"$sort": {"_id": 1}}] + cmd = {"aggregate": "c", "pipeline": pipeline, "cursor": {}} + e = col.database.command("explain", cmd, verbosity="executionStats") + return e["stages"][1].get("totalDataSizeSortedBytesEstimate") + + +@pytest.mark.backlog_fixed("RM-435") +@pytest.mark.parametrize( + "group", + [ + {"_id": "$a", "c": {"$sum": 1}}, + {"_id": "$a", "c": {"$sum": "$b"}, "m": {"$max": "$s"}}, + {"_id": {"a": "$a"}, "f": {"$first": "$s"}}, + {"_id": "$a", "p": {"$push": "$b"}}, + ], +) +def test_a_sort_after_a_pushed_down_group_sizes_each_built_group( + rust_mongo: Any, mongod: Any, group: dict +) -> None: + """Groups of several fields, a long string and a document `_id` among + them, counted as mongod 8.2.11 builds them out of SBE.""" + assert _sort_bytes_of(rust_mongo, group) == _sort_bytes_of(mongod, group) diff --git a/tests/backlog/test_rm_found_rm435.py b/tests/backlog/test_rm_found_rm435.py new file mode 100644 index 000000000..8d12f32f6 --- /dev/null +++ b/tests/backlog/test_rm_found_rm435.py @@ -0,0 +1,37 @@ +"""Divergences found working RM-435 (2026-11-14), each measured against +mongod 8.2.11.""" + +from __future__ import annotations + +from typing import Any + +import pytest + + +def _engine(client: Any, group: dict) -> Any: + client.drop_database("rm435_found") + col = client.rm435_found.c + col.insert_many([{"a": i % 3, "b": i, "s": "x" * (i + 20)} for i in range(10)]) + pipeline = [{"$group": group}, {"$sort": {"_id": 1}}] + cmd = {"aggregate": "c", "pipeline": pipeline, "cursor": {}} + e = col.database.command("explain", cmd, verbosity="executionStats") + return e.get("explainVersion"), [next(iter(s)) for s in e["stages"]] + + +@pytest.mark.backlog("RM-693") +@pytest.mark.parametrize( + "group", + [ + {"_id": {"$mod": ["$b", 2]}}, + {"_id": {"a": "$a", "b": {"$mod": ["$b", 2]}}, "n": {"$sum": 1}}, + {"_id": "$a", "q": {"$push": {"s": "$s"}}}, + {"_id": "$a", "q": {"$addToSet": {"s": "$s"}}}, + ], +) +def test_a_group_mongod_pushes_into_sbe_is_pushed( + rust_mongo: Any, mongod: Any, group: dict +) -> None: + """mongod 8.2.11 runs each of these `$group`s in SBE (`explainVersion` + "2", no `$group` among the `stages`); this server keeps them classic.""" + assert _engine(mongod, group) == ("2", ["$cursor", "$sort"]) + assert _engine(rust_mongo, group) == _engine(mongod, group) From d35154a772f735c714b4da2aedfa2f9a11f41312 Mon Sep 17 00:00:00 2001 From: Joe Drumgoole Date: Sat, 14 Nov 2026 19:21:01 +0000 Subject: [PATCH 5/5] Rust MongoDB server: $planCacheStats sizes $or and $elemMatch entries (RM-436; RM-694 filed) Co-Authored-By: Claude Opus 5.5 --- .../rm-436-plan-cache-size-or-elemmatch.md | 15 +++ .../src/explain/entry_size.rs | 122 +++++++++++++++--- crates/secantus-commands/src/explain/order.rs | 48 ++++--- crates/secantus-commands/src/explain/plan.rs | 68 ++++++++++ tasks/backlog-rust-mongodb.md | 30 +++-- .../backlog/test_rm_found_2026_11_08_rm341.py | 52 +++++++- 6 files changed, 288 insertions(+), 47 deletions(-) create mode 100644 changelog.d/rm-436-plan-cache-size-or-elemmatch.md diff --git a/changelog.d/rm-436-plan-cache-size-or-elemmatch.md b/changelog.d/rm-436-plan-cache-size-or-elemmatch.md new file mode 100644 index 000000000..dbd18a1a8 --- /dev/null +++ b/changelog.d/rm-436-plan-cache-size-or-elemmatch.md @@ -0,0 +1,15 @@ +### Rust MongoDB server: `$planCacheStats` sizes `$or` and `$elemMatch` entries + +A plan cache entry whose winner reads an `OR` of index scans beside other +clauses, or whose query holds an `$elemMatch` its index does not read, now +reports mongod 8.2.11's `estimatedSizeBytes` on macOS arm64 instead of +leaving the field out. + +#### Fixed + +- The entry's `PlanCacheIndexTree` is built for an `OR` over single-predicate + branches (each tagged with the index it reads, `tagForSort` taking the + lowest-numbered) and for an `ELEM_MATCH_VALUE` node, and an `OR` stage's + `OrStats` is sized. + +Fixes RM-436 (tests/backlog/test_rm_found_2026_11_08_rm341.py::test_entries_this_does_not_size_report_their_size) diff --git a/crates/secantus-commands/src/explain/entry_size.rs b/crates/secantus-commands/src/explain/entry_size.rs index 949c8fc8d..a6989019b 100644 --- a/crates/secantus-commands/src/explain/entry_size.rs +++ b/crates/secantus-commands/src/explain/entry_size.rs @@ -57,6 +57,8 @@ pub(crate) struct Layout { pub projection_stats: u64, pub and_sorted_stats: u64, pub merge_sort_stats: u64, + /// `OrStats` (measured 8.2.11: an `OR` winner adds 32 bytes). + pub or_stats: u64, /// `std::string::capacity()` of a string of this length, built by copy. pub string_capacity: fn(usize) -> u64, } @@ -81,6 +83,7 @@ pub(crate) const LIBCXX_ARM64: Layout = Layout { projection_stats: 24, and_sorted_stats: 32, merge_sort_stats: 40, + or_stats: 32, string_capacity: libcxx_string_capacity, }; @@ -122,6 +125,21 @@ pub(crate) struct IndexCopy { /// Per key field, how many path components are multikey; empty for an /// index without path-level multikey metadata (`_id_`). pub multikey: Vec, + /// Its position among the query's relevant indexes, the catalog's + /// order: `tagForSort` tags a node with its lowest-numbered index. + pub number: usize, +} + +/// The index tags mongod's plan put on one clause of the conjunction. +#[derive(Clone, Debug)] +pub(crate) enum ClauseTag { + /// No index answers the clause: it is rechecked by a FETCH. + None, + /// The clause's predicate set the bounds of this index. + One(IndexCopy), + /// An `$or` whose every branch is one predicate that sets the bounds + /// of the index given for it, in the order of the branches. + Or(Vec), } /// A node of the `PlanCacheIndexTree`: the match expression's node, and the @@ -228,6 +246,7 @@ fn stage(l: &Layout, s: &Document) -> Option { } // `failedAnd`, sized to the children. "AND_SORTED" => children.len() as u64 * 8 + l.and_sorted_stats, + "OR" => l.or_stats, _ => return None, }; Some(size + specific + l.stage_stats) @@ -261,17 +280,21 @@ pub(crate) fn estimate( /// `canonical` (its `parsedQuery`) when its plan assigned the clauses /// `tagged` to `index` (`QueryPlanner::cacheDataFromTaggedTree`, after /// `tagForSort`). `None` for an expression whose tree this does not build: -/// an `$or` or `$nor` (an `OR` may take index tags of its own), an -/// `$elemMatch`, a `$not` of several predicates, `$jsonSchema`. +/// an `$or` or `$nor` that is not tagged as [`ClauseTag::Or`] (an `OR` may +/// take an `OrPushdownTag` of its own), an `$elemMatch` with a tag, a +/// `$not` of several predicates, `$jsonSchema`. pub(crate) fn tagged_tree( canonical: &Document, clauses: &[Document], - tagged: &std::collections::BTreeSet, - index: &IndexCopy, + tags: &[ClauseTag], ) -> Option { let mut children = Vec::with_capacity(clauses.len()); for (i, clause) in clauses.iter().enumerate() { - children.push(clause_node(clause, tagged.contains(&i).then_some(index))?); + children.push(match tags.get(i).unwrap_or(&ClauseTag::None) { + ClauseTag::None => clause_node(clause, None)?, + ClauseTag::One(ix) => clause_node(clause, Some(ix))?, + ClauseTag::Or(branches) => or_node(clause, branches)?, + }); } let rooted_and = canonical.len() == 1 && canonical.contains_key("$and"); if !rooted_and { @@ -309,6 +332,23 @@ fn clause_node(clause: &Document, tag: Option<&IndexCopy>) -> Option { }; let ops: Vec<&str> = pred.keys().map(String::as_str).collect(); match ops.as_slice() { + // An `ELEM_MATCH_VALUE` holds one node per operator; nothing below + // it carries a tag unless the plan's index read it. The object form + // and a `$not` inside are left out: no measured figure. + ["$elemMatch"] if tag.is_none() => { + let inner = pred.get_document("$elemMatch").ok()?; + let mut children = Vec::with_capacity(inner.len()); + for (op, v) in inner { + if !op.starts_with('$') || op == "$not" { + return None; + } + children.push(clause_node(&bson::doc! { path: { op: v.clone() } }, None)?); + } + Some(TreeNode { + entry: None, + children, + }) + } ["$elemMatch"] => None, ["$not"] => { let one = match pred.get("$not")? { @@ -329,11 +369,36 @@ fn clause_node(clause: &Document, tag: Option<&IndexCopy>) -> Option { } } +/// An `$or` whose branches are single predicates, each tagged with the +/// index the plan read it from; the `OR` takes the lowest-numbered. +fn or_node(clause: &Document, branches: &[IndexCopy]) -> Option { + let list = clause.get_array("$or").ok()?; + if clause.len() != 1 || list.len() != branches.len() { + return None; + } + let mut children = Vec::with_capacity(list.len()); + for (b, ix) in list.iter().zip(branches) { + let b = b.as_document()?; + if b.len() != 1 || b.keys().any(|k| k.starts_with('$')) { + return None; + } + children.push(clause_node(b, Some(ix))?); + } + Some(sort_tagged(TreeNode { + entry: None, + children, + })) +} + /// `tagForSort` for one internal node whose children are tagged already: -/// it takes the tag of its lowest-numbered index. A tree here reads one -/// index, so that is any child's. +/// it takes the tag of its lowest-numbered index. fn sort_tagged(mut node: TreeNode) -> TreeNode { - node.entry = node.children.iter().find_map(|c| c.entry.clone()); + node.entry = node + .children + .iter() + .filter_map(|c| c.entry.as_ref()) + .min_by_key(|e| e.number) + .cloned(); node } @@ -370,6 +435,7 @@ mod tests { name: name.into(), key: doc! { "a": 1 }, multikey: vec![0], + number: 1, } } @@ -394,8 +460,8 @@ mod tests { { "a": { "$lt": 5 } }, { "b": { "$lt": 90 } }, ] }; let clauses = vec![doc! { "a": { "$lt": 5 } }, doc! { "b": { "$lt": 90 } }]; - let tagged = [0].into_iter().collect(); - let tree = tagged_tree(&canonical, &clauses, &tagged, &a_1(name)).unwrap(); + let tags = [ClauseTag::One(a_1(name)), ClauseTag::None]; + let tree = tagged_tree(&canonical, &clauses, &tags).unwrap(); estimate(&LIBCXX_ARM64, &created, &creation, &tree).unwrap() } @@ -425,8 +491,7 @@ mod tests { ]; let canonical = doc! { "a": { "$lt": 5 } }; let clauses = vec![canonical.clone()]; - let tagged = [0].into_iter().collect(); - let tree = tagged_tree(&canonical, &clauses, &tagged, &a_1("a_1")).unwrap(); + let tree = tagged_tree(&canonical, &clauses, &[ClauseTag::One(a_1("a_1"))]).unwrap(); assert_eq!( estimate(&LIBCXX_ARM64, &created, &creation, &tree), Some(2483) @@ -435,16 +500,41 @@ mod tests { #[test] fn trees_it_does_not_build() { - let ix = a_1("a_1"); - let none = std::collections::BTreeSet::new(); for clause in [ doc! { "$or": [{ "a": 1 }, { "b": 2 }] }, - doc! { "a": { "$elemMatch": { "b": 1 } } }, doc! { "a": { "$not": { "$gt": 1, "$lt": 5 } } }, doc! { "$jsonSchema": {} }, ] { - assert!(tagged_tree(&clause, std::slice::from_ref(&clause), &none, &ix).is_none()); + assert!(tagged_tree(&clause, std::slice::from_ref(&clause), &[]).is_none()); } + // An `$elemMatch` the plan's index read is not built. + let clause = doc! { "a": { "$elemMatch": { "b": 1 } } }; + let tags = [ClauseTag::One(a_1("a_1"))]; + assert!(tagged_tree(&clause, std::slice::from_ref(&clause), &tags).is_none()); + } + + #[test] + fn elem_match_and_or_trees() { + let clauses = [ + doc! { "a": { "$lt": 5 } }, + doc! { "b": { "$elemMatch": { "$gt": 1, "$lt": 90 } } }, + ]; + let canonical = doc! { "$and": clauses.to_vec() }; + let tree = tagged_tree(&canonical, &clauses, &[ClauseTag::One(a_1("a_1"))]).unwrap(); + assert_eq!(tree.children[1].children.len(), 2); + assert!(tree.children[1].entry.is_none()); + let mut b_1 = a_1("b_1"); + b_1.number = 2; + let clauses = [ + doc! { "$or": [{ "b": 5 }, { "a": 3 }] }, + doc! { "c": { "$lt": 3 } }, + ]; + let canonical = doc! { "$and": clauses.to_vec() }; + let tags = [ClauseTag::Or(vec![b_1, a_1("a_1")]), ClauseTag::None]; + let tree = tagged_tree(&canonical, &clauses, &tags).unwrap(); + // `tagForSort`: the lowest-numbered index below, at every level. + assert_eq!(tree.entry.unwrap().name, "a_1"); + assert_eq!(tree.children[0].entry.as_ref().unwrap().name, "a_1"); } #[test] diff --git a/crates/secantus-commands/src/explain/order.rs b/crates/secantus-commands/src/explain/order.rs index 2ebefc126..2f0bb07e7 100644 --- a/crates/secantus-commands/src/explain/order.rs +++ b/crates/secantus-commands/src/explain/order.rs @@ -930,7 +930,7 @@ impl plan::CacheHook for ReadCache<'_> { .unwrap_or_default(); let signature = plan::signature(winner); let solution = plan::solution(winner); - let estimated_size = estimated_size(q, winner, &created_from, &creation); + let estimated_size = estimated_size(q, winner, &self.catalog, &created_from, &creation); super::plan_cache::set( &self.place, super::plan_cache::Entry { @@ -964,26 +964,44 @@ impl plan::CacheHook for ReadCache<'_> { fn estimated_size( q: &Query, winner: &Node, + catalog: &[Arc], created_from: &Document, creation: &[Document], ) -> Option { + use super::entry_size::{ClauseTag, IndexCopy}; let layout = super::entry_size::LAYOUT?; - let (index, tagged) = plan::index_tags(q, winner)?; - if !index.plain { - return None; - } - let copy = super::entry_size::IndexCopy { - name: index.name.clone(), - key: index.key.clone(), - // The `_id` index keeps no path-level multikey metadata. - multikey: if index.name == "_id_" { - vec![] - } else { - index.multikey_paths.iter().map(Vec::len).collect() - }, + let copy = |index: &super::IndexInfo| -> Option { + if !index.plain { + return None; + } + Some(IndexCopy { + name: index.name.clone(), + key: index.key.clone(), + // The `_id` index keeps no path-level multikey metadata. + multikey: if index.name == "_id_" { + vec![] + } else { + index.multikey_paths.iter().map(Vec::len).collect() + }, + number: catalog.iter().position(|c| c.name == index.name)?, + }) }; let clauses = plan::clauses(q.canonical); - let tree = super::entry_size::tagged_tree(q.canonical, &clauses, &tagged, ©)?; + let mut tags = vec![ClauseTag::None; clauses.len()]; + if let Some((index, tagged)) = plan::index_tags(q, winner) { + let one = copy(&index)?; + for i in tagged { + *tags.get_mut(i)? = ClauseTag::One(one.clone()); + } + } else { + let (at, branches) = plan::or_tags(q, winner)?; + let branches = branches + .iter() + .map(|ix| copy(ix)) + .collect::>>()?; + *tags.get_mut(at)? = ClauseTag::Or(branches); + } + let tree = super::entry_size::tagged_tree(q.canonical, &clauses, &tags)?; let size = super::entry_size::estimate(&layout, created_from, creation, &tree)?; i64::try_from(size).ok() } diff --git a/crates/secantus-commands/src/explain/plan.rs b/crates/secantus-commands/src/explain/plan.rs index 4e2e39c68..0f8c16e66 100644 --- a/crates/secantus-commands/src/explain/plan.rs +++ b/crates/secantus-commands/src/explain/plan.rs @@ -774,6 +774,74 @@ pub(crate) fn index_tags(q: &Query, winner: &Node) -> Option<(Arc, BT Some((index.clone(), use_index(&index, &leaves, &parts, q).tagged)) } +/// For a plan that reads an `OR` of index scans beside other clauses +/// (`{$or: [{a: 3}, {b: 5}], c: {$lt: 3}}` over `a_1` and `b_1`): which +/// clause of `q.canonical` is the `$or`, and the index each of its branches +/// was assigned -- mongod's index tags under the `OR`. `None` unless the +/// query is a conjunction holding one `$or`, every branch is one predicate +/// on the leading field of exactly one scanned index, and no scanned index +/// holds a field of another clause (mongod could push that clause into the +/// branches, under an `OrPushdownTag`). +pub(crate) fn or_tags(q: &Query, winner: &Node) -> Option<(usize, Vec>)> { + let mut node = winner; + let inputs = loop { + match node { + Node::Fetch { input, .. } + | Node::Sort { input, .. } + | Node::Skip(_, input) + | Node::Limit(_, input) + | Node::Projection { input, .. } => node = input, + Node::Or(inputs) => break inputs, + _ => return None, + } + }; + let mut scanned: Vec> = Vec::new(); + for input in inputs { + let scan = match input { + Node::Fetch { input, .. } => input.as_ref(), + other => other, + }; + let Node::IxScan(ix) = scan else { return None }; + scanned.push(ix.index.clone()); + } + let parts = clauses(q.canonical); + if parts.len() < 2 { + return None; + } + let mut ors = parts + .iter() + .enumerate() + .filter(|(_, p)| p.contains_key("$or")); + let (at, or) = ors.next()?; + if ors.next().is_some() { + return None; + } + let outer: BTreeSet<&str> = parts + .iter() + .enumerate() + .filter(|(i, _)| *i != at) + .flat_map(|(_, p)| p.keys().map(String::as_str)) + .collect(); + if scanned + .iter() + .any(|ix| ix.fields().iter().any(|(f, _)| outer.contains(*f))) + { + return None; + } + let mut tags = Vec::new(); + for branch in or.get_array("$or").ok()? { + let branch = branch.as_document()?; + let (path, _) = branch.iter().next().filter(|_| branch.len() == 1)?; + let mut by = scanned.iter().filter(|ix| ix.fields()[0].0 == path); + let one = by.next()?; + if by.any(|ix| ix.name != one.name) { + return None; + } + tags.push(one.clone()); + } + Some((at, tags)) +} + /// The access under what the rest of the query puts over it. fn access_of_plan(node: &Node) -> Option<&Node> { match node { diff --git a/tasks/backlog-rust-mongodb.md b/tasks/backlog-rust-mongodb.md index 09ce07481..be0c2e024 100644 --- a/tasks/backlog-rust-mongodb.md +++ b/tasks/backlog-rust-mongodb.md @@ -2493,20 +2493,22 @@ Still different, all 77: already matches. Found 2026-11-14 (RM-435). Verify: tests/backlog/test_rm_found_rm435.py::test_a_group_mongod_pushes_into_sbe_is_pushed Backlog-ID: RM-693 -- [ ] **`estimatedSizeBytes` is left out of entries whose index tree this - server does not build.** It sizes an entry's `PlanCacheIndexTree` - from the canonical query and the clauses the winning plan's one index - was assigned (`plan::index_tags`, `entry_size::tagged_tree`). mongod - 8.2.11 sizes every entry; this server leaves the field out where the - tree's nodes or tags are not modelled: a predicate inside - `$elemMatch` (its `ELEM_MATCH_*` node and the tags under it), an - `$or` or `$nor` that is not the root (an `OR` may take an - `OrPushdownTag` of its own), a `$not` over several predicates, - `$jsonSchema`, a winner that reads two indexes (`AND_SORTED` or an - `OR`, where `tagForSort` picks the lowest-numbered index), and an - `OR` stage among the candidates (`OrStats`). Found 2026-11-08 (RM-341). - Verify: tests/backlog/test_rm_found_2026_11_08_rm341.py::test_entries_this_does_not_size_report_their_size - Backlog-ID: RM-436 +- [ ] **`estimatedSizeBytes` is still left out of some entries whose + index tree this server does not build.** It sizes an entry's + `PlanCacheIndexTree` from the canonical query and the index tags of + its winner (`plan::index_tags`, `plan::or_tags`, + `entry_size::tagged_tree`): one index, or an `OR` of single-predicate + branches beside other clauses, and an untagged `$elemMatch` over + operators. mongod 8.2.11 sizes every entry; this server leaves the + field out for a `$not` inside `$elemMatch` (3350 for `{a: {$lt: 5}, b: + {$elemMatch: {$not: {$gt: 3}}}}` over `a_1` and `b_1` on macOS + arm64), the object form of `$elemMatch` and an `$elemMatch` the + winner's index reads, an `$or` that is the root or whose clauses + mongod pushes into the branches (`OrPushdownTag`), a branch of several + predicates, `$nor`, a `$not` over several predicates, `$jsonSchema`, + and an `AND_SORTED` winner. Found 2026-11-14 (RM-436). + Verify: tests/backlog/test_rm_found_2026_11_08_rm341.py::test_a_not_inside_elem_match_entry_reports_its_size + Backlog-ID: RM-694 - [ ] **A SORT_MERGE candidate's trial counts are not mongod's.** For `{a: {$in: [1, 2]}, b: {$lt: 90}}` sorted by `c` over `a_1_c_1` and `b_1`, mongod 8.2.11's winning SORT_MERGE reports 82 `works` and its diff --git a/tests/backlog/test_rm_found_2026_11_08_rm341.py b/tests/backlog/test_rm_found_2026_11_08_rm341.py index 75d7642ae..c98b84cd6 100644 --- a/tests/backlog/test_rm_found_2026_11_08_rm341.py +++ b/tests/backlog/test_rm_found_2026_11_08_rm341.py @@ -32,10 +32,10 @@ def _stages(stage: Any) -> list: return out -@pytest.mark.backlog("RM-436") +@pytest.mark.backlog_fixed("RM-436") def test_entries_this_does_not_size_report_their_size(rust_mongo: Any, mongod: Any) -> None: """mongod 8.2.11 lists `estimatedSizeBytes` for every entry. This server - leaves it out where it does not build the entry's `PlanCacheIndexTree`: + left it out where it did not build the entry's `PlanCacheIndexTree`: a predicate inside `$elemMatch` (3215 on macOS arm64) and an `$or` beside other predicates (4912).""" only_on("darwin", why="RM-341 sizes entries with macOS arm64's layout") @@ -55,6 +55,54 @@ def sizes(client: Any) -> list: assert sizes(rust_mongo) == sizes(mongod) +@pytest.mark.backlog_fixed("RM-436") +def test_more_or_and_elem_match_entries_report_their_size(rust_mongo: Any, mongod: Any) -> None: + """mongod 8.2.11's sizes for an `OR` of three branches over two indexes + (4956), branches of ranges and `$in` (4979), and an `$elemMatch` of two + operators (3319), on macOS arm64 (measured 2026-11-14).""" + only_on("darwin", why="RM-341 sizes entries with macOS arm64's layout") + only_on("arm64", why="RM-341 sizes entries with macOS arm64's layout") + cases = [ + (["a", "b", "c"], {"$or": [{"a": 3}, {"b": 5}, {"a": 7}], "c": {"$lt": 3}}), + ( + ["a", "b", "c"], + {"$or": [{"a": {"$lt": 2}}, {"b": {"$in": [5, 6]}}], "c": {"$lt": 3}}, + ), + (["a", "b"], {"a": {"$lt": 5}, "b": {"$elemMatch": {"$gt": 1, "$lt": 90}}}), + ] + + def sizes(client: Any) -> list: + return [ + [ + e.get("estimatedSizeBytes") + for e in _entries(client, ix, lambda col, q=q: col.find(q)) + ] + for ix, q in cases + ] + + assert sizes(mongod) == [[4956], [4979], [3319]] + assert sizes(rust_mongo) == sizes(mongod) + + +@pytest.mark.backlog("RM-694") +def test_a_not_inside_elem_match_entry_reports_its_size(rust_mongo: Any, mongod: Any) -> None: + """mongod 8.2.11 sizes the entry for `{a: {$lt: 5}, b: {$elemMatch: + {$not: {$gt: 3}}}}` over `a_1` and `b_1` at 3350 on macOS arm64; this + server leaves `estimatedSizeBytes` out.""" + only_on("darwin", why="RM-341 sizes entries with macOS arm64's layout") + only_on("arm64", why="RM-341 sizes entries with macOS arm64's layout") + q = {"a": {"$lt": 5}, "b": {"$elemMatch": {"$not": {"$gt": 3}}}} + + def sizes(client: Any) -> list: + return [ + e.get("estimatedSizeBytes") + for e in _entries(client, ["a", "b"], lambda col: col.find(q)) + ] + + assert sizes(mongod) == [3350] + assert sizes(rust_mongo) == sizes(mongod) + + @pytest.mark.backlog("RM-437") def test_a_sort_merge_candidate_counts_as_mongods(rust_mongo: Any, mongod: Any) -> None: """mongod 8.2.11's winning SORT_MERGE for `{a: {$in: [1, 2]}, b: {$lt: