Rust MongoDB server: fix-one batch snapshot of rm-batch (4dbe57a2, 5 commits)

changelog.d/rm-557-or-of-one-index-sorts-keys.md

@@ -0,0 +1,12 @@
+### Rust MongoDB server: an `$or` over one index sorts its keys below the FETCH
+
+A rooted `$or` whose branches all read one index, such as
+`find({$or: [{y: 0}, {y: {$gt: 1}}]}).sort({x: 1})` over `{y: 1, x: 1}`,
+is one index scan of the branches' intervals. mongod 8.2.11 sorts its
+index keys below the FETCH (`SUBPLAN <- FETCH <- SORT default <- IXSCAN`);
+this server sorted the fetched documents above it.
+
+#### Fixed
+
+- The planner asks the merged scan's own index whether it holds the sort.
+  Fixes RM-557 (tests/backlog/test_rm_found_rm476.py::test_an_or_of_one_index_sorts_its_keys)

changelog.d/rm-559-libcxx-heap.md

@@ -0,0 +1,16 @@
+### Rust MongoDB server: on macOS, equal heap entries leave in libc++'s order
+
+A macOS arm64 build of `secantusd-rs` now pops its priority queues (mongod's
+`std::priority_queue`s) as
+libc++'s `std::pop_heap` does, as it already sorts as libc++'s
+`std::sort` (RM-526): each platform's mongod is the reference, and the
+macOS build links libc++. Linux builds keep libstdc++'s heap.
+
+#### Fixed
+
+- A SORT_MERGE whose children's heads tie on the sort key, and a `$near`
+  whose points tie on distance, hand back equal entries in macOS mongod's order on
+  macOS (`find({a: {$in: [7, "b", 5, null]}}).sort({b: 1})` over `{a: 1,
+  b: 1}` returns `[3, 14, 13, 17]` there, as mongod 8.2.11 does).
+
+Fixes RM-559 (tests/backlog/test_rm_found_rm477.py::test_a_sort_merge_beside_a_null_scan_breaks_ties_as_mongod)

changelog.d/rm-566-return-key-default-sort.md

@@ -0,0 +1,13 @@
+### Rust MongoDB server: a sort under `returnKey` is a `default` sort
+
+A blocking SORT below a RETURN_KEY now explains as `type: "default"`, as
+mongod 8.2.11's does: the sort keeps each member's index key, which
+mongod's `simple` sort would discard.
+
+#### Fixed
+
+- `find(...).sort(...).returnKey()` explained its SORT as `simple`. The
+  order of documents that tie under a limited sort is left as it was, per
+  the scope decision that retired RM-049.
+
+Fixes RM-566 (tests/backlog/test_rm_found_rm470.py::test_a_sort_under_return_key_is_a_default_sort)

changelog.d/rm-567-return-key-idhack.md

@@ -0,0 +1,14 @@
+### Rust MongoDB server: a `returnKey` lookup on `_id` is an IDHACK
+
+`find({_id: 3}).returnKey()` now plans `RETURN_KEY <- IDHACK`, as mongod
+8.2 does, where it planned `RETURN_KEY <- IXSCAN`.
+
+#### Fixed
+
+- An `_id` point lookup under `returnKey` is an IDHACK at any batch size
+  and under a limit or a sort, with a SORT_KEY_GENERATOR beneath the
+  RETURN_KEY for a projection's `{$meta: "sortKey"}`, and returns `{_id:
+  <value>}`; a hint, `showRecordId`, a skip or a collation the `_id` index
+  does not have still scans the index.
+
+Fixes RM-567 (tests/backlog/test_rm_found_rm470.py::test_return_key_of_an_id_point_lookup_is_idhack, tests/backlog/test_rm_found_rm470.py::test_return_key_id_lookup_variants).

changelog.d/rm-568-return-key-sort-key.md

@@ -0,0 +1,17 @@
+### Rust MongoDB server: `returnKey` keeps a projection's sort key
+
+A `find` with `returnKey` now keeps a projection's `{$meta: "sortKey"}`
+fields, as mongod 8.2 does, and a `sortKey` projection with no sort is
+refused.
+
+#### Fixed
+
+- `find({z: 0}, {k: {$meta: "sortKey"}}).sort({z: 1}).returnKey()`
+  returns `{k: [0]}` for each document, after the index key when the read
+  scans one, where it returned the sort's own field (`{z: 0}`) or the
+  index key alone.
+- A `{$meta: "sortKey"}` projection in a read with no sort is refused with
+  mongod's `BadValue` ("cannot use sortKey $meta projection without a
+  sort"), `returnKey` or not; it used to return the documents.
+
+Fixes RM-568 (tests/backlog/test_rm_found_rm470.py::test_return_key_keeps_a_sort_key_meta_field, tests/backlog/test_rm_found_rm568.py::test_return_key_keeps_sort_keys_beside_an_index_key, tests/backlog/test_rm_found_rm568.py::test_a_sort_key_meta_field_without_a_sort_is_refused). Files RM-703.

crates/secantus-commands/src/explain/plan.rs

@@ -2294,6 +2294,34 @@ fn find_plan(q: &Query, catalog: &[Arc<IndexInfo>]) -> Node {
     if never(q) {
         return Node::Eof("predicateEvalsToFalse");
     }
+    // An `_id` point lookup under `returnKey` is an IDHACK at any batch
+    // size, under a limit or a sort, which it does not run (measured
+    // 8.2.11, RM-567: `RETURN_KEY <- IDHACK`, and `RETURN_KEY <-
+    // SORT_KEY_GENERATOR <- IDHACK` for a projection's `{$meta: "sortKey"}`);
+    // not with a hint, a skip, `min` / `max`, `showRecordId` or a collation
+    // the `_id` index does not have, which scan the index.
+    if let (true, Some(("_id", value))) = (
+        q.return_key && q.hint == Hint::None && q.skip == 0 && !q.min_max && !q.show_record_id,
+        point_lookup(q.canonical),
+    ) {
+        let id_index = catalog.iter().any(|ix| {
+            ix.plain
+                && ix.key.len() == 1
+                && ix.position("_id") == Some(0)
+                && !collation_differs(ix, q.collation)
+        });
+        if id_index {
+            let mut node = Node::IdHack(value.clone());
+            let sort_key = q.projection.is_some_and(|p| {
+                p.values()
+                    .any(|v| secantus_core::projection::meta_spec(v) == Some("sortKey"))
+            });
+            if q.sort.is_some_and(|s| !s.is_empty()) && sort_key {
+                node = Node::SortKeyGenerator(Box::new(node));
+            }
+            return Node::ReturnKey(Box::new(node));
+        }
+    }
     let simple = q.hint == Hint::None && q.skip == 0 && !q.min_max && !q.return_key;
     if let (true, Some((path, value))) = (simple && !q.show_record_id, point_lookup(q.canonical)) {
         let index = catalog.iter().find(|ix| {
@@ -2347,7 +2375,9 @@ fn keeps_metadata(q: &Query, projection: Option<&Document>) -> bool {
             _ => false,
         }
     }
-    q.show_record_id || projection.is_some_and(|p| p.values().any(reads_metadata))
+    // `returnKey` keeps each member's index key, which the `simple` sort
+    // would discard (measured 8.2.11, RM-566: `RETURN_KEY <- SORT default`).
+    q.return_key || q.show_record_id || projection.is_some_and(|p| p.values().any(reads_metadata))
 }
 
 fn runs_below_sort(projection: &Document, sort: &Document) -> bool {
@@ -2420,7 +2450,14 @@ fn wrap(a: Access, q: &Query) -> Node {
     let sort_on_keys = blocking.is_some_and(|s| {
         a.fetch_filter.is_none()
             && match &a.scan {
-                Node::IxScan(_) => a.index.as_ref().is_some_and(|ix| in_key(s, ix)),
+                // An `$or` of one index's predicates is one scan of their
+                // intervals, planned with no `Access::index`: its keys sort
+                // below the FETCH all the same (mongod 8.2.11: `SUBPLAN <-
+                // FETCH <- SORT default <- IXSCAN`, RM-557).
+                Node::IxScan(ix) => a
+                    .index
+                    .as_ref()
+                    .map_or(in_key(s, &ix.index), |ix| in_key(s, ix)),
                 or @ Node::Or(_) => keys_hold(or, s, &in_key),
                 _ => false,
             }

crates/secantus-commands/src/find.rs

@@ -320,6 +320,19 @@ fn parse_problem(doc: &Document, ctx: &CommandContext) -> Result<Option<Document
     if let Some(err) = projection_meta_error(spec, &filter) {
         return Ok(Some(err.into_reply()));
     }
+    // mongod 8.2.11 refuses a top-level `{$meta: "sortKey"}` in a read with
+    // no sort, `returnKey` or not (`BadValue`, measured 2026-11-15, RM-567).
+    let has_sort = doc.get_document("sort").is_ok_and(|s| !s.is_empty());
+    if !has_sort && !sort_key_meta_fields(spec).is_empty() {
+        return Ok(Some(
+            CommandError::new(
+                2,
+                "BadValue",
+                "cannot use sortKey $meta projection without a sort",
+            )
+            .into_reply(),
+        ));
+    }
     if let Some(e) = projection_expression_error(spec, doc.get("let"), ctx) {
         return Err(e);
     }
@@ -808,7 +821,9 @@ fn find_body(doc: &Document, ctx: &mut CommandContext) -> HandlerResult {
         None
     };
     // A projection's `$meta` streams when it asks for what a streamed read
-    // keeps: each document's RecordId and the key it was found at.
+    // keeps: each document's RecordId and the key it was found at, and,
+    // under `returnKey`, a top-level sort key (RM-567).
+    let returns_key = bool_field(doc, "returnKey", false);
     let metas_stream = |p: &Document| {
         fn ok(v: &Bson) -> bool {
             match secantus_core::projection::meta_spec(v) {
@@ -820,7 +835,9 @@ fn find_body(doc: &Document, ctx: &mut CommandContext) -> HandlerResult {
                 },
             }
         }
-        p.values().all(ok)
+        p.values().all(|v| {
+            ok(v) || (returns_key && secantus_core::projection::meta_spec(v) == Some("sortKey"))
+        })
     };
     let streamable = !tailable
         && exists
@@ -1179,14 +1196,11 @@ fn find_body(doc: &Document, ctx: &mut CommandContext) -> HandlerResult {
                     }
                 }
             }
-            if let Some(s) = sort {
-                for k in s.keys() {
-                    if !key_fields.iter().any(|f| f == k) {
-                        key_fields.push(k.clone());
-                    }
-                }
-            }
         }
+        let sort_key_fields = match projection {
+            Some(spec) if return_key => sort_key_meta_fields(spec),
+            _ => Vec::new(),
+        };
         // `showRecordId` adds to the projected document; only `returnKey`
         // replaces it.
         let projected_first = match projection {
@@ -1213,6 +1227,17 @@ fn find_body(doc: &Document, ctx: &mut CommandContext) -> HandlerResult {
                         o.insert(f.clone(), v.clone());
                     }
                 }
+                // A projection's `{$meta: "sortKey"}` fields stay, after the
+                // index key, each the sort key as an array; one an index field
+                // already names keeps that field's place
+                // (`extractSortKeyMetaFieldsFromProjection`; measured 8.2.11,
+                // RM-567).
+                if let (false, Some(s)) = (sort_key_fields.is_empty(), sort) {
+                    let key = sort_key_array(&d, s, &ns)?;
+                    for f in &sort_key_fields {
+                        o.insert(f.clone(), key.clone());
+                    }
+                }
                 rewritten.push(o);
             } else {
                 // The document's RecordId (mongod 8.2.11: after the first of
@@ -1381,7 +1406,61 @@ fn shape_streamed(
         docs, rids, keys, ..
     } = batch;
     if cursor.return_key {
-        let out: Vec<Document> = keys.into_iter().map(Option::unwrap_or_default).collect();
+        // An IDHACK reads no index entry: its key is the document's `_id`
+        // (measured 8.2.11, RM-567: `{_id: 3}`).
+        fn id_hack(node: &crate::explain::Node) -> bool {
+            match node {
+                crate::explain::Node::IdHack(_) => true,
+                crate::explain::Node::ReturnKey(input)
+                | crate::explain::Node::SortKeyGenerator(input) => id_hack(input),
+                _ => false,
+            }
+        }
+        let by_id = id_hack(&cursor.plan);
+        let mut out: Vec<Document> = Vec::with_capacity(keys.len());
+        for (i, key) in keys.into_iter().enumerate() {
+            out.push(match (key, docs.get(i)) {
+                (Some(key), _) => key,
+                (None, Some(bytes)) if by_id => {
+                    let raw = bson::RawDocument::from_bytes(bytes.as_slice())
+                        .map_err(|e| CommandError::new(1, "InternalError", e.to_string()))?;
+                    let mut key = Document::new();
+                    if let Some(id) = raw.get("_id").ok().flatten() {
+                        let id: Bson = id.try_into().map_err(|e: bson::raw::Error| {
+                            CommandError::new(1, "InternalError", e.to_string())
+                        })?;
+                        key.insert("_id", id);
+                    }
+                    key
+                }
+                (None, _) => Document::new(),
+            });
+        }
+        // A projection's `{$meta: "sortKey"}` fields stay, after the index
+        // key, each the sort key as an array; one an index field already
+        // names keeps that field's place (`extractSortKeyMetaFieldsFromProjection`;
+        // measured 8.2.11, RM-567). A covered read has no document: its
+        // index key holds the sort's fields.
+        let fields = cursor
+            .projection
+            .as_ref()
+            .map(sort_key_meta_fields)
+            .unwrap_or_default();
+        if let (false, Some(sort)) = (fields.is_empty(), cursor.command.get_document("sort").ok()) {
+            for (i, o) in out.iter_mut().enumerate() {
+                let key = match docs.get(i) {
+                    Some(bytes) => {
+                        let d = secantus_core::bsondoc::from_bytes(bytes.as_slice())
+                            .map_err(|e| CommandError::new(1, "InternalError", e.to_string()))?;
+                        sort_key_array(&d, sort, &cursor.ns)?
+                    }
+                    None => sort_key_array(o, sort, &cursor.ns)?,
+                };
+                for f in &fields {
+                    o.insert(f.clone(), key.clone());
+                }
+            }
+        }
         return crate::util::encode_docs(out);
     }
     let mut out: Vec<Document> = match &cursor.projection {
@@ -2223,6 +2302,37 @@ fn query_has_text(query: &Document) -> bool {
 /// requires text score metadata, but it is not available`). Recognized-but-
 /// unsupported args (`indexKey` / `recordId` / `sortKey` / …) validate clean and
 /// are omitted from the result by `secantus_core::projection::apply_projection`.
+/// `d`'s `{$meta: "sortKey"}` under `sort`: the value of each sort field,
+/// as an array (mongod 8.2.11, RM-567).
+fn sort_key_array(d: &Document, sort: &Document, ns: &str) -> Result<Bson, CommandError> {
+    let fields =
+        secantus_core::aggregate::sort_fields(&Bson::Document(sort.clone())).map_err(|_| {
+            CommandError::new(2, "BadValue", "sort is not supported by the Rust server")
+        })?;
+    let key =
+        secantus_core::aggregate::sort_key_values(d, &fields).map_err(|f| match f.as_mongo() {
+            Some((code, msg)) => CommandError::new(
+                code,
+                crate::util::error_code_name(code),
+                format!("Executor error during find command: {ns} :: caused by :: {msg}"),
+            ),
+            None => CommandError::new(
+                2,
+                "BadValue",
+                "sort key is not supported by the Rust server",
+            ),
+        })?;
+    Ok(Bson::Array(key))
+}
+
+/// The fields a projection gives `{$meta: "sortKey"}`, in its order.
+fn sort_key_meta_fields(spec: &Document) -> Vec<String> {
+    spec.iter()
+        .filter(|(_, v)| secantus_core::projection::meta_spec(v) == Some("sortKey"))
+        .map(|(k, _)| k.clone())
+        .collect()
+}
+
 fn projection_meta_error(spec: &Document, filter: &Document) -> Option<CommandError> {
     for v in spec.values() {
         let Some(arg) = secantus_core::projection::meta_spec(v) else {

crates/secantus-core/src/geoorder/mod.rs

@@ -525,7 +525,8 @@ mod tests {
     fn near_ties_come_in_the_queues_order() {
         // mongod 8.2.12's Linux build (libstdc++'s heap): four points at
         // distance 1 from a 2d `$near` at the origin come back 2, 1, 3, 0.
-        // The macOS build (libc++'s) answers 2, 3, 0, 1 (RM-231).
+        // The macOS build (libc++'s) answers 2, 3, 0, 1 (RM-231), which a
+        // macOS arm64 build of this server reproduces (RM-559).
         let docs: Vec<Document> = [[1, 0], [0, 1], [-1, 0], [0, -1]]
             .iter()
             .enumerate()
@@ -537,7 +538,12 @@ mod tests {
         let ix = flat_index();
         let base = vec![FieldBounds::Full];
         let order = near::near_order(&ix, &base, &near, &docs, &[true; 4]).unwrap();
-        assert_eq!(order, [2, 1, 3, 0]);
+        let want = if crate::stl::LIBCXX {
+            [2, 3, 0, 1]
+        } else {
+            [2, 1, 3, 0]
+        };
+        assert_eq!(order, want);
     }
 
     /// A collection the test writes to between reads.

crates/secantus-core/src/heap.rs

@@ -9,7 +9,9 @@
 //! (the macOS build) and libstdc++ (the Linux build) can hand the same
 //! equal entries back in different orders. The Linux build is the exemplar
 //! where builds differ (Joe, 2026-10-21), so this is libstdc++'s
-//! `push_heap` and `pop_heap` ([`crate::stl`]).
+//! `push_heap` and `pop_heap` ([`crate::stl`]); built for macOS arm64 it pops
+//! as libc++ does, as each platform's mongod is the reference there
+//! (RM-559). (libc++'s `push_heap` moves the same elements as libstdc++'s.)
 
 pub trait HeapOrder {
     /// The queue's comparator: `true` when `self` belongs BELOW `other`
@@ -62,7 +64,11 @@ impl<T: HeapOrder> Heap<T> {
     }
 
     pub fn pop(&mut self) -> Option<T> {
-        crate::stl::pop_heap(&mut self.c, &T::heap_less);
+        if crate::stl::LIBCXX {
+            crate::stl::libcxx_pop_heap(&mut self.c, &T::heap_less);
+        } else {
+            crate::stl::pop_heap(&mut self.c, &T::heap_less);
+        }
         self.c.pop()
     }
 }
@@ -92,12 +98,18 @@ mod tests {
     #[test]
     fn equal_entries_leave_in_heap_order() {
         // Four equal distances pushed in order 0..4: libstdc++'s sift
-        // steps hand them back 0, 2, 1, 3 (libc++'s: 0, 1, 3, 2).
+        // steps hand them back 0, 2, 1, 3, libc++'s (the macOS arm64
+        // build's) 0, 1, 3, 2.
         let mut h = Heap::new();
         for i in 0..4 {
             h.push(D(1.0, i));
         }
         let out: Vec<u32> = std::iter::from_fn(|| h.pop().map(|d| d.1)).collect();
-        assert_eq!(out, vec![0, 2, 1, 3]);
+        let want = if crate::stl::LIBCXX {
+            vec![0, 1, 3, 2]
+        } else {
+            vec![0, 2, 1, 3]
+        };
+        assert_eq!(out, want);
     }
 }