feat(federation): add env var for MAX_NON_LOCAL_SELECTIONS

.changesets/feat_lrlna_router_2150_max_non_local_selections.md

@@ -0,0 +1,10 @@
+### Add a metric to track non-local selection sets observed during query planning 
+
+
+`apollo.router.query_planning.plan.non_local_selections` is a histogram of the
+number of non-local selections estimated during query planning traversal used to
+limit the number of options explored. The non-local selections limit defaults to
+100_000. Numbers observed to be close to this limit may warrant an investigation
+into complexity of the operations.
+
+By [@lrlna](https://github.com/lrlna) in https://github.com/apollographql/router/pull/10349

apollo-federation/src/query_plan/query_planner.rs

@@ -1,5 +1,6 @@
 use std::cell::Cell;
 use std::num::NonZeroU32;
+use std::num::NonZeroU64;
 use std::ops::ControlFlow;
 use std::sync::Arc;
 
@@ -158,13 +159,23 @@ pub struct QueryPlannerDebugConfig {
     ///
     /// The default value is None, which specifies no limit.
     pub paths_limit: Option<u32>,
+
+    /// As the planner traverses the query, it estimates an upper bound on the
+    /// number of "non-local" selection sets it would need to consider as
+    /// possibilities. The process is aborted if the estimate exceeds this upper
+    /// bound to prevent unbounded planning time.
+    ///
+    /// This value currently defaults to 100_000. And is intentionally not part
+    /// of configuration which can be set by users.
+    pub max_non_local_selections: NonZeroU64,
 }
 
 impl Default for QueryPlannerDebugConfig {
     fn default() -> Self {
         Self {
             max_evaluated_plans: NonZeroU32::new(10_000).unwrap(),
             paths_limit: None,
+            max_non_local_selections: NonZeroU64::new(100_000).unwrap(),
         }
     }
 }
@@ -177,6 +188,9 @@ pub struct QueryPlanningStatistics {
     /// `best_plan_cost` can be NaN, if the cost is not computed or irrelevant.
     #[serde(deserialize_with = "deserialize_f64_nullable")]
     pub best_plan_cost: f64,
+    /// `non_local_selections_count` can be `None`, if
+    /// `QueryPlanOptions::non_local_selections_limit_enabled` is `false`
+    pub non_local_selections_count: Option<u64>,
 }
 
 /// Deserialize helper for f64 that treats null as NaN.
@@ -563,6 +577,7 @@ impl QueryPlanner {
             node: root_node,
             statistics: QueryPlanningStatistics {
                 best_plan_cost: cost,
+                non_local_selections_count: non_local_selection_state.as_ref().map(|s| s.count),
                 ..statistics
             },
         };
@@ -1458,13 +1473,15 @@ type User
             evaluated_plan_count: Cell::new(10),
             evaluated_plan_paths: Cell::new(20),
             best_plan_cost: f64::NAN,
+            non_local_selections_count: Some(30),
         };
         let serialized = serde_json::to_string_pretty(&stats).expect("Serializing");
         insta::assert_snapshot!(serialized, @r###"
         {
           "evaluated_plan_count": 10,
           "evaluated_plan_paths": 20,
-          "best_plan_cost": null
+          "best_plan_cost": null,
+          "non_local_selections_count": 30
         }
         "###);
 

apollo-federation/src/query_plan/query_planning_traversal.rs

@@ -326,7 +326,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> {
             return Err(SingleFederationError::QueryPlanComplexityExceeded {
                 message: format!(
                     "Number of non-local selections exceeds limit of {}",
-                    Self::MAX_NON_LOCAL_SELECTIONS,
+                    traversal.parameters.config.debug.max_non_local_selections,
                 ),
             }
             .into());

apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs

@@ -22,8 +22,6 @@ use crate::schema::position::INTROSPECTION_TYPENAME_FIELD_NAME;
 use crate::schema::position::ObjectTypeDefinitionPosition;
 
 impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> {
-    pub(super) const MAX_NON_LOCAL_SELECTIONS: u64 = 100_000;
-
     /// This calls `check_non_local_selections_limit_exceeded()` for each of the selections in the
     /// open branches stack; see that function's doc comment for more information.
     ///
@@ -52,6 +50,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> {
                 branch.selections.len(),
                 tail_nodes_info.next_nodes.len(),
                 state,
+                self.parameters.config.debug.max_non_local_selections.get(),
             ) {
                 return Ok(true);
             }
@@ -91,8 +90,8 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> {
     /// set that wouldn't be avoided by such an optimization (i.e. the "non-local" selections), and
     /// adds it to the given count in the state. Note that the count for a given selection set is
     /// scaled by an approximate upper bound on the possible number of tail nodes for paths ending
-    /// at that selection set. If at any point, the count exceeds `Self::MAX_NON_LOCAL_SELECTIONS`,
-    /// then this function will return `true`.
+    /// at that selection set. If at any point, the count exceeds
+    /// `QueryPlannerDebugConfig::max_non_local_selections`, then this function will return `true`.
     ///
     /// This function's code is closely related to `selection_set_is_fully_local_from_all_nodes()`
     /// (which implements the aforementioned optimization). However, when it comes to traversing the
@@ -227,6 +226,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> {
                 selection_set.selections.len(),
                 parent_nodes.next_nodes.len(),
                 state,
+                self.parameters.config.debug.max_non_local_selections.get(),
             )
         {
             return Ok(true);
@@ -235,8 +235,13 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> {
     }
 
     /// Updates the non-local selection set count in the state, returning true if this causes the
-    /// count to exceed `Self::MAX_NON_LOCAL_SELECTIONS`.
-    fn update_count(num_selections: usize, num_parent_nodes: usize, state: &mut State) -> bool {
+    /// count to exceed `limit`.
+    fn update_count(
+        num_selections: usize,
+        num_parent_nodes: usize,
+        state: &mut State,
+        limit: u64,
+    ) -> bool {
         let Ok(num_selections) = u64::try_from(num_selections) else {
             return true;
         };
@@ -249,7 +254,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> {
         if let Some(new_count) = state
             .count
             .checked_add(additional_count)
-            .take_if(|v| *v <= Self::MAX_NON_LOCAL_SELECTIONS)
+            .take_if(|v| *v <= limit)
         {
             state.count = new_count;
         } else {

apollo-router/src/configuration/mod.rs

@@ -8,6 +8,7 @@ use std::iter;
 use std::net::IpAddr;
 use std::net::SocketAddr;
 use std::num::NonZeroU32;
+use std::num::NonZeroU64;
 use std::num::NonZeroUsize;
 use std::str::FromStr;
 use std::sync::Arc;
@@ -444,6 +445,13 @@ impl Configuration {
             .and_then(NonZeroU32::new)
             .unwrap_or(NonZeroU32::new(10_000).expect("it is not zero"));
 
+        // This environment variable is intentionally undocumented.
+        let max_non_local_selections = std::env::var("APOLLO_ROUTER_MAX_NON_LOCAL_SELECTIONS")
+            .ok()
+            .and_then(|value| value.parse::<u64>().ok())
+            .and_then(NonZeroU64::new)
+            .unwrap_or(NonZeroU64::new(100_000).expect("should be not zero"));
+
         QueryPlannerConfig {
             subgraph_graphql_validation: false,
             generate_query_fragments: self.supergraph.generate_query_fragments,
@@ -454,6 +462,7 @@ impl Configuration {
             debug: QueryPlannerDebugConfig {
                 max_evaluated_plans,
                 paths_limit: self.supergraph.query_planning.experimental_paths_limit,
+                max_non_local_selections,
             },
         }
     }

apollo-router/src/query_planner/query_planner_service.rs

@@ -208,6 +208,7 @@ impl QueryPlannerService {
             query_plan_root_node: root_node.map(Arc::new),
             evaluated_plan_count: plan.statistics.evaluated_plan_count.clone().into_inner() as u64,
             evaluated_plan_paths: plan.statistics.evaluated_plan_paths.clone().into_inner() as u64,
+            non_local_selections_count: plan.statistics.non_local_selections_count,
         })
     }
 
@@ -331,6 +332,7 @@ impl QueryPlannerService {
             formatted_query_plan,
             evaluated_plan_count,
             evaluated_plan_paths,
+            non_local_selections_count,
         } = plan_result;
 
         // If the query is filtered, we want to generate the signature using the original query and generate the
@@ -366,6 +368,9 @@ impl QueryPlannerService {
                 "Number of paths (including intermediate ones) considered to plan a query before starting to generate a plan",
                 evaluated_plan_paths
             );
+            if let Some(non_local_selections_count) = non_local_selections_count {
+                metric_query_planning_non_local_selections(non_local_selections_count);
+            }
 
             Ok(QueryPlannerContent::Plan {
                 plan: Arc::new(super::QueryPlan {
@@ -614,6 +619,7 @@ pub(crate) struct QueryPlanResult {
     pub(super) query_plan_root_node: Option<Arc<PlanNode>>,
     pub(super) evaluated_plan_count: u64,
     pub(super) evaluated_plan_paths: u64,
+    pub(super) non_local_selections_count: Option<u64>,
 }
 
 /// The outcome of a query-planning attempt. Shared across query-planning metrics (e.g.
@@ -673,6 +679,15 @@ pub(crate) fn metric_query_planning_plan_duration(
     );
 }
 
+pub(crate) fn metric_query_planning_non_local_selections(count: u64) {
+    u64_histogram_with_unit!(
+        "apollo.router.query_planning.plan.non_local_selections",
+        "Number of non-local selections estimated during query planning traversal, used for optimizing plan option exploration",
+        "{selection}",
+        count
+    );
+}
+
 pub(crate) fn metric_rust_qp_init(init_error_kind: Option<&'static str>) {
     if let Some(init_error_kind) = init_error_kind {
         u64_counter!(
@@ -1340,6 +1355,30 @@ mod tests {
         .await;
     }
 
+    #[test(tokio::test)]
+    async fn test_non_local_selections_histogram() {
+        async {
+            let _ = plan(
+                EXAMPLE_SCHEMA,
+                include_str!("testdata/query.graphql"),
+                include_str!("testdata/query.graphql"),
+                None,
+                PlanOptions::default(),
+            )
+            .await
+            .unwrap();
+
+            assert_histogram_exists!(
+                "apollo.router.query_planning.plan.non_local_selections",
+                u64
+            );
+            assert_histogram_count!("apollo.router.query_planning.plan.non_local_selections", 1);
+            assert_histogram_sum!("apollo.router.query_planning.plan.non_local_selections", 7);
+        }
+        .with_metrics()
+        .await;
+    }
+
     async fn plan_unauthorized_operation(compute_job_type: ComputeJobType) -> QueryPlannerContent {
         let configuration: Arc<Configuration> = Arc::default();
         let schema = Schema::parse(

apollo-router/tests/integration/query_planner/mod.rs

@@ -5,6 +5,7 @@ use crate::integration::common::graph_os_enabled;
 
 mod error_paths;
 mod max_evaluated_plans;
+mod non_local_selections;
 
 const PROMETHEUS_METRICS_CONFIG: &str =
     include_str!("../telemetry/fixtures/prometheus.router.yaml");

apollo-router/tests/integration/query_planner/non_local_selections.rs

@@ -0,0 +1,44 @@
+use serde_json::json;
+
+use crate::integration::IntegrationTest;
+use crate::integration::common::Query;
+
+const QUERY: &str = r#"{ t { v1 v2 v3 v4 } }"#;
+
+#[tokio::test(flavor = "multi_thread")]
+async fn reports_non_local_selections() {
+    let mut router = IntegrationTest::builder()
+        .config(
+            r#"
+            telemetry:
+              exporters:
+                metrics:
+                  prometheus:
+                    enabled: true
+        "#,
+        )
+        .supergraph("tests/integration/fixtures/query_planner_max_evaluated_plans.graphql")
+        .build()
+        .await;
+    router.start().await;
+    router.assert_started().await;
+    router
+        .execute_query(
+            Query::builder()
+                .body(json!({
+                    "query": QUERY,
+                    "variables": {},
+                }))
+                .build(),
+        )
+        .await;
+
+    router
+        .assert_metrics_contains(
+            r#"apollo_router_query_planning_plan_non_local_selections_sum{<any>} 10"#,
+            None,
+        )
+        .await;
+
+    router.graceful_shutdown().await;
+}

apollo-router/tests/integration/redis.rs

@@ -182,7 +182,8 @@ async fn query_planner_cache() -> Result<(), BoxError> {
     // If this test fails and the cache key format changed you'll need to update the key here.
     // Look at the top of the file for instructions on getting the new cache key.
     let known_cache_key = &format!(
-        "{namespace}:plan:router:{}:47939f0e964372951934fc662c9c2be675bc7116ec3e57029abe555284eb10a4:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:d9f7a00bc249cb51cfc8599f86b6dc5272967b37b1409dc4717f105b6939fe43",
+        //"{namespace}:plan:router:{}:47939f0e964372951934fc662c9c2be675bc7116ec3e57029abe555284eb10a4:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:d9f7a00bc249cb51cfc8599f86b6dc5272967b37b1409dc4717f105b6939fe43",
+        "{namespace}:plan:router:{}:47939f0e964372951934fc662c9c2be675bc7116ec3e57029abe555284eb10a4:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:f16bdcbfbe4cbc78bc4599b75ca159ade850734316069f5d7e65c6ac10974850",
         env!("CARGO_PKG_VERSION")
     );
 
@@ -1571,7 +1572,7 @@ async fn query_planner_redis_update_query_fragments() {
         // This configuration turns the fragment generation option *off*.
         include_str!("fixtures/query_planner_redis_config_update_query_fragments.router.yaml"),
         &format!(
-            "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:fb1a8e6e454ad6a1d0d48b24dc9c7c4dd6d9bf58b6fdaf43cd24eb77fbbb3a17",
+            "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:02e0ce725b579eff2988e98b02560a48a3d9ecce9e7d3cef3d60dcf2fb366c42",
             env!("CARGO_PKG_VERSION")
         ),
     )
@@ -1594,7 +1595,7 @@ async fn query_planner_redis_update_defer() {
     test_redis_query_plan_config_update(
         include_str!("fixtures/query_planner_redis_config_update_defer.router.yaml"),
         &format!(
-            "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:dc062fcc9cfd9582402d1e8b1fa3ee336ea1804d833443869e0b3744996716a2",
+            "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:0651b0f75af604e69a6c0f5323f1a1231e7178acd8679d8e3a154640f51c5aa2",
             env!("CARGO_PKG_VERSION")
         ),
     )
@@ -1619,7 +1620,7 @@ async fn query_planner_redis_update_type_conditional_fetching() {
             "fixtures/query_planner_redis_config_update_type_conditional_fetching.router.yaml"
         ),
         &format!(
-            "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:bdc09980aa6ef28a67f5aeb8759763d8ac5a4fc43afa8c5a89f58cc998c48db3",
+            "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:2a38e8672a9f4de5b5d88a9faa9a6243fa5aa686f394a9f0ff8a787fc5a5172a",
             env!("CARGO_PKG_VERSION")
         ),
     )
@@ -1647,7 +1648,7 @@ async fn test_redis_query_plan_config_update(updated_config: &str, new_cache_key
 
     // If the tests above are failing, this is the key that needs to be changed first.
     let starting_key = &format!(
-        "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:d9f7a00bc249cb51cfc8599f86b6dc5272967b37b1409dc4717f105b6939fe43",
+        "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:f16bdcbfbe4cbc78bc4599b75ca159ade850734316069f5d7e65c6ac10974850",
         env!("CARGO_PKG_VERSION")
     );
     assert_ne!(

docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx

@@ -195,6 +195,7 @@
 - `apollo.router.query_planning.total.duration` - Histogram of plan durations including queue time.
 - `apollo.router.query_planning.plan.evaluated_plans` - Histogram of the number of evaluated query plans.
 - `apollo.router.query_planning.plan.evaluated_paths` - Histogram of the number of paths (including intermediate ones) the planner considers before generating a plan. High values often correlate with long planning times on complex schemas or queries. Tune the limits as described in [Tuning query planner limits](/graphos/routing/query-planning/query-planning-best-practices#tuning-query-planner-limits).
+- `apollo.router.query_planning.plan.non_local_selections` - Histogram of the number of non-local selections estimated during query planning traversal used to limit the number of options explored. The non-local selections limit defaults to 100_000. Numbers observed to be close to this limit may warrant an investigation into complexity of the operations.
 - `apollo.router.query_planner.memory` - Histogram of memory allocated during query planning, in bytes. Tracks memory allocation patterns specifically for query planning operations executed in the compute job thread pool. Attributes:
   - `allocation.type`: The type of memory operation (`allocated`, `deallocated`, `zeroed`, `reallocated`)
   - `context`: The context name where the allocation occurred (e.g., `query_planning`)