From 24f70987bc607f47ddce3219b8db1de8dec737d3 Mon Sep 17 00:00:00 2001 From: Iryna Shestak Date: Thu, 1 Oct 2026 10:18:43 +0200 Subject: [PATCH 01/10] replace const MAX_NON_LOCAL_SELECTIONS with an fn --- .../query_plan/query_planning_traversal.rs | 2 +- .../non_local_selections_estimation.rs | 21 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/apollo-federation/src/query_plan/query_planning_traversal.rs b/apollo-federation/src/query_plan/query_planning_traversal.rs index 214642b3825..2e6901ee344 100644 --- a/apollo-federation/src/query_plan/query_planning_traversal.rs +++ b/apollo-federation/src/query_plan/query_planning_traversal.rs @@ -326,7 +326,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { return Err(SingleFederationError::QueryPlanComplexityExceeded { message: format!( "Number of non-local selections exceeds limit of {}", - Self::MAX_NON_LOCAL_SELECTIONS, + Self::max_non_local_selections(), ), } .into()); diff --git a/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs b/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs index e49e404ecce..b00a4690bd9 100644 --- a/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs +++ b/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs @@ -1,3 +1,5 @@ +use std::sync::OnceLock; + use apollo_compiler::Name; use apollo_compiler::collections::IndexMap; use apollo_compiler::collections::IndexSet; @@ -22,7 +24,16 @@ use crate::schema::position::INTROSPECTION_TYPENAME_FIELD_NAME; use crate::schema::position::ObjectTypeDefinitionPosition; impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { - pub(super) const MAX_NON_LOCAL_SELECTIONS: u64 = 100_000; + pub(super) fn max_non_local_selections() -> u64 { + static MAX_NON_LOCAL_SELECTIONS: OnceLock = OnceLock::new(); + *MAX_NON_LOCAL_SELECTIONS.get_or_init(|| { + // This environment variable is intentionally undocumented. + std::env::var("APOLLO_ROUTER_MAX_NON_LOCAL_SELECTIONS") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(100_000) + }) + } /// This calls `check_non_local_selections_limit_exceeded()` for each of the selections in the /// open branches stack; see that function's doc comment for more information. @@ -91,8 +102,8 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { /// set that wouldn't be avoided by such an optimization (i.e. the "non-local" selections), and /// adds it to the given count in the state. Note that the count for a given selection set is /// scaled by an approximate upper bound on the possible number of tail nodes for paths ending - /// at that selection set. If at any point, the count exceeds `Self::MAX_NON_LOCAL_SELECTIONS`, - /// then this function will return `true`. + /// at that selection set. If at any point, the count exceeds + /// `Self::max_non_local_selections()`, then this function will return `true`. /// /// This function's code is closely related to `selection_set_is_fully_local_from_all_nodes()` /// (which implements the aforementioned optimization). However, when it comes to traversing the @@ -235,7 +246,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { } /// Updates the non-local selection set count in the state, returning true if this causes the - /// count to exceed `Self::MAX_NON_LOCAL_SELECTIONS`. + /// count to exceed `Self::max_non_local_selections()`. fn update_count(num_selections: usize, num_parent_nodes: usize, state: &mut State) -> bool { let Ok(num_selections) = u64::try_from(num_selections) else { return true; @@ -249,7 +260,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { if let Some(new_count) = state .count .checked_add(additional_count) - .take_if(|v| *v <= Self::MAX_NON_LOCAL_SELECTIONS) + .take_if(|v| *v <= Self::max_non_local_selections()) { state.count = new_count; } else { From ede23dd5ff22584a6a0cf1888460d6ea0a396c6b Mon Sep 17 00:00:00 2001 From: Iryna Shestak Date: Thu, 1 Oct 2026 12:06:52 +0200 Subject: [PATCH 02/10] add a changeset --- .../feat_lrlna_router_2150_max_non_local_selections.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changesets/feat_lrlna_router_2150_max_non_local_selections.md diff --git a/.changesets/feat_lrlna_router_2150_max_non_local_selections.md b/.changesets/feat_lrlna_router_2150_max_non_local_selections.md new file mode 100644 index 00000000000..d0161c5b457 --- /dev/null +++ b/.changesets/feat_lrlna_router_2150_max_non_local_selections.md @@ -0,0 +1,7 @@ +### Add environment variable for MAX_NON_LOCAL_SELECTIONS + +Adds a environment variable for MAX_NON_LOCAL_SELECTIONS const used in query +planning traversal. This is an undocumented env var, as it's intended for +internal use. + +By [@lrlna](https://github.com/lrlna) in https://github.com/apollographql/router/pull/ From bd5952eef6646881cfe213afa4ae58b1210421b7 Mon Sep 17 00:00:00 2001 From: Iryna Shestak Date: Thu, 1 Oct 2026 12:19:45 +0200 Subject: [PATCH 03/10] add a histogram for tracking the number of non_local_selections --- ...na_router_2150_max_non_local_selections.md | 4 +++ .../src/query_plan/query_planner.rs | 8 ++++- .../query_planner/query_planner_service.rs | 36 +++++++++++++++++++ .../standard-instruments.mdx | 1 + 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/.changesets/feat_lrlna_router_2150_max_non_local_selections.md b/.changesets/feat_lrlna_router_2150_max_non_local_selections.md index d0161c5b457..fab2445b3ab 100644 --- a/.changesets/feat_lrlna_router_2150_max_non_local_selections.md +++ b/.changesets/feat_lrlna_router_2150_max_non_local_selections.md @@ -4,4 +4,8 @@ Adds a environment variable for MAX_NON_LOCAL_SELECTIONS const used in query planning traversal. This is an undocumented env var, as it's intended for internal use. +This additional adds a metric for tracking current number of non-local +selections, which is a histogram available under +`apollo.router.query_planning.plan.non_local_selections`. + By [@lrlna](https://github.com/lrlna) in https://github.com/apollographql/router/pull/ diff --git a/apollo-federation/src/query_plan/query_planner.rs b/apollo-federation/src/query_plan/query_planner.rs index 7ae6a5101e6..87e89a5f944 100644 --- a/apollo-federation/src/query_plan/query_planner.rs +++ b/apollo-federation/src/query_plan/query_planner.rs @@ -177,6 +177,9 @@ pub struct QueryPlanningStatistics { /// `best_plan_cost` can be NaN, if the cost is not computed or irrelevant. #[serde(deserialize_with = "deserialize_f64_nullable")] pub best_plan_cost: f64, + /// `non_local_selections_count` can be `None`, if + /// `QueryPlanOptions::non_local_selections_limit_enabled` is `false` + pub non_local_selections_count: Option, } /// Deserialize helper for f64 that treats null as NaN. @@ -563,6 +566,7 @@ impl QueryPlanner { node: root_node, statistics: QueryPlanningStatistics { best_plan_cost: cost, + non_local_selections_count: non_local_selection_state.as_ref().map(|s| s.count), ..statistics }, }; @@ -1458,13 +1462,15 @@ type User evaluated_plan_count: Cell::new(10), evaluated_plan_paths: Cell::new(20), best_plan_cost: f64::NAN, + non_local_selections_count: Some(30), }; let serialized = serde_json::to_string_pretty(&stats).expect("Serializing"); insta::assert_snapshot!(serialized, @r###" { "evaluated_plan_count": 10, "evaluated_plan_paths": 20, - "best_plan_cost": null + "best_plan_cost": null, + "non_local_selections_count": 30 } "###); diff --git a/apollo-router/src/query_planner/query_planner_service.rs b/apollo-router/src/query_planner/query_planner_service.rs index 93dd388204c..5073afec51f 100644 --- a/apollo-router/src/query_planner/query_planner_service.rs +++ b/apollo-router/src/query_planner/query_planner_service.rs @@ -208,6 +208,7 @@ impl QueryPlannerService { query_plan_root_node: root_node.map(Arc::new), evaluated_plan_count: plan.statistics.evaluated_plan_count.clone().into_inner() as u64, evaluated_plan_paths: plan.statistics.evaluated_plan_paths.clone().into_inner() as u64, + non_local_selections_count: plan.statistics.non_local_selections_count, }) } @@ -331,6 +332,7 @@ impl QueryPlannerService { formatted_query_plan, evaluated_plan_count, evaluated_plan_paths, + non_local_selections_count, } = plan_result; // If the query is filtered, we want to generate the signature using the original query and generate the @@ -366,6 +368,9 @@ impl QueryPlannerService { "Number of paths (including intermediate ones) considered to plan a query before starting to generate a plan", evaluated_plan_paths ); + if let Some(non_local_selections_count) = non_local_selections_count { + metric_query_planning_non_local_selections(non_local_selections_count); + } Ok(QueryPlannerContent::Plan { plan: Arc::new(super::QueryPlan { @@ -614,6 +619,7 @@ pub(crate) struct QueryPlanResult { pub(super) query_plan_root_node: Option>, pub(super) evaluated_plan_count: u64, pub(super) evaluated_plan_paths: u64, + pub(super) non_local_selections_count: Option, } /// The outcome of a query-planning attempt. Shared across query-planning metrics (e.g. @@ -673,6 +679,15 @@ pub(crate) fn metric_query_planning_plan_duration( ); } +pub(crate) fn metric_query_planning_non_local_selections(count: u64) { + u64_histogram_with_unit!( + "apollo.router.query_planning.plan.non_local_selections", + "Number of non-local selections estimated during query planning traversal, used for optimising plan option exploration", + "{selection}", + count + ); +} + pub(crate) fn metric_rust_qp_init(init_error_kind: Option<&'static str>) { if let Some(init_error_kind) = init_error_kind { u64_counter!( @@ -1340,6 +1355,27 @@ mod tests { .await; } + #[test(tokio::test)] + async fn test_non_local_selections_histogram() { + async { + let _ = plan( + EXAMPLE_SCHEMA, + include_str!("testdata/query.graphql"), + include_str!("testdata/query.graphql"), + None, + PlanOptions::default(), + ) + .await + .unwrap(); + + assert_histogram_exists!("apollo.router.query_planning.plan.non_local_selections", u64); + assert_histogram_count!("apollo.router.query_planning.plan.non_local_selections", 1 as u64); + assert_histogram_sum!("apollo.router.query_planning.plan.non_local_selections", 7 as u64); + } + .with_metrics() + .await; + } + async fn plan_unauthorized_operation(compute_job_type: ComputeJobType) -> QueryPlannerContent { let configuration: Arc = Arc::default(); let schema = Schema::parse( diff --git a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx index fa83b2f3f58..094a7a2d62f 100644 --- a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx @@ -195,6 +195,7 @@ The `apollo.router.cache.redis.errors` metric also includes an `error_type` attr - `apollo.router.query_planning.total.duration` - Histogram of plan durations including queue time. - `apollo.router.query_planning.plan.evaluated_plans` - Histogram of the number of evaluated query plans. - `apollo.router.query_planning.plan.evaluated_paths` - Histogram of the number of paths (including intermediate ones) the planner considers before generating a plan. High values often correlate with long planning times on complex schemas or queries. Tune the limits as described in [Tuning query planner limits](/graphos/routing/query-planning/query-planning-best-practices#tuning-query-planner-limits). +- `apollo.router.query_planning.plan.non_local_selections` - Histogram of the number non-local selections estimated during query planning traversal. This is used to optimise option exploration when planning an operation. - `apollo.router.query_planner.memory` - Histogram of memory allocated during query planning, in bytes. Tracks memory allocation patterns specifically for query planning operations executed in the compute job thread pool. Attributes: - `allocation.type`: The type of memory operation (`allocated`, `deallocated`, `zeroed`, `reallocated`) - `context`: The context name where the allocation occurred (e.g., `query_planning`) From 507d311a8dcebfa71316e738a3b9752df8fd4521 Mon Sep 17 00:00:00 2001 From: Iryna Shestak Date: Thu, 1 Oct 2026 12:41:05 +0200 Subject: [PATCH 04/10] update PR number in changeset --- .changesets/feat_lrlna_router_2150_max_non_local_selections.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changesets/feat_lrlna_router_2150_max_non_local_selections.md b/.changesets/feat_lrlna_router_2150_max_non_local_selections.md index fab2445b3ab..21806577d66 100644 --- a/.changesets/feat_lrlna_router_2150_max_non_local_selections.md +++ b/.changesets/feat_lrlna_router_2150_max_non_local_selections.md @@ -8,4 +8,4 @@ This additional adds a metric for tracking current number of non-local selections, which is a histogram available under `apollo.router.query_planning.plan.non_local_selections`. -By [@lrlna](https://github.com/lrlna) in https://github.com/apollographql/router/pull/ +By [@lrlna](https://github.com/lrlna) in https://github.com/apollographql/router/pull/10349 From 7c77228b325517c6e400950425058e69c4212aa7 Mon Sep 17 00:00:00 2001 From: Iryna Shestak Date: Thu, 1 Oct 2026 12:49:01 +0200 Subject: [PATCH 05/10] change british english spelling Co-authored-by: Iryna Shestak --- apollo-router/src/query_planner/query_planner_service.rs | 2 +- .../enabling-telemetry/standard-instruments.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apollo-router/src/query_planner/query_planner_service.rs b/apollo-router/src/query_planner/query_planner_service.rs index 5073afec51f..f6ed703a65f 100644 --- a/apollo-router/src/query_planner/query_planner_service.rs +++ b/apollo-router/src/query_planner/query_planner_service.rs @@ -682,7 +682,7 @@ pub(crate) fn metric_query_planning_plan_duration( pub(crate) fn metric_query_planning_non_local_selections(count: u64) { u64_histogram_with_unit!( "apollo.router.query_planning.plan.non_local_selections", - "Number of non-local selections estimated during query planning traversal, used for optimising plan option exploration", + "Number of non-local selections estimated during query planning traversal, used for optimizing plan option exploration", "{selection}", count ); diff --git a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx index 094a7a2d62f..0f18ec3a9e5 100644 --- a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx @@ -195,7 +195,7 @@ The `apollo.router.cache.redis.errors` metric also includes an `error_type` attr - `apollo.router.query_planning.total.duration` - Histogram of plan durations including queue time. - `apollo.router.query_planning.plan.evaluated_plans` - Histogram of the number of evaluated query plans. - `apollo.router.query_planning.plan.evaluated_paths` - Histogram of the number of paths (including intermediate ones) the planner considers before generating a plan. High values often correlate with long planning times on complex schemas or queries. Tune the limits as described in [Tuning query planner limits](/graphos/routing/query-planning/query-planning-best-practices#tuning-query-planner-limits). -- `apollo.router.query_planning.plan.non_local_selections` - Histogram of the number non-local selections estimated during query planning traversal. This is used to optimise option exploration when planning an operation. +- `apollo.router.query_planning.plan.non_local_selections` - Histogram of the number non-local selections estimated during query planning traversal. This is used to optimize option exploration when planning an operation. - `apollo.router.query_planner.memory` - Histogram of memory allocated during query planning, in bytes. Tracks memory allocation patterns specifically for query planning operations executed in the compute job thread pool. Attributes: - `allocation.type`: The type of memory operation (`allocated`, `deallocated`, `zeroed`, `reallocated`) - `context`: The context name where the allocation occurred (e.g., `query_planning`) From 124edbcb79e46130a653ac7e0719982e10e3efaa Mon Sep 17 00:00:00 2001 From: Iryna Shestak Date: Thu, 1 Oct 2026 15:27:59 +0200 Subject: [PATCH 06/10] cargo fmt --- apollo-federation/src/query_plan/query_planner.rs | 2 +- .../src/query_planner/query_planner_service.rs | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/apollo-federation/src/query_plan/query_planner.rs b/apollo-federation/src/query_plan/query_planner.rs index 87e89a5f944..e6b73082ef4 100644 --- a/apollo-federation/src/query_plan/query_planner.rs +++ b/apollo-federation/src/query_plan/query_planner.rs @@ -177,7 +177,7 @@ pub struct QueryPlanningStatistics { /// `best_plan_cost` can be NaN, if the cost is not computed or irrelevant. #[serde(deserialize_with = "deserialize_f64_nullable")] pub best_plan_cost: f64, - /// `non_local_selections_count` can be `None`, if + /// `non_local_selections_count` can be `None`, if /// `QueryPlanOptions::non_local_selections_limit_enabled` is `false` pub non_local_selections_count: Option, } diff --git a/apollo-router/src/query_planner/query_planner_service.rs b/apollo-router/src/query_planner/query_planner_service.rs index f6ed703a65f..1bd795ead17 100644 --- a/apollo-router/src/query_planner/query_planner_service.rs +++ b/apollo-router/src/query_planner/query_planner_service.rs @@ -1368,9 +1368,18 @@ mod tests { .await .unwrap(); - assert_histogram_exists!("apollo.router.query_planning.plan.non_local_selections", u64); - assert_histogram_count!("apollo.router.query_planning.plan.non_local_selections", 1 as u64); - assert_histogram_sum!("apollo.router.query_planning.plan.non_local_selections", 7 as u64); + assert_histogram_exists!( + "apollo.router.query_planning.plan.non_local_selections", + u64 + ); + assert_histogram_count!( + "apollo.router.query_planning.plan.non_local_selections", + 1 as u64 + ); + assert_histogram_sum!( + "apollo.router.query_planning.plan.non_local_selections", + 7 as u64 + ); } .with_metrics() .await; From 46dc665e641467579d6de1f932dc02d72171f433 Mon Sep 17 00:00:00 2001 From: Iryna Shestak Date: Fri, 2 Oct 2026 13:51:36 +0200 Subject: [PATCH 07/10] set max_non_local_selections on query planning config to make sure cache is refreshed when env var changes --- ...na_router_2150_max_non_local_selections.md | 13 ++- .../src/query_plan/query_planner.rs | 11 +++ .../query_plan/query_planning_traversal.rs | 2 +- .../non_local_selections_estimation.rs | 28 +++--- apollo-router/src/configuration/mod.rs | 9 ++ .../tests/integration/query_planner/mod.rs | 1 + .../query_planner/non_local_selections.rs | 91 +++++++++++++++++++ .../standard-instruments.mdx | 2 +- 8 files changed, 131 insertions(+), 26 deletions(-) create mode 100644 apollo-router/tests/integration/query_planner/non_local_selections.rs diff --git a/.changesets/feat_lrlna_router_2150_max_non_local_selections.md b/.changesets/feat_lrlna_router_2150_max_non_local_selections.md index 21806577d66..3bbfd418781 100644 --- a/.changesets/feat_lrlna_router_2150_max_non_local_selections.md +++ b/.changesets/feat_lrlna_router_2150_max_non_local_selections.md @@ -1,11 +1,10 @@ -### Add environment variable for MAX_NON_LOCAL_SELECTIONS +### Add a metric to track non-local selection sets observed during query planning -Adds a environment variable for MAX_NON_LOCAL_SELECTIONS const used in query -planning traversal. This is an undocumented env var, as it's intended for -internal use. -This additional adds a metric for tracking current number of non-local -selections, which is a histogram available under -`apollo.router.query_planning.plan.non_local_selections`. +`apollo.router.query_planning.plan.non_local_selections` is a histogram of the +number of non-local selections estimated during query planning traversal used to +limit the number of options explored. The non-local selections limit defaults to +100_000. Numbers observed to be close to this limit may warrant an investigation +into complexity of the operations. By [@lrlna](https://github.com/lrlna) in https://github.com/apollographql/router/pull/10349 diff --git a/apollo-federation/src/query_plan/query_planner.rs b/apollo-federation/src/query_plan/query_planner.rs index e6b73082ef4..6c014ab13f0 100644 --- a/apollo-federation/src/query_plan/query_planner.rs +++ b/apollo-federation/src/query_plan/query_planner.rs @@ -1,5 +1,6 @@ use std::cell::Cell; use std::num::NonZeroU32; +use std::num::NonZeroU64; use std::ops::ControlFlow; use std::sync::Arc; @@ -158,6 +159,15 @@ pub struct QueryPlannerDebugConfig { /// /// The default value is None, which specifies no limit. pub paths_limit: Option, + + /// As the planner traverses the query, it estimates an upper bound on the + /// number of "non-local" selection sets it would need to consider as + /// possibilities. The process is aborted if the estimate exceeds this upper + /// bound to prevent unbounded planning time. + /// + /// This value currently defaults to 100_000. And is intentionally not part + /// of configuration which can be set by users. + pub max_non_local_selections: NonZeroU64, } impl Default for QueryPlannerDebugConfig { @@ -165,6 +175,7 @@ impl Default for QueryPlannerDebugConfig { Self { max_evaluated_plans: NonZeroU32::new(10_000).unwrap(), paths_limit: None, + max_non_local_selections: NonZeroU64::new(100_000).unwrap(), } } } diff --git a/apollo-federation/src/query_plan/query_planning_traversal.rs b/apollo-federation/src/query_plan/query_planning_traversal.rs index 2e6901ee344..309c2bab250 100644 --- a/apollo-federation/src/query_plan/query_planning_traversal.rs +++ b/apollo-federation/src/query_plan/query_planning_traversal.rs @@ -326,7 +326,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { return Err(SingleFederationError::QueryPlanComplexityExceeded { message: format!( "Number of non-local selections exceeds limit of {}", - Self::max_non_local_selections(), + traversal.parameters.config.debug.max_non_local_selections, ), } .into()); diff --git a/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs b/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs index b00a4690bd9..5065080a157 100644 --- a/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs +++ b/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs @@ -1,5 +1,3 @@ -use std::sync::OnceLock; - use apollo_compiler::Name; use apollo_compiler::collections::IndexMap; use apollo_compiler::collections::IndexSet; @@ -24,17 +22,6 @@ use crate::schema::position::INTROSPECTION_TYPENAME_FIELD_NAME; use crate::schema::position::ObjectTypeDefinitionPosition; impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { - pub(super) fn max_non_local_selections() -> u64 { - static MAX_NON_LOCAL_SELECTIONS: OnceLock = OnceLock::new(); - *MAX_NON_LOCAL_SELECTIONS.get_or_init(|| { - // This environment variable is intentionally undocumented. - std::env::var("APOLLO_ROUTER_MAX_NON_LOCAL_SELECTIONS") - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(100_000) - }) - } - /// This calls `check_non_local_selections_limit_exceeded()` for each of the selections in the /// open branches stack; see that function's doc comment for more information. /// @@ -63,6 +50,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { branch.selections.len(), tail_nodes_info.next_nodes.len(), state, + self.parameters.config.debug.max_non_local_selections.get(), ) { return Ok(true); } @@ -103,7 +91,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { /// adds it to the given count in the state. Note that the count for a given selection set is /// scaled by an approximate upper bound on the possible number of tail nodes for paths ending /// at that selection set. If at any point, the count exceeds - /// `Self::max_non_local_selections()`, then this function will return `true`. + /// `QueryPlannerDebugConfig::max_non_local_selections`, then this function will return `true`. /// /// This function's code is closely related to `selection_set_is_fully_local_from_all_nodes()` /// (which implements the aforementioned optimization). However, when it comes to traversing the @@ -238,6 +226,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { selection_set.selections.len(), parent_nodes.next_nodes.len(), state, + self.parameters.config.debug.max_non_local_selections.get(), ) { return Ok(true); @@ -246,8 +235,13 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { } /// Updates the non-local selection set count in the state, returning true if this causes the - /// count to exceed `Self::max_non_local_selections()`. - fn update_count(num_selections: usize, num_parent_nodes: usize, state: &mut State) -> bool { + /// count to exceed `limit`. + fn update_count( + num_selections: usize, + num_parent_nodes: usize, + state: &mut State, + limit: u64, + ) -> bool { let Ok(num_selections) = u64::try_from(num_selections) else { return true; }; @@ -260,7 +254,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { if let Some(new_count) = state .count .checked_add(additional_count) - .take_if(|v| *v <= Self::max_non_local_selections()) + .take_if(|v| *v <= limit) { state.count = new_count; } else { diff --git a/apollo-router/src/configuration/mod.rs b/apollo-router/src/configuration/mod.rs index 5223f3f738d..ad1c60241ae 100644 --- a/apollo-router/src/configuration/mod.rs +++ b/apollo-router/src/configuration/mod.rs @@ -8,6 +8,7 @@ use std::iter; use std::net::IpAddr; use std::net::SocketAddr; use std::num::NonZeroU32; +use std::num::NonZeroU64; use std::num::NonZeroUsize; use std::str::FromStr; use std::sync::Arc; @@ -444,6 +445,13 @@ impl Configuration { .and_then(NonZeroU32::new) .unwrap_or(NonZeroU32::new(10_000).expect("it is not zero")); + // This environment variable is intentionally undocumented. + let max_non_local_selections = std::env::var("APOLLO_ROUTER_MAX_NON_LOCAL_SELECTIONS") + .ok() + .and_then(|value| value.parse::().ok()) + .and_then(NonZeroU64::new) + .unwrap_or(NonZeroU64::new(100_000).expect("should be not zero")); + QueryPlannerConfig { subgraph_graphql_validation: false, generate_query_fragments: self.supergraph.generate_query_fragments, @@ -454,6 +462,7 @@ impl Configuration { debug: QueryPlannerDebugConfig { max_evaluated_plans, paths_limit: self.supergraph.query_planning.experimental_paths_limit, + max_non_local_selections, }, } } diff --git a/apollo-router/tests/integration/query_planner/mod.rs b/apollo-router/tests/integration/query_planner/mod.rs index 35d38f1a2e4..e5f2ad923d0 100644 --- a/apollo-router/tests/integration/query_planner/mod.rs +++ b/apollo-router/tests/integration/query_planner/mod.rs @@ -5,6 +5,7 @@ use crate::integration::common::graph_os_enabled; mod error_paths; mod max_evaluated_plans; +mod non_local_selections; const PROMETHEUS_METRICS_CONFIG: &str = include_str!("../telemetry/fixtures/prometheus.router.yaml"); diff --git a/apollo-router/tests/integration/query_planner/non_local_selections.rs b/apollo-router/tests/integration/query_planner/non_local_selections.rs new file mode 100644 index 00000000000..eacdcd0cbbc --- /dev/null +++ b/apollo-router/tests/integration/query_planner/non_local_selections.rs @@ -0,0 +1,91 @@ +use std::collections::HashMap; +use std::ffi::OsString; + +use reqwest::StatusCode; +use serde_json::json; + +use crate::integration::IntegrationTest; +use crate::integration::common::Query; + +const QUERY: &str = r#"{ t { v1 v2 v3 v4 } }"#; + +#[tokio::test(flavor = "multi_thread")] +async fn reports_non_local_selections() { + let mut router = IntegrationTest::builder() + .config( + r#" + telemetry: + exporters: + metrics: + prometheus: + enabled: true + "#, + ) + .supergraph("tests/integration/fixtures/query_planner_max_evaluated_plans.graphql") + .build() + .await; + router.start().await; + router.assert_started().await; + router + .execute_query( + Query::builder() + .body(json!({ + "query": QUERY, + "variables": {}, + })) + .build(), + ) + .await; + + router + .assert_metrics_contains( + r#"apollo_router_query_planning_plan_non_local_selections_sum{} 10"#, + None, + ) + .await; + + router.graceful_shutdown().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn exceeding_max_non_local_selections_aborts_planning() { + let mut router = IntegrationTest::builder() + .config( + r#" + telemetry: + exporters: + metrics: + prometheus: + enabled: true + "#, + ) + .supergraph("tests/integration/fixtures/query_planner_max_evaluated_plans.graphql") + .env(HashMap::from([( + "APOLLO_ROUTER_MAX_NON_LOCAL_SELECTIONS".to_string(), + OsString::from("1"), + )])) + .build() + .await; + router.start().await; + router.assert_started().await; + let (_, response) = router + .execute_query( + Query::builder() + .body(json!({ + "query": QUERY, + "variables": {}, + })) + .build(), + ) + .await; + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + + router + .assert_metrics_contains( + r#"apollo_router_operations_query_planner_non_local_selections_exceeded_total{} 1"#, + None, + ) + .await; + + router.graceful_shutdown().await; +} diff --git a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx index 0f18ec3a9e5..e770f57606b 100644 --- a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx @@ -195,7 +195,7 @@ The `apollo.router.cache.redis.errors` metric also includes an `error_type` attr - `apollo.router.query_planning.total.duration` - Histogram of plan durations including queue time. - `apollo.router.query_planning.plan.evaluated_plans` - Histogram of the number of evaluated query plans. - `apollo.router.query_planning.plan.evaluated_paths` - Histogram of the number of paths (including intermediate ones) the planner considers before generating a plan. High values often correlate with long planning times on complex schemas or queries. Tune the limits as described in [Tuning query planner limits](/graphos/routing/query-planning/query-planning-best-practices#tuning-query-planner-limits). -- `apollo.router.query_planning.plan.non_local_selections` - Histogram of the number non-local selections estimated during query planning traversal. This is used to optimize option exploration when planning an operation. +- `apollo.router.query_planning.plan.non_local_selections` - Histogram of the number of non-local selections estimated during query planning traversal used to limit the number of options explored. The non-local selections limit defaults to 100_000. Numbers observed to be close to this limit may warrant an investigation into complexity of the operations. - `apollo.router.query_planner.memory` - Histogram of memory allocated during query planning, in bytes. Tracks memory allocation patterns specifically for query planning operations executed in the compute job thread pool. Attributes: - `allocation.type`: The type of memory operation (`allocated`, `deallocated`, `zeroed`, `reallocated`) - `context`: The context name where the allocation occurred (e.g., `query_planning`) From ca054d341ec940b3b92c8bcdd6b4f3a62eece54f Mon Sep 17 00:00:00 2001 From: Iryna Shestak Date: Fri, 2 Oct 2026 13:57:15 +0200 Subject: [PATCH 08/10] cargo fmt --- apollo-federation/src/query_plan/query_planner.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apollo-federation/src/query_plan/query_planner.rs b/apollo-federation/src/query_plan/query_planner.rs index 6c014ab13f0..4fb069c03cc 100644 --- a/apollo-federation/src/query_plan/query_planner.rs +++ b/apollo-federation/src/query_plan/query_planner.rs @@ -163,7 +163,7 @@ pub struct QueryPlannerDebugConfig { /// As the planner traverses the query, it estimates an upper bound on the /// number of "non-local" selection sets it would need to consider as /// possibilities. The process is aborted if the estimate exceeds this upper - /// bound to prevent unbounded planning time. + /// bound to prevent unbounded planning time. /// /// This value currently defaults to 100_000. And is intentionally not part /// of configuration which can be set by users. From 3d1bd14e48677ab620c43e6f09bb0b1a3d984c63 Mon Sep 17 00:00:00 2001 From: Iryna Shestak Date: Fri, 2 Oct 2026 14:12:29 +0200 Subject: [PATCH 09/10] xtask lint --- .../src/query_planner/query_planner_service.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/apollo-router/src/query_planner/query_planner_service.rs b/apollo-router/src/query_planner/query_planner_service.rs index 1bd795ead17..a4c488c7b48 100644 --- a/apollo-router/src/query_planner/query_planner_service.rs +++ b/apollo-router/src/query_planner/query_planner_service.rs @@ -1372,14 +1372,8 @@ mod tests { "apollo.router.query_planning.plan.non_local_selections", u64 ); - assert_histogram_count!( - "apollo.router.query_planning.plan.non_local_selections", - 1 as u64 - ); - assert_histogram_sum!( - "apollo.router.query_planning.plan.non_local_selections", - 7 as u64 - ); + assert_histogram_count!("apollo.router.query_planning.plan.non_local_selections", 1); + assert_histogram_sum!("apollo.router.query_planning.plan.non_local_selections", 7); } .with_metrics() .await; From 9c98942b6360648e8d11c87927c120afa3d48468 Mon Sep 17 00:00:00 2001 From: Iryna Shestak Date: Mon, 5 Oct 2026 14:34:20 -0700 Subject: [PATCH 10/10] fix integration tests --- .../query_planner/non_local_selections.rs | 47 ------------------- apollo-router/tests/integration/redis.rs | 11 +++-- 2 files changed, 6 insertions(+), 52 deletions(-) diff --git a/apollo-router/tests/integration/query_planner/non_local_selections.rs b/apollo-router/tests/integration/query_planner/non_local_selections.rs index eacdcd0cbbc..dc738a87b2f 100644 --- a/apollo-router/tests/integration/query_planner/non_local_selections.rs +++ b/apollo-router/tests/integration/query_planner/non_local_selections.rs @@ -1,7 +1,3 @@ -use std::collections::HashMap; -use std::ffi::OsString; - -use reqwest::StatusCode; use serde_json::json; use crate::integration::IntegrationTest; @@ -46,46 +42,3 @@ async fn reports_non_local_selections() { router.graceful_shutdown().await; } - -#[tokio::test(flavor = "multi_thread")] -async fn exceeding_max_non_local_selections_aborts_planning() { - let mut router = IntegrationTest::builder() - .config( - r#" - telemetry: - exporters: - metrics: - prometheus: - enabled: true - "#, - ) - .supergraph("tests/integration/fixtures/query_planner_max_evaluated_plans.graphql") - .env(HashMap::from([( - "APOLLO_ROUTER_MAX_NON_LOCAL_SELECTIONS".to_string(), - OsString::from("1"), - )])) - .build() - .await; - router.start().await; - router.assert_started().await; - let (_, response) = router - .execute_query( - Query::builder() - .body(json!({ - "query": QUERY, - "variables": {}, - })) - .build(), - ) - .await; - assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); - - router - .assert_metrics_contains( - r#"apollo_router_operations_query_planner_non_local_selections_exceeded_total{} 1"#, - None, - ) - .await; - - router.graceful_shutdown().await; -} diff --git a/apollo-router/tests/integration/redis.rs b/apollo-router/tests/integration/redis.rs index f254a387f89..3c03e42f59c 100644 --- a/apollo-router/tests/integration/redis.rs +++ b/apollo-router/tests/integration/redis.rs @@ -182,7 +182,8 @@ async fn query_planner_cache() -> Result<(), BoxError> { // If this test fails and the cache key format changed you'll need to update the key here. // Look at the top of the file for instructions on getting the new cache key. let known_cache_key = &format!( - "{namespace}:plan:router:{}:47939f0e964372951934fc662c9c2be675bc7116ec3e57029abe555284eb10a4:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:d9f7a00bc249cb51cfc8599f86b6dc5272967b37b1409dc4717f105b6939fe43", + //"{namespace}:plan:router:{}:47939f0e964372951934fc662c9c2be675bc7116ec3e57029abe555284eb10a4:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:d9f7a00bc249cb51cfc8599f86b6dc5272967b37b1409dc4717f105b6939fe43", + "{namespace}:plan:router:{}:47939f0e964372951934fc662c9c2be675bc7116ec3e57029abe555284eb10a4:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:f16bdcbfbe4cbc78bc4599b75ca159ade850734316069f5d7e65c6ac10974850", env!("CARGO_PKG_VERSION") ); @@ -1571,7 +1572,7 @@ async fn query_planner_redis_update_query_fragments() { // This configuration turns the fragment generation option *off*. include_str!("fixtures/query_planner_redis_config_update_query_fragments.router.yaml"), &format!( - "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:fb1a8e6e454ad6a1d0d48b24dc9c7c4dd6d9bf58b6fdaf43cd24eb77fbbb3a17", + "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:02e0ce725b579eff2988e98b02560a48a3d9ecce9e7d3cef3d60dcf2fb366c42", env!("CARGO_PKG_VERSION") ), ) @@ -1594,7 +1595,7 @@ async fn query_planner_redis_update_defer() { test_redis_query_plan_config_update( include_str!("fixtures/query_planner_redis_config_update_defer.router.yaml"), &format!( - "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:dc062fcc9cfd9582402d1e8b1fa3ee336ea1804d833443869e0b3744996716a2", + "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:0651b0f75af604e69a6c0f5323f1a1231e7178acd8679d8e3a154640f51c5aa2", env!("CARGO_PKG_VERSION") ), ) @@ -1619,7 +1620,7 @@ async fn query_planner_redis_update_type_conditional_fetching() { "fixtures/query_planner_redis_config_update_type_conditional_fetching.router.yaml" ), &format!( - "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:bdc09980aa6ef28a67f5aeb8759763d8ac5a4fc43afa8c5a89f58cc998c48db3", + "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:2a38e8672a9f4de5b5d88a9faa9a6243fa5aa686f394a9f0ff8a787fc5a5172a", env!("CARGO_PKG_VERSION") ), ) @@ -1647,7 +1648,7 @@ async fn test_redis_query_plan_config_update(updated_config: &str, new_cache_key // If the tests above are failing, this is the key that needs to be changed first. let starting_key = &format!( - "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:d9f7a00bc249cb51cfc8599f86b6dc5272967b37b1409dc4717f105b6939fe43", + "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:f16bdcbfbe4cbc78bc4599b75ca159ade850734316069f5d7e65c6ac10974850", env!("CARGO_PKG_VERSION") ); assert_ne!(