diff --git a/.changesets/feat_lrlna_router_2150_max_non_local_selections.md b/.changesets/feat_lrlna_router_2150_max_non_local_selections.md new file mode 100644 index 00000000000..3bbfd418781 --- /dev/null +++ b/.changesets/feat_lrlna_router_2150_max_non_local_selections.md @@ -0,0 +1,10 @@ +### Add a metric to track non-local selection sets observed during query planning + + +`apollo.router.query_planning.plan.non_local_selections` is a histogram of the +number of non-local selections estimated during query planning traversal used to +limit the number of options explored. The non-local selections limit defaults to +100_000. Numbers observed to be close to this limit may warrant an investigation +into complexity of the operations. + +By [@lrlna](https://github.com/lrlna) in https://github.com/apollographql/router/pull/10349 diff --git a/apollo-federation/src/query_plan/query_planner.rs b/apollo-federation/src/query_plan/query_planner.rs index 7ae6a5101e6..4fb069c03cc 100644 --- a/apollo-federation/src/query_plan/query_planner.rs +++ b/apollo-federation/src/query_plan/query_planner.rs @@ -1,5 +1,6 @@ use std::cell::Cell; use std::num::NonZeroU32; +use std::num::NonZeroU64; use std::ops::ControlFlow; use std::sync::Arc; @@ -158,6 +159,15 @@ pub struct QueryPlannerDebugConfig { /// /// The default value is None, which specifies no limit. pub paths_limit: Option, + + /// As the planner traverses the query, it estimates an upper bound on the + /// number of "non-local" selection sets it would need to consider as + /// possibilities. The process is aborted if the estimate exceeds this upper + /// bound to prevent unbounded planning time. + /// + /// This value currently defaults to 100_000. And is intentionally not part + /// of configuration which can be set by users. + pub max_non_local_selections: NonZeroU64, } impl Default for QueryPlannerDebugConfig { @@ -165,6 +175,7 @@ impl Default for QueryPlannerDebugConfig { Self { max_evaluated_plans: NonZeroU32::new(10_000).unwrap(), paths_limit: None, + max_non_local_selections: NonZeroU64::new(100_000).unwrap(), } } } @@ -177,6 +188,9 @@ pub struct QueryPlanningStatistics { /// `best_plan_cost` can be NaN, if the cost is not computed or irrelevant. #[serde(deserialize_with = "deserialize_f64_nullable")] pub best_plan_cost: f64, + /// `non_local_selections_count` can be `None`, if + /// `QueryPlanOptions::non_local_selections_limit_enabled` is `false` + pub non_local_selections_count: Option, } /// Deserialize helper for f64 that treats null as NaN. @@ -563,6 +577,7 @@ impl QueryPlanner { node: root_node, statistics: QueryPlanningStatistics { best_plan_cost: cost, + non_local_selections_count: non_local_selection_state.as_ref().map(|s| s.count), ..statistics }, }; @@ -1458,13 +1473,15 @@ type User evaluated_plan_count: Cell::new(10), evaluated_plan_paths: Cell::new(20), best_plan_cost: f64::NAN, + non_local_selections_count: Some(30), }; let serialized = serde_json::to_string_pretty(&stats).expect("Serializing"); insta::assert_snapshot!(serialized, @r###" { "evaluated_plan_count": 10, "evaluated_plan_paths": 20, - "best_plan_cost": null + "best_plan_cost": null, + "non_local_selections_count": 30 } "###); diff --git a/apollo-federation/src/query_plan/query_planning_traversal.rs b/apollo-federation/src/query_plan/query_planning_traversal.rs index 214642b3825..309c2bab250 100644 --- a/apollo-federation/src/query_plan/query_planning_traversal.rs +++ b/apollo-federation/src/query_plan/query_planning_traversal.rs @@ -326,7 +326,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { return Err(SingleFederationError::QueryPlanComplexityExceeded { message: format!( "Number of non-local selections exceeds limit of {}", - Self::MAX_NON_LOCAL_SELECTIONS, + traversal.parameters.config.debug.max_non_local_selections, ), } .into()); diff --git a/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs b/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs index e49e404ecce..5065080a157 100644 --- a/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs +++ b/apollo-federation/src/query_plan/query_planning_traversal/non_local_selections_estimation.rs @@ -22,8 +22,6 @@ use crate::schema::position::INTROSPECTION_TYPENAME_FIELD_NAME; use crate::schema::position::ObjectTypeDefinitionPosition; impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { - pub(super) const MAX_NON_LOCAL_SELECTIONS: u64 = 100_000; - /// This calls `check_non_local_selections_limit_exceeded()` for each of the selections in the /// open branches stack; see that function's doc comment for more information. /// @@ -52,6 +50,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { branch.selections.len(), tail_nodes_info.next_nodes.len(), state, + self.parameters.config.debug.max_non_local_selections.get(), ) { return Ok(true); } @@ -91,8 +90,8 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { /// set that wouldn't be avoided by such an optimization (i.e. the "non-local" selections), and /// adds it to the given count in the state. Note that the count for a given selection set is /// scaled by an approximate upper bound on the possible number of tail nodes for paths ending - /// at that selection set. If at any point, the count exceeds `Self::MAX_NON_LOCAL_SELECTIONS`, - /// then this function will return `true`. + /// at that selection set. If at any point, the count exceeds + /// `QueryPlannerDebugConfig::max_non_local_selections`, then this function will return `true`. /// /// This function's code is closely related to `selection_set_is_fully_local_from_all_nodes()` /// (which implements the aforementioned optimization). However, when it comes to traversing the @@ -227,6 +226,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { selection_set.selections.len(), parent_nodes.next_nodes.len(), state, + self.parameters.config.debug.max_non_local_selections.get(), ) { return Ok(true); @@ -235,8 +235,13 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { } /// Updates the non-local selection set count in the state, returning true if this causes the - /// count to exceed `Self::MAX_NON_LOCAL_SELECTIONS`. - fn update_count(num_selections: usize, num_parent_nodes: usize, state: &mut State) -> bool { + /// count to exceed `limit`. + fn update_count( + num_selections: usize, + num_parent_nodes: usize, + state: &mut State, + limit: u64, + ) -> bool { let Ok(num_selections) = u64::try_from(num_selections) else { return true; }; @@ -249,7 +254,7 @@ impl<'a: 'b, 'b> QueryPlanningTraversal<'a, 'b> { if let Some(new_count) = state .count .checked_add(additional_count) - .take_if(|v| *v <= Self::MAX_NON_LOCAL_SELECTIONS) + .take_if(|v| *v <= limit) { state.count = new_count; } else { diff --git a/apollo-router/src/configuration/mod.rs b/apollo-router/src/configuration/mod.rs index 5223f3f738d..ad1c60241ae 100644 --- a/apollo-router/src/configuration/mod.rs +++ b/apollo-router/src/configuration/mod.rs @@ -8,6 +8,7 @@ use std::iter; use std::net::IpAddr; use std::net::SocketAddr; use std::num::NonZeroU32; +use std::num::NonZeroU64; use std::num::NonZeroUsize; use std::str::FromStr; use std::sync::Arc; @@ -444,6 +445,13 @@ impl Configuration { .and_then(NonZeroU32::new) .unwrap_or(NonZeroU32::new(10_000).expect("it is not zero")); + // This environment variable is intentionally undocumented. + let max_non_local_selections = std::env::var("APOLLO_ROUTER_MAX_NON_LOCAL_SELECTIONS") + .ok() + .and_then(|value| value.parse::().ok()) + .and_then(NonZeroU64::new) + .unwrap_or(NonZeroU64::new(100_000).expect("should be not zero")); + QueryPlannerConfig { subgraph_graphql_validation: false, generate_query_fragments: self.supergraph.generate_query_fragments, @@ -454,6 +462,7 @@ impl Configuration { debug: QueryPlannerDebugConfig { max_evaluated_plans, paths_limit: self.supergraph.query_planning.experimental_paths_limit, + max_non_local_selections, }, } } diff --git a/apollo-router/src/query_planner/query_planner_service.rs b/apollo-router/src/query_planner/query_planner_service.rs index 93dd388204c..a4c488c7b48 100644 --- a/apollo-router/src/query_planner/query_planner_service.rs +++ b/apollo-router/src/query_planner/query_planner_service.rs @@ -208,6 +208,7 @@ impl QueryPlannerService { query_plan_root_node: root_node.map(Arc::new), evaluated_plan_count: plan.statistics.evaluated_plan_count.clone().into_inner() as u64, evaluated_plan_paths: plan.statistics.evaluated_plan_paths.clone().into_inner() as u64, + non_local_selections_count: plan.statistics.non_local_selections_count, }) } @@ -331,6 +332,7 @@ impl QueryPlannerService { formatted_query_plan, evaluated_plan_count, evaluated_plan_paths, + non_local_selections_count, } = plan_result; // If the query is filtered, we want to generate the signature using the original query and generate the @@ -366,6 +368,9 @@ impl QueryPlannerService { "Number of paths (including intermediate ones) considered to plan a query before starting to generate a plan", evaluated_plan_paths ); + if let Some(non_local_selections_count) = non_local_selections_count { + metric_query_planning_non_local_selections(non_local_selections_count); + } Ok(QueryPlannerContent::Plan { plan: Arc::new(super::QueryPlan { @@ -614,6 +619,7 @@ pub(crate) struct QueryPlanResult { pub(super) query_plan_root_node: Option>, pub(super) evaluated_plan_count: u64, pub(super) evaluated_plan_paths: u64, + pub(super) non_local_selections_count: Option, } /// The outcome of a query-planning attempt. Shared across query-planning metrics (e.g. @@ -673,6 +679,15 @@ pub(crate) fn metric_query_planning_plan_duration( ); } +pub(crate) fn metric_query_planning_non_local_selections(count: u64) { + u64_histogram_with_unit!( + "apollo.router.query_planning.plan.non_local_selections", + "Number of non-local selections estimated during query planning traversal, used for optimizing plan option exploration", + "{selection}", + count + ); +} + pub(crate) fn metric_rust_qp_init(init_error_kind: Option<&'static str>) { if let Some(init_error_kind) = init_error_kind { u64_counter!( @@ -1340,6 +1355,30 @@ mod tests { .await; } + #[test(tokio::test)] + async fn test_non_local_selections_histogram() { + async { + let _ = plan( + EXAMPLE_SCHEMA, + include_str!("testdata/query.graphql"), + include_str!("testdata/query.graphql"), + None, + PlanOptions::default(), + ) + .await + .unwrap(); + + assert_histogram_exists!( + "apollo.router.query_planning.plan.non_local_selections", + u64 + ); + assert_histogram_count!("apollo.router.query_planning.plan.non_local_selections", 1); + assert_histogram_sum!("apollo.router.query_planning.plan.non_local_selections", 7); + } + .with_metrics() + .await; + } + async fn plan_unauthorized_operation(compute_job_type: ComputeJobType) -> QueryPlannerContent { let configuration: Arc = Arc::default(); let schema = Schema::parse( diff --git a/apollo-router/tests/integration/query_planner/mod.rs b/apollo-router/tests/integration/query_planner/mod.rs index 35d38f1a2e4..e5f2ad923d0 100644 --- a/apollo-router/tests/integration/query_planner/mod.rs +++ b/apollo-router/tests/integration/query_planner/mod.rs @@ -5,6 +5,7 @@ use crate::integration::common::graph_os_enabled; mod error_paths; mod max_evaluated_plans; +mod non_local_selections; const PROMETHEUS_METRICS_CONFIG: &str = include_str!("../telemetry/fixtures/prometheus.router.yaml"); diff --git a/apollo-router/tests/integration/query_planner/non_local_selections.rs b/apollo-router/tests/integration/query_planner/non_local_selections.rs new file mode 100644 index 00000000000..dc738a87b2f --- /dev/null +++ b/apollo-router/tests/integration/query_planner/non_local_selections.rs @@ -0,0 +1,44 @@ +use serde_json::json; + +use crate::integration::IntegrationTest; +use crate::integration::common::Query; + +const QUERY: &str = r#"{ t { v1 v2 v3 v4 } }"#; + +#[tokio::test(flavor = "multi_thread")] +async fn reports_non_local_selections() { + let mut router = IntegrationTest::builder() + .config( + r#" + telemetry: + exporters: + metrics: + prometheus: + enabled: true + "#, + ) + .supergraph("tests/integration/fixtures/query_planner_max_evaluated_plans.graphql") + .build() + .await; + router.start().await; + router.assert_started().await; + router + .execute_query( + Query::builder() + .body(json!({ + "query": QUERY, + "variables": {}, + })) + .build(), + ) + .await; + + router + .assert_metrics_contains( + r#"apollo_router_query_planning_plan_non_local_selections_sum{} 10"#, + None, + ) + .await; + + router.graceful_shutdown().await; +} diff --git a/apollo-router/tests/integration/redis.rs b/apollo-router/tests/integration/redis.rs index f254a387f89..3c03e42f59c 100644 --- a/apollo-router/tests/integration/redis.rs +++ b/apollo-router/tests/integration/redis.rs @@ -182,7 +182,8 @@ async fn query_planner_cache() -> Result<(), BoxError> { // If this test fails and the cache key format changed you'll need to update the key here. // Look at the top of the file for instructions on getting the new cache key. let known_cache_key = &format!( - "{namespace}:plan:router:{}:47939f0e964372951934fc662c9c2be675bc7116ec3e57029abe555284eb10a4:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:d9f7a00bc249cb51cfc8599f86b6dc5272967b37b1409dc4717f105b6939fe43", + //"{namespace}:plan:router:{}:47939f0e964372951934fc662c9c2be675bc7116ec3e57029abe555284eb10a4:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:d9f7a00bc249cb51cfc8599f86b6dc5272967b37b1409dc4717f105b6939fe43", + "{namespace}:plan:router:{}:47939f0e964372951934fc662c9c2be675bc7116ec3e57029abe555284eb10a4:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:f16bdcbfbe4cbc78bc4599b75ca159ade850734316069f5d7e65c6ac10974850", env!("CARGO_PKG_VERSION") ); @@ -1571,7 +1572,7 @@ async fn query_planner_redis_update_query_fragments() { // This configuration turns the fragment generation option *off*. include_str!("fixtures/query_planner_redis_config_update_query_fragments.router.yaml"), &format!( - "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:fb1a8e6e454ad6a1d0d48b24dc9c7c4dd6d9bf58b6fdaf43cd24eb77fbbb3a17", + "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:02e0ce725b579eff2988e98b02560a48a3d9ecce9e7d3cef3d60dcf2fb366c42", env!("CARGO_PKG_VERSION") ), ) @@ -1594,7 +1595,7 @@ async fn query_planner_redis_update_defer() { test_redis_query_plan_config_update( include_str!("fixtures/query_planner_redis_config_update_defer.router.yaml"), &format!( - "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:dc062fcc9cfd9582402d1e8b1fa3ee336ea1804d833443869e0b3744996716a2", + "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:0651b0f75af604e69a6c0f5323f1a1231e7178acd8679d8e3a154640f51c5aa2", env!("CARGO_PKG_VERSION") ), ) @@ -1619,7 +1620,7 @@ async fn query_planner_redis_update_type_conditional_fetching() { "fixtures/query_planner_redis_config_update_type_conditional_fetching.router.yaml" ), &format!( - "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:bdc09980aa6ef28a67f5aeb8759763d8ac5a4fc43afa8c5a89f58cc998c48db3", + "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:2a38e8672a9f4de5b5d88a9faa9a6243fa5aa686f394a9f0ff8a787fc5a5172a", env!("CARGO_PKG_VERSION") ), ) @@ -1647,7 +1648,7 @@ async fn test_redis_query_plan_config_update(updated_config: &str, new_cache_key // If the tests above are failing, this is the key that needs to be changed first. let starting_key = &format!( - "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:d9f7a00bc249cb51cfc8599f86b6dc5272967b37b1409dc4717f105b6939fe43", + "plan:router:{}:14ece7260081620bb49f1f4934cf48510e5f16c3171181768bb46a5609d7dfb7:opname:3973e022e93220f9212c18d0d0c543ae7c309e46640da93a4a0314de999f5112:metadata:f16bdcbfbe4cbc78bc4599b75ca159ade850734316069f5d7e65c6ac10974850", env!("CARGO_PKG_VERSION") ); assert_ne!( diff --git a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx index fa83b2f3f58..e770f57606b 100644 --- a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx @@ -195,6 +195,7 @@ The `apollo.router.cache.redis.errors` metric also includes an `error_type` attr - `apollo.router.query_planning.total.duration` - Histogram of plan durations including queue time. - `apollo.router.query_planning.plan.evaluated_plans` - Histogram of the number of evaluated query plans. - `apollo.router.query_planning.plan.evaluated_paths` - Histogram of the number of paths (including intermediate ones) the planner considers before generating a plan. High values often correlate with long planning times on complex schemas or queries. Tune the limits as described in [Tuning query planner limits](/graphos/routing/query-planning/query-planning-best-practices#tuning-query-planner-limits). +- `apollo.router.query_planning.plan.non_local_selections` - Histogram of the number of non-local selections estimated during query planning traversal used to limit the number of options explored. The non-local selections limit defaults to 100_000. Numbers observed to be close to this limit may warrant an investigation into complexity of the operations. - `apollo.router.query_planner.memory` - Histogram of memory allocated during query planning, in bytes. Tracks memory allocation patterns specifically for query planning operations executed in the compute job thread pool. Attributes: - `allocation.type`: The type of memory operation (`allocated`, `deallocated`, `zeroed`, `reallocated`) - `context`: The context name where the allocation occurred (e.g., `query_planning`)