From df16f59c0a2315efe72e87ef0fb8c6b2f3f05576 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Fri, 28 Aug 2026 08:55:55 +0000 Subject: [PATCH 01/30] feat: Record in-progress ingress messages after a subnet merge If the `subnet_merged` marker is set, `process_batch()` now resets it and records all not yet responded ingress-induced canister call contexts as `Processing` in the ingress history, so that the corresponding ingress messages can be tracked to completion. Subnet call contexts are ignored, as subnet merging ensures that the subnets being merged have no in-progress subnet call contexts. A message that already has an ingress history entry is left alone. Its status is expected to be `Processing`; anything else raises the new `mr_unexpected_ingress_status_after_merge` critical error. Co-Authored-By: Claude Opus 5 (1M context) --- rs/messaging/src/message_routing.rs | 53 ++++++- rs/messaging/src/message_routing/tests.rs | 67 ++++++++ rs/replicated_state/src/replicated_state.rs | 83 +++++++++- rs/replicated_state/tests/replicated_state.rs | 150 +++++++++++++++++- 4 files changed, 345 insertions(+), 8 deletions(-) diff --git a/rs/messaging/src/message_routing.rs b/rs/messaging/src/message_routing.rs index 852925de4393..9d220799a989 100644 --- a/rs/messaging/src/message_routing.rs +++ b/rs/messaging/src/message_routing.rs @@ -39,13 +39,15 @@ use ic_replicated_state::{ }; use ic_types::batch::{Batch, BatchContent, BatchSummary}; use ic_types::crypto::{KeyPurpose, threshold_sig::ThresholdSigPublicKey}; +use ic_types::ingress::IngressStatus; use ic_types::malicious_flags::MaliciousFlags; +use ic_types::messages::MessageId; use ic_types::registry::RegistryClientError; use ic_types::state_manager::StateManagerError; use ic_types::xnet::{StreamHeader, StreamIndex}; use ic_types::{ - ExecutionRound, Height, NodeId, PrincipalId, PrincipalIdBlobParseError, RegistryVersion, - SubnetId, Time, + ExecutionRound, Height, NodeId, NumBytes, PrincipalId, PrincipalIdBlobParseError, + RegistryVersion, SubnetId, Time, }; use ic_types_cycles::CanisterCyclesCostSchedule; use ic_utils_thread::JoinOnDrop; @@ -133,6 +135,8 @@ pub const CRITICAL_ERROR_NON_INCREASING_BATCH_TIME: &str = "mr_non_increasing_ba pub const CRITICAL_ERROR_INDUCT_RESPONSE_FAILED: &str = "mr_induct_response_failed"; pub const CRITICAL_ERROR_ILLEGAL_ENGINE_MESSAGE: &str = "mr_illegal_engine_message"; const CRITICAL_ERROR_ILLEGAL_NON_EMPTY_SUBNET_ADMINS: &str = "mr_illegal_non_empty_subnet_admins"; +const CRITICAL_ERROR_UNEXPECTED_INGRESS_STATUS_AFTER_MERGE: &str = + "mr_unexpected_ingress_status_after_merge"; /// Records the timestamp when all messages before the given index (down to the /// previous `MessageTime`) were first added to / learned about in a stream. @@ -355,6 +359,10 @@ pub(crate) struct MessageRoutingMetrics { pub critical_error_engine_message: IntCounter, /// Critical error: a non-rental subnet has a non-empty subnet admins list. critical_error_illegal_non_empty_subnet_admins: IntCounter, + /// Critical error: an in-progress ingress message had an ingress history entry + /// with a status other than `Processing` in the first round after a subnet + /// merge. + critical_error_unexpected_ingress_status_after_merge: IntCounter, /// Metrics for query stats aggregator pub query_stats_metrics: QueryStatsAggregatorMetrics, @@ -503,6 +511,8 @@ impl MessageRoutingMetrics { .error_counter(CRITICAL_ERROR_ILLEGAL_ENGINE_MESSAGE), critical_error_illegal_non_empty_subnet_admins: metrics_registry .error_counter(CRITICAL_ERROR_ILLEGAL_NON_EMPTY_SUBNET_ADMINS), + critical_error_unexpected_ingress_status_after_merge: metrics_registry + .error_counter(CRITICAL_ERROR_UNEXPECTED_INGRESS_STATUS_AFTER_MERGE), query_stats_metrics: QueryStatsAggregatorMetrics::new(metrics_registry), @@ -543,6 +553,23 @@ impl MessageRoutingMetrics { ); } + pub fn observe_unexpected_ingress_status_after_merge( + &self, + log: &ReplicaLogger, + message_id: &MessageId, + status: &IngressStatus, + ) { + self.critical_error_unexpected_ingress_status_after_merge + .inc(); + warn!( + log, + "{}: In-progress ingress message {} has unexpected status {} after a subnet merge.", + CRITICAL_ERROR_UNEXPECTED_INGRESS_STATUS_AFTER_MERGE, + message_id, + status.as_str() + ); + } + pub fn start_phase_timer(&self, phase: &str) -> HistogramTimer { self.process_batch_phase_duration .with_label_values(&[phase]) @@ -594,6 +621,9 @@ struct BatchProcessorImpl { registry_reader: RegistryReader, metrics: MessageRoutingMetrics, log: ReplicaLogger, + /// Soft limit on the memory footprint of the ingress history; used when + /// recording in-progress ingress messages after a subnet merge. + ingress_history_memory_capacity: NumBytes, #[allow(dead_code)] malicious_flags: MaliciousFlags, } @@ -704,6 +734,7 @@ impl BatchProcessorImpl { metrics.clone(), )); + let ingress_history_memory_capacity = hypervisor_config.ingress_history_memory_capacity; let registry_reader = RegistryReader::new( registry, hypervisor_config.bitcoin, @@ -717,6 +748,7 @@ impl BatchProcessorImpl { registry_reader, metrics, log, + ingress_history_memory_capacity, malicious_flags, } } @@ -1421,6 +1453,23 @@ impl BatchProcessor for BatchProcessorImpl( ), metrics: metrics.clone(), log, + ingress_history_memory_capacity: HypervisorConfig::default() + .ingress_history_memory_capacity, malicious_flags: MaliciousFlags::default(), }; (batch_processor, metrics, state_manager, registry_settings) @@ -2419,6 +2421,71 @@ fn process_batch_resets_split_marker() { }); } +#[test] +fn process_batch_resets_merge_marker() { + with_test_replica_logger(|log| { + use Integrity::*; + + let own_subnet_id = subnet_test_id(13); + let nns_subnet_id = subnet_test_id(42); + + let own_transcript = dummy_transcript_for_tests_with_params( + vec![node_test_id(123)], // committee + NiDkgTag::HighThreshold, // dkg_tag + 2, // threshold + 3, // registry_version + ); + + let fixture = RegistryFixture::new(); + fixture + .write_test_records(&TestRecords { + subnet_ids: Valid([own_subnet_id]), + subnet_records: [Valid(&SubnetRecord::default())], + ni_dkg_transcripts: [Valid(Some(&own_transcript))], + nns_subnet_id: Valid(nns_subnet_id), + chain_key_enabled_subnets: &BTreeMap::default(), + provisional_whitelist: Valid(&ProvisionalWhitelist::All), + routing_table: Valid(&RoutingTable::new()), + canister_migrations: Valid(&CanisterMigrations::new()), + node_public_keys: &BTreeMap::default(), + api_boundary_node_records: &BTreeMap::default(), + node_records: &BTreeMap::default(), + }) + .unwrap(); + + // Reading from the registry must succeed for fully specified records. + let (batch_processor, _metrics, state_manager, _registry_settings) = + make_batch_processor(fixture.registry.clone(), log); + let (mut height, mut state) = state_manager.take_tip(); + state.metadata.own_subnet_id = own_subnet_id; + state.metadata.subnet_merged = true; + height.inc_assign(); + state_manager.commit_and_certify(state, CertificationScope::Metadata, None); + + batch_processor.process_batch(Batch { + batch_number: height.increment(), + batch_summary: None, + content: BatchContent::Data { + batch_messages: BatchMessages::default(), + consensus_responses: Vec::new(), + canister_http_spent: Default::default(), + chain_key_data: Default::default(), + requires_full_state_hash: false, + }, + randomness: Randomness::new([123; 32]), + registry_version: fixture.registry.get_latest_version(), + time: Time::from_nanos_since_unix_epoch(1), + blockmaker_metrics: BlockmakerMetrics::new_for_test(), + replica_version: test_replica_version(), + }); + + // The subnet merge marker was reset. (Which only happens in `after_merge()`, + // whose behavior is covered by the `ic-replicated-state` unit tests.) + let latest_state = state_manager.get_latest_state().take(); + assert!(!latest_state.metadata.subnet_merged); + }); +} + #[test] fn test_demux_delivers_certified_stream_slices() { struct FakeValidSetRule; diff --git a/rs/replicated_state/src/replicated_state.rs b/rs/replicated_state/src/replicated_state.rs index 672091bd51bf..ca99b9731760 100644 --- a/rs/replicated_state/src/replicated_state.rs +++ b/rs/replicated_state/src/replicated_state.rs @@ -1,7 +1,7 @@ use crate::canister_state::queues::{ CanisterInput, CanisterQueuesLoopDetector, refunds::RefundPool, }; -use crate::canister_state::system_state::{CanisterOutputQueuesIterator, push_input}; +use crate::canister_state::system_state::{CallOrigin, CanisterOutputQueuesIterator, push_input}; use crate::metadata_state::subnet_call_context_manager::{ PreSignatureStash, ReshareChainKeyContext, SignWithThresholdContext, }; @@ -30,7 +30,7 @@ use ic_types::{ CanisterId, NumBytes, SubnetId, Time, batch::{ConsensusResponse, RawQueryStats}, consensus::idkg::IDkgMasterPublicKeyId, - ingress::IngressStatus, + ingress::{IngressState, IngressStatus}, messages::{ CallbackId, Ingress, MessageId, Refund, RequestOrResponse, Response, SubnetMessage, }, @@ -1613,6 +1613,85 @@ impl ReplicatedState { *epoch_query_stats = RawQueryStats::default(); } + /// Makes adjustments to the replicated state during the first round after a + /// subnet merge: resets the "subnet was merged" marker and records all + /// not yet responded ingress-induced call contexts as `Processing` in the + /// ingress history. + /// + /// The ingress history of the merged subnet does not necessarily cover the + /// in-progress ingress messages of all merged subnets, so the corresponding + /// entries are (re)created here, ensuring that every in-progress ingress + /// message can be tracked to completion. + /// + /// Only call contexts of canisters are considered; subnet call contexts are + /// ignored, as subnet merging ensures that the subnets being merged have no + /// in-progress subnet call contexts. + /// + /// A message that already has an ingress history entry is left alone; its + /// status is expected to be `Processing`, anything else is reported via + /// `on_unexpected_ingress_status()` (as it indicates a bug). + pub fn after_merge( + &mut self, + ingress_memory_capacity: NumBytes, + on_unexpected_ingress_status: impl Fn(&MessageId, &IngressStatus), + ) { + assert!( + self.metadata.subnet_merged, + "Not a state resulting from a subnet merge" + ); + self.metadata.subnet_merged = false; + + let time = self.time(); + let ingress_statuses = self + .canisters_iter() + .flat_map(|canister_state| { + let receiver = canister_state.canister_id().get(); + canister_state + .system_state + .call_context_manager() + .into_iter() + .flat_map(|ccm| ccm.call_contexts().values()) + .filter(|call_context| !call_context.has_responded()) + .filter_map(move |call_context| match call_context.call_origin() { + CallOrigin::Ingress(user_id, message_id, _) => Some(( + message_id.clone(), + IngressStatus::Known { + receiver, + user_id: *user_id, + time, + state: IngressState::Processing, + }, + )), + CallOrigin::CanisterUpdate(..) + | CallOrigin::Query(..) + | CallOrigin::CanisterQuery(..) + | CallOrigin::SystemTask => None, + }) + }) + .collect::>(); + + for (message_id, status) in ingress_statuses { + match self.metadata.ingress_history.get(&message_id).cloned() { + // No entry yet, record the in-progress ingress message. + None => { + self.set_ingress_status(message_id, status, ingress_memory_capacity, |_| {}); + } + + // Already recorded as `Processing`, nothing to do. + Some(IngressStatus::Known { + state: IngressState::Processing, + .. + }) => {} + + // Any other status indicates a bug, report it. The existing entry is + // preserved, as overwriting a terminal status would be worse. + Some(unexpected_status) => { + on_unexpected_ingress_status(&message_id, &unexpected_status) + } + } + } + } + /// Splits the replicated state during a special DSM round, retaining only the /// canisters mapped to `subnet_id` in the loaded routing table. /// diff --git a/rs/replicated_state/tests/replicated_state.rs b/rs/replicated_state/tests/replicated_state.rs index 9a3840877009..94bca0fd8f93 100644 --- a/rs/replicated_state/tests/replicated_state.rs +++ b/rs/replicated_state/tests/replicated_state.rs @@ -12,8 +12,9 @@ use ic_management_canister_types_private::{ use ic_registry_routing_table::{CANISTER_IDS_PER_SUBNET, CanisterIdRange, RoutingTable}; use ic_registry_subnet_type::SubnetType; use ic_replicated_state::{ - CanisterQueues, CanisterState, ExecutionTask, IngressHistoryState, InputSource, OutputRequest, - RefundPool, ReplicatedState, SchedulerState, StateError, SystemMetadata, SystemState, + CallContext, CallOrigin, CanisterQueues, CanisterState, ExecutionTask, IngressHistoryState, + InputSource, OutputRequest, RefundPool, ReplicatedState, SchedulerState, StateError, + SystemMetadata, SystemState, canister_state::{ canister_snapshots::{CanisterSnapshot, CanisterSnapshots}, execution_state::{CustomSection, CustomSectionType, WasmMetadata}, @@ -41,8 +42,8 @@ use ic_test_utilities_types::messages::{RequestBuilder, ResponseBuilder}; use ic_types::batch::RawQueryStats; use ic_types::ingress::{IngressState, IngressStatus}; use ic_types::messages::{ - CallbackId, CanisterCall, CanisterMessage, MAX_RESPONSE_COUNT_BYTES, Payload, Refund, - RejectContext, Request, RequestOrResponse, Response, SubnetMessage, + CallbackId, CanisterCall, CanisterMessage, MAX_RESPONSE_COUNT_BYTES, NO_DEADLINE, Payload, + Refund, RejectContext, Request, RequestMetadata, RequestOrResponse, Response, SubnetMessage, }; use ic_types::time::{CoarseTime, UNIX_EPOCH}; use ic_types::xnet::StreamIndex; @@ -53,6 +54,7 @@ use ic_types_cycles::{ }; use maplit::btreemap; use proptest::prelude::*; +use std::cell::RefCell; use std::collections::{BTreeMap, VecDeque}; use std::mem::size_of; use std::sync::Arc; @@ -1448,6 +1450,146 @@ fn online_split() { assert_eq!(expected, state_b); } +#[test] +fn after_merge() { + const CANISTER_1: CanisterId = CanisterId::from_u64(1); + const CANISTER_2: CanisterId = CanisterId::from_u64(2); + const CANISTERS: [CanisterId; 2] = [CANISTER_1, CANISTER_2]; + + // A time different from the state time (`UNIX_EPOCH`), so that pre-existing + // ingress history entries can be told apart from newly recorded ones. + let before = Time::from_nanos_since_unix_epoch(13); + + // Makes a not yet responded call context with the given origin. + fn open_call_context(call_origin: CallOrigin) -> CallContext { + CallContext::new( + call_origin, + false, // responded + false, // deleted + Cycles::zero(), + UNIX_EPOCH, + RequestMetadata::for_new_call_tree(UNIX_EPOCH), + None, + ) + } + + // Makes an ingress call origin for the given message. + fn ingress_origin(message: u64) -> CallOrigin { + CallOrigin::Ingress( + user_test_id(message), + message_test_id(message), + "update".into(), + ) + } + + // Makes an ingress status with the given receiver, message and state. + let ingress_status = |receiver: CanisterId, message: u64, time, state| IngressStatus::Known { + receiver: receiver.get(), + user_id: user_test_id(message), + time, + state, + }; + + let mut fixture = ReplicatedStateFixture::with_canisters(&CANISTERS); + + // An in-progress ingress message to `CANISTER_1`, with no ingress history entry. + let canister_1 = fixture.state.canister_state_make_mut(&CANISTER_1).unwrap(); + canister_1 + .system_state + .with_call_context(open_call_context(ingress_origin(1))); + // An ingress message that was already responded to; and an in-progress canister + // update call. Neither should be recorded in the ingress history. + canister_1.system_state.with_call_context(CallContext::new( + ingress_origin(2), + true, // responded + false, // deleted + Cycles::zero(), + UNIX_EPOCH, + RequestMetadata::for_new_call_tree(UNIX_EPOCH), + None, + )); + canister_1 + .system_state + .with_call_context(open_call_context(CallOrigin::CanisterUpdate( + CANISTER_2, + CallbackId::from(3), + NO_DEADLINE, + "update".into(), + ))); + + // Three in-progress ingress messages to `CANISTER_2`: one with no ingress + // history entry; one already recorded as `Processing`; and one recorded with an + // unexpected status. + let canister_2 = fixture.state.canister_state_make_mut(&CANISTER_2).unwrap(); + for message in [4, 5, 6] { + canister_2 + .system_state + .with_call_context(open_call_context(ingress_origin(message))); + } + let processing_5 = ingress_status(CANISTER_2, 5, before, IngressState::Processing); + let received_6 = ingress_status(CANISTER_2, 6, before, IngressState::Received); + for (message, status) in [(5, processing_5.clone()), (6, received_6.clone())] { + fixture.state.metadata.ingress_history.insert( + message_test_id(message), + status, + before, + NumBytes::from(u64::MAX), + |_| {}, + ); + } + + let mut state = fixture.state; + state.metadata.subnet_merged = true; + + let unexpected_statuses = RefCell::new(Vec::new()); + state.after_merge(NumBytes::from(u64::MAX), |message_id, status| { + unexpected_statuses + .borrow_mut() + .push((message_id.clone(), status.clone())); + }); + + // The merge marker was reset. + assert!(!state.metadata.subnet_merged); + + // Only the `Received` entry of message 6 was reported as unexpected. + assert_eq!( + vec![(message_test_id(6), received_6.clone())], + unexpected_statuses.into_inner() + ); + + // Messages 1 and 4 were recorded as `Processing` at the state time; the existing + // entries of messages 5 and 6 were left alone; and nothing else was recorded. + let expected = BTreeMap::from([ + ( + message_test_id(1), + ingress_status(CANISTER_1, 1, UNIX_EPOCH, IngressState::Processing), + ), + ( + message_test_id(4), + ingress_status(CANISTER_2, 4, UNIX_EPOCH, IngressState::Processing), + ), + (message_test_id(5), processing_5), + (message_test_id(6), received_6), + ]); + assert_eq!( + expected, + state + .metadata + .ingress_history + .statuses() + .map(|(message_id, status)| (message_id.clone(), status.clone())) + .collect::>() + ); +} + +#[test] +#[should_panic(expected = "Not a state resulting from a subnet merge")] +fn after_merge_without_merge_marker() { + ReplicatedStateFixture::new() + .state + .after_merge(NumBytes::from(u64::MAX), |_, _| {}); +} + #[test] fn input_source_roundtrip() { use ic_protobuf::state::queues::v1::canister_queues as pb; From ef25fc8392c0f7d3fa22abcccf95c8d2a874d266 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Fri, 28 Aug 2026 09:07:44 +0000 Subject: [PATCH 02/30] feat(registry): add merge_subnets endpoint Adds a `merge_subnets` endpoint to the registry canister. It takes a source and a destination subnet ID and merges the canister ID ranges of the source subnet into the canister ID range set of the destination subnet, so that the canisters hosted by the source subnet are routed to the destination subnet afterwards. The routing table is updated via `modify_routing_table`, which assigns the source subnet's ranges to the destination subnet and coalesces the resulting adjacent ranges. Only the routing table is touched: neither subnet record is modified and the source subnet is not deleted, so decommissioning a subnet is `merge_subnets` followed by `delete_subnet`. The endpoint is restricted to governance and rejects payloads where the two subnet IDs are equal, either subnet is unknown, the source subnet hosts no canister ID range, or an ongoing canister migration overlaps the source subnet's ranges (which would leave those migrations with a host that is not on their recorded trace). Co-Authored-By: Claude Opus 5 (1M context) --- rs/registry/canister/canister/canister.rs | 19 + rs/registry/canister/canister/registry.did | 6 + .../canister/canister/registry_test.did | 6 + .../canister/src/mutations/merge_subnets.rs | 331 ++++++++++++++++++ rs/registry/canister/src/mutations/mod.rs | 1 + .../canister/src/mutations/routing_table.rs | 18 + rs/registry/canister/tests/merge_subnets.rs | 115 ++++++ rs/registry/canister/unreleased_changelog.md | 5 + 8 files changed, 501 insertions(+) create mode 100644 rs/registry/canister/src/mutations/merge_subnets.rs create mode 100644 rs/registry/canister/tests/merge_subnets.rs diff --git a/rs/registry/canister/canister/canister.rs b/rs/registry/canister/canister/canister.rs index 7112869b1b17..01568e96f077 100644 --- a/rs/registry/canister/canister/canister.rs +++ b/rs/registry/canister/canister/canister.rs @@ -79,6 +79,7 @@ use registry_canister::{ firewall::{ AddFirewallRulesPayload, RemoveFirewallRulesPayload, UpdateFirewallRulesPayload, }, + merge_subnets::MergeSubnetsPayload, node_management::{ do_remove_node_directly::RemoveNodeDirectlyPayload, do_remove_nodes::RemoveNodesPayload, @@ -1085,6 +1086,24 @@ fn reroute_canister_ranges_(payload: RerouteCanisterRangesPayload) { recertify_registry(); } +#[unsafe(export_name = "canister_update merge_subnets")] +fn merge_subnets() { + check_caller_is_governance_and_log("merge_subnets"); + over(candid_one, merge_subnets_); +} + +#[candid_method(update, rename = "merge_subnets")] +fn merge_subnets_(payload: MergeSubnetsPayload) { + registry_mut() + .merge_subnets(payload) + .unwrap_or_else(|error_message| { + trap_with(&format!( + "{LOG_PREFIX} Merge subnets failed: {error_message}" + )) + }); + recertify_registry(); +} + #[unsafe(export_name = "canister_update split_subnet")] fn split_subnet() { check_caller_is_governance_and_log("split_subnet"); diff --git a/rs/registry/canister/canister/registry.did b/rs/registry/canister/canister/registry.did index f0baf2ac5600..a3759d9ffb20 100644 --- a/rs/registry/canister/canister/registry.did +++ b/rs/registry/canister/canister/registry.did @@ -308,6 +308,11 @@ type IPv4Config = record { ip_addr : text; }; +type MergeSubnetsPayload = record { + source_subnet : principal; + destination_subnet : principal; +}; + type MigrateCanistersPayload = record { canister_ids : vec principal; target_subnet_id : principal; @@ -646,6 +651,7 @@ service : { get_node_providers_monthly_xdr_rewards : (opt GetNodeProvidersMonthlyXdrRewardsRequest) -> (GetNodeProvidersMonthlyXdrRewardsResponse) query; get_subnet : (GetSubnetRequest) -> (GetSubnetResponse) query; get_subnet_for_canister : (GetSubnetForCanisterRequest) -> (GetSubnetForCanisterResponse) query; + merge_subnets : (MergeSubnetsPayload) -> (); migrate_canisters: (MigrateCanistersPayload) -> (MigrateCanistersResponse); migrate_node_operator_directly : (MigrateNodeOperatorPayload) -> (); prepare_canister_migration : (PrepareCanisterMigrationPayload) -> (); diff --git a/rs/registry/canister/canister/registry_test.did b/rs/registry/canister/canister/registry_test.did index d635c42ecf3b..7493ae134411 100644 --- a/rs/registry/canister/canister/registry_test.did +++ b/rs/registry/canister/canister/registry_test.did @@ -308,6 +308,11 @@ type IPv4Config = record { ip_addr : text; }; +type MergeSubnetsPayload = record { + source_subnet : principal; + destination_subnet : principal; +}; + type MigrateCanistersPayload = record { canister_ids : vec principal; target_subnet_id : principal; @@ -646,6 +651,7 @@ service : { get_node_providers_monthly_xdr_rewards : (opt GetNodeProvidersMonthlyXdrRewardsRequest) -> (GetNodeProvidersMonthlyXdrRewardsResponse) query; get_subnet : (GetSubnetRequest) -> (GetSubnetResponse) query; get_subnet_for_canister : (GetSubnetForCanisterRequest) -> (GetSubnetForCanisterResponse) query; + merge_subnets : (MergeSubnetsPayload) -> (); migrate_canisters: (MigrateCanistersPayload) -> (MigrateCanistersResponse); migrate_node_operator_directly : (MigrateNodeOperatorPayload) -> (); prepare_canister_migration : (PrepareCanisterMigrationPayload) -> (); diff --git a/rs/registry/canister/src/mutations/merge_subnets.rs b/rs/registry/canister/src/mutations/merge_subnets.rs new file mode 100644 index 000000000000..a907ac83af77 --- /dev/null +++ b/rs/registry/canister/src/mutations/merge_subnets.rs @@ -0,0 +1,331 @@ +use crate::{common::LOG_PREFIX, registry::Registry}; +use candid::CandidType; +#[cfg(target_arch = "wasm32")] +use dfn_core::println; +use ic_base_types::SubnetId; +use ic_registry_keys::make_subnet_record_key; +use ic_registry_routing_table::are_disjoint; +use serde::{Deserialize, Serialize}; + +impl Registry { + /// Merges the canister ID ranges of the source subnet into the canister ID + /// range set of the destination subnet. + /// + /// After this operation, all canisters that used to be hosted by the source + /// subnet are routed to the destination subnet and the source subnet does + /// not host any canister ID range anymore. + /// + /// Note that only the routing table is updated: neither subnet record is + /// modified and, in particular, the source subnet is not deleted. + pub fn merge_subnets(&mut self, payload: MergeSubnetsPayload) -> Result<(), String> { + println!("{LOG_PREFIX}merge_subnets: {payload:?}"); + + let MergeSubnetsPayload { + source_subnet, + destination_subnet, + } = payload; + + if source_subnet == destination_subnet { + return Err(format!( + "source subnet {source_subnet} and destination subnet {destination_subnet} must be different subnets" + )); + } + + let version = self.latest_version(); + + self.get(&make_subnet_record_key(source_subnet).into_bytes(), version) + .ok_or_else(|| format!("source {source_subnet} is not a known subnet"))?; + self.get( + &make_subnet_record_key(destination_subnet).into_bytes(), + version, + ) + .ok_or_else(|| format!("destination {destination_subnet} is not a known subnet"))?; + + let routing_table = self.get_routing_table_or_panic(version); + let source_ranges = routing_table.ranges(source_subnet); + if source_ranges.is_empty() { + return Err(format!( + "source subnet {source_subnet} does not host any canister ID range" + )); + } + + // Rerouting the canister ID ranges of the source subnet would break any ongoing canister + // migration out of those ranges: the migrated ranges would end up being hosted by the + // destination subnet, which is not on the recorded migration trace. + if let Some(canister_migrations) = self.get_canister_migrations(version) + && !are_disjoint(canister_migrations.ranges(), source_ranges.iter()) + { + return Err(format!( + "source subnet {source_subnet} hosts canister ID ranges with ongoing canister migrations" + )); + } + + self.maybe_apply_mutation_internal(self.merge_subnets_mutation( + version, + source_subnet, + destination_subnet, + )); + + Ok(()) + } +} + +/// The argument for the `merge_subnets` update call. +#[derive(Clone, Eq, PartialEq, Debug, CandidType, Deserialize, Serialize)] +pub struct MergeSubnetsPayload { + /// The subnet whose canister ID ranges are merged into the canister ID range + /// set of `destination_subnet`. + pub source_subnet: SubnetId, + /// The subnet that hosts the canister ID ranges of `source_subnet` after the + /// merge. + pub destination_subnet: SubnetId, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + common::test_helpers::{ + add_fake_subnet, get_invariant_compliant_subnet_record, invariant_compliant_registry, + prepare_registry_with_nodes, + }, + mutations::{ + prepare_canister_migration::PrepareCanisterMigrationPayload, + routing_table::routing_table_into_registry_mutation, + }, + }; + use ic_registry_routing_table::{CanisterIdRange, RoutingTable}; + use ic_types::CanisterId; + use ic_types_test_utils::ids::{SUBNET_1, SUBNET_2, SUBNET_3}; + use maplit::btreemap; + + fn range(start: u64, end: u64) -> CanisterIdRange { + CanisterIdRange { + start: CanisterId::from_u64(start), + end: CanisterId::from_u64(end), + } + } + + /// Returns a registry with two subnets, `SUBNET_1` and `SUBNET_2`, where + /// `SUBNET_1` hosts the canister ID ranges `[10, 19]` and `[30, 39]` and + /// `SUBNET_2` hosts the canister ID range `[20, 29]`. + fn new_two_subnets_fixture_registry() -> Registry { + let mut registry = invariant_compliant_registry(0); + + let (mutate_request, node_ids_and_dkg_pks) = + prepare_registry_with_nodes(/* start_mutation_id = */ 1, /* nodes = */ 2); + registry.maybe_apply_mutation_internal(mutate_request.mutations); + + let mut subnet_list_record = registry.get_subnet_list_record(); + let subnet_ids_and_nodes = [SUBNET_1, SUBNET_2].into_iter().zip(node_ids_and_dkg_pks); + + for (subnet_id, (node_id, dkg_pk)) in subnet_ids_and_nodes { + let subnet_record = get_invariant_compliant_subnet_record(vec![node_id]); + let subnet_mutations = add_fake_subnet( + subnet_id, + &mut subnet_list_record, + subnet_record, + &btreemap! { node_id => dkg_pk }, + ); + registry.maybe_apply_mutation_internal(subnet_mutations); + } + + let mut routing_table = RoutingTable::new(); + routing_table.insert(range(10, 19), SUBNET_1).unwrap(); + routing_table.insert(range(20, 29), SUBNET_2).unwrap(); + routing_table.insert(range(30, 39), SUBNET_1).unwrap(); + registry.maybe_apply_mutation_internal(routing_table_into_registry_mutation( + ®istry, + routing_table, + )); + + registry + } + + fn get_routing_table_entries(registry: &Registry) -> Vec<(CanisterIdRange, SubnetId)> { + registry + .get_routing_table_or_panic(registry.latest_version()) + .into_iter() + .collect::>() + } + + #[test] + fn test_merge_subnets() { + // Step 1: Prepare the world. + let mut registry = new_two_subnets_fixture_registry(); + + // Step 2: Run the code under test. + let result = registry.merge_subnets(MergeSubnetsPayload { + source_subnet: SUBNET_1, + destination_subnet: SUBNET_2, + }); + + // Step 3: Verify results. + + // Step 3.1: Inspect the return value. + assert_eq!(result, Ok(())); + + // Step 3.2: The canister ID ranges of both subnets are now hosted by the + // destination subnet, and the three adjacent ranges got merged into one. + assert_eq!( + get_routing_table_entries(®istry), + vec![(range(10, 39), SUBNET_2)], + ); + } + + #[test] + fn test_merge_subnets_into_subnet_without_canister_id_ranges() { + // Step 1: Prepare the world: let the destination subnet host no canister ID + // range at all, so that the merge has to add the destination subnet to the + // routing table. The two ranges of the source subnet are not adjacent, so + // they must stay two separate entries. + let mut registry = new_two_subnets_fixture_registry(); + let mut routing_table = RoutingTable::new(); + routing_table.insert(range(10, 19), SUBNET_1).unwrap(); + routing_table.insert(range(30, 39), SUBNET_1).unwrap(); + registry.maybe_apply_mutation_internal(routing_table_into_registry_mutation( + ®istry, + routing_table, + )); + + // Step 2: Run the code under test. + let result = registry.merge_subnets(MergeSubnetsPayload { + source_subnet: SUBNET_1, + destination_subnet: SUBNET_2, + }); + + // Step 3: Verify results. Both ranges are hosted by the destination subnet + // and, not being adjacent, did not get merged into a single entry. + assert_eq!(result, Ok(())); + assert_eq!( + get_routing_table_entries(®istry), + vec![(range(10, 19), SUBNET_2), (range(30, 39), SUBNET_2)], + ); + } + + #[test] + fn test_merge_subnets_fails_when_subnets_are_equal() { + // Step 1: Prepare the world. + let mut registry = new_two_subnets_fixture_registry(); + + // Step 2: Run the code under test. + let result = registry.merge_subnets(MergeSubnetsPayload { + source_subnet: SUBNET_1, + destination_subnet: SUBNET_1, + }); + + // Step 3: Verify results. + let error_message = result.unwrap_err(); + assert!( + error_message.contains("must be different subnets"), + "{error_message}" + ); + assert_eq!( + get_routing_table_entries(®istry), + vec![ + (range(10, 19), SUBNET_1), + (range(20, 29), SUBNET_2), + (range(30, 39), SUBNET_1), + ], + ); + } + + #[test] + fn test_merge_subnets_fails_when_source_subnet_is_unknown() { + // Step 1: Prepare the world. + let mut registry = new_two_subnets_fixture_registry(); + + // Step 2: Run the code under test. + let result = registry.merge_subnets(MergeSubnetsPayload { + source_subnet: SUBNET_3, + destination_subnet: SUBNET_2, + }); + + // Step 3: Verify results. + let error_message = result.unwrap_err(); + assert!( + error_message.contains(&format!("source {SUBNET_3} is not a known subnet")), + "{error_message}" + ); + } + + #[test] + fn test_merge_subnets_fails_when_destination_subnet_is_unknown() { + // Step 1: Prepare the world. + let mut registry = new_two_subnets_fixture_registry(); + + // Step 2: Run the code under test. + let result = registry.merge_subnets(MergeSubnetsPayload { + source_subnet: SUBNET_1, + destination_subnet: SUBNET_3, + }); + + // Step 3: Verify results. + let error_message = result.unwrap_err(); + assert!( + error_message.contains(&format!("destination {SUBNET_3} is not a known subnet")), + "{error_message}" + ); + } + + #[test] + fn test_merge_subnets_fails_when_source_subnet_hosts_no_canister_id_range() { + // Step 1: Prepare the world: only the destination subnet hosts a canister + // ID range. + let mut registry = new_two_subnets_fixture_registry(); + let mut routing_table = RoutingTable::new(); + routing_table.insert(range(20, 29), SUBNET_2).unwrap(); + registry.maybe_apply_mutation_internal(routing_table_into_registry_mutation( + ®istry, + routing_table, + )); + + // Step 2: Run the code under test. + let result = registry.merge_subnets(MergeSubnetsPayload { + source_subnet: SUBNET_1, + destination_subnet: SUBNET_2, + }); + + // Step 3: Verify results. + let error_message = result.unwrap_err(); + assert!( + error_message.contains("does not host any canister ID range"), + "{error_message}" + ); + } + + #[test] + fn test_merge_subnets_fails_with_ongoing_canister_migration() { + // Step 1: Prepare the world: start migrating a canister ID range away from + // the source subnet. + let mut registry = new_two_subnets_fixture_registry(); + registry + .prepare_canister_migration(PrepareCanisterMigrationPayload { + canister_id_ranges: vec![range(10, 11)], + source_subnet: SUBNET_1, + destination_subnet: SUBNET_2, + }) + .unwrap(); + + // Step 2: Run the code under test. + let result = registry.merge_subnets(MergeSubnetsPayload { + source_subnet: SUBNET_1, + destination_subnet: SUBNET_2, + }); + + // Step 3: Verify results. + let error_message = result.unwrap_err(); + assert!( + error_message.contains("ongoing canister migrations"), + "{error_message}" + ); + assert_eq!( + get_routing_table_entries(®istry), + vec![ + (range(10, 19), SUBNET_1), + (range(20, 29), SUBNET_2), + (range(30, 39), SUBNET_1), + ], + ); + } +} diff --git a/rs/registry/canister/src/mutations/mod.rs b/rs/registry/canister/src/mutations/mod.rs index 846fdbac93cf..73de4d92300a 100644 --- a/rs/registry/canister/src/mutations/mod.rs +++ b/rs/registry/canister/src/mutations/mod.rs @@ -37,6 +37,7 @@ pub mod do_update_subnet; pub mod do_update_subnet_admins; pub mod do_update_unassigned_nodes_config; pub mod firewall; +pub mod merge_subnets; mod node; pub mod node_management; pub mod prepare_canister_migration; diff --git a/rs/registry/canister/src/mutations/routing_table.rs b/rs/registry/canister/src/mutations/routing_table.rs index cbb4734ce976..d026e4b6f6dc 100644 --- a/rs/registry/canister/src/mutations/routing_table.rs +++ b/rs/registry/canister/src/mutations/routing_table.rs @@ -378,6 +378,24 @@ impl Registry { }) } + /// Makes a registry mutation that merges all canister ID ranges currently + /// assigned to the `source` subnet into the canister ID range set of the + /// `destination` subnet. After the mutation, `source` does not host any + /// canister ID range anymore. + pub fn merge_subnets_mutation( + &self, + version: u64, + source: SubnetId, + destination: SubnetId, + ) -> Vec { + self.modify_routing_table(version, |routing_table| { + let source_ranges = routing_table.ranges(source); + routing_table + .assign_ranges(source_ranges, destination) + .unwrap(); + }) + } + /// Retrieves the canister migrations if the key exists. pub fn get_canister_migrations(&self, version: u64) -> Option { self.get(make_canister_migrations_record_key().as_bytes(), version) diff --git a/rs/registry/canister/tests/merge_subnets.rs b/rs/registry/canister/tests/merge_subnets.rs new file mode 100644 index 000000000000..16139835f211 --- /dev/null +++ b/rs/registry/canister/tests/merge_subnets.rs @@ -0,0 +1,115 @@ +use candid::Encode; +use ic_nns_test_utils::{ + itest_helpers::{ + set_up_registry_canister, set_up_universal_canister, state_machine_test_on_nns_subnet, + try_call_via_universal_canister, + }, + registry::{initial_routing_table_mutations, prepare_registry_with_two_node_sets}, +}; +use ic_registry_routing_table::{CanisterIdRange, RoutingTable}; +use ic_registry_transport::pb::v1::RegistryAtomicMutateRequest; +use ic_types::CanisterId; +use registry_canister::{ + init::RegistryCanisterInitPayloadBuilder, mutations::merge_subnets::MergeSubnetsPayload, +}; + +mod common; +use common::test_helpers::{check_error_message, check_subnet_for_canisters}; + +/// Exercises the `merge_subnets` endpoint end to end. The payload validation +/// itself is covered by the unit tests of `Registry::merge_subnets`, so this test +/// only covers what those cannot: that the endpoint is reachable with a Candid +/// encoded payload, that only governance may call it, and that the resulting +/// routing table is visible through the canister's query API. +#[test] +fn test_merge_subnets() { + state_machine_test_on_nns_subnet(|runtime| { + async move { + // Step 1: Prepare the world: two subnets where subnet 2 hosts the canister + // ID range [0, 255] and subnet 1 hosts [256, 511]. + let (subnet_1_mutation, subnet_id_1, subnet_id_2_option, _, _) = + prepare_registry_with_two_node_sets( + /* num_nodes_in_subnet = */ 4, /* num_unassigned_nodes = */ 4, true, + ); + let subnet_id_2 = subnet_id_2_option.unwrap(); + let rt_mutation = { + fn range(start: u64, end: u64) -> CanisterIdRange { + CanisterIdRange { + start: CanisterId::from(start), + end: CanisterId::from(end), + } + } + + let mut rt = RoutingTable::new(); + rt.insert(range(0, 255), subnet_id_2) + .expect("failed to update the routing table"); + rt.insert(range(256, 511), subnet_id_1) + .expect("failed to update the routing table"); + + RegistryAtomicMutateRequest { + mutations: initial_routing_table_mutations(&rt), + preconditions: vec![], + } + }; + + let registry = set_up_registry_canister( + &runtime, + RegistryCanisterInitPayloadBuilder::new() + .push_init_mutate_request(subnet_1_mutation) + .push_init_mutate_request(rt_mutation) + .build(), + ) + .await; + + let governance_fake = set_up_universal_canister(&runtime).await; + assert_eq!( + governance_fake.canister_id(), + ic_nns_constants::GOVERNANCE_CANISTER_ID + ); + + // Step 2: A caller other than governance may not merge subnets. + check_error_message( + registry + .update_( + "merge_subnets", + dfn_candid::candid_one, + MergeSubnetsPayload { + source_subnet: subnet_id_1, + destination_subnet: subnet_id_2, + }, + ) + .await as Result<(), String>, + "not authorized", + ); + + // Step 3: Run the code under test: merge subnet 1 into subnet 2. + try_call_via_universal_canister( + &governance_fake, + ®istry, + "merge_subnets", + Encode!(&MergeSubnetsPayload { + source_subnet: subnet_id_1, + destination_subnet: subnet_id_2, + }) + .unwrap(), + ) + .await + .unwrap(); + + // Step 4: Verify results: the canisters formerly hosted by subnet 1 are now + // hosted by subnet 2, and the canisters of subnet 2 stay put. + check_subnet_for_canisters( + ®istry, + vec![ + (CanisterId::from(0), subnet_id_2), + (CanisterId::from(255), subnet_id_2), + (CanisterId::from(256), subnet_id_2), + (CanisterId::from(511), subnet_id_2), + ], + ) + .await; + + Ok(()) + } + }); +} diff --git a/rs/registry/canister/unreleased_changelog.md b/rs/registry/canister/unreleased_changelog.md index 06e503d26a58..e83c068094e7 100644 --- a/rs/registry/canister/unreleased_changelog.md +++ b/rs/registry/canister/unreleased_changelog.md @@ -14,6 +14,11 @@ on the process that this file is part of, see be set on mainnet before the replica version rejecting ingress messages to cooling down subnets has been rolled out to all subnets. +* `merge_subnets` endpoint. It takes a source and a destination subnet ID, and merges the canister + ID ranges of the source subnet into the canister ID range set of the destination subnet, i.e., the + canisters hosted by the source subnet are routed to the destination subnet afterwards. Only the + routing table is updated: neither subnet record is modified and the source subnet is not deleted. + ## Changed ## Deprecated From 608b73f37029297a2b9921b40b8392296b1c156a Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Fri, 28 Aug 2026 15:32:26 +0000 Subject: [PATCH 03/30] test: drain a subnet that is "cooling down" A system test covering the `cooling_down` subnet record flag end to end: while two universal canisters keep calling each other across subnets in a loop, labeling one of their subnets as "cooling down" must bring that subnet to a standstill. The IC consists of an NNS subnet (with the NNS canisters installed) and two Application subnets S and T, holding one universal canister each. Both canisters are made to call the one on the other subnet in a loop, where the reply (or reject) callback of every call fires a new call. A payload cannot contain itself, so each canister holds the loop body in its global data and the continuation calls the canister itself, passing the global data as the payload to execute; the loop body replies as soon as it has fired the cross-subnet call, which keeps the number of open call contexts bounded. An `UpdateConfigOfSubnet` proposal then labels S as "cooling down" and the test waits until S rejects ingress messages and, subsequently, until it is "merge ready" according to the `Merge readiness` panel of the `Subnet merging` dashboard (`bases/apps/ic-dashboards/core/subnet-merging.json` in dfinity-ops/k8s) for `V` = the registry version created by the proposal and `R` = 0 cycles: every subnet has reached V, no stream in either direction holds a message (loopback included), the ingress history holds nothing but `processing` entries, S's subnet input and output queues are empty, its subnet call context manager holds no call context, and the pending anonymous refunds are worth at most R. As in the dashboard, each term is evaluated on the median across the replicas reporting it, and missing data reads as zero. Finally, the test checks that both loops are stalled, so that readiness is the consequence of S cooling down rather than of the loops having stopped. Only the HEAD NNS variant runs: the mainnet NNS canisters do not know the `cooling_down` field of `UpdateSubnetPayload`, so Candid would silently drop it and the proposal would be a no-op. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 3 + rs/tests/message_routing/BUILD.bazel | 24 + rs/tests/message_routing/Cargo.toml | 7 + .../subnet_cooling_down_test.rs | 657 ++++++++++++++++++ 4 files changed, 691 insertions(+) create mode 100644 rs/tests/message_routing/subnet_cooling_down_test.rs diff --git a/Cargo.lock b/Cargo.lock index 2669f3ed0ea2..537c07c9f2bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17683,13 +17683,16 @@ dependencies = [ "ic-agent", "ic-base-types", "ic-management-canister-types 0.8.0", + "ic-nns-governance-api", "ic-registry-subnet-type", "ic-system-test-driver", "ic-types", + "ic-universal-canister", "ic-utils 0.49.1", "itertools 0.12.1", "rand 0.8.6", "rand_chacha 0.3.1", + "registry-canister", "rejoin-test-lib", "slog", "tempfile", diff --git a/rs/tests/message_routing/BUILD.bazel b/rs/tests/message_routing/BUILD.bazel index da7d9d4d57d9..0b64dc7336d5 100644 --- a/rs/tests/message_routing/BUILD.bazel +++ b/rs/tests/message_routing/BUILD.bazel @@ -139,6 +139,30 @@ system_test_nns( ], ) +system_test_nns( + name = "subnet_cooling_down_test", + cpus = MIN_LOCAL_CPUS + 3 * DEFAULT_VCPUS_PER_VM, # 1 System + 2 Application fast-single-node subnets = 3 IC Node VMs * 6 vCPUs. + enable_mainnet_nns_variant = False, # The `cooling_down` field of the subnet record is not supported by the mainnet NNS canisters. + tags = [ + "long_test", # since the subnet only quiesces once the ingress history is pruned. + ], + test_timeout = "eternal", + runtime_deps = UNIVERSAL_CANISTER_RUNTIME_DEPS, + deps = [ + # Keep sorted. + "//rs/nns/governance/api", + "//rs/registry/canister", + "//rs/registry/subnet_type", + "//rs/tests/driver:ic-system-test-driver", + "//rs/types/types", + "//rs/universal_canister/lib", + "@crate_index//:anyhow", + "@crate_index//:candid", + "@crate_index//:slog", + "@crate_index//:tokio", + ], +) + system_test( name = "queues_compatibility_test", cpus = MIN_LOCAL_CPUS, # setup is `|_| ()`, no VMs deployed. diff --git a/rs/tests/message_routing/Cargo.toml b/rs/tests/message_routing/Cargo.toml index bc9bcfefb901..aba3de1f93ec 100644 --- a/rs/tests/message_routing/Cargo.toml +++ b/rs/tests/message_routing/Cargo.toml @@ -14,13 +14,16 @@ dfn_candid = { path = "../../rust_canisters/dfn_candid" } ic-agent = { workspace = true } ic-base-types = { path = "../../types/base_types" } ic-management-canister-types = { workspace = true } +ic-nns-governance-api = { path = "../../nns/governance/api" } ic-registry-subnet-type = { path = "../../registry/subnet_type" } ic-system-test-driver = { path = "../driver" } ic-types = { path = "../../types/types" } +ic-universal-canister = { path = "../../universal_canister/lib" } ic-utils = { workspace = true } itertools = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } +registry-canister = { path = "../../registry/canister" } rejoin-test-lib = { path = "./rejoin_test_lib" } slog = { workspace = true } tempfile = { workspace = true } @@ -54,3 +57,7 @@ path = "rejoin_test.rs" [[bin]] name = "state_sync_malicious_chunk_test" path = "state_sync_malicious_chunk_test.rs" + +[[bin]] +name = "subnet_cooling_down_test" +path = "subnet_cooling_down_test.rs" diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs new file mode 100644 index 000000000000..597151f5d410 --- /dev/null +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -0,0 +1,657 @@ +/* tag::catalog[] +Title:: Draining a subnet that is "cooling down". + +Goal:: Verify that a subnet labeled "cooling down" in its subnet record quiesces +while its canisters are busy making cross-subnet calls in a loop, i.e. that it +reaches the "merge readiness" condition of the `Subnet merging` dashboard (see +`bases/apps/ic-dashboards/core/subnet-merging.json` on branch +`mraszyk/subnet-merging-dashboard` of `dfinity/k8s`) for `V` = the registry +version at which the subnet was labeled "cooling down" and `R` = 0 cycles. + +Runbook:: +0. Set up an IC with an NNS subnet (with the NNS canisters installed) and two + Application subnets S and T. +1. Install a universal canister on each Application subnet: US on S, UT on T. +2. Make an ingress call to each of US and UT with a payload that calls the + universal canister on the other subnet in a loop: the reply (or reject) + callback of every call fires a new call. +3. Wait until both loops have completed a few iterations, i.e. messages are + actually flowing between S and T in both directions. +4. Submit (and adopt) an `UpdateConfigOfSubnet` NNS proposal labeling S as + "cooling down" in its subnet record, and record the registry version V it + creates. +5. Wait until S rejects ingress messages, i.e. the replicas of S observed the + "cooling down" label. +6. Wait until S is "merge ready" according to the dashboard's condition for V + and R = 0: all subnets have reached registry version V, no stream in either + direction holds a message (loopback included), the ingress history holds + nothing but `processing` entries, S's subnet input and output queues are + empty, S's subnet call context manager holds no call context, and the + pending anonymous refunds are worth at most R. +7. Check that both loops are indeed stalled (their iteration counters, read via + queries, no longer advance): while S is cooling down, neither S nor T routes + any message to or from S, so the messages of both loops are retained in + their senders' output queues. + +Success:: +S becomes "merge ready" while both loops are stalled. + +end::catalog[] */ + +use anyhow::{Result, anyhow, bail}; +use candid::Principal; +use ic_nns_governance_api::NnsFunction; +use ic_registry_subnet_type::SubnetType; +use ic_system_test_driver::driver::group::SystemTestGroup; +use ic_system_test_driver::driver::ic::{InternetComputer, Subnet}; +use ic_system_test_driver::driver::test_env::TestEnv; +use ic_system_test_driver::driver::test_env_api::{ + HasPublicApiUrl, HasRegistryVersion, HasTopologySnapshot, IcNodeContainer, + NnsInstallationBuilder, READY_WAIT_TIMEOUT, RETRY_BACKOFF, SubnetSnapshot, TopologySnapshot, +}; +use ic_system_test_driver::nns::{ + get_governance_canister, submit_external_proposal_with_test_id, + vote_execute_proposal_assert_executed, +}; +use ic_system_test_driver::retry_with_msg_async; +use ic_system_test_driver::systest; +use ic_system_test_driver::util::{ + MetricsFetcher, UniversalCanister, assert_create_agent, block_on, runtime_from_url, +}; +use ic_types::SubnetId; +use ic_universal_canister::{call_args, wasm}; +use registry_canister::mutations::do_update_subnet::UpdateSubnetPayload; +use slog::{Logger, info}; +use std::collections::BTreeMap; +use std::time::Duration; + +/// Metrics making up the "merge readiness" condition. +const METRIC_REGISTRY_VERSION: &str = "mr_registry_version"; +const METRIC_STREAM_MESSAGES: &str = "mr_stream_messages"; +const METRIC_INGRESS_HISTORY_BY_STATE: &str = "replicated_state_ingress_history_length_by_state"; +const METRIC_SUBNET_INPUT_QUEUE_MESSAGES: &str = "execution_subnet_input_queue_messages"; +const METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES: &str = "execution_subnet_output_queue_messages"; +const METRIC_SUBNET_CALL_CONTEXTS: &str = "replicated_state_subnet_call_contexts"; +const METRIC_PENDING_REFUNDS_CYCLES: &str = "replicated_state_pending_refunds_cycles"; + +/// `R` in the dashboard's readiness condition: the maximum total value in +/// cycles of the pending anonymous refunds of the cooling down subnet. +const MAX_REFUND_VALUE_CYCLES: f64 = 0.0; + +/// Number of loop iterations each universal canister must have completed before +/// the subnet is labeled "cooling down", so that the loops are known to be +/// making cross-subnet calls when the label takes effect. +const MIN_LOOP_ITERATIONS: u64 = 3; + +/// Timeout for the subnet to become "merge ready". The binding term is the +/// ingress history, which only becomes free of terminal statuses once the +/// entries of the ingress messages submitted before the subnet started cooling +/// down are pruned, i.e. at their (up to `MAX_INGRESS_TTL` = 5 minutes away) +/// expiry times. +const MERGE_READY_TIMEOUT: Duration = Duration::from_secs(600); +/// Backoff between two evaluations of the readiness condition. Longer than the +/// default because every evaluation scrapes the metrics of all subnets. +const MERGE_READY_BACKOFF: Duration = Duration::from_secs(10); + +/// How long the loops are observed to be stalled (Step 7). +const STALL_OBSERVATION_PERIOD: Duration = Duration::from_secs(15); + +/// Timeouts of the test itself: the ingress history pruning above dominates, +/// the rest of the scenario takes a couple of minutes. The overall timeout +/// additionally covers the setup (booting the IC and installing the NNS). +const PER_TEST_TIMEOUT: Duration = Duration::from_secs(900); +const OVERALL_TIMEOUT: Duration = Duration::from_secs(1500); + +fn main() -> Result<()> { + SystemTestGroup::new() + .with_setup(setup) + .add_test(systest!(test)) + .with_timeout_per_test(PER_TEST_TIMEOUT) + .with_overall_timeout(OVERALL_TIMEOUT) + .execute_from_args()?; + Ok(()) +} + +pub fn setup(env: TestEnv) { + InternetComputer::new() + .add_subnet(Subnet::fast_single_node(SubnetType::System)) + .add_subnet(Subnet::fast_single_node(SubnetType::Application)) + .add_subnet(Subnet::fast_single_node(SubnetType::Application)) + .setup_and_start(&env) + .expect("failed to setup IC under test"); + env.topology_snapshot().subnets().for_each(|subnet| { + subnet + .nodes() + .for_each(|node| node.await_status_is_healthy().unwrap()) + }); + let nns_node = env + .topology_snapshot() + .root_subnet() + .nodes() + .next() + .unwrap(); + NnsInstallationBuilder::new() + .install(&nns_node, &env) + .expect("failed to install NNS canisters"); +} + +pub fn test(env: TestEnv) { + block_on(run(env)); +} + +async fn run(env: TestEnv) { + let logger = env.logger(); + let topology = env.topology_snapshot(); + + // The two Application subnets: S is the one that will be labeled "cooling + // down", T is the one it exchanges messages with. + let app_subnets: Vec<_> = topology + .subnets() + .filter(|subnet| subnet.subnet_type() == SubnetType::Application) + .collect(); + assert_eq!( + app_subnets.len(), + 2, + "expected exactly 2 Application subnets" + ); + let s_subnet = app_subnets[0].clone(); + let t_subnet = app_subnets[1].clone(); + let s_node = s_subnet.nodes().next().unwrap(); + let t_node = t_subnet.nodes().next().unwrap(); + let s_agent = assert_create_agent(s_node.get_public_url().as_str()).await; + let t_agent = assert_create_agent(t_node.get_public_url().as_str()).await; + + // Step 1: Install a universal canister on each Application subnet. + info!( + logger, + "Step 1: Installing universal canisters on S ({}) and T ({})", + s_subnet.subnet_id, + t_subnet.subnet_id, + ); + let us = UniversalCanister::new_with_retries(&s_agent, s_node.effective_canister_id(), &logger) + .await; + let ut = UniversalCanister::new_with_retries(&t_agent, t_node.effective_canister_id(), &logger) + .await; + info!( + logger, + "Step 1 done: US={}, UT={}", + us.canister_id(), + ut.canister_id(), + ); + + // Step 2: Start a loop of calls to the canister on the other subnet on both + // universal canisters. + info!(logger, "Step 2: Starting the US <-> UT call loops"); + start_call_loop(&us, ut.canister_id()).await; + start_call_loop(&ut, us.canister_id()).await; + info!(logger, "Step 2 done: both call loops started"); + + // Step 3: Wait until both loops have completed a few iterations. + info!( + logger, + "Step 3: Waiting for {MIN_LOOP_ITERATIONS} iterations of both call loops" + ); + for (canister, name) in [(&us, "US"), (&ut, "UT")] { + retry_with_msg_async!( + format!("waiting for {MIN_LOOP_ITERATIONS} iterations of {name}'s call loop"), + &logger, + READY_WAIT_TIMEOUT, + RETRY_BACKOFF, + || async { + let iterations = loop_iterations(canister).await?; + if iterations < MIN_LOOP_ITERATIONS { + bail!("{name}'s call loop is at iteration {iterations}"); + } + Ok(()) + } + ) + .await + .unwrap_or_else(|e| panic!("{name}'s call loop did not make progress: {e}")); + } + info!(logger, "Step 3 done: both call loops are making progress"); + + // Step 4: Label S as "cooling down" in its subnet record. + info!( + logger, + "Step 4: Labeling subnet S ({}) as \"cooling down\"", s_subnet.subnet_id, + ); + let registry_version = set_subnet_cooling_down(&env, s_subnet.subnet_id, &logger).await; + info!( + logger, + "Step 4 done: subnet S is labeled \"cooling down\" as of registry version \ + {registry_version} (V)", + ); + + // Step 5: Wait until the replicas of S observed the "cooling down" label, + // i.e. until S rejects ingress messages. + info!( + logger, + "Step 5: Waiting until subnet S rejects ingress messages" + ); + retry_with_msg_async!( + "waiting until subnet S rejects ingress messages", + &logger, + READY_WAIT_TIMEOUT, + RETRY_BACKOFF, + || async { + match us.update(wasm().reply_data(&[])).await { + Ok(_) => bail!("ingress message to US was still accepted"), + Err(err) => { + let err = err.to_string(); + if !err.contains("cooling down") { + bail!("ingress message to US failed unexpectedly: {err}"); + } + Ok(()) + } + } + } + ) + .await + .expect("subnet S did not start rejecting ingress messages"); + info!( + logger, + "Step 5 done: subnet S rejects ingress messages, so it is cooling down" + ); + + // Step 6: Wait until S is "merge ready". + info!( + logger, + "Step 6: Waiting until subnet S is \"merge ready\" for V={registry_version} and \ + R={MAX_REFUND_VALUE_CYCLES} cycles", + ); + retry_with_msg_async!( + format!( + "waiting until subnet {} is \"merge ready\"", + s_subnet.subnet_id + ), + &logger, + MERGE_READY_TIMEOUT, + MERGE_READY_BACKOFF, + || async { + let terms = evaluate_merge_readiness( + &topology, + &s_subnet, + registry_version, + MAX_REFUND_VALUE_CYCLES, + ) + .await?; + let unsatisfied: Vec<_> = terms + .iter() + .filter(|(_, satisfied)| !satisfied) + .map(|(term, _)| term.as_str()) + .collect(); + if !unsatisfied.is_empty() { + bail!("not merge ready: {}", unsatisfied.join("; ")); + } + for (term, _) in &terms { + info!(logger, "Step 6: merge readiness term holds: {term}"); + } + Ok(()) + } + ) + .await + .unwrap_or_else(|e| panic!("subnet S did not become \"merge ready\": {e}")); + info!(logger, "Step 6 done: subnet S is \"merge ready\""); + + // Step 7: Check that both call loops are stalled, i.e. that S became + // "merge ready" because it is cooling down and not because the loops + // stopped making calls. + info!( + logger, + "Step 7: Checking that both call loops are stalled over {STALL_OBSERVATION_PERIOD:?}" + ); + let before = [ + loop_iterations(&us).await.unwrap(), + loop_iterations(&ut).await.unwrap(), + ]; + tokio::time::sleep(STALL_OBSERVATION_PERIOD).await; + for ((canister, name), before) in [(&us, "US"), (&ut, "UT")].into_iter().zip(before) { + let after = loop_iterations(canister).await.unwrap(); + assert_eq!( + before, after, + "{name}'s call loop advanced from iteration {before} to {after} while subnet S was \ + cooling down", + ); + } + info!( + logger, + "Step 7 done: both call loops are stalled at iterations {before:?}" + ); +} + +/// Starts an endless loop of calls from `canister` to `peer` (on another +/// subnet): `canister` is made to execute the loop body below once, via an +/// ingress message; from there on the reply (or reject) callback of every call +/// to `peer` fires a new call. +/// +/// The loop body cannot contain itself, so its continuation re-enters it +/// indirectly: `canister` holds the loop body in its global data and the +/// continuation calls `canister` itself, passing the global data as the payload +/// for the callee (i.e. `canister`) to execute. +/// +/// The loop body replies (to the ingress message or to the self-call that +/// triggered this iteration) as soon as it has fired the call to `peer`. This +/// keeps the number of open call contexts bounded (had it not replied, every +/// iteration would have left behind one open call context) and it makes each +/// iteration consist of one loopback call plus one cross-subnet call. +async fn start_call_loop(canister: &UniversalCanister<'_>, peer: Principal) { + // The continuation, executed by the reply and reject callbacks of the call + // to `peer`: call `canister` itself with the loop body it holds in its + // global data as the payload. Neither callback of this call may reply, as + // the call context it is made in was already responded by the loop body. + let continuation = wasm() + .call_simple( + canister.canister_id(), + "update", + call_args() + .eval_other_side(wasm().get_global_data()) + .on_reply(wasm().noop()) + .on_reject(wasm().noop()), + ) + .build(); + // The loop body: bump the iteration counter, fire a call to `peer` (which + // merely replies) with the continuation as both callbacks, then reply. + let loop_body = wasm() + .inc_global_counter() + .call_simple( + peer, + "update", + call_args() + .other_side(wasm().reply_data(&[])) + .on_reply(continuation.clone()) + .on_reject(continuation), + ) + .reply_data(&[]) + .build(); + + canister + .update(wasm().set_global_data(&loop_body).reply_data(&[])) + .await + .expect("setting the loop body as the global data should succeed"); + canister + .update(loop_body) + .await + .expect("starting the call loop should succeed"); +} + +/// Returns the number of loop iterations `canister` has executed so far, i.e. +/// its global counter, read via a query (an ingress message would be rejected +/// by a subnet that is cooling down). +async fn loop_iterations(canister: &UniversalCanister<'_>) -> Result { + let reply = canister + .query(wasm().get_global_counter().reply_int64()) + .await + .map_err(|e| anyhow!("failed to read the loop iteration counter: {e}"))?; + let reply: [u8; 8] = reply + .as_slice() + .try_into() + .map_err(|_| anyhow!("expected 8 bytes, got {} bytes: {reply:?}", reply.len()))?; + Ok(u64::from_le_bytes(reply)) +} + +/// Submits and adopts an `UpdateConfigOfSubnet` proposal labeling `subnet_id` as +/// "cooling down" in its subnet record. Returns the registry version created by +/// the proposal, i.e. `V` in the dashboard's readiness condition. +async fn set_subnet_cooling_down(env: &TestEnv, subnet_id: SubnetId, logger: &Logger) -> u64 { + let topology = env.topology_snapshot(); + let nns_node = topology.root_subnet().nodes().next().unwrap(); + let nns_runtime = runtime_from_url(nns_node.get_public_url(), nns_node.effective_canister_id()); + let governance = get_governance_canister(&nns_runtime); + + let payload = UpdateSubnetPayload { + subnet_id, + cooling_down: Some(true), + max_ingress_bytes_per_message: None, + max_ingress_messages_per_block: None, + max_ingress_bytes_per_block: None, + max_block_payload_size: None, + unit_delay_millis: None, + initial_notary_delay_millis: None, + dkg_interval_length: None, + dkg_dealings_per_block: None, + start_as_nns: None, + subnet_type: None, + is_halted: None, + halt_at_cup_height: None, + features: None, + resource_limits: None, + chain_key_config: None, + chain_key_signing_enable: None, + chain_key_signing_disable: None, + max_number_of_canisters: None, + ssh_readonly_access: None, + ssh_backup_access: None, + subnet_admins: None, + // Deprecated/unused values follow + max_artifact_streams_per_peer: None, + max_chunk_wait_ms: None, + max_duplicity: None, + max_chunk_size: None, + receive_check_cache_size: None, + pfn_evaluation_period_ms: None, + registry_poll_period_ms: None, + retransmission_request_ms: None, + set_gossip_config_to_default: false, + }; + let proposal_id = submit_external_proposal_with_test_id( + &governance, + NnsFunction::UpdateConfigOfSubnet, + payload, + ) + .await; + info!(logger, "Submitted proposal {proposal_id}"); + vote_execute_proposal_assert_executed(&governance, proposal_id).await; + + // The proposal's single registry mutation is the newest registry version. + // The snapshot above was taken before the proposal was submitted, so this + // cannot miss the version the mutation created. + topology + .block_for_newer_registry_version() + .await + .expect("the registry should have a newer version after the proposal executed") + .get_registry_version() + .get() +} + +/// Evaluates the terms of the "merge readiness" condition of the `Subnet +/// merging` dashboard for `subnet` (the subnet that is cooling down), +/// `registry_version` (`V`) and `max_refund_value_cycles` (`R`). Returns one +/// (description, satisfied) pair per term, in the order the terms appear in the +/// dashboard's readiness expression. +/// +/// As in the dashboard, every term is evaluated on the median across the +/// replicas reporting the respective series, and missing data reads as zero +/// (the dashboard's `or vector(0)` fallback). +async fn evaluate_merge_readiness( + topology: &TopologySnapshot, + subnet: &SubnetSnapshot, + registry_version: u64, + max_refund_value_cycles: f64, +) -> Result> { + let subnet_id = subnet.subnet_id; + let own_metrics = fetch_metrics( + subnet, + &[ + METRIC_REGISTRY_VERSION, + METRIC_STREAM_MESSAGES, + METRIC_INGRESS_HISTORY_BY_STATE, + METRIC_SUBNET_INPUT_QUEUE_MESSAGES, + METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES, + METRIC_SUBNET_CALL_CONTEXTS, + METRIC_PENDING_REFUNDS_CYCLES, + ], + ) + .await?; + + // Terms 1 and 2 range over all subnets: the registry version of every + // subnet and the streams of all remote subnets towards this one. + let remote_label = format!("remote=\"{subnet_id}\""); + let mut min_registry_version = None; + let mut incoming_stream_messages = 0.0; + for other in topology.subnets() { + let metrics = if other.subnet_id == subnet_id { + own_metrics.clone() + } else { + fetch_metrics(&other, &[METRIC_REGISTRY_VERSION, METRIC_STREAM_MESSAGES]).await? + }; + let version = + median_across_replicas(&metrics, METRIC_REGISTRY_VERSION, |_| true).unwrap_or(0.0); + min_registry_version = Some(min_registry_version.map_or(version, |v: f64| v.min(version))); + if other.subnet_id != subnet_id { + incoming_stream_messages += + sum_of_medians(&metrics, METRIC_STREAM_MESSAGES, |labels| { + labels.contains(&remote_label) + }); + } + } + let min_registry_version = min_registry_version.unwrap_or(0.0); + + let outgoing_stream_messages = sum_of_medians(&own_metrics, METRIC_STREAM_MESSAGES, |_| true); + let ingress_history_messages = + sum_of_medians(&own_metrics, METRIC_INGRESS_HISTORY_BY_STATE, |labels| { + !labels.contains("state=\"processing\"") + }); + let subnet_input_queue_messages = + sum_of_medians(&own_metrics, METRIC_SUBNET_INPUT_QUEUE_MESSAGES, |_| true); + let subnet_output_queue_messages = + median_across_replicas(&own_metrics, METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES, |_| true) + .unwrap_or(0.0); + let subnet_call_contexts = sum_of_medians(&own_metrics, METRIC_SUBNET_CALL_CONTEXTS, |_| true); + let pending_refunds_cycles = + median_across_replicas(&own_metrics, METRIC_PENDING_REFUNDS_CYCLES, |_| true) + .unwrap_or(0.0); + + Ok(vec![ + ( + format!( + "every subnet has reached registry version {registry_version} (the lowest one is \ + at {min_registry_version})" + ), + min_registry_version >= registry_version as f64, + ), + ( + format!( + "no remote subnet holds a message in its stream to subnet {subnet_id} \ + ({incoming_stream_messages} messages)" + ), + incoming_stream_messages == 0.0, + ), + ( + format!( + "subnet {subnet_id} holds no message in any of its streams, loopback included \ + ({outgoing_stream_messages} messages)" + ), + outgoing_stream_messages == 0.0, + ), + ( + format!( + "the ingress history holds nothing but `processing` entries \ + ({ingress_history_messages} other entries)" + ), + ingress_history_messages == 0.0, + ), + ( + format!("the subnet input queues are empty ({subnet_input_queue_messages} messages)"), + subnet_input_queue_messages == 0.0, + ), + ( + format!("the subnet output queues are empty ({subnet_output_queue_messages} messages)"), + subnet_output_queue_messages == 0.0, + ), + ( + format!( + "the subnet call context manager holds no call context ({subnet_call_contexts} \ + call contexts)" + ), + subnet_call_contexts == 0.0, + ), + ( + format!( + "the pending anonymous refunds are worth at most {max_refund_value_cycles} cycles \ + ({pending_refunds_cycles} cycles)" + ), + pending_refunds_cycles <= max_refund_value_cycles, + ), + ]) +} + +/// Fetches the given metrics from all nodes of `subnet`, keyed by series (i.e. +/// metric name plus labels), with one value per node reporting the series. +async fn fetch_metrics( + subnet: &SubnetSnapshot, + metrics: &[&str], +) -> Result>> { + MetricsFetcher::new( + subnet.nodes(), + metrics.iter().map(|metric| metric.to_string()).collect(), + ) + .fetch::() + .await + .map_err(|e| { + anyhow!( + "failed to fetch the metrics of subnet {}: {e}", + subnet.subnet_id + ) + }) +} + +/// The per-node values of every series of `metric` whose labels (`{...}`, or the +/// empty string for an unlabeled series) match `labels_match`. +/// +/// `MetricsFetcher` matches metric names by prefix, so this also filters out +/// the series of any other metric that `metric` happens to be a prefix of. +fn matching_series<'a>( + metrics: &'a BTreeMap>, + metric: &str, + labels_match: impl Fn(&str) -> bool, +) -> Vec<&'a Vec> { + metrics + .iter() + .filter(|(series, _)| match series.strip_prefix(metric) { + Some(labels) if labels.is_empty() || labels.starts_with('{') => labels_match(labels), + _ => false, + }) + .map(|(_, values)| values) + .collect() +} + +/// Prometheus' `quantile(0.5, ...)`: the median of `values`, interpolating +/// between the two middle values if there is an even number of them. `None` iff +/// `values` is empty. +fn median(values: &[f64]) -> Option { + if values.is_empty() { + return None; + } + let mut values = values.to_vec(); + values.sort_by(|a, b| a.partial_cmp(b).expect("metric value should not be NaN")); + let middle = (values.len() - 1) as f64 / 2.0; + Some((values[middle.floor() as usize] + values[middle.ceil() as usize]) / 2.0) +} + +/// `sum(quantile by () (0.5, {}))`: the median across +/// the replicas reporting each matching series, summed over those series. +fn sum_of_medians( + metrics: &BTreeMap>, + metric: &str, + labels_match: impl Fn(&str) -> bool, +) -> f64 { + matching_series(metrics, metric, labels_match) + .into_iter() + .filter_map(|values| median(values)) + .sum() +} + +/// `quantile(0.5, {})`: the median across all replicas +/// reporting any matching series. `None` if there is no such series. +fn median_across_replicas( + metrics: &BTreeMap>, + metric: &str, + labels_match: impl Fn(&str) -> bool, +) -> Option { + let values: Vec = matching_series(metrics, metric, labels_match) + .into_iter() + .flatten() + .copied() + .collect(); + median(&values) +} From f16e39695eb0c4d8c16b026c45009116e88be329 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 31 Aug 2026 08:50:25 +0000 Subject: [PATCH 04/30] test: drain long-running calls from a subnet that is "cooling down" Extend the `cooling_down` system test with the kinds of work a subnet may be busy with when it is labeled "cooling down", and rename the subnets after the roles they would play in a merge: `M` is the subnet being merged, `R` the root subnet it would be merged into, `T` the subnet `M` exchanges messages with in a loop, and `S` a third Application subnet, newly added. Besides the two universal canisters calling each other across the `M`/`T` boundary in a loop, `M` now also holds: - `U1`, which makes five `install_code` calls to the management canister, one per empty canister `U2a` .. `U2e` it controls, installing the universal canister module with an `arg` that makes `canister_init` burn 295B instructions. Only one long-running `install_code` makes progress per round, so these five calls alone keep `M` busy for minutes and are what merge readiness now waits for, rather than the pruning of the ingress history. - `U3`, looping until its global data is set to a value this test never sets. Every iteration is a self-targeted `canister_status` call, so the loop stalls as soon as `M` stops routing messages out of its canisters' output queues. - `U5`, running that same loop in a call made from `U4` on `S`, i.e. in a call from another subnet that it can never respond to. - `U6`, waiting for the response of a call to `U7` on `S` that never arrives. The test then asserts that `U2a` .. `U2e` have been installed once `M` is merge ready, i.e. that cooling down retains messages in canister output queues without losing the management calls `M` had already accepted. The endless loops leave nothing behind but `processing` ingress history entries and open canister call contexts, neither of which is part of the readiness condition. Two ordering constraints the test has to respect, both of which would otherwise make it hang or install nothing: - Every canister has to be installed before `U1`'s calls start, as no `install_code` is executed at all while another one is long-running on the same subnet. - `U1`'s requests have to leave its output queue before the proposal lands: a request still sitting there would never be routed once `M` cools down, not even into the loopback stream. `canister_init` burns 295B rather than the full 300B an `install_code` message may consume, because that budget also covers compiling the module: 6_000 instructions per byte of the decompressed (~350 KB) universal canister module, i.e. ~2.2B instructions, plus a 20M base cost. The reduced compilation cost only applies while the module is in `expected_compiled_wasms`, which is cleared at every checkpoint, so an `install_code` aborted at a checkpoint and restarted afterwards pays the full amount. Supporting changes: `CandidCallBuilder` grows a `with_arg`, as there was no way to set the argument passed to `canister_init`, and `UniversalCanister` grows a `submit_update` that submits an ingress message without waiting for it to complete, which most of the calls above need. Co-Authored-By: Claude Opus 5 --- rs/tests/driver/src/util.rs | 12 + rs/tests/message_routing/BUILD.bazel | 4 +- .../subnet_cooling_down_test.rs | 458 +++++++++++++++--- rs/universal_canister/lib/src/management.rs | 8 + 4 files changed, 401 insertions(+), 81 deletions(-) diff --git a/rs/tests/driver/src/util.rs b/rs/tests/driver/src/util.rs index 87a88c314645..e165c020ff9d 100644 --- a/rs/tests/driver/src/util.rs +++ b/rs/tests/driver/src/util.rs @@ -593,6 +593,18 @@ impl<'a> UniversalCanister<'a> { .call_and_wait() .await } + + /// Submits `payload` as an ingress message to the canister's `update` + /// method without waiting for the call to complete. Useful for update calls + /// that are not expected to ever complete. + pub async fn submit_update>>(&self, payload: P) -> Result<(), AgentError> { + self.agent + .update(&self.canister_id, "update") + .with_arg(payload.into()) + .call() + .await + .map(|_| ()) + } } /// Provides an abstraction to the message canister. diff --git a/rs/tests/message_routing/BUILD.bazel b/rs/tests/message_routing/BUILD.bazel index 0b64dc7336d5..a93deef5c6e3 100644 --- a/rs/tests/message_routing/BUILD.bazel +++ b/rs/tests/message_routing/BUILD.bazel @@ -141,10 +141,10 @@ system_test_nns( system_test_nns( name = "subnet_cooling_down_test", - cpus = MIN_LOCAL_CPUS + 3 * DEFAULT_VCPUS_PER_VM, # 1 System + 2 Application fast-single-node subnets = 3 IC Node VMs * 6 vCPUs. + cpus = MIN_LOCAL_CPUS + 4 * DEFAULT_VCPUS_PER_VM, # 1 System + 3 Application fast-single-node subnets = 4 IC Node VMs * 6 vCPUs. enable_mainnet_nns_variant = False, # The `cooling_down` field of the subnet record is not supported by the mainnet NNS canisters. tags = [ - "long_test", # since the subnet only quiesces once the ingress history is pruned. + "long_test", # since the subnet only quiesces once its long-running `install_code` calls are done and the ingress history is pruned. ], test_timeout = "eternal", runtime_deps = UNIVERSAL_CANISTER_RUNTIME_DEPS, diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs index 597151f5d410..cd13195af02e 100644 --- a/rs/tests/message_routing/subnet_cooling_down_test.rs +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -2,39 +2,77 @@ Title:: Draining a subnet that is "cooling down". Goal:: Verify that a subnet labeled "cooling down" in its subnet record quiesces -while its canisters are busy making cross-subnet calls in a loop, i.e. that it +while its canisters are busy making cross-subnet calls in a loop, installing +code on one another and waiting for responses that never arrive, i.e. that it reaches the "merge readiness" condition of the `Subnet merging` dashboard (see `bases/apps/ic-dashboards/core/subnet-merging.json` on branch `mraszyk/subnet-merging-dashboard` of `dfinity/k8s`) for `V` = the registry -version at which the subnet was labeled "cooling down" and `R` = 0 cycles. +version at which the subnet was labeled "cooling down" and a pending refund +budget (the dashboard's `R`) of 0 cycles. + +The subnet that is cooling down, i.e. the one that would be merged, is called +`M`; `R` is the root (NNS) subnet it would be merged into, which holds nothing +but the NNS canisters; `T` and `S` are two further Application subnets holding +the canisters at the other end of `M`'s cross-subnet calls. + +"Executing" an update call below always means submitting it as an ingress +message without waiting for it to complete: most of the calls of this test are +never meant to complete. Runbook:: -0. Set up an IC with an NNS subnet (with the NNS canisters installed) and two - Application subnets S and T. -1. Install a universal canister on each Application subnet: US on S, UT on T. -2. Make an ingress call to each of US and UT with a payload that calls the +0. Set up an IC with an NNS subnet `R` (with the NNS canisters installed) and + three Application subnets `M`, `T` and `S`. +1. Install a universal canister on each of `M` and `T`: `US` on `M`, `UT` on `T`. +2. Make an ingress call to each of `US` and `UT` with a payload that calls the universal canister on the other subnet in a loop: the reply (or reject) callback of every call fires a new call. 3. Wait until both loops have completed a few iterations, i.e. messages are - actually flowing between S and T in both directions. -4. Submit (and adopt) an `UpdateConfigOfSubnet` NNS proposal labeling S as - "cooling down" in its subnet record, and record the registry version V it + actually flowing between `M` and `T` in both directions. +4. Install the universal canisters of the steps below (`U1`, `U3`, `U5` and + `U6` on `M`, `U4` and `U7` on `S`) and create five empty canisters + `U2a` .. `U2e` on `M`, controlled by `U1`. All of this has to happen before + step 5: a long-running `install_code` blocks every other `install_code` on + the same subnet. +5. Execute an update call on `U1` that makes five calls to the management + canister's `install_code` method, one per `U2x`, in mode `install`, with the + universal canister module and an `arg` that makes `canister_init` burn + `INIT_INSTRUCTIONS` instructions. Wait until all five requests have left + `U1`'s output queue: a request still sitting there when `M` starts cooling + down would never be routed, not even into the loopback stream, and the code + would never be installed. +6. Start three endless loops, each of which runs until the global data of the + looping canister is set to `LOOP_BREAK_TRIGGER`, which this test never does: + a. execute an update call on `U3` that loops on `U3` itself; + b. execute an update call on `U4` (on `S`) that calls `U5` (on `M`) with the + loop as its payload, so that `M` holds a canister looping in a call from + another subnet that it can never respond to; + c. execute an update call on `U6` (on `M`) that calls `U7` (on `S`) with the + loop as its payload, so that `M` holds a canister waiting for a response + from another subnet that never arrives. + Wait until all three loops are running. +7. Submit (and adopt) an `UpdateConfigOfSubnet` NNS proposal labeling `M` as + "cooling down" in its subnet record, and record the registry version `V` it creates. -5. Wait until S rejects ingress messages, i.e. the replicas of S observed the - "cooling down" label. -6. Wait until S is "merge ready" according to the dashboard's condition for V - and R = 0: all subnets have reached registry version V, no stream in either - direction holds a message (loopback included), the ingress history holds - nothing but `processing` entries, S's subnet input and output queues are - empty, S's subnet call context manager holds no call context, and the - pending anonymous refunds are worth at most R. -7. Check that both loops are indeed stalled (their iteration counters, read via - queries, no longer advance): while S is cooling down, neither S nor T routes - any message to or from S, so the messages of both loops are retained in - their senders' output queues. +8. Wait until `M` rejects ingress messages, i.e. the replicas of `M` observed + the "cooling down" label. +9. Wait until `M` is "merge ready" according to the dashboard's condition for + `V` and 0 cycles of pending refunds: all subnets have reached registry + version `V`, no stream in either direction holds a message (loopback + included), the ingress history holds nothing but `processing` entries, `M`'s + subnet input and output queues are empty, `M`'s subnet call context manager + holds no call context, and the pending anonymous refunds are worth at most 0 + cycles. +10. Check that `U2a` .. `U2e` have been installed, i.e. that the `install_code` + calls of step 5 ran to completion rather than being lost or rejected while + `M` was cooling down. +11. Check that the two loops of step 2 are indeed stalled (their iteration + counters, read via queries, no longer advance): while `M` is cooling down, + neither `M` nor `T` routes any message to or from `M`, so the messages of + both loops are retained in their senders' output queues. Success:: -S becomes "merge ready" while both loops are stalled. +`M` becomes "merge ready", with `U2a` .. `U2e` installed, while both loops of +step 2 are stalled. end::catalog[] */ @@ -56,10 +94,14 @@ use ic_system_test_driver::nns::{ use ic_system_test_driver::retry_with_msg_async; use ic_system_test_driver::systest; use ic_system_test_driver::util::{ - MetricsFetcher, UniversalCanister, assert_create_agent, block_on, runtime_from_url, + MetricsFetcher, UniversalCanister, assert_create_agent, block_on, create_canister, + runtime_from_url, set_controller, }; use ic_types::SubnetId; -use ic_universal_canister::{call_args, wasm}; +use ic_universal_canister::management::InstallMode; +use ic_universal_canister::{ + CallInterface, call_args, get_universal_canister_wasm, management, wasm, +}; use registry_canister::mutations::do_update_subnet::UpdateSubnetPayload; use slog::{Logger, info}; use std::collections::BTreeMap; @@ -74,8 +116,13 @@ const METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES: &str = "execution_subnet_output_queue const METRIC_SUBNET_CALL_CONTEXTS: &str = "replicated_state_subnet_call_contexts"; const METRIC_PENDING_REFUNDS_CYCLES: &str = "replicated_state_pending_refunds_cycles"; +/// The label selecting the `install_code` call contexts of +/// `METRIC_SUBNET_CALL_CONTEXTS`. +const LABEL_INSTALL_CODE: &str = "type=\"install_code\""; + /// `R` in the dashboard's readiness condition: the maximum total value in -/// cycles of the pending anonymous refunds of the cooling down subnet. +/// cycles of the pending anonymous refunds of the cooling down subnet. (Not to +/// be confused with the subnet `R` of the runbook above.) const MAX_REFUND_VALUE_CYCLES: f64 = 0.0; /// Number of loop iterations each universal canister must have completed before @@ -83,24 +130,52 @@ const MAX_REFUND_VALUE_CYCLES: f64 = 0.0; /// making cross-subnet calls when the label takes effect. const MIN_LOOP_ITERATIONS: u64 = 3; -/// Timeout for the subnet to become "merge ready". The binding term is the -/// ingress history, which only becomes free of terminal statuses once the -/// entries of the ingress messages submitted before the subnet started cooling -/// down are pruned, i.e. at their (up to `MAX_INGRESS_TTL` = 5 minutes away) -/// expiry times. -const MERGE_READY_TIMEOUT: Duration = Duration::from_secs(600); +/// One billion, the unit `INIT_INSTRUCTIONS` is expressed in. +const B: u64 = 1_000_000_000; + +/// The names of the canisters `U1` installs code on. +const INSTALL_CODE_TARGETS: [&str; 5] = ["U2a", "U2b", "U2c", "U2d", "U2e"]; + +/// Instructions the `canister_init` of every canister installed by `U1` burns, +/// i.e. how long each of `U1`'s `install_code` calls runs. The point is to make +/// them as long-running as possible, so that the subnet has to drain +/// `install_code` calls that span hundreds of rounds before it can be merged. +/// +/// An `install_code` message may consume at most +/// `MAX_INSTRUCTIONS_PER_INSTALL_CODE` = 300B instructions on an Application +/// subnet (`rs/config/src/subnet_config.rs`) and that budget also has to cover +/// compiling the module: 6_000 instructions per byte of the decompressed +/// (~350 KB) universal canister module, i.e. ~2.2B instructions, plus a 20M +/// base cost. The full compilation cost is charged whenever the module is not +/// in `expected_compiled_wasms`, which is cleared at every checkpoint, so an +/// `install_code` that is aborted at a checkpoint and restarted afterwards pays +/// it; hence the budget for `canister_init` has to leave room for it. +const INIT_INSTRUCTIONS: u64 = 295 * B; + +/// The global data value that would end the endless loops of `U3`, `U5` and +/// `U7`. The test never sets it, so those loops never end. +const LOOP_BREAK_TRIGGER: &[u8] = b"break"; + +/// Timeout for the subnet to become "merge ready". The binding terms are the +/// `install_code` calls of step 5, which take a couple of hundred rounds each +/// (and are executed one at a time, as at most one long-running `install_code` +/// makes progress per round), and the ingress history, which only becomes free +/// of terminal statuses once the entries of the ingress messages submitted +/// before the subnet started cooling down are pruned, i.e. at their (up to +/// `MAX_INGRESS_TTL` = 5 minutes away) expiry times. +const MERGE_READY_TIMEOUT: Duration = Duration::from_secs(2400); /// Backoff between two evaluations of the readiness condition. Longer than the /// default because every evaluation scrapes the metrics of all subnets. const MERGE_READY_BACKOFF: Duration = Duration::from_secs(10); -/// How long the loops are observed to be stalled (Step 7). +/// How long the loops are observed to be stalled (step 11). const STALL_OBSERVATION_PERIOD: Duration = Duration::from_secs(15); -/// Timeouts of the test itself: the ingress history pruning above dominates, +/// Timeouts of the test itself: draining the `install_code` calls dominates, /// the rest of the scenario takes a couple of minutes. The overall timeout /// additionally covers the setup (booting the IC and installing the NNS). -const PER_TEST_TIMEOUT: Duration = Duration::from_secs(900); -const OVERALL_TIMEOUT: Duration = Duration::from_secs(1500); +const PER_TEST_TIMEOUT: Duration = Duration::from_secs(3300); +const OVERALL_TIMEOUT: Duration = Duration::from_secs(3900); fn main() -> Result<()> { SystemTestGroup::new() @@ -117,6 +192,7 @@ pub fn setup(env: TestEnv) { .add_subnet(Subnet::fast_single_node(SubnetType::System)) .add_subnet(Subnet::fast_single_node(SubnetType::Application)) .add_subnet(Subnet::fast_single_node(SubnetType::Application)) + .add_subnet(Subnet::fast_single_node(SubnetType::Application)) .setup_and_start(&env) .expect("failed to setup IC under test"); env.topology_snapshot().subnets().for_each(|subnet| { @@ -143,32 +219,40 @@ async fn run(env: TestEnv) { let logger = env.logger(); let topology = env.topology_snapshot(); - // The two Application subnets: S is the one that will be labeled "cooling - // down", T is the one it exchanges messages with. + // The three Application subnets: `M` is the one that will be labeled + // "cooling down", `T` is the one it exchanges messages with in a loop, and + // `S` is the one holding the canisters at the other end of the cross-subnet + // calls of the endless loops of step 6. let app_subnets: Vec<_> = topology .subnets() .filter(|subnet| subnet.subnet_type() == SubnetType::Application) .collect(); assert_eq!( app_subnets.len(), - 2, - "expected exactly 2 Application subnets" + 3, + "expected exactly 3 Application subnets" ); - let s_subnet = app_subnets[0].clone(); + let m_subnet = app_subnets[0].clone(); let t_subnet = app_subnets[1].clone(); - let s_node = s_subnet.nodes().next().unwrap(); + let s_subnet = app_subnets[2].clone(); + let m_node = m_subnet.nodes().next().unwrap(); let t_node = t_subnet.nodes().next().unwrap(); - let s_agent = assert_create_agent(s_node.get_public_url().as_str()).await; + let s_node = s_subnet.nodes().next().unwrap(); + let m_agent = assert_create_agent(m_node.get_public_url().as_str()).await; let t_agent = assert_create_agent(t_node.get_public_url().as_str()).await; - - // Step 1: Install a universal canister on each Application subnet. + let s_agent = assert_create_agent(s_node.get_public_url().as_str()).await; info!( logger, - "Step 1: Installing universal canisters on S ({}) and T ({})", - s_subnet.subnet_id, + "Subnets under test: M={}, T={}, S={} (R={})", + m_subnet.subnet_id, t_subnet.subnet_id, + s_subnet.subnet_id, + topology.root_subnet_id(), ); - let us = UniversalCanister::new_with_retries(&s_agent, s_node.effective_canister_id(), &logger) + + // Step 1: Install a universal canister on each of `M` and `T`. + info!(logger, "Step 1: Installing universal canisters US and UT"); + let us = UniversalCanister::new_with_retries(&m_agent, m_node.effective_canister_id(), &logger) .await; let ut = UniversalCanister::new_with_retries(&t_agent, t_node.effective_canister_id(), &logger) .await; @@ -198,7 +282,7 @@ async fn run(env: TestEnv) { READY_WAIT_TIMEOUT, RETRY_BACKOFF, || async { - let iterations = loop_iterations(canister).await?; + let iterations = global_counter(canister).await?; if iterations < MIN_LOOP_ITERATIONS { bail!("{name}'s call loop is at iteration {iterations}"); } @@ -210,26 +294,96 @@ async fn run(env: TestEnv) { } info!(logger, "Step 3 done: both call loops are making progress"); - // Step 4: Label S as "cooling down" in its subnet record. + // Step 4: Install all the canisters of the steps below. Every installation + // has to happen before step 5 starts `U1`'s `install_code` calls: at most + // one long-running `install_code` makes progress per round, and while one + // is in progress no other `install_code` on the same subnet is executed at + // all, so any installation attempted here later would be stuck behind + // `U1`'s calls for as long as they run. + info!( + logger, + "Step 4: Installing U1, U3, U5, U6 on M and U4, U7 on S, and creating {} canisters for U1 \ + to install code on", + INSTALL_CODE_TARGETS.len(), + ); + let m_id = m_node.effective_canister_id(); + let s_id = s_node.effective_canister_id(); + let u1 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; + let u3 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; + let u4 = UniversalCanister::new_with_retries(&s_agent, s_id, &logger).await; + let u5 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; + let u6 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; + let u7 = UniversalCanister::new_with_retries(&s_agent, s_id, &logger).await; info!( logger, - "Step 4: Labeling subnet S ({}) as \"cooling down\"", s_subnet.subnet_id, + "Step 4: U1={}, U3={}, U5={}, U6={} on M; U4={}, U7={} on S", + u1.canister_id(), + u3.canister_id(), + u5.canister_id(), + u6.canister_id(), + u4.canister_id(), + u7.canister_id(), ); - let registry_version = set_subnet_cooling_down(&env, s_subnet.subnet_id, &logger).await; + let mut targets = Vec::new(); + for name in INSTALL_CODE_TARGETS { + let target = create_canister(&m_agent, m_id).await; + set_controller(&target, &u1.canister_id(), &m_agent).await; + info!(logger, "Step 4: {name}={target} on M, controlled by U1"); + targets.push(target); + } + info!(logger, "Step 4 done: all canisters installed"); + + // Step 5: Have `U1` install code on the five canisters it controls. info!( logger, - "Step 4 done: subnet S is labeled \"cooling down\" as of registry version \ + "Step 5: Executing the update call on U1 making the {} `install_code` calls", + targets.len(), + ); + u1.submit_update(install_code_payload(&targets)) + .await + .expect("submitting U1's `install_code` calls should succeed"); + await_install_code_requests_inducted(&m_subnet, &logger).await; + info!( + logger, + "Step 5 done: all of U1's `install_code` requests left its output queue" + ); + + // Step 6: Start the three endless loops. + info!(logger, "Step 6: Starting the three endless loops"); + u3.submit_update(endless_loop()) + .await + .expect("submitting U3's endless loop should succeed"); + submit_endless_loop_call(&u4, u5.canister_id()).await; + submit_endless_loop_call(&u6, u7.canister_id()).await; + for (canister, name) in [(&u3, "U3"), (&u5, "U5"), (&u7, "U7")] { + await_loop_started(canister, name, &logger).await; + } + info!( + logger, + "Step 6 done: U3 is looping, U5 is looping in a call from U4, and U7 is looping in a call \ + from U6" + ); + + // Step 7: Label `M` as "cooling down" in its subnet record. + info!( + logger, + "Step 7: Labeling subnet M ({}) as \"cooling down\"", m_subnet.subnet_id, + ); + let registry_version = set_subnet_cooling_down(&env, m_subnet.subnet_id, &logger).await; + info!( + logger, + "Step 7 done: subnet M is labeled \"cooling down\" as of registry version \ {registry_version} (V)", ); - // Step 5: Wait until the replicas of S observed the "cooling down" label, - // i.e. until S rejects ingress messages. + // Step 8: Wait until the replicas of `M` observed the "cooling down" label, + // i.e. until `M` rejects ingress messages. info!( logger, - "Step 5: Waiting until subnet S rejects ingress messages" + "Step 8: Waiting until subnet M rejects ingress messages" ); retry_with_msg_async!( - "waiting until subnet S rejects ingress messages", + "waiting until subnet M rejects ingress messages", &logger, READY_WAIT_TIMEOUT, RETRY_BACKOFF, @@ -247,22 +401,22 @@ async fn run(env: TestEnv) { } ) .await - .expect("subnet S did not start rejecting ingress messages"); + .expect("subnet M did not start rejecting ingress messages"); info!( logger, - "Step 5 done: subnet S rejects ingress messages, so it is cooling down" + "Step 8 done: subnet M rejects ingress messages, so it is cooling down" ); - // Step 6: Wait until S is "merge ready". + // Step 9: Wait until `M` is "merge ready". info!( logger, - "Step 6: Waiting until subnet S is \"merge ready\" for V={registry_version} and \ - R={MAX_REFUND_VALUE_CYCLES} cycles", + "Step 9: Waiting until subnet M is \"merge ready\" for V={registry_version} and at most \ + {MAX_REFUND_VALUE_CYCLES} cycles of pending refunds", ); retry_with_msg_async!( format!( "waiting until subnet {} is \"merge ready\"", - s_subnet.subnet_id + m_subnet.subnet_id ), &logger, MERGE_READY_TIMEOUT, @@ -270,7 +424,7 @@ async fn run(env: TestEnv) { || async { let terms = evaluate_merge_readiness( &topology, - &s_subnet, + &m_subnet, registry_version, MAX_REFUND_VALUE_CYCLES, ) @@ -284,38 +438,65 @@ async fn run(env: TestEnv) { bail!("not merge ready: {}", unsatisfied.join("; ")); } for (term, _) in &terms { - info!(logger, "Step 6: merge readiness term holds: {term}"); + info!(logger, "Step 9: merge readiness term holds: {term}"); } Ok(()) } ) .await - .unwrap_or_else(|e| panic!("subnet S did not become \"merge ready\": {e}")); - info!(logger, "Step 6 done: subnet S is \"merge ready\""); + .unwrap_or_else(|e| panic!("subnet M did not become \"merge ready\": {e}")); + info!(logger, "Step 9 done: subnet M is \"merge ready\""); + + // Step 10: Check that `U1`'s `install_code` calls did install the universal + // canister module: a canister that has no module rejects every query. + info!( + logger, + "Step 10: Checking that {} have been installed", + INSTALL_CODE_TARGETS.join(", "), + ); + for (&target, name) in targets.iter().zip(INSTALL_CODE_TARGETS) { + let canister = UniversalCanister::from_canister_id(&m_agent, target); + let reply = canister + .query(wasm().reply_data(name.as_bytes())) + .await + .unwrap_or_else(|e| { + panic!("{name} ({target}) does not answer queries, so it was not installed: {e}") + }); + assert_eq!( + reply, + name.as_bytes(), + "{name} ({target}) answered a query with an unexpected reply", + ); + } + info!( + logger, + "Step 10 done: {} have been installed", + INSTALL_CODE_TARGETS.join(", "), + ); - // Step 7: Check that both call loops are stalled, i.e. that S became + // Step 11: Check that both call loops are stalled, i.e. that `M` became // "merge ready" because it is cooling down and not because the loops // stopped making calls. info!( logger, - "Step 7: Checking that both call loops are stalled over {STALL_OBSERVATION_PERIOD:?}" + "Step 11: Checking that both call loops are stalled over {STALL_OBSERVATION_PERIOD:?}" ); let before = [ - loop_iterations(&us).await.unwrap(), - loop_iterations(&ut).await.unwrap(), + global_counter(&us).await.unwrap(), + global_counter(&ut).await.unwrap(), ]; tokio::time::sleep(STALL_OBSERVATION_PERIOD).await; for ((canister, name), before) in [(&us, "US"), (&ut, "UT")].into_iter().zip(before) { - let after = loop_iterations(canister).await.unwrap(); + let after = global_counter(canister).await.unwrap(); assert_eq!( before, after, - "{name}'s call loop advanced from iteration {before} to {after} while subnet S was \ + "{name}'s call loop advanced from iteration {before} to {after} while subnet M was \ cooling down", ); } info!( logger, - "Step 7 done: both call loops are stalled at iterations {before:?}" + "Step 11 done: both call loops are stalled at iterations {before:?}" ); } @@ -374,14 +555,132 @@ async fn start_call_loop(canister: &UniversalCanister<'_>, peer: Principal) { .expect("starting the call loop should succeed"); } -/// Returns the number of loop iterations `canister` has executed so far, i.e. -/// its global counter, read via a query (an ingress message would be rejected -/// by a subnet that is cooling down). -async fn loop_iterations(canister: &UniversalCanister<'_>) -> Result { +/// The payload of the endless loops of `U3`, `U5` and `U7`: bump the global +/// counter, so that the test can observe (via a query) that the payload started +/// executing, and then loop until the global data is set to +/// `LOOP_BREAK_TRIGGER`, which this test never does. +/// +/// Every iteration of the loop is a management canister `canister_status` call +/// for the executing canister itself, so the loop stalls as soon as the subnet +/// holding that canister stops routing messages out of its canisters' output +/// queues, i.e. as soon as it is cooling down. The canister never responds to +/// the call it is executing, so that call context stays open forever. +fn endless_loop() -> Vec { + wasm() + .inc_global_counter() + .loop_until_global_data_set(LOOP_BREAK_TRIGGER, &[]) + .build() +} + +/// Executes an update call on `caller` that calls `callee` with `endless_loop()` +/// as the payload for `callee` to execute. As `callee` never responds, neither +/// does `caller`, so its ingress message stays `processing` forever. +async fn submit_endless_loop_call(caller: &UniversalCanister<'_>, callee: Principal) { + caller + .submit_update(wasm().call_simple(callee, "update", call_args().other_side(endless_loop()))) + .await + .expect("submitting the call starting the endless loop should succeed"); +} + +/// The payload of the update call on `U1`: one management canister +/// `install_code` call per canister in `targets`, installing the universal +/// canister module with an `arg` that makes `canister_init` burn +/// `INIT_INSTRUCTIONS` instructions, followed by a reply. +/// +/// `U1` replies as soon as all the calls have been made, which keeps its +/// ingress message from lingering in the ingress history as a `processing` +/// entry; neither callback of the `install_code` calls may respond to that +/// already responded call context, so both are no-ops. +fn install_code_payload(targets: &[Principal]) -> Vec { + let init = wasm() + .instruction_counter_is_at_least(INIT_INSTRUCTIONS) + .build(); + let module = get_universal_canister_wasm(); + let mut payload = wasm(); + for target in targets { + payload = payload.call( + management::install_code(target.as_slice(), &module) + .with_mode(InstallMode::Install) + .with_arg(init.clone()) + .on_reply(wasm().noop()) + .on_reject(wasm().noop()), + ); + } + payload.reply_data(&[]).build() +} + +/// Waits until all of `U1`'s `install_code` requests have left `U1`'s output +/// queue, i.e. are either enqueued in `subnet`'s subnet input queues or already +/// executing (and hence hold a call context in the subnet call context +/// manager). +/// +/// Only then may `subnet` start cooling down: a request still sitting in `U1`'s +/// output queue would never be routed, not even into the loopback stream, and +/// the code would never be installed. +/// +/// Every subnet message this test made before `U1`'s calls (creating and +/// installing canisters, setting their controller) was waited for, so the subnet +/// queues hold nothing but those calls by the time they are inducted. +async fn await_install_code_requests_inducted(subnet: &SubnetSnapshot, logger: &Logger) { + let expected = INSTALL_CODE_TARGETS.len() as f64; + retry_with_msg_async!( + format!( + "waiting until all {} `install_code` requests are inducted on subnet {}", + INSTALL_CODE_TARGETS.len(), + subnet.subnet_id + ), + logger, + READY_WAIT_TIMEOUT, + RETRY_BACKOFF, + || async { + let metrics = fetch_metrics( + subnet, + &[ + METRIC_SUBNET_INPUT_QUEUE_MESSAGES, + METRIC_SUBNET_CALL_CONTEXTS, + ], + ) + .await?; + let enqueued = sum_of_medians(&metrics, METRIC_SUBNET_INPUT_QUEUE_MESSAGES, |_| true); + let executing = sum_of_medians(&metrics, METRIC_SUBNET_CALL_CONTEXTS, |labels| { + labels.contains(LABEL_INSTALL_CODE) + }); + if enqueued + executing < expected { + bail!("{enqueued} request(s) enqueued and {executing} executing"); + } + Ok(()) + } + ) + .await + .unwrap_or_else(|e| panic!("U1's `install_code` requests were not inducted: {e}")); +} + +/// Waits until `canister`'s global counter is non-zero, i.e. until the payload +/// of `endless_loop()` started executing on it. +async fn await_loop_started(canister: &UniversalCanister<'_>, name: &str, logger: &Logger) { + retry_with_msg_async!( + format!("waiting until {name}'s endless loop started"), + logger, + READY_WAIT_TIMEOUT, + RETRY_BACKOFF, + || async { + if global_counter(canister).await? == 0 { + bail!("{name} has not started executing the endless loop yet"); + } + Ok(()) + } + ) + .await + .unwrap_or_else(|e| panic!("{name}'s endless loop did not start: {e}")); +} + +/// Returns `canister`'s global counter, read via a query (an ingress message +/// would be rejected by a subnet that is cooling down). +async fn global_counter(canister: &UniversalCanister<'_>) -> Result { let reply = canister .query(wasm().get_global_counter().reply_int64()) .await - .map_err(|e| anyhow!("failed to read the loop iteration counter: {e}"))?; + .map_err(|e| anyhow!("failed to read the global counter: {e}"))?; let reply: [u8; 8] = reply .as_slice() .try_into() @@ -455,7 +754,8 @@ async fn set_subnet_cooling_down(env: &TestEnv, subnet_id: SubnetId, logger: &Lo /// Evaluates the terms of the "merge readiness" condition of the `Subnet /// merging` dashboard for `subnet` (the subnet that is cooling down), -/// `registry_version` (`V`) and `max_refund_value_cycles` (`R`). Returns one +/// `registry_version` (`V`) and `max_refund_value_cycles` (the dashboard's +/// `R`, not to be confused with the subnet `R`). Returns one /// (description, satisfied) pair per term, in the order the terms appear in the /// dashboard's readiness expression. /// diff --git a/rs/universal_canister/lib/src/management.rs b/rs/universal_canister/lib/src/management.rs index d4b76648372a..ad54df6b425b 100644 --- a/rs/universal_canister/lib/src/management.rs +++ b/rs/universal_canister/lib/src/management.rs @@ -226,6 +226,14 @@ impl CandidCallBuilder { self } + /// The argument passed to the installed canister's `canister_init` (or + /// `canister_post_upgrade`), i.e. the payload the universal canister + /// evaluates there. + pub fn with_arg>>(mut self, arg: A) -> Self { + self.args.arg = arg.into(); + self + } + pub fn with_compute_allocation(mut self, allocation: u64) -> Self { self.args.compute_allocation = Some(candid::Nat::from(allocation)); self From 46f16bd4e80dc651f8a744a4e8545e6bfeefbdb0 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 31 Aug 2026 09:26:31 +0000 Subject: [PATCH 05/30] feat: a MergeSubnets proposal performs the registry side of a subnet merge Add the `MergeSubnets` NNS function, mapped to the `merge_subnets` method of the registry canister under the `SubnetManagement` topic, and extend that method so that a single such proposal performs the whole registry-side part of a subnet merge, all in one registry version, so that the destination subnet never observes a state where only some of it took effect: 1. the canister ID ranges of the source subnet are merged into the canister ID range set of the destination subnet, so that all canisters that used to be hosted by the source subnet are routed to the destination subnet; 2. a recovery catch-up package is created for the destination subnet, at the height, time and state hash of the merged state, running a fresh DKG for the destination subnet's membership; and 3. the destination subnet is brought back online. The payload therefore grows the `height`, `time_ns` and `state_hash` of the merged state, plus an `initial_dkg_subnet_id` naming the subnet that handles the `setup_initial_dkg` call: it must not be the destination subnet, which is offline while the merge is in progress. The caller is expected to have taken both subnets offline and to have extended the state of the destination subnet with the state of the canisters of the source subnet beforehand; `state_hash` is the hash of the manifest of the result. The source subnet record is not modified and the source subnet is not deleted: it merely does not host any canister ID range anymore. The recovery catch-up package needs fresh DKG transcripts because its `initial_ni_dkg_transcript_{low,high}_threshold` become the current transcripts of the DKG summary that bootstraps consensus at the recovery height. The registry only holds the destination subnet's genesis (or last recovery) catch-up package, whose transcripts its nodes may no longer have the secret key shares for, so reusing those would risk restarting a subnet that cannot sign. Merging into a subnet holding chain keys is rejected: recovering such a subnet requires resharing its keys onto the recovery catch-up package, which this method does not do, and silently leaving the destination subnet unable to sign would be worse than refusing. As the method now makes an inter-canister call half way through, it follows `do_recover_subnet` in checking that none of the records it is about to overwrite -- and none of the canister ID ranges it validated -- changed while that call was in flight. Unlike `do_recover_subnet`, it reports the reject code and message if that call fails, rather than an opaque `unwrap` panic. `StateMachine` only answered `setup_initial_dkg` requests from `do_execute_round`, which `tick()` (and hence `await_ingress`) does not go through, so any canister awaiting such a call hung there forever. Factor the fake responses out into `setup_initial_dkg_responses` and produce them from `tick_with_config` as well, next to the threshold signing requests it already answers. Without this the integration test of the success path cannot run, which is why it is part of this commit. The two success unit tests of `merge_subnets`, which can no longer drive the method to completion, now exercise the routing table part directly; the success path as a whole, including the recovery catch-up package and the unhalting, is covered by the integration test. Co-Authored-By: Claude Opus 5 --- rs/nns/governance/api/src/types.rs | 8 + .../ic_nns_governance/pb/v1/governance.proto | 7 + .../src/gen/ic_nns_governance.pb.v1.rs | 8 + rs/nns/governance/src/pb/conversions/mod.rs | 2 + .../src/proposals/execute_nns_function.rs | 16 +- rs/registry/canister/canister/canister.rs | 7 +- rs/registry/canister/canister/registry.did | 4 + .../canister/canister/registry_test.did | 4 + .../src/mutations/do_recover_subnet.rs | 2 +- .../canister/src/mutations/merge_subnets.rs | 311 +++++++++++++++--- rs/registry/canister/tests/merge_subnets.rs | 62 +++- rs/registry/canister/unreleased_changelog.md | 20 +- rs/state_machine_tests/src/lib.rs | 73 ++-- 13 files changed, 436 insertions(+), 88 deletions(-) diff --git a/rs/nns/governance/api/src/types.rs b/rs/nns/governance/api/src/types.rs index 895abc43b262..0b81400536bc 100644 --- a/rs/nns/governance/api/src/types.rs +++ b/rs/nns/governance/api/src/types.rs @@ -4333,6 +4333,12 @@ pub enum NnsFunction { /// `SetupInitialDKG` requests without an explicit subnet id are routed to the /// calling subnet (NNS). SetDefaultInitialDkgSubnet = 58, + /// Merge a subnet into another subnet: the canister ID ranges of the source + /// subnet are merged into the canister ID range set of the destination subnet, + /// a recovery catch-up package is created for the destination subnet (whose + /// state is expected to have been extended with the state of the canisters of + /// the source subnet) and the destination subnet is brought back online. + MergeSubnets = 59, } impl NnsFunction { /// String value of the enum field names used in the ProtoBuf definition. @@ -4421,6 +4427,7 @@ impl NnsFunction { NnsFunction::SetDefaultInitialDkgSubnet => { "NNS_FUNCTION_SET_DEFAULT_INITIAL_DKG_SUBNET" } + NnsFunction::MergeSubnets => "NNS_FUNCTION_MERGE_SUBNETS", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -4504,6 +4511,7 @@ impl NnsFunction { "NNS_FUNCTION_SPLIT_SUBNET" => Some(Self::SplitSubnet), "NNS_FUNCTION_DELETE_SUBNET" => Some(Self::DeleteSubnet), "NNS_FUNCTION_SET_DEFAULT_INITIAL_DKG_SUBNET" => Some(Self::SetDefaultInitialDkgSubnet), + "NNS_FUNCTION_MERGE_SUBNETS" => Some(Self::MergeSubnets), _ => None, } } diff --git a/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto b/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto index 811308539c83..115184a2b76d 100644 --- a/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto +++ b/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto @@ -513,6 +513,13 @@ enum NnsFunction { // `SetupInitialDKG` requests without an explicit subnet id are routed to the // calling subnet (NNS). NNS_FUNCTION_SET_DEFAULT_INITIAL_DKG_SUBNET = 58; + + // Merge a subnet into another subnet: the canister ID ranges of the source + // subnet are merged into the canister ID range set of the destination subnet, + // a recovery catch-up package is created for the destination subnet (whose + // state is expected to have been extended with the state of the canisters of + // the source subnet) and the destination subnet is brought back online. + NNS_FUNCTION_MERGE_SUBNETS = 59; } // Payload of a proposal that calls a function on another NNS diff --git a/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs b/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs index 78cc3c203c61..3e89c08852d3 100644 --- a/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs +++ b/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs @@ -5546,6 +5546,12 @@ pub enum NnsFunction { /// `SetupInitialDKG` requests without an explicit subnet id are routed to the /// calling subnet (NNS). SetDefaultInitialDkgSubnet = 58, + /// Merge a subnet into another subnet: the canister ID ranges of the source + /// subnet are merged into the canister ID range set of the destination subnet, + /// a recovery catch-up package is created for the destination subnet (whose + /// state is expected to have been extended with the state of the canisters of + /// the source subnet) and the destination subnet is brought back online. + MergeSubnets = 59, } impl NnsFunction { /// String value of the enum field names used in the ProtoBuf definition. @@ -5622,6 +5628,7 @@ impl NnsFunction { Self::SplitSubnet => "NNS_FUNCTION_SPLIT_SUBNET", Self::DeleteSubnet => "NNS_FUNCTION_DELETE_SUBNET", Self::SetDefaultInitialDkgSubnet => "NNS_FUNCTION_SET_DEFAULT_INITIAL_DKG_SUBNET", + Self::MergeSubnets => "NNS_FUNCTION_MERGE_SUBNETS", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -5705,6 +5712,7 @@ impl NnsFunction { "NNS_FUNCTION_SPLIT_SUBNET" => Some(Self::SplitSubnet), "NNS_FUNCTION_DELETE_SUBNET" => Some(Self::DeleteSubnet), "NNS_FUNCTION_SET_DEFAULT_INITIAL_DKG_SUBNET" => Some(Self::SetDefaultInitialDkgSubnet), + "NNS_FUNCTION_MERGE_SUBNETS" => Some(Self::MergeSubnets), _ => None, } } diff --git a/rs/nns/governance/src/pb/conversions/mod.rs b/rs/nns/governance/src/pb/conversions/mod.rs index 280e49e2c7f9..0f8c33efd8e1 100644 --- a/rs/nns/governance/src/pb/conversions/mod.rs +++ b/rs/nns/governance/src/pb/conversions/mod.rs @@ -3940,6 +3940,7 @@ impl From for api::NnsFunction { api::NnsFunction::SetSubnetOperationalLevel } pb::NnsFunction::SplitSubnet => api::NnsFunction::SplitSubnet, + pb::NnsFunction::MergeSubnets => api::NnsFunction::MergeSubnets, pb::NnsFunction::DeleteSubnet => api::NnsFunction::DeleteSubnet, pb::NnsFunction::SetDefaultInitialDkgSubnet => { api::NnsFunction::SetDefaultInitialDkgSubnet @@ -4040,6 +4041,7 @@ impl From for pb::NnsFunction { pb::NnsFunction::SetSubnetOperationalLevel } api::NnsFunction::SplitSubnet => pb::NnsFunction::SplitSubnet, + api::NnsFunction::MergeSubnets => pb::NnsFunction::MergeSubnets, api::NnsFunction::DeleteSubnet => pb::NnsFunction::DeleteSubnet, api::NnsFunction::SetDefaultInitialDkgSubnet => { pb::NnsFunction::SetDefaultInitialDkgSubnet diff --git a/rs/nns/governance/src/proposals/execute_nns_function.rs b/rs/nns/governance/src/proposals/execute_nns_function.rs index 7aba6eaa7d39..cd50c8a6215d 100644 --- a/rs/nns/governance/src/proposals/execute_nns_function.rs +++ b/rs/nns/governance/src/proposals/execute_nns_function.rs @@ -459,6 +459,7 @@ pub enum ValidNnsFunction { SplitSubnet, DeleteSubnet, SetDefaultInitialDkgSubnet, + MergeSubnets, } impl ValidNnsFunction { @@ -592,6 +593,7 @@ impl ValidNnsFunction { ValidNnsFunction::SetDefaultInitialDkgSubnet => { (REGISTRY_CANISTER_ID, "set_default_initial_dkg_subnet") } + ValidNnsFunction::MergeSubnets => (REGISTRY_CANISTER_ID, "merge_subnets"), } } @@ -623,7 +625,8 @@ impl ValidNnsFunction { | ValidNnsFunction::SetSubnetOperationalLevel | ValidNnsFunction::SplitSubnet | ValidNnsFunction::DeleteSubnet - | ValidNnsFunction::SetDefaultInitialDkgSubnet => Topic::SubnetManagement, + | ValidNnsFunction::SetDefaultInitialDkgSubnet + | ValidNnsFunction::MergeSubnets => Topic::SubnetManagement, ValidNnsFunction::ReviseElectedGuestosVersions | ValidNnsFunction::ReviseElectedHostosVersions => Topic::IcOsVersionElection, @@ -714,6 +717,7 @@ impl ValidNnsFunction { ValidNnsFunction::SplitSubnet => "Split subnet", ValidNnsFunction::DeleteSubnet => "Delete Subnet", ValidNnsFunction::SetDefaultInitialDkgSubnet => "Set Default Initial DKG Subnet", + ValidNnsFunction::MergeSubnets => "Merge subnets", } } @@ -944,6 +948,15 @@ impl ValidNnsFunction { calls are routed when no subnet is specified explicitly in the request. If unset, \ such requests are routed to the calling subnet (NNS)." } + ValidNnsFunction::MergeSubnets => { + "Merge a subnet into another subnet. The canister ID ranges of the source subnet \ + are merged into the canister ID range set of the destination subnet, so that all \ + canisters that used to be hosted by the source subnet are routed to the \ + destination subnet; a recovery catch-up package is created for the destination \ + subnet, whose state is expected to have been extended with the state of the \ + canisters of the source subnet while both subnets were offline; and the \ + destination subnet is brought back online." + } } } } @@ -1033,6 +1046,7 @@ impl TryFrom for ValidNnsFunction { Ok(ValidNnsFunction::SetSubnetOperationalLevel) } NnsFunction::SplitSubnet => Ok(ValidNnsFunction::SplitSubnet), + NnsFunction::MergeSubnets => Ok(ValidNnsFunction::MergeSubnets), NnsFunction::DeleteSubnet => Ok(ValidNnsFunction::DeleteSubnet), NnsFunction::SetDefaultInitialDkgSubnet => { Ok(ValidNnsFunction::SetDefaultInitialDkgSubnet) diff --git a/rs/registry/canister/canister/canister.rs b/rs/registry/canister/canister/canister.rs index 01568e96f077..a342f824e13c 100644 --- a/rs/registry/canister/canister/canister.rs +++ b/rs/registry/canister/canister/canister.rs @@ -1089,13 +1089,16 @@ fn reroute_canister_ranges_(payload: RerouteCanisterRangesPayload) { #[unsafe(export_name = "canister_update merge_subnets")] fn merge_subnets() { check_caller_is_governance_and_log("merge_subnets"); - over(candid_one, merge_subnets_); + over_async(candid_one, |payload: MergeSubnetsPayload| async move { + merge_subnets_(payload).await + }); } #[candid_method(update, rename = "merge_subnets")] -fn merge_subnets_(payload: MergeSubnetsPayload) { +async fn merge_subnets_(payload: MergeSubnetsPayload) { registry_mut() .merge_subnets(payload) + .await .unwrap_or_else(|error_message| { trap_with(&format!( "{LOG_PREFIX} Merge subnets failed: {error_message}" diff --git a/rs/registry/canister/canister/registry.did b/rs/registry/canister/canister/registry.did index a3759d9ffb20..7183998d0222 100644 --- a/rs/registry/canister/canister/registry.did +++ b/rs/registry/canister/canister/registry.did @@ -311,6 +311,10 @@ type IPv4Config = record { type MergeSubnetsPayload = record { source_subnet : principal; destination_subnet : principal; + height : nat64; + time_ns : nat64; + state_hash : blob; + initial_dkg_subnet_id : opt principal; }; type MigrateCanistersPayload = record { diff --git a/rs/registry/canister/canister/registry_test.did b/rs/registry/canister/canister/registry_test.did index 7493ae134411..2be881b0da74 100644 --- a/rs/registry/canister/canister/registry_test.did +++ b/rs/registry/canister/canister/registry_test.did @@ -311,6 +311,10 @@ type IPv4Config = record { type MergeSubnetsPayload = record { source_subnet : principal; destination_subnet : principal; + height : nat64; + time_ns : nat64; + state_hash : blob; + initial_dkg_subnet_id : opt principal; }; type MigrateCanistersPayload = record { diff --git a/rs/registry/canister/src/mutations/do_recover_subnet.rs b/rs/registry/canister/src/mutations/do_recover_subnet.rs index cf393eec3a7d..b07e8bfc9296 100644 --- a/rs/registry/canister/src/mutations/do_recover_subnet.rs +++ b/rs/registry/canister/src/mutations/do_recover_subnet.rs @@ -488,7 +488,7 @@ impl TryFrom for KeyConfigRequestInternal { } } -fn panic_if_record_changed_across_versions( +pub(crate) fn panic_if_record_changed_across_versions( registry: &Registry, key: &str, initial_registry_version: Version, diff --git a/rs/registry/canister/src/mutations/merge_subnets.rs b/rs/registry/canister/src/mutations/merge_subnets.rs index a907ac83af77..3d141d13a2f7 100644 --- a/rs/registry/canister/src/mutations/merge_subnets.rs +++ b/rs/registry/canister/src/mutations/merge_subnets.rs @@ -1,28 +1,71 @@ -use crate::{common::LOG_PREFIX, registry::Registry}; -use candid::CandidType; +//! Contains the method to merge a subnet into another subnet. +//! +//! Merging the source subnet into the destination subnet reroutes all canisters +//! of the source subnet to the destination subnet and lets the destination +//! subnet resume from a state that was extended, while both subnets were +//! offline, with the state of those canisters. The state extension itself +//! happens outside of the registry: this method only records its outcome, as the +//! state hash of a recovery catch-up package for the destination subnet. + +use crate::{ + common::LOG_PREFIX, mutations::do_recover_subnet::panic_if_record_changed_across_versions, + registry::Registry, +}; +use candid::{CandidType, Encode}; +use dfn_core::api::{CanisterId, call}; #[cfg(target_arch = "wasm32")] use dfn_core::println; -use ic_base_types::SubnetId; -use ic_registry_keys::make_subnet_record_key; +use ic_base_types::{NodeId, PrincipalId, RegistryVersion, SubnetId}; +use ic_management_canister_types_private::{SetupInitialDKGArgs, SetupInitialDKGResponse}; +use ic_protobuf::registry::subnet::v1::{RecoveryArgs, catch_up_package_contents::CupType}; +use ic_registry_keys::{ + make_catch_up_package_contents_key, make_crypto_threshold_signing_pubkey_key, + make_subnet_record_key, +}; use ic_registry_routing_table::are_disjoint; +use ic_registry_transport::{ + pb::v1::{RegistryMutation, registry_mutation}, + upsert, +}; +use on_wire::bytes; +use prost::Message; use serde::{Deserialize, Serialize}; +use std::convert::TryFrom; impl Registry { - /// Merges the canister ID ranges of the source subnet into the canister ID - /// range set of the destination subnet. + /// Merges the source subnet into the destination subnet. /// - /// After this operation, all canisters that used to be hosted by the source - /// subnet are routed to the destination subnet and the source subnet does - /// not host any canister ID range anymore. + /// Three things happen, all in a single registry version, so that the + /// destination subnet never observes a state where only some of them took + /// effect: /// - /// Note that only the routing table is updated: neither subnet record is - /// modified and, in particular, the source subnet is not deleted. - pub fn merge_subnets(&mut self, payload: MergeSubnetsPayload) -> Result<(), String> { + /// 1. the canister ID ranges of the source subnet are merged into the + /// canister ID range set of the destination subnet, so that all + /// canisters that used to be hosted by the source subnet are routed to + /// the destination subnet; + /// 2. a recovery catch-up package is created for the destination subnet, + /// at the height, time and state hash of the merged state, running a + /// fresh DKG for the destination subnet's membership; and + /// 3. the destination subnet is brought back online. + /// + /// The caller is expected to have taken both subnets offline and to have + /// extended the state of the destination subnet with the state of the + /// canisters of the source subnet beforehand; `state_hash` is the hash of + /// the manifest of the resulting merged state. + /// + /// Note that neither subnet record is deleted and, in particular, the source + /// subnet is not deleted: it merely does not host any canister ID range + /// anymore. + pub async fn merge_subnets(&mut self, payload: MergeSubnetsPayload) -> Result<(), String> { println!("{LOG_PREFIX}merge_subnets: {payload:?}"); let MergeSubnetsPayload { source_subnet, destination_subnet, + height, + time_ns, + state_hash, + initial_dkg_subnet_id, } = payload; if source_subnet == destination_subnet { @@ -31,17 +74,18 @@ impl Registry { )); } - let version = self.latest_version(); + let pre_call_registry_version = self.latest_version(); - self.get(&make_subnet_record_key(source_subnet).into_bytes(), version) - .ok_or_else(|| format!("source {source_subnet} is not a known subnet"))?; self.get( - &make_subnet_record_key(destination_subnet).into_bytes(), - version, + &make_subnet_record_key(source_subnet).into_bytes(), + pre_call_registry_version, ) - .ok_or_else(|| format!("destination {destination_subnet} is not a known subnet"))?; + .ok_or_else(|| format!("source {source_subnet} is not a known subnet"))?; + let destination_record = self + .get_subnet(destination_subnet, pre_call_registry_version) + .map_err(|_| format!("destination {destination_subnet} is not a known subnet"))?; - let routing_table = self.get_routing_table_or_panic(version); + let routing_table = self.get_routing_table_or_panic(pre_call_registry_version); let source_ranges = routing_table.ranges(source_subnet); if source_ranges.is_empty() { return Err(format!( @@ -52,7 +96,7 @@ impl Registry { // Rerouting the canister ID ranges of the source subnet would break any ongoing canister // migration out of those ranges: the migrated ranges would end up being hosted by the // destination subnet, which is not on the recorded migration trace. - if let Some(canister_migrations) = self.get_canister_migrations(version) + if let Some(canister_migrations) = self.get_canister_migrations(pre_call_registry_version) && !are_disjoint(canister_migrations.ranges(), source_ranges.iter()) { return Err(format!( @@ -60,12 +104,148 @@ impl Registry { )); } - self.maybe_apply_mutation_internal(self.merge_subnets_mutation( - version, + // Recovering a subnet holding chain keys requires resharing those keys onto the recovery + // CUP, which this method does not do; rather than silently leave the destination subnet + // unable to sign, refuse to merge into it. + if destination_record + .chain_key_config + .as_ref() + .is_some_and(|config| !config.key_configs.is_empty()) + { + return Err(format!( + "destination subnet {destination_subnet} holds chain keys, which merging does not reshare" + )); + } + + // `setup_initial_dkg` must not be handled by the subnet being recovered, as that subnet is + // offline and could not respond. + if let Some(initial_dkg_subnet_id) = initial_dkg_subnet_id { + if initial_dkg_subnet_id == destination_subnet { + return Err(format!( + "initial DKG subnet {initial_dkg_subnet_id} must be different from the destination subnet" + )); + } + self.get( + &make_subnet_record_key(initial_dkg_subnet_id).into_bytes(), + pre_call_registry_version, + ) + .ok_or_else(|| { + format!("initial DKG subnet {initial_dkg_subnet_id} is not a known subnet") + })?; + } + + let mut cup_contents = self + .get_subnet_catch_up_package(destination_subnet, Some(pre_call_registry_version)) + .map_err(|err| format!("failed to get the CUP of {destination_subnet}: {err}"))?; + cup_contents.registry_store_uri = None; + + let mut subnet_record = destination_record; + + // Bring the destination subnet back online. Consensus looks at the registry version from + // the highest CUP when considering `halt_at_cup_height`, so clearing both flags is what + // makes the subnet resume from the recovery CUP created below. + subnet_record.halt_at_cup_height = false; + subnet_record.is_halted = false; + + let dkg_nodes: Vec = subnet_record + .membership + .iter() + .map(|bytes| NodeId::from(PrincipalId::try_from(bytes).unwrap())) + .collect(); + + let request = SetupInitialDKGArgs::new( + dkg_nodes, + RegistryVersion::new(pre_call_registry_version), + initial_dkg_subnet_id, + ); + let response_bytes = call( + CanisterId::ic_00(), + "setup_initial_dkg", + bytes, + Encode!(&request).unwrap(), + ) + .await + .unwrap_or_else(|(code, msg)| { + panic!("{LOG_PREFIX}`setup_initial_dkg` failed with code {code:?}: {msg}") + }); + + let post_call_registry_version = self.latest_version(); + + // Check that the records this method is about to overwrite, and the routing table it based + // its validation on, did not change while `setup_initial_dkg` was in flight. + for (key, what) in [ + ( + make_subnet_record_key(destination_subnet), + format!("Subnet with ID {destination_subnet}"), + ), + ( + make_crypto_threshold_signing_pubkey_key(destination_subnet), + format!("Threshold Signing Pubkey for Subnet {destination_subnet}"), + ), + ( + make_catch_up_package_contents_key(destination_subnet), + format!("CUP for Subnet {destination_subnet}"), + ), + ( + make_subnet_record_key(source_subnet), + format!("Subnet with ID {source_subnet}"), + ), + ] { + panic_if_record_changed_across_versions( + self, + &key, + pre_call_registry_version, + post_call_registry_version, + format!("{what} was updated during the `setup_initial_dkg` call"), + ); + } + assert_eq!( + self.get_routing_table_or_panic(post_call_registry_version) + .ranges(source_subnet), + source_ranges, + "{LOG_PREFIX}The canister ID ranges of subnet {source_subnet} were updated during the \ + `setup_initial_dkg` call", + ); + + let dkg_response = SetupInitialDKGResponse::decode(&response_bytes).unwrap(); + + cup_contents.initial_ni_dkg_transcript_low_threshold = + Some(dkg_response.low_threshold_transcript_record); + cup_contents.initial_ni_dkg_transcript_high_threshold = + Some(dkg_response.high_threshold_transcript_record); + cup_contents.height = height; + cup_contents.time = time_ns; + cup_contents.state_hash = state_hash.clone(); + cup_contents.cup_type = Some(CupType::Recovery(RecoveryArgs { + height, + time: time_ns, + state_hash, + })); + + let mut mutations = vec![ + RegistryMutation { + mutation_type: registry_mutation::Type::Update as i32, + key: make_crypto_threshold_signing_pubkey_key(destination_subnet).into_bytes(), + value: dkg_response.subnet_threshold_public_key.encode_to_vec(), + }, + RegistryMutation { + mutation_type: registry_mutation::Type::Update as i32, + key: make_catch_up_package_contents_key(destination_subnet).into_bytes(), + value: cup_contents.encode_to_vec(), + }, + upsert( + make_subnet_record_key(destination_subnet), + subnet_record.encode_to_vec(), + ), + ]; + mutations.append(&mut self.merge_subnets_mutation( + post_call_registry_version, source_subnet, destination_subnet, )); + self.maybe_apply_mutation_internal(mutations); + Ok(()) } } @@ -77,8 +257,22 @@ pub struct MergeSubnetsPayload { /// set of `destination_subnet`. pub source_subnet: SubnetId, /// The subnet that hosts the canister ID ranges of `source_subnet` after the - /// merge. + /// merge, and that is recovered at the merged state and brought back online. pub destination_subnet: SubnetId, + /// The height of the recovery CUP of `destination_subnet`, i.e. the height + /// of the checkpoint holding the merged state. + pub height: u64, + /// The block time the recovered `destination_subnet` starts from, in + /// nanoseconds since the Epoch. Must be larger than the times of the + /// checkpoints at which both subnets were taken offline. + pub time_ns: u64, + /// The hash of the manifest of the merged state. + pub state_hash: Vec, + /// The subnet that should handle the `setup_initial_dkg` call producing the + /// DKG transcripts of the recovery CUP. Must be different from + /// `destination_subnet`, which is offline while the merge is in progress. If + /// unset, the request is handled by the NNS subnet. + pub initial_dkg_subnet_id: Option, } #[cfg(test)] @@ -94,11 +288,25 @@ mod tests { routing_table::routing_table_into_registry_mutation, }, }; + use futures::executor::block_on; use ic_registry_routing_table::{CanisterIdRange, RoutingTable}; use ic_types::CanisterId; use ic_types_test_utils::ids::{SUBNET_1, SUBNET_2, SUBNET_3}; use maplit::btreemap; + /// The recovery CUP fields of the payload, which the validation-failure tests + /// below are not about: they all fail before the CUP is even looked at. + fn default_cup_args() -> MergeSubnetsPayload { + MergeSubnetsPayload { + source_subnet: SUBNET_1, + destination_subnet: SUBNET_2, + height: 100, + time_ns: 1_000_000_000, + state_hash: vec![1; 32], + initial_dkg_subnet_id: Some(SUBNET_3), + } + } + fn range(start: u64, end: u64) -> CanisterIdRange { CanisterIdRange { start: CanisterId::from_u64(start), @@ -149,24 +357,28 @@ mod tests { .collect::>() } + /// Applies just the routing table part of a merge. `Registry::merge_subnets` + /// itself cannot be driven to completion in a unit test, as it calls + /// `setup_initial_dkg` on the management canister half way through; the + /// success path as a whole is covered by the integration test in + /// `rs/registry/canister/tests/merge_subnets.rs`. + fn merge_routing_table(registry: &mut Registry, source: SubnetId, destination: SubnetId) { + let mutations = + registry.merge_subnets_mutation(registry.latest_version(), source, destination); + registry.maybe_apply_mutation_internal(mutations); + } + #[test] fn test_merge_subnets() { // Step 1: Prepare the world. let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { - source_subnet: SUBNET_1, - destination_subnet: SUBNET_2, - }); - - // Step 3: Verify results. - - // Step 3.1: Inspect the return value. - assert_eq!(result, Ok(())); + merge_routing_table(&mut registry, SUBNET_1, SUBNET_2); - // Step 3.2: The canister ID ranges of both subnets are now hosted by the - // destination subnet, and the three adjacent ranges got merged into one. + // Step 3: Verify results: the canister ID ranges of both subnets are now + // hosted by the destination subnet, and the three adjacent ranges got + // merged into one. assert_eq!( get_routing_table_entries(®istry), vec![(range(10, 39), SUBNET_2)], @@ -189,14 +401,10 @@ mod tests { )); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { - source_subnet: SUBNET_1, - destination_subnet: SUBNET_2, - }); + merge_routing_table(&mut registry, SUBNET_1, SUBNET_2); // Step 3: Verify results. Both ranges are hosted by the destination subnet // and, not being adjacent, did not get merged into a single entry. - assert_eq!(result, Ok(())); assert_eq!( get_routing_table_entries(®istry), vec![(range(10, 19), SUBNET_2), (range(30, 39), SUBNET_2)], @@ -209,10 +417,11 @@ mod tests { let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { + let result = block_on(registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_1, - }); + ..default_cup_args() + })); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -236,10 +445,11 @@ mod tests { let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { + let result = block_on(registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_3, destination_subnet: SUBNET_2, - }); + ..default_cup_args() + })); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -255,10 +465,11 @@ mod tests { let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { + let result = block_on(registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_3, - }); + ..default_cup_args() + })); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -281,10 +492,11 @@ mod tests { )); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { + let result = block_on(registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_2, - }); + ..default_cup_args() + })); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -308,10 +520,11 @@ mod tests { .unwrap(); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { + let result = block_on(registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_2, - }); + ..default_cup_args() + })); // Step 3: Verify results. let error_message = result.unwrap_err(); diff --git a/rs/registry/canister/tests/merge_subnets.rs b/rs/registry/canister/tests/merge_subnets.rs index 16139835f211..119fb6c1337e 100644 --- a/rs/registry/canister/tests/merge_subnets.rs +++ b/rs/registry/canister/tests/merge_subnets.rs @@ -6,6 +6,7 @@ use ic_nns_test_utils::{ }, registry::{initial_routing_table_mutations, prepare_registry_with_two_node_sets}, }; +use ic_protobuf::registry::subnet::v1::{RecoveryArgs, catch_up_package_contents::CupType}; use ic_registry_routing_table::{CanisterIdRange, RoutingTable}; use ic_registry_transport::pb::v1::RegistryAtomicMutateRequest; use ic_types::CanisterId; @@ -14,13 +15,25 @@ use registry_canister::{ }; mod common; -use common::test_helpers::{check_error_message, check_subnet_for_canisters}; +use common::test_helpers::{ + check_error_message, check_subnet_for_canisters, get_cup_contents, get_subnet_record, +}; + +/// The recovery CUP the merge creates for the destination subnet. The values are +/// arbitrary: this test does not run a subnet from the resulting CUP, it only +/// checks that the endpoint accepts them and records them. +const MERGE_HEIGHT: u64 = 100; +const MERGE_TIME_NS: u64 = 1_234_567_890; +const MERGED_STATE_HASH: &[u8] = &[42; 32]; /// Exercises the `merge_subnets` endpoint end to end. The payload validation /// itself is covered by the unit tests of `Registry::merge_subnets`, so this test /// only covers what those cannot: that the endpoint is reachable with a Candid -/// encoded payload, that only governance may call it, and that the resulting -/// routing table is visible through the canister's query API. +/// encoded payload, that only governance may call it, that the canisters of the +/// source subnet end up routed to the destination subnet, and that the recovery +/// CUP of the destination subnet -- whose DKG transcripts come from the +/// `setup_initial_dkg` call the endpoint makes half way through -- is recorded +/// and the destination subnet is brought back online. #[test] fn test_merge_subnets() { state_machine_test_on_nns_subnet(|runtime| { @@ -76,6 +89,10 @@ fn test_merge_subnets() { MergeSubnetsPayload { source_subnet: subnet_id_1, destination_subnet: subnet_id_2, + height: MERGE_HEIGHT, + time_ns: MERGE_TIME_NS, + state_hash: MERGED_STATE_HASH.to_vec(), + initial_dkg_subnet_id: None, }, ) .await as Result<(), String>, @@ -90,6 +107,10 @@ fn test_merge_subnets() { Encode!(&MergeSubnetsPayload { source_subnet: subnet_id_1, destination_subnet: subnet_id_2, + height: MERGE_HEIGHT, + time_ns: MERGE_TIME_NS, + state_hash: MERGED_STATE_HASH.to_vec(), + initial_dkg_subnet_id: None, }) .unwrap(), ) @@ -109,6 +130,41 @@ fn test_merge_subnets() { ) .await; + // Step 5: Verify results: the destination subnet got a recovery CUP at the + // height, time and state hash of the merged state, with fresh DKG + // transcripts, and is no longer halted. + let cup_contents = get_cup_contents(®istry, subnet_id_2).await; + assert_eq!(cup_contents.height, MERGE_HEIGHT); + assert_eq!(cup_contents.time, MERGE_TIME_NS); + assert_eq!(cup_contents.state_hash, MERGED_STATE_HASH); + assert_eq!( + cup_contents.cup_type, + Some(CupType::Recovery(RecoveryArgs { + height: MERGE_HEIGHT, + time: MERGE_TIME_NS, + state_hash: MERGED_STATE_HASH.to_vec(), + })), + ); + assert!( + cup_contents + .initial_ni_dkg_transcript_low_threshold + .is_some(), + "the recovery CUP should hold a low threshold DKG transcript", + ); + assert!( + cup_contents + .initial_ni_dkg_transcript_high_threshold + .is_some(), + "the recovery CUP should hold a high threshold DKG transcript", + ); + + let subnet_record = get_subnet_record(®istry, subnet_id_2).await; + assert!( + !subnet_record.is_halted, + "the destination subnet should have been brought back online", + ); + assert!(!subnet_record.halt_at_cup_height); + Ok(()) } }); diff --git a/rs/registry/canister/unreleased_changelog.md b/rs/registry/canister/unreleased_changelog.md index e83c068094e7..0c084bccda81 100644 --- a/rs/registry/canister/unreleased_changelog.md +++ b/rs/registry/canister/unreleased_changelog.md @@ -14,10 +14,22 @@ on the process that this file is part of, see be set on mainnet before the replica version rejecting ingress messages to cooling down subnets has been rolled out to all subnets. -* `merge_subnets` endpoint. It takes a source and a destination subnet ID, and merges the canister - ID ranges of the source subnet into the canister ID range set of the destination subnet, i.e., the - canisters hosted by the source subnet are routed to the destination subnet afterwards. Only the - routing table is updated: neither subnet record is modified and the source subnet is not deleted. +* `merge_subnets` endpoint, callable through a `MergeSubnets` proposal. It takes a source and a + destination subnet ID, plus the height, time and state hash of the merged state, and performs the + whole registry-side part of a subnet merge in a single registry version: + + * the canister ID ranges of the source subnet are merged into the canister ID range set of the + destination subnet, i.e., the canisters hosted by the source subnet are routed to the + destination subnet afterwards; + * a recovery catch-up package is created for the destination subnet, at the given height, time + and state hash, running a fresh DKG for the destination subnet's membership; and + * the destination subnet is brought back online. + + The caller is expected to have taken both subnets offline and to have extended the state of the + destination subnet with the state of the canisters of the source subnet beforehand. The source + subnet record is not modified and the source subnet is not deleted: it merely does not host any + canister ID range anymore. Merging into a subnet holding chain keys is rejected, as the recovery + catch-up package does not reshare them. ## Changed diff --git a/rs/state_machine_tests/src/lib.rs b/rs/state_machine_tests/src/lib.rs index 2bf6c8f96f16..b1f51d275d71 100644 --- a/rs/state_machine_tests/src/lib.rs +++ b/rs/state_machine_tests/src/lib.rs @@ -1846,6 +1846,42 @@ impl Default for StateMachineBuilder { } } +/// The responses consensus would produce for the pending `setup_initial_dkg` +/// requests of `state`: a dummy transcript per request, derived from `seed` so +/// that the result stays deterministic. +/// +/// `setup_initial_dkg` can only be called on the NNS subnet, so the seed does +/// not need to depend on the subnet ID. +fn setup_initial_dkg_responses(state: &ReplicatedState, seed: u64) -> Vec { + let mut rng = StdRng::seed_from_u64(seed); + state + .metadata + .subnet_call_context_manager + .setup_initial_dkg_contexts + .keys() + .map(|callback_id| { + let ni_dkg_transcript = dummy_initial_dkg_transcript_with_master_key(&mut rng).0; + let public_key = (&ni_dkg_transcript).try_into().unwrap(); + let public_key_der = threshold_sig_public_key_to_der(public_key).unwrap(); + let subnet_id = PrincipalId::new_self_authenticating(&public_key_der).into(); + let mut high_threshold_transcript_record = ni_dkg_transcript.clone(); + high_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::HighThreshold; + let mut low_threshold_transcript_record = ni_dkg_transcript; + low_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::LowThreshold; + let initial_transcript_records = SetupInitialDKGResponse { + low_threshold_transcript_record: high_threshold_transcript_record.into(), + high_threshold_transcript_record: low_threshold_transcript_record.into(), + fresh_subnet_id: subnet_id, + subnet_threshold_public_key: public_key.into(), + }; + ConsensusResponse::new( + *callback_id, + MsgPayload::Data(initial_transcript_records.encode()), + ) + }) + .collect() +} + impl StateMachine { /// Provides the implicit time increment for a single round of execution /// if time does not advance between consecutive rounds. @@ -1960,34 +1996,7 @@ impl StateMachine { } let self_validating = Some(batch_payload.self_validating); let mut consensus_responses = http_responses; - // `setup_initial_dkg` can only be called on the NNS subnet - // and thus the seed does not need to depend on the subnet ID - let mut rng = StdRng::seed_from_u64(certified_height.get()); - for callback_id in state - .metadata - .subnet_call_context_manager - .setup_initial_dkg_contexts - .keys() - { - let ni_dkg_transcript = dummy_initial_dkg_transcript_with_master_key(&mut rng).0; - let public_key = (&ni_dkg_transcript).try_into().unwrap(); - let public_key_der = threshold_sig_public_key_to_der(public_key).unwrap(); - let subnet_id = PrincipalId::new_self_authenticating(&public_key_der).into(); - let mut high_threshold_transcript_record = ni_dkg_transcript.clone(); - high_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::HighThreshold; - let mut low_threshold_transcript_record = ni_dkg_transcript; - low_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::LowThreshold; - let initial_transcript_records = SetupInitialDKGResponse { - low_threshold_transcript_record: high_threshold_transcript_record.into(), - high_threshold_transcript_record: low_threshold_transcript_record.into(), - fresh_subnet_id: subnet_id, - subnet_threshold_public_key: public_key.into(), - }; - consensus_responses.push(ConsensusResponse::new( - *callback_id, - MsgPayload::Data(initial_transcript_records.encode()), - )); - } + consensus_responses.extend(setup_initial_dkg_responses(&state, certified_height.get())); let mut payload = PayloadBuilder::new() .with_ingress_messages(ingress_messages) .with_xnet_payload(xnet_payload) @@ -3035,6 +3044,14 @@ impl StateMachine { self.process_threshold_signing_request(id, context, &mut payload_builder); } + // Process `setup_initial_dkg` requests, which consensus would answer. + payload_builder + .consensus_responses + .extend(setup_initial_dkg_responses( + &state, + self.state_manager.latest_state_height().get(), + )); + self.execute_payload(payload_builder); } From 691f033ce3d2c59f16de8499c8a3be2841aca297 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 31 Aug 2026 10:09:26 +0000 Subject: [PATCH 06/30] feat(state_tool): print the batch time of a checkpoint Add a `checkpoint_time` subcommand printing the `batch_time_nanos` of the system metadata of a checkpoint, i.e. the IC time the subnet had reached when it wrote that checkpoint. Merging a subnet into another one has to pick the block time the destination subnet resumes from, which must be larger than the times of the checkpoints at which both subnets were taken offline. Nothing printed a checkpoint's time so far, so there was no way to check that from outside the replica. Co-Authored-By: Claude Opus 5 --- rs/state_tool/src/commands.rs | 1 + rs/state_tool/src/commands/checkpoint_time.rs | 24 +++++++++++++++++++ rs/state_tool/src/main.rs | 9 +++++++ 3 files changed, 34 insertions(+) create mode 100644 rs/state_tool/src/commands/checkpoint_time.rs diff --git a/rs/state_tool/src/commands.rs b/rs/state_tool/src/commands.rs index 20d529f1d2a0..dd847c79d3bc 100644 --- a/rs/state_tool/src/commands.rs +++ b/rs/state_tool/src/commands.rs @@ -2,6 +2,7 @@ pub mod canister_metrics; pub mod cdiff; pub mod chash; +pub mod checkpoint_time; pub mod convert_ids; pub mod copy; pub mod decode; diff --git a/rs/state_tool/src/commands/checkpoint_time.rs b/rs/state_tool/src/commands/checkpoint_time.rs new file mode 100644 index 000000000000..e32c3d206b14 --- /dev/null +++ b/rs/state_tool/src/commands/checkpoint_time.rs @@ -0,0 +1,24 @@ +//! Prints the batch time of a checkpoint. + +use ic_state_layout::CompleteCheckpointLayout; +use ic_types::Height; +use std::path::PathBuf; + +const HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED: Height = Height::new(0); + +/// Prints the batch time of the checkpoint rooted at `path`, in nanoseconds +/// since the Epoch, i.e. the IC time the subnet had reached when it wrote the +/// checkpoint. +pub fn do_print_checkpoint_time(path: PathBuf) -> Result<(), String> { + let checkpoint_layout = + CompleteCheckpointLayout::new_untracked(path, HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED) + .map_err(|err| format!("Failed to create CheckpointLayout: {err:?}"))?; + let system_metadata = checkpoint_layout + .system_metadata() + .deserialize() + .map_err(|err| format!("Failed to read the system metadata: {err:?}"))?; + + println!("{}", system_metadata.batch_time_nanos); + + Ok(()) +} diff --git a/rs/state_tool/src/main.rs b/rs/state_tool/src/main.rs index 06c8dfcbb346..18d3f8437f58 100644 --- a/rs/state_tool/src/main.rs +++ b/rs/state_tool/src/main.rs @@ -68,6 +68,14 @@ enum Opt { path: PathBuf, }, + /// Prints the batch time of a checkpoint, in nanoseconds since the Epoch. + #[clap(name = "checkpoint_time")] + CheckpointTime { + /// Path to a checkpoint. + #[clap(long = "state")] + path: PathBuf, + }, + /// Verifies whether the textual representation /// of a manifest matches its root hash. #[clap(name = "verify_manifest")] @@ -256,6 +264,7 @@ pub(crate) fn main_inner(args: Vec) { heights, } => commands::copy::do_copy(source, destination, heights.into()), Opt::Manifest { path } => commands::manifest::do_compute_manifest(path), + Opt::CheckpointTime { path } => commands::checkpoint_time::do_print_checkpoint_time(path), Opt::VerifyManifest { file } => commands::verify_manifest::do_verify_manifest(&file), Opt::ListStates { config } => commands::list::do_list(config), Opt::Decode { file } => commands::decode::do_decode(file), From e28d5aec6a2f29bcc3b08d03bb454449a1d6db5b Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 31 Aug 2026 13:37:52 +0000 Subject: [PATCH 07/30] feat(consensus): report at info level that a halted subnet delivers no batches A subnet halting because of the `halt_at_cup_height` flag of its subnet record logged nothing observable: the only trace it left was a `debug!`, which the nodes' log level drops, so nothing outside the replica could tell that such a subnet had come to a stop. The `is_halted` flag, in contrast, is reported at info level by `ConsensusImpl::on_state_change`. Raise that message to info level, so that both ways of halting a subnet can be observed the same way. It is already emitted at most once every five seconds, so this does not make the logs noticeably busier, and it names the height of the first batch that is not delivered, i.e. one past the state the subnet stopped in. Co-Authored-By: Claude Opus 5 --- rs/consensus/src/consensus/batch_delivery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/consensus/src/consensus/batch_delivery.rs b/rs/consensus/src/consensus/batch_delivery.rs index 162703ad4dff..039f319e1b3c 100644 --- a/rs/consensus/src/consensus/batch_delivery.rs +++ b/rs/consensus/src/consensus/batch_delivery.rs @@ -167,7 +167,7 @@ pub(crate) fn deliver_batches_with_result_processor( log, ) { Some(Status::Halting | Status::Halted) => { - debug!( + info!( every_n_seconds => 5, log, "Batch of height {} is not delivered because replica is halted", From bdd0bc9579ebf1c1e66cecdbe3978d783bbe4065 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 31 Aug 2026 13:38:13 +0000 Subject: [PATCH 08/30] test: merge a subnet that is "cooling down" into another subnet Extend the `cooling_down` system test past the point where the subnet is "merge ready": it now goes on to merge that subnet away and to delete it, which is what the cooling down and the merge readiness condition exist for. The destination is a third Application subnet `R`; the NNS subnet stays available throughout, as it is where the proposals of the test are executed, including the one recovering `R` at the merged state. The merge itself, once both subnets are "merge ready": - Both subnets are told to halt at their next CUP and are waited for. The wait reads each node's journal, where a halted replica reports that it delivers no batches, rather than watching for the height of the latest checkpoint to stop advancing: checkpoints are written once per DKG interval, i.e. minutes apart, so that height looks stable long before the subnet halts. Its checkpoint is then hundreds of rounds behind the state the merge readiness was established on, and holds, for instance, an `install_code` aborted at that checkpoint and only completed afterwards -- whose call context and reserved response slot the merge then drops, so that the destination subnet hits `[EXC-BUG] Could not find any install code call ...` and panics on resuming it. - The state of the merge is assembled from the two checkpoints the subnets halted at, as a new checkpoint of `R` at the next multiple of its DKG interval, so that `R`'s own checkpoint stays untouched: the canisters and canister snapshots of `M` are added to those of `R`, and the result is marked as the product of a subnet merge. Taking the system metadata and subnet queues of `R` wholesale is sound because those of `M` are empty, which is what the merge readiness established and what `M` cooling down preserves until it halts. - The block time of the merged state is picked past both checkpoint times, read with `state-tool checkpoint_time`, and its manifest hash with `state-tool manifest`. - The merged state is added to `R`'s checkpoints while its replica is stopped, and the replica is only started after the `MergeSubnets` proposal created the recovery CUP; the test then waits for `R` to report exactly that CUP, so that a subnet resuming from the wrong state fails here rather than much later. - The three endless loops are ended and every ingress message that was in progress across the merge is checked to have completed. This is what the `subnet_merged` marker is for: the ingress history of `M` is deliberately not merged in, the marker makes the replica re-register those messages instead. - Finally, once every subnet observed the merge, `M` -- which hosts no canister ID range anymore -- is deleted, and is checked to be gone from the registry. `UniversalCanister::submit_update` now returns the ID of the message it submitted, which is what lets the test track the ingress messages that outlive the merge. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 4 + rs/tests/driver/src/util.rs | 26 +- rs/tests/message_routing/BUILD.bazel | 9 +- rs/tests/message_routing/Cargo.toml | 4 + .../subnet_cooling_down_test.rs | 976 +++++++++++++++++- 5 files changed, 956 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61ff5a1bf87c..bb2c3ad9cabd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17681,11 +17681,14 @@ dependencies = [ "candid", "canister-test", "dfn_candid", + "hex", "ic-agent", "ic-base-types", "ic-management-canister-types 0.8.0", "ic-nns-governance-api", + "ic-recovery", "ic-registry-subnet-type", + "ic-state-layout", "ic-system-test-driver", "ic-types", "ic-universal-canister", @@ -17698,6 +17701,7 @@ dependencies = [ "slog", "tempfile", "tokio", + "url", "xnet-test", ] diff --git a/rs/tests/driver/src/util.rs b/rs/tests/driver/src/util.rs index e165c020ff9d..42eaa8aaf5b7 100644 --- a/rs/tests/driver/src/util.rs +++ b/rs/tests/driver/src/util.rs @@ -16,7 +16,7 @@ use futures::{ future::{join_all, select_all, try_join_all}, }; use ic_agent::{ - Agent, AgentError, Identity, Signature, + Agent, AgentError, Identity, RequestId, Signature, agent::{ CallResponse, EnvelopeContent, RejectCode, RejectResponse, http_transport::reqwest_transport::reqwest, @@ -595,15 +595,27 @@ impl<'a> UniversalCanister<'a> { } /// Submits `payload` as an ingress message to the canister's `update` - /// method without waiting for the call to complete. Useful for update calls - /// that are not expected to ever complete. - pub async fn submit_update>>(&self, payload: P) -> Result<(), AgentError> { - self.agent + /// method without waiting for the call to complete, and returns the ID of + /// the submitted message, so that its status can be polled later. `None` if + /// the call happened to complete before the submission returned, in which + /// case there is nothing left to poll for. + /// + /// Useful for update calls that are not expected to complete for a long + /// time, or at all. + pub async fn submit_update>>( + &self, + payload: P, + ) -> Result, AgentError> { + let response = self + .agent .update(&self.canister_id, "update") .with_arg(payload.into()) .call() - .await - .map(|_| ()) + .await?; + Ok(match response { + CallResponse::Response(_) => None, + CallResponse::Poll(request_id) => Some(request_id), + }) } } diff --git a/rs/tests/message_routing/BUILD.bazel b/rs/tests/message_routing/BUILD.bazel index a93deef5c6e3..bf1dfd1efa4f 100644 --- a/rs/tests/message_routing/BUILD.bazel +++ b/rs/tests/message_routing/BUILD.bazel @@ -147,19 +147,26 @@ system_test_nns( "long_test", # since the subnet only quiesces once its long-running `install_code` calls are done and the ingress history is pruned. ], test_timeout = "eternal", - runtime_deps = UNIVERSAL_CANISTER_RUNTIME_DEPS, + runtime_deps = UNIVERSAL_CANISTER_RUNTIME_DEPS | { + "ENV_DEPS__STATE_TOOL": "//rs/state_tool:state-tool", + }, deps = [ # Keep sorted. "//rs/nns/governance/api", + "//rs/recovery", "//rs/registry/canister", "//rs/registry/subnet_type", + "//rs/state_layout", "//rs/tests/driver:ic-system-test-driver", "//rs/types/types", "//rs/universal_canister/lib", "@crate_index//:anyhow", "@crate_index//:candid", + "@crate_index//:hex", + "@crate_index//:ic-agent", "@crate_index//:slog", "@crate_index//:tokio", + "@crate_index//:url", ], ) diff --git a/rs/tests/message_routing/Cargo.toml b/rs/tests/message_routing/Cargo.toml index aba3de1f93ec..0e5c860aa3af 100644 --- a/rs/tests/message_routing/Cargo.toml +++ b/rs/tests/message_routing/Cargo.toml @@ -8,6 +8,7 @@ documentation.workspace = true [dependencies] anyhow = { workspace = true } +hex = { workspace = true } candid = { workspace = true } canister-test = { path = "../../rust_canisters/canister_test" } dfn_candid = { path = "../../rust_canisters/dfn_candid" } @@ -15,7 +16,9 @@ ic-agent = { workspace = true } ic-base-types = { path = "../../types/base_types" } ic-management-canister-types = { workspace = true } ic-nns-governance-api = { path = "../../nns/governance/api" } +ic-recovery = { path = "../../recovery" } ic-registry-subnet-type = { path = "../../registry/subnet_type" } +ic-state-layout = { path = "../../state_layout" } ic-system-test-driver = { path = "../driver" } ic-types = { path = "../../types/types" } ic-universal-canister = { path = "../../universal_canister/lib" } @@ -28,6 +31,7 @@ rejoin-test-lib = { path = "./rejoin_test_lib" } slog = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } +url = { workspace = true } xnet-test = { path = "../../rust_canisters/xnet_test" } [[bin]] diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs index cd13195af02e..dd32a6af94ce 100644 --- a/rs/tests/message_routing/subnet_cooling_down_test.rs +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -10,18 +10,20 @@ reaches the "merge readiness" condition of the `Subnet merging` dashboard (see version at which the subnet was labeled "cooling down" and a pending refund budget (the dashboard's `R`) of 0 cycles. -The subnet that is cooling down, i.e. the one that would be merged, is called -`M`; `R` is the root (NNS) subnet it would be merged into, which holds nothing -but the NNS canisters; `T` and `S` are two further Application subnets holding -the canisters at the other end of `M`'s cross-subnet calls. +The subnet that is cooling down, i.e. the one that is merged away, is called `M` +and the Application subnet it is merged into is called `R`. A third Application +subnet `T` holds the canisters at the other end of `M`'s cross-subnet calls. The +NNS subnet is none of these: it has to stay available throughout, as it is where +the proposals of this test are executed, including the one recovering `R` at the +merged state. "Executing" an update call below always means submitting it as an ingress message without waiting for it to complete: most of the calls of this test are never meant to complete. Runbook:: -0. Set up an IC with an NNS subnet `R` (with the NNS canisters installed) and - three Application subnets `M`, `T` and `S`. +0. Set up an IC with an NNS subnet (with the NNS canisters installed) and three + Application subnets `M`, `T` and `R`. 1. Install a universal canister on each of `M` and `T`: `US` on `M`, `UT` on `T`. 2. Make an ingress call to each of `US` and `UT` with a payload that calls the universal canister on the other subnet in a loop: the reply (or reject) @@ -29,7 +31,7 @@ Runbook:: 3. Wait until both loops have completed a few iterations, i.e. messages are actually flowing between `M` and `T` in both directions. 4. Install the universal canisters of the steps below (`U1`, `U3`, `U5` and - `U6` on `M`, `U4` and `U7` on `S`) and create five empty canisters + `U6` on `M`, `U4` and `U7` on `T`) and create five empty canisters `U2a` .. `U2e` on `M`, controlled by `U1`. All of this has to happen before step 5: a long-running `install_code` blocks every other `install_code` on the same subnet. @@ -43,10 +45,10 @@ Runbook:: 6. Start three endless loops, each of which runs until the global data of the looping canister is set to `LOOP_BREAK_TRIGGER`, which this test never does: a. execute an update call on `U3` that loops on `U3` itself; - b. execute an update call on `U4` (on `S`) that calls `U5` (on `M`) with the + b. execute an update call on `U4` (on `T`) that calls `U5` (on `M`) with the loop as its payload, so that `M` holds a canister looping in a call from another subnet that it can never respond to; - c. execute an update call on `U6` (on `M`) that calls `U7` (on `S`) with the + c. execute an update call on `U6` (on `M`) that calls `U7` (on `T`) with the loop as its payload, so that `M` holds a canister waiting for a response from another subnet that never arrives. Wait until all three loops are running. @@ -69,23 +71,70 @@ Runbook:: counters, read via queries, no longer advance): while `M` is cooling down, neither `M` nor `T` routes any message to or from `M`, so the messages of both loops are retained in their senders' output queues. +12. Submit (and adopt) `UpdateConfigOfSubnet` NNS proposals setting the + `halt_at_cup_height` flag of both `M` and `R`, and wait until each of their + nodes reports in its journal that it is halted. Record the heights of the + checkpoints they halted at. +13. Stop the replicas of both subnets and download the states they halted at. + Assemble the merged state as a new checkpoint of `R`, at the next multiple of + the DKG interval after the height `R` halted at, so that `R`'s own checkpoint + is left untouched: the canisters and canister snapshots of `M` are added to + those of `R`, and the result is marked as the product of a subnet merge. The + ingress history of `M` is deliberately not merged in: the marker makes the + replica re-register the ingress messages of the merged-in canisters that are + still in progress. Compute the block time the merged state starts from, which + must be larger than the times of both checkpoints, and the hash of its + manifest. +14. Add the merged state to the checkpoints of `R`'s node, leaving its replica + stopped. +15. Submit (and adopt) a `MergeSubnets` NNS proposal for `M` and `R`, which + reroutes the canister ID ranges of `M` to `R`, creates a recovery CUP for `R` + at the merged state and brings `R` back online, all in one registry version. +16. Start `R`'s replica and wait until it is healthy. Only now: a replica started + before the recovery CUP exists resumes from the checkpoint `R` halted at, + which does not hold the canisters of `M`. +17. Set the global data of `U3`, `U5` and `U7` to `LOOP_BREAK_TRIGGER`, ending + the three endless loops, and check that every ingress message that was in + progress across the merge completed. `U3` and `U5` are reached through `R`, + which serves the canisters of `M` after the merge. +18. Wait until every subnet other than `M` has reached the registry version the + merge created, i.e. routes the canisters that used to be hosted by `M` to + `R`. `M` itself is excluded: its replica was stopped for the merge and it is + about to be deleted. +19. Submit (and adopt) a `DeleteSubnet` NNS proposal deleting `M`, which hosts no + canister ID range anymore, and check that it is gone from the registry. Success:: `M` becomes "merge ready", with `U2a` .. `U2e` installed, while both loops of -step 2 are stalled. +step 2 are stalled; the merge moves its canisters to `R`, where every ingress +message that was in progress across the merge completes; and `M` can then be +deleted. end::catalog[] */ use anyhow::{Result, anyhow, bail}; use candid::Principal; +use ic_agent::{Agent, RequestId, agent::RequestStatusResponse}; use ic_nns_governance_api::NnsFunction; +use ic_recovery::registry_helper::RegistryPollingStrategy; +use ic_recovery::ssh_helper::SshHelper; +use ic_recovery::steps::Step; +use ic_recovery::util::SshUser; +use ic_recovery::{IC_STATE_DIR, Recovery, RecoveryArgs}; use ic_registry_subnet_type::SubnetType; +use ic_state_layout::{ + CANISTER_STATES_DIR, SNAPSHOTS_DIR, SUBNET_MERGED_FILE, StateLayout, + UNVERIFIED_CHECKPOINT_MARKER, +}; +use ic_system_test_driver::driver::constants::SSH_USERNAME; +use ic_system_test_driver::driver::driver_setup::SSH_AUTHORIZED_PRIV_KEYS_DIR; use ic_system_test_driver::driver::group::SystemTestGroup; use ic_system_test_driver::driver::ic::{InternetComputer, Subnet}; use ic_system_test_driver::driver::test_env::TestEnv; use ic_system_test_driver::driver::test_env_api::{ - HasPublicApiUrl, HasRegistryVersion, HasTopologySnapshot, IcNodeContainer, - NnsInstallationBuilder, READY_WAIT_TIMEOUT, RETRY_BACKOFF, SubnetSnapshot, TopologySnapshot, + HasPublicApiUrl, HasRegistryVersion, HasTopologySnapshot, IcNodeContainer, IcNodeSnapshot, + NnsInstallationBuilder, READY_WAIT_TIMEOUT, RETRY_BACKOFF, SshSession, SubnetSnapshot, + TopologySnapshot, get_dependency_path_from_env, }; use ic_system_test_driver::nns::{ get_governance_canister, submit_external_proposal_with_test_id, @@ -94,18 +143,24 @@ use ic_system_test_driver::nns::{ use ic_system_test_driver::retry_with_msg_async; use ic_system_test_driver::systest; use ic_system_test_driver::util::{ - MetricsFetcher, UniversalCanister, assert_create_agent, block_on, create_canister, - runtime_from_url, set_controller, + JournalStreamer, MetricsFetcher, UniversalCanister, assert_create_agent, block_on, + create_canister, runtime_from_url, set_controller, }; -use ic_types::SubnetId; +use ic_types::{Height, SubnetId}; use ic_universal_canister::management::InstallMode; use ic_universal_canister::{ CallInterface, call_args, get_universal_canister_wasm, management, wasm, }; +use registry_canister::mutations::do_delete_subnet::DeleteSubnetPayload; use registry_canister::mutations::do_update_subnet::UpdateSubnetPayload; +use registry_canister::mutations::merge_subnets::MergeSubnetsPayload; use slog::{Logger, info}; use std::collections::BTreeMap; +use std::net::IpAddr; +use std::path::{Path, PathBuf}; +use std::process::Command; use std::time::Duration; +use url::Url; /// Metrics making up the "merge readiness" condition. const METRIC_REGISTRY_VERSION: &str = "mr_registry_version"; @@ -115,12 +170,21 @@ const METRIC_SUBNET_INPUT_QUEUE_MESSAGES: &str = "execution_subnet_input_queue_m const METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES: &str = "execution_subnet_output_queue_messages"; const METRIC_SUBNET_CALL_CONTEXTS: &str = "replicated_state_subnet_call_contexts"; const METRIC_PENDING_REFUNDS_CYCLES: &str = "replicated_state_pending_refunds_cycles"; +/// Timeout for a subnet to halt at its next CUP, which is up to a full DKG +/// interval away. +const HALT_TIMEOUT: Duration = Duration::from_secs(900); +/// Backoff between two searches of a node's journal for the halt message. +const HALT_BACKOFF: Duration = Duration::from_secs(10); + +/// What a halted replica logs, once every few seconds, instead of delivering the +/// batches it would otherwise deliver (see `rs/consensus/src/consensus/batch_delivery.rs`). +const HALTED_LOG_PATTERN: &str = "is not delivered because replica is halted"; /// The label selecting the `install_code` call contexts of /// `METRIC_SUBNET_CALL_CONTEXTS`. const LABEL_INSTALL_CODE: &str = "type=\"install_code\""; -/// `R` in the dashboard's readiness condition: the maximum total value in +/// The dashboard's `R` in the readiness condition: the maximum total value in /// cycles of the pending anonymous refunds of the cooling down subnet. (Not to /// be confused with the subnet `R` of the runbook above.) const MAX_REFUND_VALUE_CYCLES: f64 = 0.0; @@ -156,6 +220,27 @@ const INIT_INSTRUCTIONS: u64 = 295 * B; /// `U7`. The test never sets it, so those loops never end. const LOOP_BREAK_TRIGGER: &[u8] = b"break"; +/// The DKG interval length of the Application subnets, i.e. one less than the +/// distance between two consecutive checkpoints (and CUPs). The default is long +/// enough for an `install_code` burning `INIT_INSTRUCTIONS` to complete within +/// one interval, which matters because a paused `install_code` is aborted at +/// every checkpoint and has to start over afterwards. +const DKG_INTERVAL_LENGTH: u64 = 499; +/// The distance between two consecutive checkpoint (and CUP) heights. +const CHECKPOINT_INTERVAL: u64 = DKG_INTERVAL_LENGTH + 1; + +/// How much later than the checkpoints it is assembled from the merged state +/// starts, i.e. the block time of the recovery CUP of `R` minus the larger of +/// the two checkpoint times. +const MERGED_STATE_TIME_MARGIN: Duration = Duration::from_secs(60); + +/// Timeout for an ingress message that was in progress across the merge to +/// complete once the loop it is waiting for is broken. Generous because the +/// destination subnet has just resumed from the merged state and is busy +/// recomputing its manifest and draining the message loops of step 2 at the same +/// time: this has been observed to take up to four minutes. +const INGRESS_COMPLETION_TIMEOUT: Duration = Duration::from_secs(900); + /// Timeout for the subnet to become "merge ready". The binding terms are the /// `install_code` calls of step 5, which take a couple of hundred rounds each /// (and are executed one at a time, as at most one long-running `install_code` @@ -183,16 +268,33 @@ fn main() -> Result<()> { .add_test(systest!(test)) .with_timeout_per_test(PER_TEST_TIMEOUT) .with_overall_timeout(OVERALL_TIMEOUT) + // The merge stops and starts the replicas of the two subnets being merged, + // so their nodes legitimately start the replica more than once. The + // metrics to check have to stay prefix-free, so this updates the entry of + // the default set rather than adding a more specific one. + .update_orchestrator_metrics_to_check("orchestrator_processes_start_attempts_total", 2) .execute_from_args()?; Ok(()) } pub fn setup(env: TestEnv) { InternetComputer::new() - .add_subnet(Subnet::fast_single_node(SubnetType::System)) - .add_subnet(Subnet::fast_single_node(SubnetType::Application)) - .add_subnet(Subnet::fast_single_node(SubnetType::Application)) - .add_subnet(Subnet::fast_single_node(SubnetType::Application)) + .add_subnet( + Subnet::fast_single_node(SubnetType::System) + .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), + ) + .add_subnet( + Subnet::fast_single_node(SubnetType::Application) + .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), + ) + .add_subnet( + Subnet::fast_single_node(SubnetType::Application) + .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), + ) + .add_subnet( + Subnet::fast_single_node(SubnetType::Application) + .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), + ) .setup_and_start(&env) .expect("failed to setup IC under test"); env.topology_snapshot().subnets().for_each(|subnet| { @@ -220,9 +322,9 @@ async fn run(env: TestEnv) { let topology = env.topology_snapshot(); // The three Application subnets: `M` is the one that will be labeled - // "cooling down", `T` is the one it exchanges messages with in a loop, and - // `S` is the one holding the canisters at the other end of the cross-subnet - // calls of the endless loops of step 6. + // "cooling down" and merged away, `R` is the one it is merged into, and `T` + // is the one `M` exchanges messages with in a loop and that holds the + // canisters at the other end of the cross-subnet endless loops of step 6. let app_subnets: Vec<_> = topology .subnets() .filter(|subnet| subnet.subnet_type() == SubnetType::Application) @@ -234,20 +336,29 @@ async fn run(env: TestEnv) { ); let m_subnet = app_subnets[0].clone(); let t_subnet = app_subnets[1].clone(); - let s_subnet = app_subnets[2].clone(); + let r_subnet = app_subnets[2].clone(); let m_node = m_subnet.nodes().next().unwrap(); let t_node = t_subnet.nodes().next().unwrap(); - let s_node = s_subnet.nodes().next().unwrap(); + let r_node = r_subnet.nodes().next().unwrap(); let m_agent = assert_create_agent(m_node.get_public_url().as_str()).await; let t_agent = assert_create_agent(t_node.get_public_url().as_str()).await; - let s_agent = assert_create_agent(s_node.get_public_url().as_str()).await; + let r_agent = assert_create_agent(r_node.get_public_url().as_str()).await; + let nns_node = topology.root_subnet().nodes().next().unwrap(); info!( logger, - "Subnets under test: M={}, T={}, S={} (R={})", + "Subnets under test, with their (single) nodes:\n \ + M={} on {}\n \ + R={} on {}\n \ + T={} on {}\n \ + NNS={} on {}", m_subnet.subnet_id, + m_node.node_id, + r_subnet.subnet_id, + r_node.node_id, t_subnet.subnet_id, - s_subnet.subnet_id, + t_node.node_id, topology.root_subnet_id(), + nns_node.node_id, ); // Step 1: Install a universal canister on each of `M` and `T`. @@ -302,21 +413,21 @@ async fn run(env: TestEnv) { // `U1`'s calls for as long as they run. info!( logger, - "Step 4: Installing U1, U3, U5, U6 on M and U4, U7 on S, and creating {} canisters for U1 \ + "Step 4: Installing U1, U3, U5, U6 on M and U4, U7 on T, and creating {} canisters for U1 \ to install code on", INSTALL_CODE_TARGETS.len(), ); let m_id = m_node.effective_canister_id(); - let s_id = s_node.effective_canister_id(); + let t_id = t_node.effective_canister_id(); let u1 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; let u3 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; - let u4 = UniversalCanister::new_with_retries(&s_agent, s_id, &logger).await; + let u4 = UniversalCanister::new_with_retries(&t_agent, t_id, &logger).await; let u5 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; let u6 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; - let u7 = UniversalCanister::new_with_retries(&s_agent, s_id, &logger).await; + let u7 = UniversalCanister::new_with_retries(&t_agent, t_id, &logger).await; info!( logger, - "Step 4: U1={}, U3={}, U5={}, U6={} on M; U4={}, U7={} on S", + "Step 4: U1={}, U3={}, U5={}, U6={} on M; U4={}, U7={} on T", u1.canister_id(), u3.canister_id(), u5.canister_id(), @@ -350,11 +461,38 @@ async fn run(env: TestEnv) { // Step 6: Start the three endless loops. info!(logger, "Step 6: Starting the three endless loops"); - u3.submit_update(endless_loop()) - .await - .expect("submitting U3's endless loop should succeed"); - submit_endless_loop_call(&u4, u5.canister_id()).await; - submit_endless_loop_call(&u6, u7.canister_id()).await; + // The IDs of the ingress messages that stay in progress across the merge, so + // that step 16 can check that all of them eventually completed. `U3` and + // `U6` are on `M` and thus served by `R` after the merge; `U4` stays on `T`. + let mut pending_ingress_messages: Vec<(String, Agent, Principal, RequestId)> = Vec::new(); + for (canister, name, agent) in [ + (&u3, "U3", &m_agent), + (&u4, "U4", &t_agent), + (&u6, "U6", &m_agent), + ] { + let payload = if name == "U3" { + endless_loop() + } else { + let callee = if name == "U4" { &u5 } else { &u7 }; + endless_loop_call(callee.canister_id()) + }; + let request_id = canister + .submit_update(payload) + .await + .unwrap_or_else(|e| panic!("submitting {name}'s update call should succeed: {e}")) + .unwrap_or_else(|| panic!("{name}'s update call should not have completed already")); + let agent = if name == "U4" { + agent.clone() + } else { + r_agent.clone() + }; + pending_ingress_messages.push(( + name.to_string(), + agent, + canister.canister_id(), + request_id, + )); + } for (canister, name) in [(&u3, "U3"), (&u5, "U5"), (&u7, "U7")] { await_loop_started(canister, name, &logger).await; } @@ -498,6 +636,567 @@ async fn run(env: TestEnv) { logger, "Step 11 done: both call loops are stalled at iterations {before:?}" ); + + // Step 12: Halt both `M` and `R` at their next CUP, i.e. at a checkpoint + // whose state is certified, so that the merged state can be assembled from + // states both subnets agree on. + info!( + logger, + "Step 12: Halting subnets M and R at their next checkpoint" + ); + for (subnet, name) in [(&m_subnet, "M"), (&r_subnet, "R")] { + let version = halt_subnet_at_cup_height(&env, subnet.subnet_id, &logger).await; + info!( + logger, + "Step 12: subnet {name} is set to halt at its next CUP as of registry version {version}" + ); + } + let m_height = await_halted_at_checkpoint(&m_node, "M", &logger).await; + let r_height = await_halted_at_checkpoint(&r_node, "R", &logger).await; + info!( + logger, + "Step 12 done: M halted at checkpoint {m_height}, R halted at checkpoint {r_height}" + ); + + // Step 13: Assemble the merged state: `R`'s state at the checkpoint it + // halted at, with the canisters (and canister snapshots) of `M` added to it, + // as a new checkpoint at the next multiple of the DKG interval, so that + // `R`'s own checkpoint is left untouched. + // + // Taking `R`'s system metadata and subnet queues wholesale, i.e. dropping + // `M`'s, is only sound because `M`'s were empty, which is what the merge + // readiness of step 9 established. That they are *still* empty at the + // checkpoint `M` halted at, minutes later, is due to `M` cooling down: no + // message is routed out of any of its canisters' output queues, not even + // into the loopback stream, so no management call can be inducted, no subnet + // call context can be created and no `install_code` can start in between. + let merged_height = r_height + CHECKPOINT_INTERVAL; + info!( + logger, + "Step 13: Assembling the merged state as checkpoint {merged_height} of R" + ); + + // The replicas have to be stopped before their states are touched: the state + // manager of a running replica owns its state directory, even while + // consensus is halted. + for (node, name) in [(&m_node, "M"), (&r_node, "R")] { + node.block_on_bash_script("sudo systemctl stop ic-replica") + .unwrap_or_else(|e| panic!("failed to stop the replica of subnet {name}: {e}")); + info!(logger, "Step 13: stopped the replica of subnet {name}"); + } + + // `ic-recovery` is a synchronous library that blocks on its own runtime + // internally (registry polling, rsync steps), which cannot be done from a + // thread that is driving this runtime, so all of it runs on a blocking one. + let merge = MergeStateArgs { + logger: logger.clone(), + admin_key_file: env + .get_path(SSH_AUTHORIZED_PRIV_KEYS_DIR) + .join(SSH_USERNAME), + nns_url: topology + .root_subnet() + .nodes() + .next() + .unwrap() + .get_public_url(), + m_dir: env.get_path("recovery_m"), + r_dir: env.get_path("recovery_r"), + m_node_ip: m_node.get_ip_addr(), + r_node_ip: r_node.get_ip_addr(), + m_height, + r_height, + merged_height, + }; + let (merged_time, state_hash) = tokio::task::spawn_blocking(move || merge.exec()) + .await + .expect("the state merging task panicked"); + info!( + logger, + "Step 14 done: R holds the merged state, which hashes to {} and starts at {merged_time}", + hex::encode(&state_hash), + ); + + // Step 15: Merge `M` into `R`: reroute `M`'s canister ID ranges to `R`, + // recover `R` at the merged state and bring it back online, all in one + // proposal. + info!( + logger, + "Step 15: Submitting the MergeSubnets proposal for M -> R" + ); + let merge_registry_version = merge_subnets( + &env, + m_subnet.subnet_id, + r_subnet.subnet_id, + merged_height, + merged_time, + state_hash.clone(), + &logger, + ) + .await; + info!( + logger, + "Step 15 done: M is merged into R as of registry version {merge_registry_version}" + ); + + // Step 16: Start `R`'s replica, now that the recovery CUP exists. Starting it + // any earlier would have it resume from its own checkpoint, which does not + // hold the canisters of `M`. + info!(logger, "Step 16: Starting the replica of subnet R"); + r_node + .block_on_bash_script("sudo systemctl start ic-replica") + .expect("failed to start the replica of subnet R"); + // Whether `R` resumes from the merged state or from the checkpoint it halted + // at is not something to leave to chance: a replica that started before its + // node had synced the registry version holding the recovery CUP would come + // up on the latter, silently serving a state without the canisters of `M`. + // Wait for the node to report exactly the recovery CUP, so that this fails + // loudly and promptly instead. + { + let logger = logger.clone(); + let node_ip = r_node.get_ip_addr(); + let state_hash = hex::encode(&state_hash); + tokio::task::spawn_blocking(move || { + Recovery::wait_for_recovery_cup( + &logger, + node_ip, + Height::from(merged_height), + state_hash, + ) + }) + .await + .expect("the recovery CUP waiting task panicked") + .expect("subnet R did not adopt the recovery CUP holding the merged state"); + } + r_node + .await_status_is_healthy() + .expect("subnet R did not become healthy after the merge"); + info!( + logger, + "Step 16 done: subnet R adopted the recovery CUP at height {merged_height} and is healthy" + ); + + // Step 17: Let the endless loops finish and check that every ingress message + // that was still in progress when the merge happened completed. + // + // The canisters of `M` now live on `R`, which serves them under the same + // canister IDs, so the agent for `R` is what reaches them. `U4` and `U7` did + // not move: they are on `T`. + info!( + logger, + "Step 17: Breaking the endless loops and waiting for the pending ingress messages" + ); + let u3 = UniversalCanister::from_canister_id(&r_agent, u3.canister_id()); + let u5 = UniversalCanister::from_canister_id(&r_agent, u5.canister_id()); + for (canister, name) in [(&u3, "U3"), (&u5, "U5"), (&u7, "U7")] { + retry_with_msg_async!( + format!("setting the global data of {name} to {LOOP_BREAK_TRIGGER:?}"), + &logger, + READY_WAIT_TIMEOUT, + RETRY_BACKOFF, + || async { + canister + .update(wasm().set_global_data(LOOP_BREAK_TRIGGER).reply_data(&[])) + .await + .map(|_| ()) + .map_err(|e| anyhow!("failed to break {name}'s loop: {e}")) + } + ) + .await + .unwrap_or_else(|e| panic!("could not break {name}'s endless loop: {e}")); + info!(logger, "Step 17: broke {name}'s endless loop"); + } + + for (name, agent, canister_id, request_id) in pending_ingress_messages { + await_ingress_message_replied(&agent, canister_id, &request_id, &name, &logger).await; + info!(logger, "Step 17: {name}'s ingress message completed"); + } + info!( + logger, + "Step 17 done: all the ingress messages that were pending across the merge completed" + ); + + // Step 18: Wait until every subnet observed the merge, i.e. routes the + // canisters that used to be hosted by `M` to `R`. Only then may `M` be + // deleted: a subnet still on an older registry version would keep routing + // messages to a subnet that no longer exists. + info!( + logger, + "Step 18: Waiting until all subnets reached registry version \ + {merge_registry_version}, which holds the merge" + ); + await_registry_version_on_all_subnets( + &topology, + m_subnet.subnet_id, + merge_registry_version, + &logger, + ) + .await; + info!(logger, "Step 18 done: all subnets observed the merge"); + + // Step 19: Delete the merged subnet, which hosts no canister ID range + // anymore, and check that it is gone from the registry. + info!( + logger, + "Step 19: Deleting subnet M ({})", m_subnet.subnet_id + ); + let topology = delete_subnet(&env, m_subnet.subnet_id, &logger).await; + let remaining: Vec<_> = topology.subnets().map(|subnet| subnet.subnet_id).collect(); + assert!( + !remaining.contains(&m_subnet.subnet_id), + "subnet M ({}) is still in the registry at version {}: {remaining:?}", + m_subnet.subnet_id, + topology.get_registry_version(), + ); + info!( + logger, + "Step 19 done: subnet M is gone as of registry version {}; the remaining subnets are \ + {remaining:?}", + topology.get_registry_version(), + ); +} + +/// Everything the synchronous, `ic-recovery` driven part of the merge needs: it +/// downloads the states of both subnets, assembles the merged state as a new +/// checkpoint of the destination subnet, and puts it on the destination node. +/// +/// This is a plain struct of owned data rather than a closure over the test's +/// state because it has to be moved onto a blocking thread: `ic-recovery` blocks +/// on its own runtime, which a thread driving the test's runtime cannot do. +struct MergeStateArgs { + logger: Logger, + admin_key_file: PathBuf, + nns_url: Url, + m_dir: PathBuf, + r_dir: PathBuf, + m_node_ip: IpAddr, + r_node_ip: IpAddr, + m_height: u64, + r_height: u64, + merged_height: u64, +} + +impl MergeStateArgs { + /// Returns the block time the recovered destination subnet should start from + /// and the hash of the manifest of the merged state. + fn exec(self) -> (u64, Vec) { + let m_recovery = self.recovery(self.m_dir.clone()); + let r_recovery = self.recovery(self.r_dir.clone()); + + for (recovery, node_ip, height, name) in [ + (&m_recovery, self.m_node_ip, self.m_height, "M"), + (&r_recovery, self.r_node_ip, self.r_height, "R"), + ] { + info!(self.logger, "Downloading the state of subnet {name}"); + recovery + .get_download_state_step( + node_ip, + SshUser::Admin, + Some(self.admin_key_file.clone()), + /* keep_downloaded_state= */ false, + Some(height), + ) + .expect("failed to build the download step") + .exec() + .unwrap_or_else(|e| panic!("failed to download the state of subnet {name}: {e}")); + } + + let m_checkpoints = m_recovery.work_dir.join(IC_STATE_DIR).join("checkpoints"); + let r_checkpoints = r_recovery.work_dir.join(IC_STATE_DIR).join("checkpoints"); + let m_checkpoint = + m_checkpoints.join(StateLayout::checkpoint_name(Height::from(self.m_height))); + let r_checkpoint = + r_checkpoints.join(StateLayout::checkpoint_name(Height::from(self.r_height))); + let merged_checkpoint = r_checkpoints.join(StateLayout::checkpoint_name(Height::from( + self.merged_height, + ))); + + // The block time the recovered subnet starts from has to be larger than + // the times of both checkpoints the merged state is assembled from. + let m_time = checkpoint_time_nanos(&m_checkpoint); + let r_time = checkpoint_time_nanos(&r_checkpoint); + let merged_time = m_time.max(r_time) + MERGED_STATE_TIME_MARGIN.as_nanos() as u64; + info!( + self.logger, + "M halted at time {m_time}, R at {r_time}; the merged state starts at {merged_time}" + ); + + assemble_merged_checkpoint( + &r_checkpoint, + &m_checkpoint, + &merged_checkpoint, + &self.logger, + ); + let state_hash = manifest_root_hash(&merged_checkpoint); + + self.upload_merged_checkpoint(&merged_checkpoint); + + (merged_time, state_hash) + } + + /// Adds `merged_checkpoint` to the checkpoints of the destination node, + /// leaving its replica stopped. + /// + /// Not `Recovery::get_upload_state_and_restart_step`: that one replaces the + /// whole state directory (and insists that it hold a single checkpoint), + /// which would delete the checkpoint the destination subnet halted at. That + /// checkpoint is meant to survive the merge untouched, so the merged state is + /// added next to it instead. + fn upload_merged_checkpoint(&self, merged_checkpoint: &Path) { + let ssh_helper = SshHelper::new( + self.logger.clone(), + SshUser::Admin, + self.r_node_ip, + /* require_confirmation= */ false, + Some(self.admin_key_file.clone()), + ); + let staging = PathBuf::from("/var/lib/ic/data/merged_state"); + + info!( + self.logger, + "Uploading the merged state to {}", + staging.display() + ); + // `/var/lib/ic/data` is not writable by the SSH user, so the staging + // directory has to be created with `sudo` and then handed over to it, or + // the `rsync` below (which runs as that user) cannot write into it. + ssh_helper + .ssh(format!( + "set -e; + sudo rm -rf {staging}; + sudo mkdir -p {staging}; + sudo chown -R {ssh_user} {staging};", + staging = staging.display(), + ssh_user = SshUser::Admin, + )) + .expect("failed to prepare the staging directory on R"); + ssh_helper + .rsync( + format!("{}/", merged_checkpoint.display()), + ssh_helper.remote_path(staging.join("")), + ) + .expect("failed to rsync the merged state to R"); + + info!(self.logger, "Installing the merged state on R"); + let name = StateLayout::checkpoint_name(Height::from(self.merged_height)); + ssh_helper + .ssh(format!( + "set -e; + CHECKPOINTS={NODE_IC_STATE_DIR}/checkpoints; + OWNER_UID=$(sudo stat -c '%u' $CHECKPOINTS); + GROUP_UID=$(sudo stat -c '%g' $CHECKPOINTS); + sudo mv {staging} $CHECKPOINTS/{name}; + sudo chown -R \"$OWNER_UID:$GROUP_UID\" $CHECKPOINTS/{name}; + sudo chmod -R a-w $CHECKPOINTS/{name}; + sudo systemctl restart setup-permissions;", + staging = staging.display(), + )) + .expect("failed to install the merged state on R"); + } + + fn recovery(&self, dir: PathBuf) -> Recovery { + Recovery::new( + self.logger.clone(), + RecoveryArgs { + dir, + nns_url: self.nns_url.clone(), + replica_version: None, + admin_key_file: Some(self.admin_key_file.clone()), + test_mode: true, + skip_prompts: true, + }, + /* neuron_args= */ None, + self.nns_url.clone(), + RegistryPollingStrategy::OnlyOnInit, + ) + .expect("failed to init recovery") + } +} + +/// Copies the checkpoint at `base` to `merged`, replacing its canisters and +/// canister snapshots with the union of those of `base` and of `source`, and +/// marks the result as the product of a subnet merge. +/// +/// Only the canisters and their snapshots are taken over from `source`: its +/// ingress history is not, as the `subnet_merged` marker makes the replica +/// re-register the ingress messages of the merged-in canisters that are still in +/// progress. Everything else (system metadata, subnet queues, ...) is `base`'s. +fn assemble_merged_checkpoint(base: &Path, source: &Path, merged: &Path, logger: &Logger) { + // Checkpoints are read-only and `rsync` preserved that, so the downloaded + // trees have to be made writable before anything can be assembled in them. + for path in [base, source] { + run_local(&format!( + "chmod -R u+w {}", + path.parent().expect("a checkpoint has a parent").display() + )); + } + // `cp -al` hard links the file contents rather than copying them, which + // keeps this cheap. The links are only ever read afterwards, except for the + // marker written below, which is a fresh file. + run_local(&format!("cp -al {} {}", base.display(), merged.display())); + run_local(&format!("chmod -R u+w {}", merged.display())); + for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { + let source_dir = source.join(dir); + if !source_dir.exists() { + info!(logger, "{} holds no {dir}", source.display()); + continue; + } + run_local(&format!( + "mkdir -p {merged_dir} && cp -al {source_dir}/. {merged_dir}/", + merged_dir = merged.join(dir).display(), + source_dir = source_dir.display(), + )); + } + // A `SubnetMerged` message with `merged` (field 1) set to `true`. + std::fs::write(merged.join(SUBNET_MERGED_FILE), [0x08, 0x01]) + .expect("failed to write the subnet merged marker"); + // The uploaded checkpoint must not look unverified to the state manager. + let _ = std::fs::remove_file(merged.join(UNVERIFIED_CHECKPOINT_MARKER)); +} + +/// Runs `script` locally, panicking with its output if it fails. +fn run_local(script: &str) { + let output = Command::new("bash") + .arg("-c") + .arg(script) + .output() + .unwrap_or_else(|e| panic!("failed to run {script:?}: {e}")); + assert!( + output.status.success(), + "{script:?} failed: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} + +/// Submits (and adopts) the `MergeSubnets` proposal merging `source_subnet` into +/// `destination_subnet`, recovering the latter at the merged state. +async fn merge_subnets( + env: &TestEnv, + source_subnet: SubnetId, + destination_subnet: SubnetId, + height: u64, + time_ns: u64, + state_hash: Vec, + logger: &Logger, +) -> u64 { + let topology = env.topology_snapshot(); + let nns_node = topology.root_subnet().nodes().next().unwrap(); + let nns_runtime = runtime_from_url(nns_node.get_public_url(), nns_node.effective_canister_id()); + let governance = get_governance_canister(&nns_runtime); + + let payload = MergeSubnetsPayload { + source_subnet, + destination_subnet, + height, + time_ns, + state_hash, + // The NNS subnet, which is neither of the two subnets being merged and + // stays available throughout, handles the DKG of the recovery CUP. + initial_dkg_subnet_id: None, + }; + let proposal_id = + submit_external_proposal_with_test_id(&governance, NnsFunction::MergeSubnets, payload) + .await; + info!(logger, "Submitted {proposal_id}"); + vote_execute_proposal_assert_executed(&governance, proposal_id).await; + + topology + .block_for_newer_registry_version() + .await + .expect("the registry should have a newer version after the proposal executed") + .get_registry_version() + .get() +} + +/// Submits (and adopts) the `DeleteSubnet` proposal deleting `subnet_id`, and +/// returns a topology snapshot taken after its mutations were applied. +async fn delete_subnet(env: &TestEnv, subnet_id: SubnetId, logger: &Logger) -> TopologySnapshot { + let topology = env.topology_snapshot(); + let nns_node = topology.root_subnet().nodes().next().unwrap(); + let nns_runtime = runtime_from_url(nns_node.get_public_url(), nns_node.effective_canister_id()); + let governance = get_governance_canister(&nns_runtime); + + let payload = DeleteSubnetPayload { + subnet_id: subnet_id.get().into(), + }; + let proposal_id = + submit_external_proposal_with_test_id(&governance, NnsFunction::DeleteSubnet, payload) + .await; + info!(logger, "Submitted {proposal_id}"); + vote_execute_proposal_assert_executed(&governance, proposal_id).await; + + topology + .block_for_newer_registry_version() + .await + .expect("the registry should have a newer version after the proposal executed") +} + +/// Waits until every subnet other than `stopped` has reached `registry_version`. +/// +/// `stopped` is the merged subnet, whose replica this test stopped for the merge +/// and which therefore does not report metrics anymore. It is also the subnet +/// about to be deleted, so what matters is that every *other* subnet already +/// routes its canisters to the destination subnet. +async fn await_registry_version_on_all_subnets( + topology: &TopologySnapshot, + stopped: SubnetId, + registry_version: u64, + logger: &Logger, +) { + retry_with_msg_async!( + format!("waiting until all subnets reached registry version {registry_version}"), + logger, + READY_WAIT_TIMEOUT, + RETRY_BACKOFF, + || async { + for subnet in topology.subnets().filter(|s| s.subnet_id != stopped) { + let metrics = fetch_metrics(&subnet, &[METRIC_REGISTRY_VERSION]).await?; + let version = median_across_replicas(&metrics, METRIC_REGISTRY_VERSION, |_| true) + .unwrap_or(0.0); + if version < registry_version as f64 { + bail!( + "subnet {} is at registry version {version}", + subnet.subnet_id + ); + } + } + Ok(()) + } + ) + .await + .unwrap_or_else(|e| panic!("not all subnets reached registry version {registry_version}: {e}")); +} + +/// Waits until the ingress message `request_id` sent to `canister_id` is +/// replied, i.e. until the update call it carries completed successfully. +async fn await_ingress_message_replied( + agent: &Agent, + canister_id: Principal, + request_id: &RequestId, + name: &str, + logger: &Logger, +) { + retry_with_msg_async!( + format!("waiting for {name}'s ingress message to complete"), + logger, + INGRESS_COMPLETION_TIMEOUT, + RETRY_BACKOFF, + || async { + let (status, _) = agent + .request_status_raw(request_id, canister_id) + .await + .map_err(|e| anyhow!("failed to read the status of {name}'s message: {e}"))?; + match status { + RequestStatusResponse::Replied(_) => Ok(()), + RequestStatusResponse::Rejected(reject) => { + panic!("{name}'s ingress message was rejected: {reject:?}") + } + other => bail!("{name}'s ingress message is {other:?}"), + } + } + ) + .await + .unwrap_or_else(|e| panic!("{name}'s ingress message did not complete: {e}")); } /// Starts an endless loop of calls from `canister` to `peer` (on another @@ -572,14 +1271,14 @@ fn endless_loop() -> Vec { .build() } -/// Executes an update call on `caller` that calls `callee` with `endless_loop()` -/// as the payload for `callee` to execute. As `callee` never responds, neither -/// does `caller`, so its ingress message stays `processing` forever. -async fn submit_endless_loop_call(caller: &UniversalCanister<'_>, callee: Principal) { - caller - .submit_update(wasm().call_simple(callee, "update", call_args().other_side(endless_loop()))) - .await - .expect("submitting the call starting the endless loop should succeed"); +/// The payload of an update call that calls `callee` with `endless_loop()` as the +/// payload for `callee` to execute. As `callee` does not respond until its global +/// data is set, neither does the caller, so the caller's ingress message stays +/// `processing` until then. +fn endless_loop_call(callee: Principal) -> Vec { + wasm() + .call_simple(callee, "update", call_args().other_side(endless_loop())) + .build() } /// The payload of the update call on `U1`: one management canister @@ -623,6 +1322,12 @@ fn install_code_payload(targets: &[Principal]) -> Vec { /// queues hold nothing but those calls by the time they are inducted. async fn await_install_code_requests_inducted(subnet: &SubnetSnapshot, logger: &Logger) { let expected = INSTALL_CODE_TARGETS.len() as f64; + // The number of requests enqueued plus executing only reaches `expected` + // between the moment the last one is inducted and the moment the first one + // completes, so waiting for the current value to reach it would be waiting + // for a condition that stops holding. Remember the highest value seen + // instead, which only grows. + let highest = std::cell::Cell::new(0.0_f64); retry_with_msg_async!( format!( "waiting until all {} `install_code` requests are inducted on subnet {}", @@ -645,8 +1350,13 @@ async fn await_install_code_requests_inducted(subnet: &SubnetSnapshot, logger: & let executing = sum_of_medians(&metrics, METRIC_SUBNET_CALL_CONTEXTS, |labels| { labels.contains(LABEL_INSTALL_CODE) }); - if enqueued + executing < expected { - bail!("{enqueued} request(s) enqueued and {executing} executing"); + highest.set(highest.get().max(enqueued + executing)); + if highest.get() < expected { + bail!( + "{enqueued} request(s) enqueued and {executing} executing, at most {} of them \ + at once so far", + highest.get(), + ); } Ok(()) } @@ -692,14 +1402,19 @@ async fn global_counter(canister: &UniversalCanister<'_>) -> Result { /// "cooling down" in its subnet record. Returns the registry version created by /// the proposal, i.e. `V` in the dashboard's readiness condition. async fn set_subnet_cooling_down(env: &TestEnv, subnet_id: SubnetId, logger: &Logger) -> u64 { - let topology = env.topology_snapshot(); - let nns_node = topology.root_subnet().nodes().next().unwrap(); - let nns_runtime = runtime_from_url(nns_node.get_public_url(), nns_node.effective_canister_id()); - let governance = get_governance_canister(&nns_runtime); - let payload = UpdateSubnetPayload { - subnet_id, cooling_down: Some(true), + ..empty_update_subnet_payload(subnet_id) + }; + submit_and_adopt_update_subnet_proposal(env, payload, logger).await +} + +/// An `UpdateSubnetPayload` for `subnet_id` that changes nothing, to be used as +/// the base of a payload changing a single field. +fn empty_update_subnet_payload(subnet_id: SubnetId) -> UpdateSubnetPayload { + UpdateSubnetPayload { + subnet_id, + cooling_down: None, max_ingress_bytes_per_message: None, max_ingress_messages_per_block: None, max_ingress_bytes_per_block: None, @@ -731,7 +1446,21 @@ async fn set_subnet_cooling_down(env: &TestEnv, subnet_id: SubnetId, logger: &Lo registry_poll_period_ms: None, retransmission_request_ms: None, set_gossip_config_to_default: false, - }; + } +} + +/// Submits (and adopts) `payload` as an `UpdateConfigOfSubnet` proposal, and +/// returns the registry version its mutation created. +async fn submit_and_adopt_update_subnet_proposal( + env: &TestEnv, + payload: UpdateSubnetPayload, + logger: &Logger, +) -> u64 { + let topology = env.topology_snapshot(); + let nns_node = topology.root_subnet().nodes().next().unwrap(); + let nns_runtime = runtime_from_url(nns_node.get_public_url(), nns_node.effective_canister_id()); + let governance = get_governance_canister(&nns_runtime); + let proposal_id = submit_external_proposal_with_test_id( &governance, NnsFunction::UpdateConfigOfSubnet, @@ -955,3 +1684,140 @@ fn median_across_replicas( .collect(); median(&values) } + +// --------------------------------------------------------------------------- +// Merging subnet M into subnet R. +// --------------------------------------------------------------------------- + +/// The name of the directory the replica keeps its states in, on a node. +const NODE_IC_STATE_DIR: &str = "/var/lib/ic/data/ic_state"; + +/// Waits until `node`'s subnet is halted, and returns the height of the +/// checkpoint it halted at, i.e. of the state it stopped in. +/// +/// Whether the subnet halted is read off the node's journal, which is where a +/// halted replica says that it stops delivering batches. Waiting for the +/// *checkpoint* height to stop advancing instead would not do: while the subnet +/// is running, its state runs ahead of its latest checkpoint by up to a whole DKG +/// interval, which is minutes of wall clock time, so the checkpoint height looks +/// stable long before the subnet halts. The state of that checkpoint is then +/// hundreds of rounds behind the state the merge readiness of step 9 was +/// established on, and may hold, say, an `install_code` that was aborted at the +/// checkpoint and only completed afterwards. +/// +/// The batch heights of a subnet halting because of `halt_at_cup_height` stop at +/// a CUP height: the flag is read at the registry version of the summary block +/// active at a height, and that version only changes at a summary, so batch +/// delivery stops exactly when the summary carrying it becomes active. As +/// checkpoints are written at CUP heights, the latest checkpoint of a halted +/// subnet holds precisely the state it stopped in. +async fn await_halted_at_checkpoint(node: &IcNodeSnapshot, name: &str, logger: &Logger) -> u64 { + info!(logger, "Waiting until subnet {name} is halted"); + // Polling the journal rather than following it: `follow()` blocks in + // `journalctl --follow | grep -m 1` until the line shows up, with no timeout + // of its own, so a line that never comes (because it was reworded, say) + // would hang the test until the whole test times out. The cursor of + // `from_now()` makes every poll search all entries since this point, so the + // condition, once true, stays true. + let journal = JournalStreamer::new( + node.block_on_ssh_session() + .unwrap_or_else(|e| panic!("failed to open an SSH session to subnet {name}: {e}")), + ) + .from_now() + .unwrap_or_else(|e| panic!("failed to create a journal streamer for subnet {name}: {e}")); + retry_with_msg_async!( + format!("waiting until subnet {name} reports that it is halted"), + logger, + HALT_TIMEOUT, + HALT_BACKOFF, + || async { + if !journal + .contains(HALTED_LOG_PATTERN) + .map_err(|e| anyhow!("failed to search the journal of subnet {name}: {e}"))? + { + bail!("subnet {name} has not reported that it is halted yet"); + } + Ok(()) + } + ) + .await + .unwrap_or_else(|e| panic!("subnet {name} did not report that it is halted: {e}")); + + let height = latest_checkpoint_height(node) + .await + .unwrap_or_else(|e| panic!("failed to read the checkpoint of subnet {name}: {e}")); + assert_eq!( + height % CHECKPOINT_INTERVAL, + 0, + "subnet {name} halted at checkpoint {height}, which is not a CUP height", + ); + height +} + +/// The height of the highest checkpoint `node` holds. Checkpoint directories are +/// named after their height, in hexadecimal. +async fn latest_checkpoint_height(node: &IcNodeSnapshot) -> Result { + let output = node + .block_on_bash_script_async(&format!("sudo ls -1 {NODE_IC_STATE_DIR}/checkpoints")) + .await + .map_err(|e| anyhow!("failed to list the checkpoints: {e}"))?; + output + .split_whitespace() + .map(|name| { + u64::from_str_radix(name, 16) + .map_err(|e| anyhow!("checkpoint name {name} is not a hex height: {e}")) + }) + .collect::>>()? + .into_iter() + .max() + .ok_or_else(|| anyhow!("no checkpoint yet")) +} + +/// Submits (and adopts) an `UpdateConfigOfSubnet` proposal setting the +/// `halt_at_cup_height` flag of `subnet_id`, so that the subnet halts once it +/// reaches its next CUP, i.e. at a checkpoint whose state is certified. +/// Returns the registry version the proposal created, which is the version at +/// which the subnet is instructed to halt. +async fn halt_subnet_at_cup_height(env: &TestEnv, subnet_id: SubnetId, logger: &Logger) -> u64 { + let payload = UpdateSubnetPayload { + halt_at_cup_height: Some(true), + ..empty_update_subnet_payload(subnet_id) + }; + submit_and_adopt_update_subnet_proposal(env, payload, logger).await +} + +/// Runs `state-tool` with the given arguments and returns its standard output. +fn state_tool(args: &[&str]) -> String { + let binary = get_dependency_path_from_env("ENV_DEPS__STATE_TOOL"); + let output = Command::new(&binary) + .args(args) + .output() + .unwrap_or_else(|e| panic!("failed to run {}: {e}", binary.display())); + assert!( + output.status.success(), + "{} {args:?} failed: {}", + binary.display(), + String::from_utf8_lossy(&output.stderr), + ); + String::from_utf8(output.stdout).expect("state-tool output should be UTF-8") +} + +/// The batch time of the checkpoint at `path`, in nanoseconds since the Epoch. +fn checkpoint_time_nanos(path: &Path) -> u64 { + let output = state_tool(&["checkpoint_time", "--state", &path.display().to_string()]); + output + .trim() + .parse() + .unwrap_or_else(|e| panic!("failed to parse the checkpoint time {output:?}: {e}")) +} + +/// The root hash of the manifest of the checkpoint at `path`. +fn manifest_root_hash(path: &Path) -> Vec { + let output = state_tool(&["manifest", "--state", &path.display().to_string()]); + let hash = output + .lines() + .find_map(|line| line.strip_prefix("ROOT HASH: ")) + .unwrap_or_else(|| panic!("no root hash in the manifest of {}", path.display())) + .trim(); + hex::decode(hash).unwrap_or_else(|e| panic!("root hash {hash} is not hex: {e}")) +} From 2e69e0d952d40b9df255aa5a44551af5e5ca7d5a Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 31 Aug 2026 16:29:13 +0000 Subject: [PATCH 09/30] test: cover more of what a subnet merge has to carry over Strengthen the subnet merge system test along the axes it was blind to. Every subnet now has four nodes rather than one, so that the merge has to get the merged state to the other nodes of the destination subnet the way a recovery does, i.e. by state sync from the one node it is uploaded to, and so that the medians the merge readiness condition is made of are medians of more than one value. The destination subnet is no longer empty: it hosts `UR`, which the merge has to leave alone even though the canisters of the merged subnet are added right next to it, and which has to be able to call one of those canisters afterwards, both of them being on the same subnet from then on. `U8` carries the state that has to survive the merge: a blob in its stable memory, a canister snapshot, and a cycles balance, all three of which are checked once the destination subnet serves it. The balance is compared against a share of itself rather than an amount, as the two readings are however many minutes apart the waits in between take, and what this catches is a balance that was not carried over at all rather than the resource charges of an idle canister. The merge readiness condition is now checked to *not* hold right after the subnet starts cooling down. A condition that held from the start would be satisfied by a subnet that never had anything to drain, so waiting for it would prove nothing; six of its eight terms are in fact unsatisfied at that point. Finally, a best effort call from `U9` on `T` to `U10` on `M`, which `U10` never answers, puts a best effort message in flight across the merge: its deadline passes while `M` is cooling down, and the cycles it carries are held by the open call context of its callee, which the merge has to carry over like any other canister state. The test had only guaranteed response calls until now. The one term of the readiness condition that stays untested is the pending anonymous refunds, which requires a cycle bearing message to be dropped from one of the subnet's queues while owed to a canister of another subnet; the cycles of the best effort call above are not it, as that call is picked up rather than dropped. See the comment on `MAX_REFUND_VALUE_CYCLES`. Co-Authored-By: Claude Opus 5 --- rs/tests/message_routing/BUILD.bazel | 4 +- .../subnet_cooling_down_test.rs | 329 +++++++++++++++--- 2 files changed, 278 insertions(+), 55 deletions(-) diff --git a/rs/tests/message_routing/BUILD.bazel b/rs/tests/message_routing/BUILD.bazel index bf1dfd1efa4f..3bb4b1b534c3 100644 --- a/rs/tests/message_routing/BUILD.bazel +++ b/rs/tests/message_routing/BUILD.bazel @@ -141,7 +141,7 @@ system_test_nns( system_test_nns( name = "subnet_cooling_down_test", - cpus = MIN_LOCAL_CPUS + 4 * DEFAULT_VCPUS_PER_VM, # 1 System + 3 Application fast-single-node subnets = 4 IC Node VMs * 6 vCPUs. + cpus = MIN_LOCAL_CPUS + 16 * DEFAULT_VCPUS_PER_VM, # 1 System + 3 Application subnets of 4 nodes each = 16 IC Node VMs * 6 vCPUs. enable_mainnet_nns_variant = False, # The `cooling_down` field of the subnet record is not supported by the mainnet NNS canisters. tags = [ "long_test", # since the subnet only quiesces once its long-running `install_code` calls are done and the ingress history is pruned. @@ -164,6 +164,8 @@ system_test_nns( "@crate_index//:candid", "@crate_index//:hex", "@crate_index//:ic-agent", + "@crate_index//:ic-management-canister-types", + "@crate_index//:ic-utils", "@crate_index//:slog", "@crate_index//:tokio", "@crate_index//:url", diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs index dd32a6af94ce..210581ab0cc6 100644 --- a/rs/tests/message_routing/subnet_cooling_down_test.rs +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -23,18 +23,20 @@ never meant to complete. Runbook:: 0. Set up an IC with an NNS subnet (with the NNS canisters installed) and three - Application subnets `M`, `T` and `R`. + Application subnets `M`, `T` and `R`, of `SUBNET_SIZE` nodes each. 1. Install a universal canister on each of `M` and `T`: `US` on `M`, `UT` on `T`. 2. Make an ingress call to each of `US` and `UT` with a payload that calls the universal canister on the other subnet in a loop: the reply (or reject) callback of every call fires a new call. 3. Wait until both loops have completed a few iterations, i.e. messages are actually flowing between `M` and `T` in both directions. -4. Install the universal canisters of the steps below (`U1`, `U3`, `U5` and - `U6` on `M`, `U4` and `U7` on `T`) and create five empty canisters - `U2a` .. `U2e` on `M`, controlled by `U1`. All of this has to happen before - step 5: a long-running `install_code` blocks every other `install_code` on - the same subnet. +4. Install the universal canisters of the steps below (`U1`, `U3`, `U5`, `U6`, + `U8` and `U10` on `M`, `U4`, `U7` and `U9` on `T`, `UR` on `R`) and create + five empty canisters `U2a` .. `U2e` on `M`, controlled by `U1`. All of this + has to happen before step 5: a long-running `install_code` blocks every other + `install_code` on the same subnet. Then give `U8` the state that has to + survive the merge: a blob in its stable memory, a canister snapshot, and a + cycles balance to compare against later. 5. Execute an update call on `U1` that makes five calls to the management canister's `install_code` method, one per `U2x`, in mode `install`, with the universal canister module and an `arg` that makes `canister_init` burn @@ -42,8 +44,12 @@ Runbook:: `U1`'s output queue: a request still sitting there when `M` starts cooling down would never be routed, not even into the loopback stream, and the code would never be installed. -6. Start three endless loops, each of which runs until the global data of the - looping canister is set to `LOOP_BREAK_TRIGGER`, which this test never does: +6. Execute an update call on `U9` (on `T`) making a best effort call to `U10` + (on `M`) that carries cycles and that `U10` never answers, so that a best + effort message, whose deadline passes while `M` is cooling down, is in flight + across the merge. Then start three endless loops, each of which runs + until the global data of the looping canister is set to `LOOP_BREAK_TRIGGER`, + which this test never does: a. execute an update call on `U3` that loops on `U3` itself; b. execute an update call on `U4` (on `T`) that calls `U5` (on `M`) with the loop as its payload, so that `M` holds a canister looping in a call from @@ -57,13 +63,14 @@ Runbook:: creates. 8. Wait until `M` rejects ingress messages, i.e. the replicas of `M` observed the "cooling down" label. -9. Wait until `M` is "merge ready" according to the dashboard's condition for - `V` and 0 cycles of pending refunds: all subnets have reached registry - version `V`, no stream in either direction holds a message (loopback - included), the ingress history holds nothing but `processing` entries, `M`'s - subnet input and output queues are empty, `M`'s subnet call context manager - holds no call context, and the pending anonymous refunds are worth at most 0 - cycles. +9. Check that `M` is not "merge ready" yet, so that the wait below is known to + be waiting for something, and then wait until it is, according to the + dashboard's condition for `V` and `MAX_REFUND_VALUE_CYCLES`: all subnets have + reached registry version `V`, no stream in either direction holds a message + (loopback included), the ingress history holds nothing but `processing` + entries, `M`'s subnet input and output queues are empty, `M`'s subnet call + context manager holds no call context, and the pending anonymous refunds are + worth at most `MAX_REFUND_VALUE_CYCLES`. 10. Check that `U2a` .. `U2e` have been installed, i.e. that the `install_code` calls of step 5 ran to completion rather than being lost or rejected while `M` was cooling down. @@ -93,15 +100,19 @@ Runbook:: 16. Start `R`'s replica and wait until it is healthy. Only now: a replica started before the recovery CUP exists resumes from the checkpoint `R` halted at, which does not hold the canisters of `M`. -17. Set the global data of `U3`, `U5` and `U7` to `LOOP_BREAK_TRIGGER`, ending +17. Check that `U8`, now served by `R`, kept the stable memory, the snapshot and + (up to what an idle canister burns) the cycles balance of step 4, and that + `UR`, which `R` hosted all along, is undisturbed and can call `U8` now that + both are on the same subnet. +18. Set the global data of `U3`, `U5` and `U7` to `LOOP_BREAK_TRIGGER`, ending the three endless loops, and check that every ingress message that was in progress across the merge completed. `U3` and `U5` are reached through `R`, which serves the canisters of `M` after the merge. -18. Wait until every subnet other than `M` has reached the registry version the +19. Wait until every subnet other than `M` has reached the registry version the merge created, i.e. routes the canisters that used to be hosted by `M` to `R`. `M` itself is excluded: its replica was stopped for the merge and it is about to be deleted. -19. Submit (and adopt) a `DeleteSubnet` NNS proposal deleting `M`, which hosts no +20. Submit (and adopt) a `DeleteSubnet` NNS proposal deleting `M`, which hosts no canister ID range anymore, and check that it is gone from the registry. Success:: @@ -115,6 +126,7 @@ end::catalog[] */ use anyhow::{Result, anyhow, bail}; use candid::Principal; use ic_agent::{Agent, RequestId, agent::RequestStatusResponse}; +use ic_management_canister_types::{SnapshotId, TakeCanisterSnapshotArgs}; use ic_nns_governance_api::NnsFunction; use ic_recovery::registry_helper::RegistryPollingStrategy; use ic_recovery::ssh_helper::SshHelper; @@ -151,6 +163,8 @@ use ic_universal_canister::management::InstallMode; use ic_universal_canister::{ CallInterface, call_args, get_universal_canister_wasm, management, wasm, }; +use ic_utils::call::AsyncCall; +use ic_utils::interfaces::ManagementCanister; use registry_canister::mutations::do_delete_subnet::DeleteSubnetPayload; use registry_canister::mutations::do_update_subnet::UpdateSubnetPayload; use registry_canister::mutations::merge_subnets::MergeSubnetsPayload; @@ -187,6 +201,17 @@ const LABEL_INSTALL_CODE: &str = "type=\"install_code\""; /// The dashboard's `R` in the readiness condition: the maximum total value in /// cycles of the pending anonymous refunds of the cooling down subnet. (Not to /// be confused with the subnet `R` of the runbook above.) +/// +/// This test leaves no pending refunds behind, so it requires them to be worth +/// nothing at all. Making it hold non-zero ones would take a cycle bearing +/// message that is dropped from one of the subnet's queues while owed to a +/// canister of another subnet: the cycles of the best effort call of step 6 are +/// not it, as that call is picked up and its cycles are held by the open call +/// context of its callee rather than by a queued message. Which is just as well: +/// a cooling down subnet routes no refunds either (see `route_refunds` in +/// `rs/messaging/src/routing/stream_builder.rs`), so any refund it does hold +/// stays pending until it is merged, and is then lost -- the merged state takes +/// the refunds of the destination subnet, not those of the merged one. const MAX_REFUND_VALUE_CYCLES: f64 = 0.0; /// Number of loop iterations each universal canister must have completed before @@ -216,10 +241,46 @@ const INSTALL_CODE_TARGETS: [&str; 5] = ["U2a", "U2b", "U2c", "U2d", "U2e"]; /// it; hence the budget for `canister_init` has to leave room for it. const INIT_INSTRUCTIONS: u64 = 295 * B; +/// Where in the stable memory of `U8` the blob that has to survive the merge is +/// stored, and the blob itself. +const STABLE_MEMORY_OFFSET: u32 = 0; +const STABLE_MEMORY_BLOB: &[u8] = b"this blob has to survive the subnet merge"; + +/// The cycles the best effort call of step 6 carries, and its timeout. The point +/// of the call is that a best effort message is in flight across the merge: its +/// callee never responds, so its deadline passes while `M` is cooling down, and +/// the cycles it carries are held by the callee's open call context, which the +/// merge has to carry over like any other canister state. +const BEST_EFFORT_CALL_CYCLES: u128 = 1_000_000_000; +const BEST_EFFORT_CALL_TIMEOUT_SECONDS: u32 = 60; + +/// What the canister of the destination subnet gets back from the canister that +/// the merge moved onto it. +const MERGED_CALL_REPLY: &[u8] = b"hello from the merged subnet"; + +/// The fraction of its cycles balance that the canister holding the state that +/// has to survive the merge may have burned in between the two readings, as one +/// in `MAX_BURNED_CYCLES_FRACTION`. +/// +/// A share rather than an amount because the two readings are however many +/// minutes apart the waits of the steps in between take, and generous because +/// what this is meant to catch is a balance that the merge did not carry over at +/// all, which would be a loss of everything, rather than the resource charges of +/// an idle canister, which have been observed to be some three billion cycles of +/// a hundred trillion. +const MAX_BURNED_CYCLES_FRACTION: u128 = 1_000; + /// The global data value that would end the endless loops of `U3`, `U5` and /// `U7`. The test never sets it, so those loops never end. const LOOP_BREAK_TRIGGER: &[u8] = b"break"; +/// The number of nodes of every subnet. More than one so that the merge has to +/// get the merged state to the other nodes of the destination subnet the way a +/// recovery does, i.e. by state sync from the one node it was uploaded to, and so +/// that the medians the merge readiness condition is made of are medians of more +/// than one value. +const SUBNET_SIZE: usize = 4; + /// The DKG interval length of the Application subnets, i.e. one less than the /// distance between two consecutive checkpoints (and CUPs). The default is long /// enough for an `install_code` burning `INIT_INSTRUCTIONS` to complete within @@ -278,23 +339,15 @@ fn main() -> Result<()> { } pub fn setup(env: TestEnv) { + let subnet = |subnet_type| { + Subnet::fast(subnet_type, SUBNET_SIZE) + .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)) + }; InternetComputer::new() - .add_subnet( - Subnet::fast_single_node(SubnetType::System) - .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), - ) - .add_subnet( - Subnet::fast_single_node(SubnetType::Application) - .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), - ) - .add_subnet( - Subnet::fast_single_node(SubnetType::Application) - .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), - ) - .add_subnet( - Subnet::fast_single_node(SubnetType::Application) - .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), - ) + .add_subnet(subnet(SubnetType::System)) + .add_subnet(subnet(SubnetType::Application)) + .add_subnet(subnet(SubnetType::Application)) + .add_subnet(subnet(SubnetType::Application)) .setup_and_start(&env) .expect("failed to setup IC under test"); env.topology_snapshot().subnets().for_each(|subnet| { @@ -419,21 +472,34 @@ async fn run(env: TestEnv) { ); let m_id = m_node.effective_canister_id(); let t_id = t_node.effective_canister_id(); + let r_id = r_node.effective_canister_id(); let u1 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; let u3 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; let u4 = UniversalCanister::new_with_retries(&t_agent, t_id, &logger).await; let u5 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; let u6 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; let u7 = UniversalCanister::new_with_retries(&t_agent, t_id, &logger).await; + // `U8` carries the state that has to survive the merge; `U9` on `T` and + // `U10` on `M` are the two ends of the best effort call of step 6; and `UR` + // is a canister of the destination subnet, which the merge must leave alone. + let u8 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; + let u9 = UniversalCanister::new_with_retries(&t_agent, t_id, &logger).await; + let u10 = UniversalCanister::new_with_retries(&m_agent, m_id, &logger).await; + let ur = UniversalCanister::new_with_retries(&r_agent, r_id, &logger).await; info!( logger, - "Step 4: U1={}, U3={}, U5={}, U6={} on M; U4={}, U7={} on T", + "Step 4: U1={}, U3={}, U5={}, U6={}, U8={}, U10={} on M; U4={}, U7={}, U9={} on T; UR={} \ + on R", u1.canister_id(), u3.canister_id(), u5.canister_id(), u6.canister_id(), + u8.canister_id(), + u10.canister_id(), u4.canister_id(), u7.canister_id(), + u9.canister_id(), + ur.canister_id(), ); let mut targets = Vec::new(); for name in INSTALL_CODE_TARGETS { @@ -442,7 +508,19 @@ async fn run(env: TestEnv) { info!(logger, "Step 4: {name}={target} on M, controlled by U1"); targets.push(target); } - info!(logger, "Step 4 done: all canisters installed"); + // The state of `U8` that the merge has to carry over: a blob in its stable + // memory, a canister snapshot, and its cycles balance. + u8.store_to_stable(STABLE_MEMORY_OFFSET, STABLE_MEMORY_BLOB) + .await; + let u8_snapshot = take_canister_snapshot(&m_agent, u8.canister_id()).await; + let u8_cycles_before = cycles_balance(&u8).await.unwrap(); + info!( + logger, + "Step 4 done: all canisters installed; U8 holds {} bytes in its stable memory, snapshot \ + {} and {u8_cycles_before} cycles", + STABLE_MEMORY_BLOB.len(), + hex::encode(&u8_snapshot), + ); // Step 5: Have `U1` install code on the five canisters it controls. info!( @@ -459,8 +537,25 @@ async fn run(env: TestEnv) { "Step 5 done: all of U1's `install_code` requests left its output queue" ); - // Step 6: Start the three endless loops. - info!(logger, "Step 6: Starting the three endless loops"); + // Step 6: Start the three endless loops, and the best effort call whose + // cycles end up as a pending anonymous refund of `M`. + info!( + logger, + "Step 6: Starting the three endless loops and U9's best effort call to U10" + ); + // `U10` never responds, so the call is still in flight when `M` starts + // cooling down, and its deadline passes while it is. Dropping it leaves `M` + // owing its cycles to `U9`, which is on `T`: a refund `M` cannot route while + // it is cooling down, and hence one that is still pending when it is merged. + u9.submit_update(wasm().call_simple_with_cycles_and_best_effort_response( + u10.canister_id(), + "update", + call_args().other_side(endless_loop()), + BEST_EFFORT_CALL_CYCLES, + BEST_EFFORT_CALL_TIMEOUT_SECONDS, + )) + .await + .expect("submitting U9's best effort call should succeed"); // The IDs of the ingress messages that stay in progress across the merge, so // that step 16 can check that all of them eventually completed. `U3` and // `U6` are on `M` and thus served by `R` after the merge; `U4` stays on `T`. @@ -545,7 +640,34 @@ async fn run(env: TestEnv) { "Step 8 done: subnet M rejects ingress messages, so it is cooling down" ); - // Step 9: Wait until `M` is "merge ready". + // Step 9: Check that `M` is *not* "merge ready" yet, so that the wait below + // is known to be waiting for something: a readiness condition that held from + // the start would be satisfied by a subnet that never had anything to drain. + let terms = evaluate_merge_readiness( + &topology, + &m_subnet, + registry_version, + MAX_REFUND_VALUE_CYCLES, + ) + .await + .expect("failed to evaluate the merge readiness of subnet M"); + let unsatisfied: Vec<_> = terms + .iter() + .filter(|(_, satisfied)| !satisfied) + .map(|(term, _)| term.as_str()) + .collect(); + assert!( + !unsatisfied.is_empty(), + "subnet M was already \"merge ready\" right after it started cooling down, so the wait \ + below would prove nothing", + ); + info!( + logger, + "Step 9: subnet M is not \"merge ready\" yet: {}", + unsatisfied.join("; "), + ); + + // Step 9 (continued): Wait until `M` is "merge ready". info!( logger, "Step 9: Waiting until subnet M is \"merge ready\" for V={registry_version} and at most \ @@ -775,7 +897,61 @@ async fn run(env: TestEnv) { "Step 16 done: subnet R adopted the recovery CUP at height {merged_height} and is healthy" ); - // Step 17: Let the endless loops finish and check that every ingress message + // Step 17: Check that the merge carried the state of `M`'s canisters over and + // left `R`'s own canister alone. + info!( + logger, + "Step 17: Checking the state of U8 and UR after the merge" + ); + let u8_on_r = UniversalCanister::from_canister_id(&r_agent, u8.canister_id()); + assert_eq!( + u8_on_r + .try_read_stable( + STABLE_MEMORY_OFFSET, + STABLE_MEMORY_BLOB.len().try_into().unwrap() + ) + .await, + STABLE_MEMORY_BLOB, + "the stable memory of U8 did not survive the merge", + ); + assert!( + canister_snapshot_ids(&r_agent, u8.canister_id()) + .await + .contains(&u8_snapshot), + "the snapshot of U8 did not survive the merge", + ); + let u8_cycles_after = cycles_balance(&u8_on_r) + .await + .expect("failed to read the cycles balance of U8 after the merge"); + assert!( + u8_cycles_after <= u8_cycles_before + && u8_cycles_before - u8_cycles_after <= u8_cycles_before / MAX_BURNED_CYCLES_FRACTION, + "the cycles balance of U8 went from {u8_cycles_before} to {u8_cycles_after} across the \ + merge, a difference of more than the one in {MAX_BURNED_CYCLES_FRACTION} an idle canister \ + is expected to burn", + ); + + // `UR` was hosted by `R` all along: adding the canisters of `M` to `R`'s + // state must not have disturbed it. And now that both are on `R`, they must + // be able to call each other. + let ur_reply = ur + .update(wasm().call_simple( + u8.canister_id(), + "update", + call_args().other_side(wasm().push_bytes(MERGED_CALL_REPLY).append_and_reply()), + )) + .await + .expect("UR should be able to call a canister that was hosted by M"); + assert_eq!( + ur_reply, MERGED_CALL_REPLY, + "UR got an unexpected reply from U8", + ); + info!( + logger, + "Step 17 done: U8 kept its stable memory, snapshot and cycles, and UR can call it" + ); + + // Step 18: Let the endless loops finish and check that every ingress message // that was still in progress when the merge happened completed. // // The canisters of `M` now live on `R`, which serves them under the same @@ -783,7 +959,7 @@ async fn run(env: TestEnv) { // not move: they are on `T`. info!( logger, - "Step 17: Breaking the endless loops and waiting for the pending ingress messages" + "Step 18: Breaking the endless loops and waiting for the pending ingress messages" ); let u3 = UniversalCanister::from_canister_id(&r_agent, u3.canister_id()); let u5 = UniversalCanister::from_canister_id(&r_agent, u5.canister_id()); @@ -803,16 +979,16 @@ async fn run(env: TestEnv) { ) .await .unwrap_or_else(|e| panic!("could not break {name}'s endless loop: {e}")); - info!(logger, "Step 17: broke {name}'s endless loop"); + info!(logger, "Step 18: broke {name}'s endless loop"); } for (name, agent, canister_id, request_id) in pending_ingress_messages { await_ingress_message_replied(&agent, canister_id, &request_id, &name, &logger).await; - info!(logger, "Step 17: {name}'s ingress message completed"); + info!(logger, "Step 18: {name}'s ingress message completed"); } info!( logger, - "Step 17 done: all the ingress messages that were pending across the merge completed" + "Step 18 done: all the ingress messages that were pending across the merge completed" ); // Step 18: Wait until every subnet observed the merge, i.e. routes the @@ -821,7 +997,7 @@ async fn run(env: TestEnv) { // messages to a subnet that no longer exists. info!( logger, - "Step 18: Waiting until all subnets reached registry version \ + "Step 19: Waiting until all subnets reached registry version \ {merge_registry_version}, which holds the merge" ); await_registry_version_on_all_subnets( @@ -831,13 +1007,13 @@ async fn run(env: TestEnv) { &logger, ) .await; - info!(logger, "Step 18 done: all subnets observed the merge"); + info!(logger, "Step 19 done: all subnets observed the merge"); // Step 19: Delete the merged subnet, which hosts no canister ID range // anymore, and check that it is gone from the registry. info!( logger, - "Step 19: Deleting subnet M ({})", m_subnet.subnet_id + "Step 20: Deleting subnet M ({})", m_subnet.subnet_id ); let topology = delete_subnet(&env, m_subnet.subnet_id, &logger).await; let remaining: Vec<_> = topology.subnets().map(|subnet| subnet.subnet_id).collect(); @@ -849,7 +1025,7 @@ async fn run(env: TestEnv) { ); info!( logger, - "Step 19 done: subnet M is gone as of registry version {}; the remaining subnets are \ + "Step 20 done: subnet M is gone as of registry version {}; the remaining subnets are \ {remaining:?}", topology.get_registry_version(), ); @@ -1384,6 +1560,45 @@ async fn await_loop_started(canister: &UniversalCanister<'_>, name: &str, logger .unwrap_or_else(|e| panic!("{name}'s endless loop did not start: {e}")); } +/// Takes a snapshot of `canister_id` and returns its ID. +async fn take_canister_snapshot(agent: &Agent, canister_id: Principal) -> SnapshotId { + let (snapshot,) = ManagementCanister::create(agent) + .take_canister_snapshot(&TakeCanisterSnapshotArgs { + canister_id, + replace_snapshot: None, + sender_canister_version: None, + uninstall_code: None, + }) + .call_and_wait() + .await + .expect("taking a canister snapshot should succeed"); + snapshot.id +} + +/// The IDs of the snapshots `canister_id` holds. +async fn canister_snapshot_ids(agent: &Agent, canister_id: Principal) -> Vec { + let (snapshots,) = ManagementCanister::create(agent) + .list_canister_snapshots(&canister_id) + .call_and_wait() + .await + .expect("listing the canister snapshots should succeed"); + snapshots.into_iter().map(|snapshot| snapshot.id).collect() +} + +/// `canister`'s cycles balance, read via a query (an ingress message would be +/// rejected by a subnet that is cooling down). +async fn cycles_balance(canister: &UniversalCanister<'_>) -> Result { + let reply = canister + .query(wasm().cycles_balance128().append_and_reply()) + .await + .map_err(|e| anyhow!("failed to read the cycles balance: {e}"))?; + let reply: [u8; 16] = reply + .as_slice() + .try_into() + .map_err(|_| anyhow!("expected 16 bytes, got {} bytes: {reply:?}", reply.len()))?; + Ok(u128::from_le_bytes(reply)) +} + /// Returns `canister`'s global counter, read via a query (an ingress message /// would be rejected by a subnet that is cooling down). async fn global_counter(canister: &UniversalCanister<'_>) -> Result { @@ -1731,13 +1946,19 @@ async fn await_halted_at_checkpoint(node: &IcNodeSnapshot, name: &str, logger: & HALT_TIMEOUT, HALT_BACKOFF, || async { - if !journal - .contains(HALTED_LOG_PATTERN) - .map_err(|e| anyhow!("failed to search the journal of subnet {name}: {e}"))? - { - bail!("subnet {name} has not reported that it is halted yet"); + // `contains` runs `journalctl | grep`, and `grep` exits non-zero when + // it matches nothing, which the SSH helper in turn reports as an + // error: an error here is indistinguishable from the line not being + // there yet, so both mean "keep waiting". A journal that cannot be + // searched at all therefore surfaces as the timeout below. + match journal.contains(HALTED_LOG_PATTERN) { + Ok(true) => Ok(()), + Ok(false) => bail!("subnet {name} has not reported that it is halted yet"), + Err(e) => bail!( + "subnet {name} has not reported that it is halted yet (or its journal could \ + not be searched: {e})" + ), } - Ok(()) } ) .await From 76ecda4e38ede0c9813e75433916546db20f9844 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 31 Aug 2026 09:26:31 +0000 Subject: [PATCH 10/30] feat: a MergeSubnets proposal performs the registry side of a subnet merge Add the `MergeSubnets` NNS function, mapped to the `merge_subnets` method of the registry canister under the `SubnetManagement` topic, and extend that method so that a single such proposal performs the whole registry-side part of a subnet merge, all in one registry version, so that the destination subnet never observes a state where only some of it took effect: 1. the canister ID ranges of the source subnet are merged into the canister ID range set of the destination subnet, so that all canisters that used to be hosted by the source subnet are routed to the destination subnet; 2. a recovery catch-up package is created for the destination subnet, at the height, time and state hash of the merged state, running a fresh DKG for the destination subnet's membership; and 3. the destination subnet is brought back online. The payload therefore grows the `height`, `time_ns` and `state_hash` of the merged state, plus an `initial_dkg_subnet_id` naming the subnet that handles the `setup_initial_dkg` call: it must not be the destination subnet, which is offline while the merge is in progress. The caller is expected to have taken both subnets offline and to have extended the state of the destination subnet with the state of the canisters of the source subnet beforehand; `state_hash` is the hash of the manifest of the result. The source subnet record is not modified and the source subnet is not deleted: it merely does not host any canister ID range anymore. The recovery catch-up package needs fresh DKG transcripts because its `initial_ni_dkg_transcript_{low,high}_threshold` become the current transcripts of the DKG summary that bootstraps consensus at the recovery height. The registry only holds the destination subnet's genesis (or last recovery) catch-up package, whose transcripts its nodes may no longer have the secret key shares for, so reusing those would risk restarting a subnet that cannot sign. Merging into a subnet holding chain keys is rejected: recovering such a subnet requires resharing its keys onto the recovery catch-up package, which this method does not do, and silently leaving the destination subnet unable to sign would be worse than refusing. As the method now makes an inter-canister call half way through, it follows `do_recover_subnet` in checking that none of the records it is about to overwrite -- and none of the canister ID ranges it validated -- changed while that call was in flight. Unlike `do_recover_subnet`, it reports the reject code and message if that call fails, rather than an opaque `unwrap` panic. `StateMachine` only answered `setup_initial_dkg` requests from `do_execute_round`, which `tick()` (and hence `await_ingress`) does not go through, so any canister awaiting such a call hung there forever. Factor the fake responses out into `setup_initial_dkg_responses` and produce them from `tick_with_config` as well, next to the threshold signing requests it already answers. Without this the integration test of the success path cannot run, which is why it is part of this commit. The two success unit tests of `merge_subnets`, which can no longer drive the method to completion, now exercise the routing table part directly; the success path as a whole, including the recovery catch-up package and the unhalting, is covered by the integration test. Co-Authored-By: Claude Opus 5 --- rs/nns/governance/api/src/types.rs | 8 + .../ic_nns_governance/pb/v1/governance.proto | 7 + .../src/gen/ic_nns_governance.pb.v1.rs | 8 + rs/nns/governance/src/pb/conversions/mod.rs | 2 + .../src/proposals/execute_nns_function.rs | 16 +- rs/registry/canister/canister/canister.rs | 7 +- rs/registry/canister/canister/registry.did | 4 + .../canister/canister/registry_test.did | 4 + .../src/mutations/do_recover_subnet.rs | 2 +- .../canister/src/mutations/merge_subnets.rs | 311 +++++++++++++++--- rs/registry/canister/tests/merge_subnets.rs | 62 +++- rs/registry/canister/unreleased_changelog.md | 20 +- rs/state_machine_tests/src/lib.rs | 73 ++-- 13 files changed, 436 insertions(+), 88 deletions(-) diff --git a/rs/nns/governance/api/src/types.rs b/rs/nns/governance/api/src/types.rs index 895abc43b262..0b81400536bc 100644 --- a/rs/nns/governance/api/src/types.rs +++ b/rs/nns/governance/api/src/types.rs @@ -4333,6 +4333,12 @@ pub enum NnsFunction { /// `SetupInitialDKG` requests without an explicit subnet id are routed to the /// calling subnet (NNS). SetDefaultInitialDkgSubnet = 58, + /// Merge a subnet into another subnet: the canister ID ranges of the source + /// subnet are merged into the canister ID range set of the destination subnet, + /// a recovery catch-up package is created for the destination subnet (whose + /// state is expected to have been extended with the state of the canisters of + /// the source subnet) and the destination subnet is brought back online. + MergeSubnets = 59, } impl NnsFunction { /// String value of the enum field names used in the ProtoBuf definition. @@ -4421,6 +4427,7 @@ impl NnsFunction { NnsFunction::SetDefaultInitialDkgSubnet => { "NNS_FUNCTION_SET_DEFAULT_INITIAL_DKG_SUBNET" } + NnsFunction::MergeSubnets => "NNS_FUNCTION_MERGE_SUBNETS", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -4504,6 +4511,7 @@ impl NnsFunction { "NNS_FUNCTION_SPLIT_SUBNET" => Some(Self::SplitSubnet), "NNS_FUNCTION_DELETE_SUBNET" => Some(Self::DeleteSubnet), "NNS_FUNCTION_SET_DEFAULT_INITIAL_DKG_SUBNET" => Some(Self::SetDefaultInitialDkgSubnet), + "NNS_FUNCTION_MERGE_SUBNETS" => Some(Self::MergeSubnets), _ => None, } } diff --git a/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto b/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto index 811308539c83..115184a2b76d 100644 --- a/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto +++ b/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto @@ -513,6 +513,13 @@ enum NnsFunction { // `SetupInitialDKG` requests without an explicit subnet id are routed to the // calling subnet (NNS). NNS_FUNCTION_SET_DEFAULT_INITIAL_DKG_SUBNET = 58; + + // Merge a subnet into another subnet: the canister ID ranges of the source + // subnet are merged into the canister ID range set of the destination subnet, + // a recovery catch-up package is created for the destination subnet (whose + // state is expected to have been extended with the state of the canisters of + // the source subnet) and the destination subnet is brought back online. + NNS_FUNCTION_MERGE_SUBNETS = 59; } // Payload of a proposal that calls a function on another NNS diff --git a/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs b/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs index 78cc3c203c61..3e89c08852d3 100644 --- a/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs +++ b/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs @@ -5546,6 +5546,12 @@ pub enum NnsFunction { /// `SetupInitialDKG` requests without an explicit subnet id are routed to the /// calling subnet (NNS). SetDefaultInitialDkgSubnet = 58, + /// Merge a subnet into another subnet: the canister ID ranges of the source + /// subnet are merged into the canister ID range set of the destination subnet, + /// a recovery catch-up package is created for the destination subnet (whose + /// state is expected to have been extended with the state of the canisters of + /// the source subnet) and the destination subnet is brought back online. + MergeSubnets = 59, } impl NnsFunction { /// String value of the enum field names used in the ProtoBuf definition. @@ -5622,6 +5628,7 @@ impl NnsFunction { Self::SplitSubnet => "NNS_FUNCTION_SPLIT_SUBNET", Self::DeleteSubnet => "NNS_FUNCTION_DELETE_SUBNET", Self::SetDefaultInitialDkgSubnet => "NNS_FUNCTION_SET_DEFAULT_INITIAL_DKG_SUBNET", + Self::MergeSubnets => "NNS_FUNCTION_MERGE_SUBNETS", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -5705,6 +5712,7 @@ impl NnsFunction { "NNS_FUNCTION_SPLIT_SUBNET" => Some(Self::SplitSubnet), "NNS_FUNCTION_DELETE_SUBNET" => Some(Self::DeleteSubnet), "NNS_FUNCTION_SET_DEFAULT_INITIAL_DKG_SUBNET" => Some(Self::SetDefaultInitialDkgSubnet), + "NNS_FUNCTION_MERGE_SUBNETS" => Some(Self::MergeSubnets), _ => None, } } diff --git a/rs/nns/governance/src/pb/conversions/mod.rs b/rs/nns/governance/src/pb/conversions/mod.rs index 280e49e2c7f9..0f8c33efd8e1 100644 --- a/rs/nns/governance/src/pb/conversions/mod.rs +++ b/rs/nns/governance/src/pb/conversions/mod.rs @@ -3940,6 +3940,7 @@ impl From for api::NnsFunction { api::NnsFunction::SetSubnetOperationalLevel } pb::NnsFunction::SplitSubnet => api::NnsFunction::SplitSubnet, + pb::NnsFunction::MergeSubnets => api::NnsFunction::MergeSubnets, pb::NnsFunction::DeleteSubnet => api::NnsFunction::DeleteSubnet, pb::NnsFunction::SetDefaultInitialDkgSubnet => { api::NnsFunction::SetDefaultInitialDkgSubnet @@ -4040,6 +4041,7 @@ impl From for pb::NnsFunction { pb::NnsFunction::SetSubnetOperationalLevel } api::NnsFunction::SplitSubnet => pb::NnsFunction::SplitSubnet, + api::NnsFunction::MergeSubnets => pb::NnsFunction::MergeSubnets, api::NnsFunction::DeleteSubnet => pb::NnsFunction::DeleteSubnet, api::NnsFunction::SetDefaultInitialDkgSubnet => { pb::NnsFunction::SetDefaultInitialDkgSubnet diff --git a/rs/nns/governance/src/proposals/execute_nns_function.rs b/rs/nns/governance/src/proposals/execute_nns_function.rs index 7aba6eaa7d39..cd50c8a6215d 100644 --- a/rs/nns/governance/src/proposals/execute_nns_function.rs +++ b/rs/nns/governance/src/proposals/execute_nns_function.rs @@ -459,6 +459,7 @@ pub enum ValidNnsFunction { SplitSubnet, DeleteSubnet, SetDefaultInitialDkgSubnet, + MergeSubnets, } impl ValidNnsFunction { @@ -592,6 +593,7 @@ impl ValidNnsFunction { ValidNnsFunction::SetDefaultInitialDkgSubnet => { (REGISTRY_CANISTER_ID, "set_default_initial_dkg_subnet") } + ValidNnsFunction::MergeSubnets => (REGISTRY_CANISTER_ID, "merge_subnets"), } } @@ -623,7 +625,8 @@ impl ValidNnsFunction { | ValidNnsFunction::SetSubnetOperationalLevel | ValidNnsFunction::SplitSubnet | ValidNnsFunction::DeleteSubnet - | ValidNnsFunction::SetDefaultInitialDkgSubnet => Topic::SubnetManagement, + | ValidNnsFunction::SetDefaultInitialDkgSubnet + | ValidNnsFunction::MergeSubnets => Topic::SubnetManagement, ValidNnsFunction::ReviseElectedGuestosVersions | ValidNnsFunction::ReviseElectedHostosVersions => Topic::IcOsVersionElection, @@ -714,6 +717,7 @@ impl ValidNnsFunction { ValidNnsFunction::SplitSubnet => "Split subnet", ValidNnsFunction::DeleteSubnet => "Delete Subnet", ValidNnsFunction::SetDefaultInitialDkgSubnet => "Set Default Initial DKG Subnet", + ValidNnsFunction::MergeSubnets => "Merge subnets", } } @@ -944,6 +948,15 @@ impl ValidNnsFunction { calls are routed when no subnet is specified explicitly in the request. If unset, \ such requests are routed to the calling subnet (NNS)." } + ValidNnsFunction::MergeSubnets => { + "Merge a subnet into another subnet. The canister ID ranges of the source subnet \ + are merged into the canister ID range set of the destination subnet, so that all \ + canisters that used to be hosted by the source subnet are routed to the \ + destination subnet; a recovery catch-up package is created for the destination \ + subnet, whose state is expected to have been extended with the state of the \ + canisters of the source subnet while both subnets were offline; and the \ + destination subnet is brought back online." + } } } } @@ -1033,6 +1046,7 @@ impl TryFrom for ValidNnsFunction { Ok(ValidNnsFunction::SetSubnetOperationalLevel) } NnsFunction::SplitSubnet => Ok(ValidNnsFunction::SplitSubnet), + NnsFunction::MergeSubnets => Ok(ValidNnsFunction::MergeSubnets), NnsFunction::DeleteSubnet => Ok(ValidNnsFunction::DeleteSubnet), NnsFunction::SetDefaultInitialDkgSubnet => { Ok(ValidNnsFunction::SetDefaultInitialDkgSubnet) diff --git a/rs/registry/canister/canister/canister.rs b/rs/registry/canister/canister/canister.rs index 01568e96f077..a342f824e13c 100644 --- a/rs/registry/canister/canister/canister.rs +++ b/rs/registry/canister/canister/canister.rs @@ -1089,13 +1089,16 @@ fn reroute_canister_ranges_(payload: RerouteCanisterRangesPayload) { #[unsafe(export_name = "canister_update merge_subnets")] fn merge_subnets() { check_caller_is_governance_and_log("merge_subnets"); - over(candid_one, merge_subnets_); + over_async(candid_one, |payload: MergeSubnetsPayload| async move { + merge_subnets_(payload).await + }); } #[candid_method(update, rename = "merge_subnets")] -fn merge_subnets_(payload: MergeSubnetsPayload) { +async fn merge_subnets_(payload: MergeSubnetsPayload) { registry_mut() .merge_subnets(payload) + .await .unwrap_or_else(|error_message| { trap_with(&format!( "{LOG_PREFIX} Merge subnets failed: {error_message}" diff --git a/rs/registry/canister/canister/registry.did b/rs/registry/canister/canister/registry.did index a3759d9ffb20..7183998d0222 100644 --- a/rs/registry/canister/canister/registry.did +++ b/rs/registry/canister/canister/registry.did @@ -311,6 +311,10 @@ type IPv4Config = record { type MergeSubnetsPayload = record { source_subnet : principal; destination_subnet : principal; + height : nat64; + time_ns : nat64; + state_hash : blob; + initial_dkg_subnet_id : opt principal; }; type MigrateCanistersPayload = record { diff --git a/rs/registry/canister/canister/registry_test.did b/rs/registry/canister/canister/registry_test.did index 7493ae134411..2be881b0da74 100644 --- a/rs/registry/canister/canister/registry_test.did +++ b/rs/registry/canister/canister/registry_test.did @@ -311,6 +311,10 @@ type IPv4Config = record { type MergeSubnetsPayload = record { source_subnet : principal; destination_subnet : principal; + height : nat64; + time_ns : nat64; + state_hash : blob; + initial_dkg_subnet_id : opt principal; }; type MigrateCanistersPayload = record { diff --git a/rs/registry/canister/src/mutations/do_recover_subnet.rs b/rs/registry/canister/src/mutations/do_recover_subnet.rs index cf393eec3a7d..b07e8bfc9296 100644 --- a/rs/registry/canister/src/mutations/do_recover_subnet.rs +++ b/rs/registry/canister/src/mutations/do_recover_subnet.rs @@ -488,7 +488,7 @@ impl TryFrom for KeyConfigRequestInternal { } } -fn panic_if_record_changed_across_versions( +pub(crate) fn panic_if_record_changed_across_versions( registry: &Registry, key: &str, initial_registry_version: Version, diff --git a/rs/registry/canister/src/mutations/merge_subnets.rs b/rs/registry/canister/src/mutations/merge_subnets.rs index a907ac83af77..3d141d13a2f7 100644 --- a/rs/registry/canister/src/mutations/merge_subnets.rs +++ b/rs/registry/canister/src/mutations/merge_subnets.rs @@ -1,28 +1,71 @@ -use crate::{common::LOG_PREFIX, registry::Registry}; -use candid::CandidType; +//! Contains the method to merge a subnet into another subnet. +//! +//! Merging the source subnet into the destination subnet reroutes all canisters +//! of the source subnet to the destination subnet and lets the destination +//! subnet resume from a state that was extended, while both subnets were +//! offline, with the state of those canisters. The state extension itself +//! happens outside of the registry: this method only records its outcome, as the +//! state hash of a recovery catch-up package for the destination subnet. + +use crate::{ + common::LOG_PREFIX, mutations::do_recover_subnet::panic_if_record_changed_across_versions, + registry::Registry, +}; +use candid::{CandidType, Encode}; +use dfn_core::api::{CanisterId, call}; #[cfg(target_arch = "wasm32")] use dfn_core::println; -use ic_base_types::SubnetId; -use ic_registry_keys::make_subnet_record_key; +use ic_base_types::{NodeId, PrincipalId, RegistryVersion, SubnetId}; +use ic_management_canister_types_private::{SetupInitialDKGArgs, SetupInitialDKGResponse}; +use ic_protobuf::registry::subnet::v1::{RecoveryArgs, catch_up_package_contents::CupType}; +use ic_registry_keys::{ + make_catch_up_package_contents_key, make_crypto_threshold_signing_pubkey_key, + make_subnet_record_key, +}; use ic_registry_routing_table::are_disjoint; +use ic_registry_transport::{ + pb::v1::{RegistryMutation, registry_mutation}, + upsert, +}; +use on_wire::bytes; +use prost::Message; use serde::{Deserialize, Serialize}; +use std::convert::TryFrom; impl Registry { - /// Merges the canister ID ranges of the source subnet into the canister ID - /// range set of the destination subnet. + /// Merges the source subnet into the destination subnet. /// - /// After this operation, all canisters that used to be hosted by the source - /// subnet are routed to the destination subnet and the source subnet does - /// not host any canister ID range anymore. + /// Three things happen, all in a single registry version, so that the + /// destination subnet never observes a state where only some of them took + /// effect: /// - /// Note that only the routing table is updated: neither subnet record is - /// modified and, in particular, the source subnet is not deleted. - pub fn merge_subnets(&mut self, payload: MergeSubnetsPayload) -> Result<(), String> { + /// 1. the canister ID ranges of the source subnet are merged into the + /// canister ID range set of the destination subnet, so that all + /// canisters that used to be hosted by the source subnet are routed to + /// the destination subnet; + /// 2. a recovery catch-up package is created for the destination subnet, + /// at the height, time and state hash of the merged state, running a + /// fresh DKG for the destination subnet's membership; and + /// 3. the destination subnet is brought back online. + /// + /// The caller is expected to have taken both subnets offline and to have + /// extended the state of the destination subnet with the state of the + /// canisters of the source subnet beforehand; `state_hash` is the hash of + /// the manifest of the resulting merged state. + /// + /// Note that neither subnet record is deleted and, in particular, the source + /// subnet is not deleted: it merely does not host any canister ID range + /// anymore. + pub async fn merge_subnets(&mut self, payload: MergeSubnetsPayload) -> Result<(), String> { println!("{LOG_PREFIX}merge_subnets: {payload:?}"); let MergeSubnetsPayload { source_subnet, destination_subnet, + height, + time_ns, + state_hash, + initial_dkg_subnet_id, } = payload; if source_subnet == destination_subnet { @@ -31,17 +74,18 @@ impl Registry { )); } - let version = self.latest_version(); + let pre_call_registry_version = self.latest_version(); - self.get(&make_subnet_record_key(source_subnet).into_bytes(), version) - .ok_or_else(|| format!("source {source_subnet} is not a known subnet"))?; self.get( - &make_subnet_record_key(destination_subnet).into_bytes(), - version, + &make_subnet_record_key(source_subnet).into_bytes(), + pre_call_registry_version, ) - .ok_or_else(|| format!("destination {destination_subnet} is not a known subnet"))?; + .ok_or_else(|| format!("source {source_subnet} is not a known subnet"))?; + let destination_record = self + .get_subnet(destination_subnet, pre_call_registry_version) + .map_err(|_| format!("destination {destination_subnet} is not a known subnet"))?; - let routing_table = self.get_routing_table_or_panic(version); + let routing_table = self.get_routing_table_or_panic(pre_call_registry_version); let source_ranges = routing_table.ranges(source_subnet); if source_ranges.is_empty() { return Err(format!( @@ -52,7 +96,7 @@ impl Registry { // Rerouting the canister ID ranges of the source subnet would break any ongoing canister // migration out of those ranges: the migrated ranges would end up being hosted by the // destination subnet, which is not on the recorded migration trace. - if let Some(canister_migrations) = self.get_canister_migrations(version) + if let Some(canister_migrations) = self.get_canister_migrations(pre_call_registry_version) && !are_disjoint(canister_migrations.ranges(), source_ranges.iter()) { return Err(format!( @@ -60,12 +104,148 @@ impl Registry { )); } - self.maybe_apply_mutation_internal(self.merge_subnets_mutation( - version, + // Recovering a subnet holding chain keys requires resharing those keys onto the recovery + // CUP, which this method does not do; rather than silently leave the destination subnet + // unable to sign, refuse to merge into it. + if destination_record + .chain_key_config + .as_ref() + .is_some_and(|config| !config.key_configs.is_empty()) + { + return Err(format!( + "destination subnet {destination_subnet} holds chain keys, which merging does not reshare" + )); + } + + // `setup_initial_dkg` must not be handled by the subnet being recovered, as that subnet is + // offline and could not respond. + if let Some(initial_dkg_subnet_id) = initial_dkg_subnet_id { + if initial_dkg_subnet_id == destination_subnet { + return Err(format!( + "initial DKG subnet {initial_dkg_subnet_id} must be different from the destination subnet" + )); + } + self.get( + &make_subnet_record_key(initial_dkg_subnet_id).into_bytes(), + pre_call_registry_version, + ) + .ok_or_else(|| { + format!("initial DKG subnet {initial_dkg_subnet_id} is not a known subnet") + })?; + } + + let mut cup_contents = self + .get_subnet_catch_up_package(destination_subnet, Some(pre_call_registry_version)) + .map_err(|err| format!("failed to get the CUP of {destination_subnet}: {err}"))?; + cup_contents.registry_store_uri = None; + + let mut subnet_record = destination_record; + + // Bring the destination subnet back online. Consensus looks at the registry version from + // the highest CUP when considering `halt_at_cup_height`, so clearing both flags is what + // makes the subnet resume from the recovery CUP created below. + subnet_record.halt_at_cup_height = false; + subnet_record.is_halted = false; + + let dkg_nodes: Vec = subnet_record + .membership + .iter() + .map(|bytes| NodeId::from(PrincipalId::try_from(bytes).unwrap())) + .collect(); + + let request = SetupInitialDKGArgs::new( + dkg_nodes, + RegistryVersion::new(pre_call_registry_version), + initial_dkg_subnet_id, + ); + let response_bytes = call( + CanisterId::ic_00(), + "setup_initial_dkg", + bytes, + Encode!(&request).unwrap(), + ) + .await + .unwrap_or_else(|(code, msg)| { + panic!("{LOG_PREFIX}`setup_initial_dkg` failed with code {code:?}: {msg}") + }); + + let post_call_registry_version = self.latest_version(); + + // Check that the records this method is about to overwrite, and the routing table it based + // its validation on, did not change while `setup_initial_dkg` was in flight. + for (key, what) in [ + ( + make_subnet_record_key(destination_subnet), + format!("Subnet with ID {destination_subnet}"), + ), + ( + make_crypto_threshold_signing_pubkey_key(destination_subnet), + format!("Threshold Signing Pubkey for Subnet {destination_subnet}"), + ), + ( + make_catch_up_package_contents_key(destination_subnet), + format!("CUP for Subnet {destination_subnet}"), + ), + ( + make_subnet_record_key(source_subnet), + format!("Subnet with ID {source_subnet}"), + ), + ] { + panic_if_record_changed_across_versions( + self, + &key, + pre_call_registry_version, + post_call_registry_version, + format!("{what} was updated during the `setup_initial_dkg` call"), + ); + } + assert_eq!( + self.get_routing_table_or_panic(post_call_registry_version) + .ranges(source_subnet), + source_ranges, + "{LOG_PREFIX}The canister ID ranges of subnet {source_subnet} were updated during the \ + `setup_initial_dkg` call", + ); + + let dkg_response = SetupInitialDKGResponse::decode(&response_bytes).unwrap(); + + cup_contents.initial_ni_dkg_transcript_low_threshold = + Some(dkg_response.low_threshold_transcript_record); + cup_contents.initial_ni_dkg_transcript_high_threshold = + Some(dkg_response.high_threshold_transcript_record); + cup_contents.height = height; + cup_contents.time = time_ns; + cup_contents.state_hash = state_hash.clone(); + cup_contents.cup_type = Some(CupType::Recovery(RecoveryArgs { + height, + time: time_ns, + state_hash, + })); + + let mut mutations = vec![ + RegistryMutation { + mutation_type: registry_mutation::Type::Update as i32, + key: make_crypto_threshold_signing_pubkey_key(destination_subnet).into_bytes(), + value: dkg_response.subnet_threshold_public_key.encode_to_vec(), + }, + RegistryMutation { + mutation_type: registry_mutation::Type::Update as i32, + key: make_catch_up_package_contents_key(destination_subnet).into_bytes(), + value: cup_contents.encode_to_vec(), + }, + upsert( + make_subnet_record_key(destination_subnet), + subnet_record.encode_to_vec(), + ), + ]; + mutations.append(&mut self.merge_subnets_mutation( + post_call_registry_version, source_subnet, destination_subnet, )); + self.maybe_apply_mutation_internal(mutations); + Ok(()) } } @@ -77,8 +257,22 @@ pub struct MergeSubnetsPayload { /// set of `destination_subnet`. pub source_subnet: SubnetId, /// The subnet that hosts the canister ID ranges of `source_subnet` after the - /// merge. + /// merge, and that is recovered at the merged state and brought back online. pub destination_subnet: SubnetId, + /// The height of the recovery CUP of `destination_subnet`, i.e. the height + /// of the checkpoint holding the merged state. + pub height: u64, + /// The block time the recovered `destination_subnet` starts from, in + /// nanoseconds since the Epoch. Must be larger than the times of the + /// checkpoints at which both subnets were taken offline. + pub time_ns: u64, + /// The hash of the manifest of the merged state. + pub state_hash: Vec, + /// The subnet that should handle the `setup_initial_dkg` call producing the + /// DKG transcripts of the recovery CUP. Must be different from + /// `destination_subnet`, which is offline while the merge is in progress. If + /// unset, the request is handled by the NNS subnet. + pub initial_dkg_subnet_id: Option, } #[cfg(test)] @@ -94,11 +288,25 @@ mod tests { routing_table::routing_table_into_registry_mutation, }, }; + use futures::executor::block_on; use ic_registry_routing_table::{CanisterIdRange, RoutingTable}; use ic_types::CanisterId; use ic_types_test_utils::ids::{SUBNET_1, SUBNET_2, SUBNET_3}; use maplit::btreemap; + /// The recovery CUP fields of the payload, which the validation-failure tests + /// below are not about: they all fail before the CUP is even looked at. + fn default_cup_args() -> MergeSubnetsPayload { + MergeSubnetsPayload { + source_subnet: SUBNET_1, + destination_subnet: SUBNET_2, + height: 100, + time_ns: 1_000_000_000, + state_hash: vec![1; 32], + initial_dkg_subnet_id: Some(SUBNET_3), + } + } + fn range(start: u64, end: u64) -> CanisterIdRange { CanisterIdRange { start: CanisterId::from_u64(start), @@ -149,24 +357,28 @@ mod tests { .collect::>() } + /// Applies just the routing table part of a merge. `Registry::merge_subnets` + /// itself cannot be driven to completion in a unit test, as it calls + /// `setup_initial_dkg` on the management canister half way through; the + /// success path as a whole is covered by the integration test in + /// `rs/registry/canister/tests/merge_subnets.rs`. + fn merge_routing_table(registry: &mut Registry, source: SubnetId, destination: SubnetId) { + let mutations = + registry.merge_subnets_mutation(registry.latest_version(), source, destination); + registry.maybe_apply_mutation_internal(mutations); + } + #[test] fn test_merge_subnets() { // Step 1: Prepare the world. let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { - source_subnet: SUBNET_1, - destination_subnet: SUBNET_2, - }); - - // Step 3: Verify results. - - // Step 3.1: Inspect the return value. - assert_eq!(result, Ok(())); + merge_routing_table(&mut registry, SUBNET_1, SUBNET_2); - // Step 3.2: The canister ID ranges of both subnets are now hosted by the - // destination subnet, and the three adjacent ranges got merged into one. + // Step 3: Verify results: the canister ID ranges of both subnets are now + // hosted by the destination subnet, and the three adjacent ranges got + // merged into one. assert_eq!( get_routing_table_entries(®istry), vec![(range(10, 39), SUBNET_2)], @@ -189,14 +401,10 @@ mod tests { )); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { - source_subnet: SUBNET_1, - destination_subnet: SUBNET_2, - }); + merge_routing_table(&mut registry, SUBNET_1, SUBNET_2); // Step 3: Verify results. Both ranges are hosted by the destination subnet // and, not being adjacent, did not get merged into a single entry. - assert_eq!(result, Ok(())); assert_eq!( get_routing_table_entries(®istry), vec![(range(10, 19), SUBNET_2), (range(30, 39), SUBNET_2)], @@ -209,10 +417,11 @@ mod tests { let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { + let result = block_on(registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_1, - }); + ..default_cup_args() + })); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -236,10 +445,11 @@ mod tests { let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { + let result = block_on(registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_3, destination_subnet: SUBNET_2, - }); + ..default_cup_args() + })); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -255,10 +465,11 @@ mod tests { let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { + let result = block_on(registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_3, - }); + ..default_cup_args() + })); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -281,10 +492,11 @@ mod tests { )); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { + let result = block_on(registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_2, - }); + ..default_cup_args() + })); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -308,10 +520,11 @@ mod tests { .unwrap(); // Step 2: Run the code under test. - let result = registry.merge_subnets(MergeSubnetsPayload { + let result = block_on(registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_2, - }); + ..default_cup_args() + })); // Step 3: Verify results. let error_message = result.unwrap_err(); diff --git a/rs/registry/canister/tests/merge_subnets.rs b/rs/registry/canister/tests/merge_subnets.rs index 16139835f211..119fb6c1337e 100644 --- a/rs/registry/canister/tests/merge_subnets.rs +++ b/rs/registry/canister/tests/merge_subnets.rs @@ -6,6 +6,7 @@ use ic_nns_test_utils::{ }, registry::{initial_routing_table_mutations, prepare_registry_with_two_node_sets}, }; +use ic_protobuf::registry::subnet::v1::{RecoveryArgs, catch_up_package_contents::CupType}; use ic_registry_routing_table::{CanisterIdRange, RoutingTable}; use ic_registry_transport::pb::v1::RegistryAtomicMutateRequest; use ic_types::CanisterId; @@ -14,13 +15,25 @@ use registry_canister::{ }; mod common; -use common::test_helpers::{check_error_message, check_subnet_for_canisters}; +use common::test_helpers::{ + check_error_message, check_subnet_for_canisters, get_cup_contents, get_subnet_record, +}; + +/// The recovery CUP the merge creates for the destination subnet. The values are +/// arbitrary: this test does not run a subnet from the resulting CUP, it only +/// checks that the endpoint accepts them and records them. +const MERGE_HEIGHT: u64 = 100; +const MERGE_TIME_NS: u64 = 1_234_567_890; +const MERGED_STATE_HASH: &[u8] = &[42; 32]; /// Exercises the `merge_subnets` endpoint end to end. The payload validation /// itself is covered by the unit tests of `Registry::merge_subnets`, so this test /// only covers what those cannot: that the endpoint is reachable with a Candid -/// encoded payload, that only governance may call it, and that the resulting -/// routing table is visible through the canister's query API. +/// encoded payload, that only governance may call it, that the canisters of the +/// source subnet end up routed to the destination subnet, and that the recovery +/// CUP of the destination subnet -- whose DKG transcripts come from the +/// `setup_initial_dkg` call the endpoint makes half way through -- is recorded +/// and the destination subnet is brought back online. #[test] fn test_merge_subnets() { state_machine_test_on_nns_subnet(|runtime| { @@ -76,6 +89,10 @@ fn test_merge_subnets() { MergeSubnetsPayload { source_subnet: subnet_id_1, destination_subnet: subnet_id_2, + height: MERGE_HEIGHT, + time_ns: MERGE_TIME_NS, + state_hash: MERGED_STATE_HASH.to_vec(), + initial_dkg_subnet_id: None, }, ) .await as Result<(), String>, @@ -90,6 +107,10 @@ fn test_merge_subnets() { Encode!(&MergeSubnetsPayload { source_subnet: subnet_id_1, destination_subnet: subnet_id_2, + height: MERGE_HEIGHT, + time_ns: MERGE_TIME_NS, + state_hash: MERGED_STATE_HASH.to_vec(), + initial_dkg_subnet_id: None, }) .unwrap(), ) @@ -109,6 +130,41 @@ fn test_merge_subnets() { ) .await; + // Step 5: Verify results: the destination subnet got a recovery CUP at the + // height, time and state hash of the merged state, with fresh DKG + // transcripts, and is no longer halted. + let cup_contents = get_cup_contents(®istry, subnet_id_2).await; + assert_eq!(cup_contents.height, MERGE_HEIGHT); + assert_eq!(cup_contents.time, MERGE_TIME_NS); + assert_eq!(cup_contents.state_hash, MERGED_STATE_HASH); + assert_eq!( + cup_contents.cup_type, + Some(CupType::Recovery(RecoveryArgs { + height: MERGE_HEIGHT, + time: MERGE_TIME_NS, + state_hash: MERGED_STATE_HASH.to_vec(), + })), + ); + assert!( + cup_contents + .initial_ni_dkg_transcript_low_threshold + .is_some(), + "the recovery CUP should hold a low threshold DKG transcript", + ); + assert!( + cup_contents + .initial_ni_dkg_transcript_high_threshold + .is_some(), + "the recovery CUP should hold a high threshold DKG transcript", + ); + + let subnet_record = get_subnet_record(®istry, subnet_id_2).await; + assert!( + !subnet_record.is_halted, + "the destination subnet should have been brought back online", + ); + assert!(!subnet_record.halt_at_cup_height); + Ok(()) } }); diff --git a/rs/registry/canister/unreleased_changelog.md b/rs/registry/canister/unreleased_changelog.md index e83c068094e7..0c084bccda81 100644 --- a/rs/registry/canister/unreleased_changelog.md +++ b/rs/registry/canister/unreleased_changelog.md @@ -14,10 +14,22 @@ on the process that this file is part of, see be set on mainnet before the replica version rejecting ingress messages to cooling down subnets has been rolled out to all subnets. -* `merge_subnets` endpoint. It takes a source and a destination subnet ID, and merges the canister - ID ranges of the source subnet into the canister ID range set of the destination subnet, i.e., the - canisters hosted by the source subnet are routed to the destination subnet afterwards. Only the - routing table is updated: neither subnet record is modified and the source subnet is not deleted. +* `merge_subnets` endpoint, callable through a `MergeSubnets` proposal. It takes a source and a + destination subnet ID, plus the height, time and state hash of the merged state, and performs the + whole registry-side part of a subnet merge in a single registry version: + + * the canister ID ranges of the source subnet are merged into the canister ID range set of the + destination subnet, i.e., the canisters hosted by the source subnet are routed to the + destination subnet afterwards; + * a recovery catch-up package is created for the destination subnet, at the given height, time + and state hash, running a fresh DKG for the destination subnet's membership; and + * the destination subnet is brought back online. + + The caller is expected to have taken both subnets offline and to have extended the state of the + destination subnet with the state of the canisters of the source subnet beforehand. The source + subnet record is not modified and the source subnet is not deleted: it merely does not host any + canister ID range anymore. Merging into a subnet holding chain keys is rejected, as the recovery + catch-up package does not reshare them. ## Changed diff --git a/rs/state_machine_tests/src/lib.rs b/rs/state_machine_tests/src/lib.rs index 2bf6c8f96f16..b1f51d275d71 100644 --- a/rs/state_machine_tests/src/lib.rs +++ b/rs/state_machine_tests/src/lib.rs @@ -1846,6 +1846,42 @@ impl Default for StateMachineBuilder { } } +/// The responses consensus would produce for the pending `setup_initial_dkg` +/// requests of `state`: a dummy transcript per request, derived from `seed` so +/// that the result stays deterministic. +/// +/// `setup_initial_dkg` can only be called on the NNS subnet, so the seed does +/// not need to depend on the subnet ID. +fn setup_initial_dkg_responses(state: &ReplicatedState, seed: u64) -> Vec { + let mut rng = StdRng::seed_from_u64(seed); + state + .metadata + .subnet_call_context_manager + .setup_initial_dkg_contexts + .keys() + .map(|callback_id| { + let ni_dkg_transcript = dummy_initial_dkg_transcript_with_master_key(&mut rng).0; + let public_key = (&ni_dkg_transcript).try_into().unwrap(); + let public_key_der = threshold_sig_public_key_to_der(public_key).unwrap(); + let subnet_id = PrincipalId::new_self_authenticating(&public_key_der).into(); + let mut high_threshold_transcript_record = ni_dkg_transcript.clone(); + high_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::HighThreshold; + let mut low_threshold_transcript_record = ni_dkg_transcript; + low_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::LowThreshold; + let initial_transcript_records = SetupInitialDKGResponse { + low_threshold_transcript_record: high_threshold_transcript_record.into(), + high_threshold_transcript_record: low_threshold_transcript_record.into(), + fresh_subnet_id: subnet_id, + subnet_threshold_public_key: public_key.into(), + }; + ConsensusResponse::new( + *callback_id, + MsgPayload::Data(initial_transcript_records.encode()), + ) + }) + .collect() +} + impl StateMachine { /// Provides the implicit time increment for a single round of execution /// if time does not advance between consecutive rounds. @@ -1960,34 +1996,7 @@ impl StateMachine { } let self_validating = Some(batch_payload.self_validating); let mut consensus_responses = http_responses; - // `setup_initial_dkg` can only be called on the NNS subnet - // and thus the seed does not need to depend on the subnet ID - let mut rng = StdRng::seed_from_u64(certified_height.get()); - for callback_id in state - .metadata - .subnet_call_context_manager - .setup_initial_dkg_contexts - .keys() - { - let ni_dkg_transcript = dummy_initial_dkg_transcript_with_master_key(&mut rng).0; - let public_key = (&ni_dkg_transcript).try_into().unwrap(); - let public_key_der = threshold_sig_public_key_to_der(public_key).unwrap(); - let subnet_id = PrincipalId::new_self_authenticating(&public_key_der).into(); - let mut high_threshold_transcript_record = ni_dkg_transcript.clone(); - high_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::HighThreshold; - let mut low_threshold_transcript_record = ni_dkg_transcript; - low_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::LowThreshold; - let initial_transcript_records = SetupInitialDKGResponse { - low_threshold_transcript_record: high_threshold_transcript_record.into(), - high_threshold_transcript_record: low_threshold_transcript_record.into(), - fresh_subnet_id: subnet_id, - subnet_threshold_public_key: public_key.into(), - }; - consensus_responses.push(ConsensusResponse::new( - *callback_id, - MsgPayload::Data(initial_transcript_records.encode()), - )); - } + consensus_responses.extend(setup_initial_dkg_responses(&state, certified_height.get())); let mut payload = PayloadBuilder::new() .with_ingress_messages(ingress_messages) .with_xnet_payload(xnet_payload) @@ -3035,6 +3044,14 @@ impl StateMachine { self.process_threshold_signing_request(id, context, &mut payload_builder); } + // Process `setup_initial_dkg` requests, which consensus would answer. + payload_builder + .consensus_responses + .extend(setup_initial_dkg_responses( + &state, + self.state_manager.latest_state_height().get(), + )); + self.execute_payload(payload_builder); } From 4b43061883e896b1b2c6c3d57bdaa3eb761fac5f Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Tue, 1 Sep 2026 06:43:40 +0000 Subject: [PATCH 11/30] chore(governance): changelog entry for the MergeSubnets proposal type Co-Authored-By: Claude Opus 5 --- rs/nns/governance/unreleased_changelog.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rs/nns/governance/unreleased_changelog.md b/rs/nns/governance/unreleased_changelog.md index 94126a0ff421..a7ef4662e34d 100644 --- a/rs/nns/governance/unreleased_changelog.md +++ b/rs/nns/governance/unreleased_changelog.md @@ -9,6 +9,13 @@ on the process that this file is part of, see ## Added +* Added a new `NnsFunction` variant `MergeSubnets`, which proposes to merge a + subnet into another subnet: the canister ID ranges of the source subnet are + merged into the canister ID range set of the destination subnet, a recovery + catch-up package is created for the destination subnet (whose state is + expected to have been extended with the state of the canisters of the source + subnet) and the destination subnet is brought back online. + ## Changed ## Deprecated From 155414e1cbed7e34d37fa5b899541eb2351e0b86 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Tue, 1 Sep 2026 12:11:48 +0000 Subject: [PATCH 12/30] fix: address Copilot review on the merge_subnets PR Revalidate, after `setup_initial_dkg` returns, that no canister migration overlapping the canister ID ranges of the source subnet was prepared while the call was in flight: `prepare_canister_migration` does not touch the routing table, so the existing post-call check could not catch it. Also fix the low/high threshold transcript records being swapped in the `SetupInitialDKGResponse` that `StateMachine` synthesizes for pending `setup_initial_dkg` requests. Co-Authored-By: Claude Opus 5 --- rs/registry/canister/src/mutations/merge_subnets.rs | 12 ++++++++++++ rs/state_machine_tests/src/lib.rs | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/rs/registry/canister/src/mutations/merge_subnets.rs b/rs/registry/canister/src/mutations/merge_subnets.rs index 3d141d13a2f7..dfc3e9d871e0 100644 --- a/rs/registry/canister/src/mutations/merge_subnets.rs +++ b/rs/registry/canister/src/mutations/merge_subnets.rs @@ -206,6 +206,18 @@ impl Registry { "{LOG_PREFIX}The canister ID ranges of subnet {source_subnet} were updated during the \ `setup_initial_dkg` call", ); + // A canister migration overlapping the canister ID ranges of the source subnet could have + // been prepared, without changing the routing table, while `setup_initial_dkg` was in + // flight; rerouting those ranges would break it, just like it would have before the call. + assert!( + self.get_canister_migrations(post_call_registry_version) + .is_none_or(|canister_migrations| are_disjoint( + canister_migrations.ranges(), + source_ranges.iter() + )), + "{LOG_PREFIX}Canister migrations overlapping the canister ID ranges of subnet \ + {source_subnet} were added during the `setup_initial_dkg` call", + ); let dkg_response = SetupInitialDKGResponse::decode(&response_bytes).unwrap(); diff --git a/rs/state_machine_tests/src/lib.rs b/rs/state_machine_tests/src/lib.rs index dff9dc6d55fd..e31216f4983a 100644 --- a/rs/state_machine_tests/src/lib.rs +++ b/rs/state_machine_tests/src/lib.rs @@ -1888,8 +1888,8 @@ fn setup_initial_dkg_responses(state: &ReplicatedState, seed: u64) -> Vec Date: Tue, 1 Sep 2026 12:39:33 +0000 Subject: [PATCH 13/30] fix: clear stale chain key initializations from the merge recovery CUP Chain key initializations in a CUP take precedence over the chain key configuration of the subnet record, so carrying over the ones of the CUP being replaced would make the destination subnet bootstrap obsolete key material -- contradicting the endpoint's refusal to merge into a subnet holding chain keys. Clear both initialization fields, as recovering a subnet without an initial chain key configuration does. Co-Authored-By: Claude Opus 5 --- .../canister/src/mutations/merge_subnets.rs | 7 +++++ rs/registry/canister/tests/merge_subnets.rs | 31 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/rs/registry/canister/src/mutations/merge_subnets.rs b/rs/registry/canister/src/mutations/merge_subnets.rs index dfc3e9d871e0..96879a25802f 100644 --- a/rs/registry/canister/src/mutations/merge_subnets.rs +++ b/rs/registry/canister/src/mutations/merge_subnets.rs @@ -138,6 +138,13 @@ impl Registry { .get_subnet_catch_up_package(destination_subnet, Some(pre_call_registry_version)) .map_err(|err| format!("failed to get the CUP of {destination_subnet}: {err}"))?; cup_contents.registry_store_uri = None; + // Chain key initializations in a CUP take precedence over the chain key configuration of + // the subnet record, so carrying over the ones of the CUP being replaced would make the + // destination subnet bootstrap stale key material. The destination subnet holds no chain + // keys (checked above) and merging reshares none, so both fields are cleared, just like + // recovering a subnet without an initial chain key configuration does. + cup_contents.chain_key_initializations = vec![]; + cup_contents.ecdsa_initializations = vec![]; let mut subnet_record = destination_record; diff --git a/rs/registry/canister/tests/merge_subnets.rs b/rs/registry/canister/tests/merge_subnets.rs index 119fb6c1337e..f552c7801201 100644 --- a/rs/registry/canister/tests/merge_subnets.rs +++ b/rs/registry/canister/tests/merge_subnets.rs @@ -6,10 +6,15 @@ use ic_nns_test_utils::{ }, registry::{initial_routing_table_mutations, prepare_registry_with_two_node_sets}, }; -use ic_protobuf::registry::subnet::v1::{RecoveryArgs, catch_up_package_contents::CupType}; +use ic_protobuf::registry::subnet::v1::{ + CatchUpPackageContents, ChainKeyInitialization, EcdsaInitialization, RecoveryArgs, + catch_up_package_contents::CupType, +}; +use ic_registry_keys::make_catch_up_package_contents_key; use ic_registry_routing_table::{CanisterIdRange, RoutingTable}; use ic_registry_transport::pb::v1::RegistryAtomicMutateRequest; use ic_types::CanisterId; +use prost::Message; use registry_canister::{ init::RegistryCanisterInitPayloadBuilder, mutations::merge_subnets::MergeSubnetsPayload, }; @@ -45,6 +50,25 @@ fn test_merge_subnets() { /* num_nodes_in_subnet = */ 4, /* num_unassigned_nodes = */ 4, true, ); let subnet_id_2 = subnet_id_2_option.unwrap(); + // Give the destination subnet a CUP holding chain key initializations, as it + // would if it had once been recovered while holding chain keys. Merging must + // not carry them over into the recovery CUP it creates. + let subnet_1_mutation = { + let mut subnet_1_mutation = subnet_1_mutation; + let cup_contents_key = make_catch_up_package_contents_key(subnet_id_2).into_bytes(); + let cup_contents_mutation = subnet_1_mutation + .mutations + .iter_mut() + .find(|mutation| mutation.key == cup_contents_key) + .expect("the destination subnet should have CUP contents"); + let mut cup_contents = + CatchUpPackageContents::decode(&cup_contents_mutation.value[..]) + .expect("failed to decode the CUP contents"); + cup_contents.ecdsa_initializations = vec![EcdsaInitialization::default()]; + cup_contents.chain_key_initializations = vec![ChainKeyInitialization::default()]; + cup_contents_mutation.value = cup_contents.encode_to_vec(); + subnet_1_mutation + }; let rt_mutation = { fn range(start: u64, end: u64) -> CanisterIdRange { CanisterIdRange { @@ -157,6 +181,11 @@ fn test_merge_subnets() { .is_some(), "the recovery CUP should hold a high threshold DKG transcript", ); + // Chain key initializations in a CUP take precedence over the chain key + // configuration of the subnet record, so the stale ones seeded above must be + // gone: merging reshares no chain key. + assert_eq!(cup_contents.ecdsa_initializations, vec![]); + assert_eq!(cup_contents.chain_key_initializations, vec![]); let subnet_record = get_subnet_record(®istry, subnet_id_2).await; assert!( From 7c8fe642a9955f176d4a45eb8d2cb92dccb70238 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Tue, 1 Sep 2026 13:03:35 +0000 Subject: [PATCH 14/30] feat: restrict merge_subnets to the routing table change Revert `merge_subnets` to what it originally did: merge the canister ID ranges of the source subnet into the canister ID range set of the destination subnet, and nothing else. The recovery catch-up package, the fresh DKG for the destination subnet's membership and the unhalting are dropped again, and with them the `height`, `time_ns`, `state_hash` and `initial_dkg_subnet_id` payload fields, the inter-canister call and the guards around it. Recovering the destination subnet at the merged state stays with `recover_subnet`. The `MergeSubnets` NNS function is kept -- without it the endpoint would be unreachable -- with its description narrowed accordingly. The `StateMachine` change that answered `setup_initial_dkg` requests from `tick_with_config` goes away with the inter-canister call that needed it. Its fix of the swapped `SetupInitialDKGResponse` fields stays, as that is a bug of its own: the `HighThreshold`-tagged transcript was encoded as the low-threshold record and vice versa. Co-Authored-By: Claude Opus 5 --- rs/nns/governance/api/src/types.rs | 5 +- .../ic_nns_governance/pb/v1/governance.proto | 5 +- .../src/gen/ic_nns_governance.pb.v1.rs | 5 +- .../src/proposals/execute_nns_function.rs | 6 +- rs/nns/governance/unreleased_changelog.md | 8 +- rs/registry/canister/canister/canister.rs | 7 +- rs/registry/canister/canister/registry.did | 4 - .../canister/canister/registry_test.did | 4 - .../src/mutations/do_recover_subnet.rs | 2 +- .../canister/src/mutations/merge_subnets.rs | 330 +++--------------- rs/registry/canister/tests/merge_subnets.rs | 91 +---- rs/registry/canister/unreleased_changelog.md | 19 +- rs/state_machine_tests/src/lib.rs | 73 ++-- 13 files changed, 99 insertions(+), 460 deletions(-) diff --git a/rs/nns/governance/api/src/types.rs b/rs/nns/governance/api/src/types.rs index d7b1c59c3fda..e3d633ad0e07 100644 --- a/rs/nns/governance/api/src/types.rs +++ b/rs/nns/governance/api/src/types.rs @@ -4336,9 +4336,8 @@ pub enum NnsFunction { SetDefaultInitialDkgSubnet = 58, /// Merge a subnet into another subnet: the canister ID ranges of the source /// subnet are merged into the canister ID range set of the destination subnet, - /// a recovery catch-up package is created for the destination subnet (whose - /// state is expected to have been extended with the state of the canisters of - /// the source subnet) and the destination subnet is brought back online. + /// so that all canisters that used to be hosted by the source subnet are routed + /// to the destination subnet. MergeSubnets = 59, } impl NnsFunction { diff --git a/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto b/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto index 16957cf9184b..c3423c86abcc 100644 --- a/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto +++ b/rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto @@ -516,9 +516,8 @@ enum NnsFunction { // Merge a subnet into another subnet: the canister ID ranges of the source // subnet are merged into the canister ID range set of the destination subnet, - // a recovery catch-up package is created for the destination subnet (whose - // state is expected to have been extended with the state of the canisters of - // the source subnet) and the destination subnet is brought back online. + // so that all canisters that used to be hosted by the source subnet are routed + // to the destination subnet. NNS_FUNCTION_MERGE_SUBNETS = 59; } diff --git a/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs b/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs index 8ce5e5c64343..d72208764af4 100644 --- a/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs +++ b/rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs @@ -5550,9 +5550,8 @@ pub enum NnsFunction { SetDefaultInitialDkgSubnet = 58, /// Merge a subnet into another subnet: the canister ID ranges of the source /// subnet are merged into the canister ID range set of the destination subnet, - /// a recovery catch-up package is created for the destination subnet (whose - /// state is expected to have been extended with the state of the canisters of - /// the source subnet) and the destination subnet is brought back online. + /// so that all canisters that used to be hosted by the source subnet are routed + /// to the destination subnet. MergeSubnets = 59, } impl NnsFunction { diff --git a/rs/nns/governance/src/proposals/execute_nns_function.rs b/rs/nns/governance/src/proposals/execute_nns_function.rs index cd50c8a6215d..9db38f6a8356 100644 --- a/rs/nns/governance/src/proposals/execute_nns_function.rs +++ b/rs/nns/governance/src/proposals/execute_nns_function.rs @@ -952,10 +952,8 @@ impl ValidNnsFunction { "Merge a subnet into another subnet. The canister ID ranges of the source subnet \ are merged into the canister ID range set of the destination subnet, so that all \ canisters that used to be hosted by the source subnet are routed to the \ - destination subnet; a recovery catch-up package is created for the destination \ - subnet, whose state is expected to have been extended with the state of the \ - canisters of the source subnet while both subnets were offline; and the \ - destination subnet is brought back online." + destination subnet. Only the routing table is updated: neither subnet record is \ + modified and the source subnet is not deleted." } } } diff --git a/rs/nns/governance/unreleased_changelog.md b/rs/nns/governance/unreleased_changelog.md index a7ef4662e34d..eccd5226d599 100644 --- a/rs/nns/governance/unreleased_changelog.md +++ b/rs/nns/governance/unreleased_changelog.md @@ -11,10 +11,10 @@ on the process that this file is part of, see * Added a new `NnsFunction` variant `MergeSubnets`, which proposes to merge a subnet into another subnet: the canister ID ranges of the source subnet are - merged into the canister ID range set of the destination subnet, a recovery - catch-up package is created for the destination subnet (whose state is - expected to have been extended with the state of the canisters of the source - subnet) and the destination subnet is brought back online. + merged into the canister ID range set of the destination subnet, so that all + canisters that used to be hosted by the source subnet are routed to the + destination subnet. Only the routing table is updated: neither subnet record + is modified and the source subnet is not deleted. ## Changed diff --git a/rs/registry/canister/canister/canister.rs b/rs/registry/canister/canister/canister.rs index a342f824e13c..01568e96f077 100644 --- a/rs/registry/canister/canister/canister.rs +++ b/rs/registry/canister/canister/canister.rs @@ -1089,16 +1089,13 @@ fn reroute_canister_ranges_(payload: RerouteCanisterRangesPayload) { #[unsafe(export_name = "canister_update merge_subnets")] fn merge_subnets() { check_caller_is_governance_and_log("merge_subnets"); - over_async(candid_one, |payload: MergeSubnetsPayload| async move { - merge_subnets_(payload).await - }); + over(candid_one, merge_subnets_); } #[candid_method(update, rename = "merge_subnets")] -async fn merge_subnets_(payload: MergeSubnetsPayload) { +fn merge_subnets_(payload: MergeSubnetsPayload) { registry_mut() .merge_subnets(payload) - .await .unwrap_or_else(|error_message| { trap_with(&format!( "{LOG_PREFIX} Merge subnets failed: {error_message}" diff --git a/rs/registry/canister/canister/registry.did b/rs/registry/canister/canister/registry.did index 7183998d0222..a3759d9ffb20 100644 --- a/rs/registry/canister/canister/registry.did +++ b/rs/registry/canister/canister/registry.did @@ -311,10 +311,6 @@ type IPv4Config = record { type MergeSubnetsPayload = record { source_subnet : principal; destination_subnet : principal; - height : nat64; - time_ns : nat64; - state_hash : blob; - initial_dkg_subnet_id : opt principal; }; type MigrateCanistersPayload = record { diff --git a/rs/registry/canister/canister/registry_test.did b/rs/registry/canister/canister/registry_test.did index 2be881b0da74..7493ae134411 100644 --- a/rs/registry/canister/canister/registry_test.did +++ b/rs/registry/canister/canister/registry_test.did @@ -311,10 +311,6 @@ type IPv4Config = record { type MergeSubnetsPayload = record { source_subnet : principal; destination_subnet : principal; - height : nat64; - time_ns : nat64; - state_hash : blob; - initial_dkg_subnet_id : opt principal; }; type MigrateCanistersPayload = record { diff --git a/rs/registry/canister/src/mutations/do_recover_subnet.rs b/rs/registry/canister/src/mutations/do_recover_subnet.rs index b07e8bfc9296..cf393eec3a7d 100644 --- a/rs/registry/canister/src/mutations/do_recover_subnet.rs +++ b/rs/registry/canister/src/mutations/do_recover_subnet.rs @@ -488,7 +488,7 @@ impl TryFrom for KeyConfigRequestInternal { } } -pub(crate) fn panic_if_record_changed_across_versions( +fn panic_if_record_changed_across_versions( registry: &Registry, key: &str, initial_registry_version: Version, diff --git a/rs/registry/canister/src/mutations/merge_subnets.rs b/rs/registry/canister/src/mutations/merge_subnets.rs index 96879a25802f..a907ac83af77 100644 --- a/rs/registry/canister/src/mutations/merge_subnets.rs +++ b/rs/registry/canister/src/mutations/merge_subnets.rs @@ -1,71 +1,28 @@ -//! Contains the method to merge a subnet into another subnet. -//! -//! Merging the source subnet into the destination subnet reroutes all canisters -//! of the source subnet to the destination subnet and lets the destination -//! subnet resume from a state that was extended, while both subnets were -//! offline, with the state of those canisters. The state extension itself -//! happens outside of the registry: this method only records its outcome, as the -//! state hash of a recovery catch-up package for the destination subnet. - -use crate::{ - common::LOG_PREFIX, mutations::do_recover_subnet::panic_if_record_changed_across_versions, - registry::Registry, -}; -use candid::{CandidType, Encode}; -use dfn_core::api::{CanisterId, call}; +use crate::{common::LOG_PREFIX, registry::Registry}; +use candid::CandidType; #[cfg(target_arch = "wasm32")] use dfn_core::println; -use ic_base_types::{NodeId, PrincipalId, RegistryVersion, SubnetId}; -use ic_management_canister_types_private::{SetupInitialDKGArgs, SetupInitialDKGResponse}; -use ic_protobuf::registry::subnet::v1::{RecoveryArgs, catch_up_package_contents::CupType}; -use ic_registry_keys::{ - make_catch_up_package_contents_key, make_crypto_threshold_signing_pubkey_key, - make_subnet_record_key, -}; +use ic_base_types::SubnetId; +use ic_registry_keys::make_subnet_record_key; use ic_registry_routing_table::are_disjoint; -use ic_registry_transport::{ - pb::v1::{RegistryMutation, registry_mutation}, - upsert, -}; -use on_wire::bytes; -use prost::Message; use serde::{Deserialize, Serialize}; -use std::convert::TryFrom; impl Registry { - /// Merges the source subnet into the destination subnet. + /// Merges the canister ID ranges of the source subnet into the canister ID + /// range set of the destination subnet. /// - /// Three things happen, all in a single registry version, so that the - /// destination subnet never observes a state where only some of them took - /// effect: + /// After this operation, all canisters that used to be hosted by the source + /// subnet are routed to the destination subnet and the source subnet does + /// not host any canister ID range anymore. /// - /// 1. the canister ID ranges of the source subnet are merged into the - /// canister ID range set of the destination subnet, so that all - /// canisters that used to be hosted by the source subnet are routed to - /// the destination subnet; - /// 2. a recovery catch-up package is created for the destination subnet, - /// at the height, time and state hash of the merged state, running a - /// fresh DKG for the destination subnet's membership; and - /// 3. the destination subnet is brought back online. - /// - /// The caller is expected to have taken both subnets offline and to have - /// extended the state of the destination subnet with the state of the - /// canisters of the source subnet beforehand; `state_hash` is the hash of - /// the manifest of the resulting merged state. - /// - /// Note that neither subnet record is deleted and, in particular, the source - /// subnet is not deleted: it merely does not host any canister ID range - /// anymore. - pub async fn merge_subnets(&mut self, payload: MergeSubnetsPayload) -> Result<(), String> { + /// Note that only the routing table is updated: neither subnet record is + /// modified and, in particular, the source subnet is not deleted. + pub fn merge_subnets(&mut self, payload: MergeSubnetsPayload) -> Result<(), String> { println!("{LOG_PREFIX}merge_subnets: {payload:?}"); let MergeSubnetsPayload { source_subnet, destination_subnet, - height, - time_ns, - state_hash, - initial_dkg_subnet_id, } = payload; if source_subnet == destination_subnet { @@ -74,18 +31,17 @@ impl Registry { )); } - let pre_call_registry_version = self.latest_version(); + let version = self.latest_version(); + self.get(&make_subnet_record_key(source_subnet).into_bytes(), version) + .ok_or_else(|| format!("source {source_subnet} is not a known subnet"))?; self.get( - &make_subnet_record_key(source_subnet).into_bytes(), - pre_call_registry_version, + &make_subnet_record_key(destination_subnet).into_bytes(), + version, ) - .ok_or_else(|| format!("source {source_subnet} is not a known subnet"))?; - let destination_record = self - .get_subnet(destination_subnet, pre_call_registry_version) - .map_err(|_| format!("destination {destination_subnet} is not a known subnet"))?; + .ok_or_else(|| format!("destination {destination_subnet} is not a known subnet"))?; - let routing_table = self.get_routing_table_or_panic(pre_call_registry_version); + let routing_table = self.get_routing_table_or_panic(version); let source_ranges = routing_table.ranges(source_subnet); if source_ranges.is_empty() { return Err(format!( @@ -96,7 +52,7 @@ impl Registry { // Rerouting the canister ID ranges of the source subnet would break any ongoing canister // migration out of those ranges: the migrated ranges would end up being hosted by the // destination subnet, which is not on the recorded migration trace. - if let Some(canister_migrations) = self.get_canister_migrations(pre_call_registry_version) + if let Some(canister_migrations) = self.get_canister_migrations(version) && !are_disjoint(canister_migrations.ranges(), source_ranges.iter()) { return Err(format!( @@ -104,167 +60,12 @@ impl Registry { )); } - // Recovering a subnet holding chain keys requires resharing those keys onto the recovery - // CUP, which this method does not do; rather than silently leave the destination subnet - // unable to sign, refuse to merge into it. - if destination_record - .chain_key_config - .as_ref() - .is_some_and(|config| !config.key_configs.is_empty()) - { - return Err(format!( - "destination subnet {destination_subnet} holds chain keys, which merging does not reshare" - )); - } - - // `setup_initial_dkg` must not be handled by the subnet being recovered, as that subnet is - // offline and could not respond. - if let Some(initial_dkg_subnet_id) = initial_dkg_subnet_id { - if initial_dkg_subnet_id == destination_subnet { - return Err(format!( - "initial DKG subnet {initial_dkg_subnet_id} must be different from the destination subnet" - )); - } - self.get( - &make_subnet_record_key(initial_dkg_subnet_id).into_bytes(), - pre_call_registry_version, - ) - .ok_or_else(|| { - format!("initial DKG subnet {initial_dkg_subnet_id} is not a known subnet") - })?; - } - - let mut cup_contents = self - .get_subnet_catch_up_package(destination_subnet, Some(pre_call_registry_version)) - .map_err(|err| format!("failed to get the CUP of {destination_subnet}: {err}"))?; - cup_contents.registry_store_uri = None; - // Chain key initializations in a CUP take precedence over the chain key configuration of - // the subnet record, so carrying over the ones of the CUP being replaced would make the - // destination subnet bootstrap stale key material. The destination subnet holds no chain - // keys (checked above) and merging reshares none, so both fields are cleared, just like - // recovering a subnet without an initial chain key configuration does. - cup_contents.chain_key_initializations = vec![]; - cup_contents.ecdsa_initializations = vec![]; - - let mut subnet_record = destination_record; - - // Bring the destination subnet back online. Consensus looks at the registry version from - // the highest CUP when considering `halt_at_cup_height`, so clearing both flags is what - // makes the subnet resume from the recovery CUP created below. - subnet_record.halt_at_cup_height = false; - subnet_record.is_halted = false; - - let dkg_nodes: Vec = subnet_record - .membership - .iter() - .map(|bytes| NodeId::from(PrincipalId::try_from(bytes).unwrap())) - .collect(); - - let request = SetupInitialDKGArgs::new( - dkg_nodes, - RegistryVersion::new(pre_call_registry_version), - initial_dkg_subnet_id, - ); - let response_bytes = call( - CanisterId::ic_00(), - "setup_initial_dkg", - bytes, - Encode!(&request).unwrap(), - ) - .await - .unwrap_or_else(|(code, msg)| { - panic!("{LOG_PREFIX}`setup_initial_dkg` failed with code {code:?}: {msg}") - }); - - let post_call_registry_version = self.latest_version(); - - // Check that the records this method is about to overwrite, and the routing table it based - // its validation on, did not change while `setup_initial_dkg` was in flight. - for (key, what) in [ - ( - make_subnet_record_key(destination_subnet), - format!("Subnet with ID {destination_subnet}"), - ), - ( - make_crypto_threshold_signing_pubkey_key(destination_subnet), - format!("Threshold Signing Pubkey for Subnet {destination_subnet}"), - ), - ( - make_catch_up_package_contents_key(destination_subnet), - format!("CUP for Subnet {destination_subnet}"), - ), - ( - make_subnet_record_key(source_subnet), - format!("Subnet with ID {source_subnet}"), - ), - ] { - panic_if_record_changed_across_versions( - self, - &key, - pre_call_registry_version, - post_call_registry_version, - format!("{what} was updated during the `setup_initial_dkg` call"), - ); - } - assert_eq!( - self.get_routing_table_or_panic(post_call_registry_version) - .ranges(source_subnet), - source_ranges, - "{LOG_PREFIX}The canister ID ranges of subnet {source_subnet} were updated during the \ - `setup_initial_dkg` call", - ); - // A canister migration overlapping the canister ID ranges of the source subnet could have - // been prepared, without changing the routing table, while `setup_initial_dkg` was in - // flight; rerouting those ranges would break it, just like it would have before the call. - assert!( - self.get_canister_migrations(post_call_registry_version) - .is_none_or(|canister_migrations| are_disjoint( - canister_migrations.ranges(), - source_ranges.iter() - )), - "{LOG_PREFIX}Canister migrations overlapping the canister ID ranges of subnet \ - {source_subnet} were added during the `setup_initial_dkg` call", - ); - - let dkg_response = SetupInitialDKGResponse::decode(&response_bytes).unwrap(); - - cup_contents.initial_ni_dkg_transcript_low_threshold = - Some(dkg_response.low_threshold_transcript_record); - cup_contents.initial_ni_dkg_transcript_high_threshold = - Some(dkg_response.high_threshold_transcript_record); - cup_contents.height = height; - cup_contents.time = time_ns; - cup_contents.state_hash = state_hash.clone(); - cup_contents.cup_type = Some(CupType::Recovery(RecoveryArgs { - height, - time: time_ns, - state_hash, - })); - - let mut mutations = vec![ - RegistryMutation { - mutation_type: registry_mutation::Type::Update as i32, - key: make_crypto_threshold_signing_pubkey_key(destination_subnet).into_bytes(), - value: dkg_response.subnet_threshold_public_key.encode_to_vec(), - }, - RegistryMutation { - mutation_type: registry_mutation::Type::Update as i32, - key: make_catch_up_package_contents_key(destination_subnet).into_bytes(), - value: cup_contents.encode_to_vec(), - }, - upsert( - make_subnet_record_key(destination_subnet), - subnet_record.encode_to_vec(), - ), - ]; - mutations.append(&mut self.merge_subnets_mutation( - post_call_registry_version, + self.maybe_apply_mutation_internal(self.merge_subnets_mutation( + version, source_subnet, destination_subnet, )); - self.maybe_apply_mutation_internal(mutations); - Ok(()) } } @@ -276,22 +77,8 @@ pub struct MergeSubnetsPayload { /// set of `destination_subnet`. pub source_subnet: SubnetId, /// The subnet that hosts the canister ID ranges of `source_subnet` after the - /// merge, and that is recovered at the merged state and brought back online. + /// merge. pub destination_subnet: SubnetId, - /// The height of the recovery CUP of `destination_subnet`, i.e. the height - /// of the checkpoint holding the merged state. - pub height: u64, - /// The block time the recovered `destination_subnet` starts from, in - /// nanoseconds since the Epoch. Must be larger than the times of the - /// checkpoints at which both subnets were taken offline. - pub time_ns: u64, - /// The hash of the manifest of the merged state. - pub state_hash: Vec, - /// The subnet that should handle the `setup_initial_dkg` call producing the - /// DKG transcripts of the recovery CUP. Must be different from - /// `destination_subnet`, which is offline while the merge is in progress. If - /// unset, the request is handled by the NNS subnet. - pub initial_dkg_subnet_id: Option, } #[cfg(test)] @@ -307,25 +94,11 @@ mod tests { routing_table::routing_table_into_registry_mutation, }, }; - use futures::executor::block_on; use ic_registry_routing_table::{CanisterIdRange, RoutingTable}; use ic_types::CanisterId; use ic_types_test_utils::ids::{SUBNET_1, SUBNET_2, SUBNET_3}; use maplit::btreemap; - /// The recovery CUP fields of the payload, which the validation-failure tests - /// below are not about: they all fail before the CUP is even looked at. - fn default_cup_args() -> MergeSubnetsPayload { - MergeSubnetsPayload { - source_subnet: SUBNET_1, - destination_subnet: SUBNET_2, - height: 100, - time_ns: 1_000_000_000, - state_hash: vec![1; 32], - initial_dkg_subnet_id: Some(SUBNET_3), - } - } - fn range(start: u64, end: u64) -> CanisterIdRange { CanisterIdRange { start: CanisterId::from_u64(start), @@ -376,28 +149,24 @@ mod tests { .collect::>() } - /// Applies just the routing table part of a merge. `Registry::merge_subnets` - /// itself cannot be driven to completion in a unit test, as it calls - /// `setup_initial_dkg` on the management canister half way through; the - /// success path as a whole is covered by the integration test in - /// `rs/registry/canister/tests/merge_subnets.rs`. - fn merge_routing_table(registry: &mut Registry, source: SubnetId, destination: SubnetId) { - let mutations = - registry.merge_subnets_mutation(registry.latest_version(), source, destination); - registry.maybe_apply_mutation_internal(mutations); - } - #[test] fn test_merge_subnets() { // Step 1: Prepare the world. let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - merge_routing_table(&mut registry, SUBNET_1, SUBNET_2); + let result = registry.merge_subnets(MergeSubnetsPayload { + source_subnet: SUBNET_1, + destination_subnet: SUBNET_2, + }); - // Step 3: Verify results: the canister ID ranges of both subnets are now - // hosted by the destination subnet, and the three adjacent ranges got - // merged into one. + // Step 3: Verify results. + + // Step 3.1: Inspect the return value. + assert_eq!(result, Ok(())); + + // Step 3.2: The canister ID ranges of both subnets are now hosted by the + // destination subnet, and the three adjacent ranges got merged into one. assert_eq!( get_routing_table_entries(®istry), vec![(range(10, 39), SUBNET_2)], @@ -420,10 +189,14 @@ mod tests { )); // Step 2: Run the code under test. - merge_routing_table(&mut registry, SUBNET_1, SUBNET_2); + let result = registry.merge_subnets(MergeSubnetsPayload { + source_subnet: SUBNET_1, + destination_subnet: SUBNET_2, + }); // Step 3: Verify results. Both ranges are hosted by the destination subnet // and, not being adjacent, did not get merged into a single entry. + assert_eq!(result, Ok(())); assert_eq!( get_routing_table_entries(®istry), vec![(range(10, 19), SUBNET_2), (range(30, 39), SUBNET_2)], @@ -436,11 +209,10 @@ mod tests { let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = block_on(registry.merge_subnets(MergeSubnetsPayload { + let result = registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_1, - ..default_cup_args() - })); + }); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -464,11 +236,10 @@ mod tests { let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = block_on(registry.merge_subnets(MergeSubnetsPayload { + let result = registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_3, destination_subnet: SUBNET_2, - ..default_cup_args() - })); + }); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -484,11 +255,10 @@ mod tests { let mut registry = new_two_subnets_fixture_registry(); // Step 2: Run the code under test. - let result = block_on(registry.merge_subnets(MergeSubnetsPayload { + let result = registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_3, - ..default_cup_args() - })); + }); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -511,11 +281,10 @@ mod tests { )); // Step 2: Run the code under test. - let result = block_on(registry.merge_subnets(MergeSubnetsPayload { + let result = registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_2, - ..default_cup_args() - })); + }); // Step 3: Verify results. let error_message = result.unwrap_err(); @@ -539,11 +308,10 @@ mod tests { .unwrap(); // Step 2: Run the code under test. - let result = block_on(registry.merge_subnets(MergeSubnetsPayload { + let result = registry.merge_subnets(MergeSubnetsPayload { source_subnet: SUBNET_1, destination_subnet: SUBNET_2, - ..default_cup_args() - })); + }); // Step 3: Verify results. let error_message = result.unwrap_err(); diff --git a/rs/registry/canister/tests/merge_subnets.rs b/rs/registry/canister/tests/merge_subnets.rs index f552c7801201..16139835f211 100644 --- a/rs/registry/canister/tests/merge_subnets.rs +++ b/rs/registry/canister/tests/merge_subnets.rs @@ -6,39 +6,21 @@ use ic_nns_test_utils::{ }, registry::{initial_routing_table_mutations, prepare_registry_with_two_node_sets}, }; -use ic_protobuf::registry::subnet::v1::{ - CatchUpPackageContents, ChainKeyInitialization, EcdsaInitialization, RecoveryArgs, - catch_up_package_contents::CupType, -}; -use ic_registry_keys::make_catch_up_package_contents_key; use ic_registry_routing_table::{CanisterIdRange, RoutingTable}; use ic_registry_transport::pb::v1::RegistryAtomicMutateRequest; use ic_types::CanisterId; -use prost::Message; use registry_canister::{ init::RegistryCanisterInitPayloadBuilder, mutations::merge_subnets::MergeSubnetsPayload, }; mod common; -use common::test_helpers::{ - check_error_message, check_subnet_for_canisters, get_cup_contents, get_subnet_record, -}; - -/// The recovery CUP the merge creates for the destination subnet. The values are -/// arbitrary: this test does not run a subnet from the resulting CUP, it only -/// checks that the endpoint accepts them and records them. -const MERGE_HEIGHT: u64 = 100; -const MERGE_TIME_NS: u64 = 1_234_567_890; -const MERGED_STATE_HASH: &[u8] = &[42; 32]; +use common::test_helpers::{check_error_message, check_subnet_for_canisters}; /// Exercises the `merge_subnets` endpoint end to end. The payload validation /// itself is covered by the unit tests of `Registry::merge_subnets`, so this test /// only covers what those cannot: that the endpoint is reachable with a Candid -/// encoded payload, that only governance may call it, that the canisters of the -/// source subnet end up routed to the destination subnet, and that the recovery -/// CUP of the destination subnet -- whose DKG transcripts come from the -/// `setup_initial_dkg` call the endpoint makes half way through -- is recorded -/// and the destination subnet is brought back online. +/// encoded payload, that only governance may call it, and that the resulting +/// routing table is visible through the canister's query API. #[test] fn test_merge_subnets() { state_machine_test_on_nns_subnet(|runtime| { @@ -50,25 +32,6 @@ fn test_merge_subnets() { /* num_nodes_in_subnet = */ 4, /* num_unassigned_nodes = */ 4, true, ); let subnet_id_2 = subnet_id_2_option.unwrap(); - // Give the destination subnet a CUP holding chain key initializations, as it - // would if it had once been recovered while holding chain keys. Merging must - // not carry them over into the recovery CUP it creates. - let subnet_1_mutation = { - let mut subnet_1_mutation = subnet_1_mutation; - let cup_contents_key = make_catch_up_package_contents_key(subnet_id_2).into_bytes(); - let cup_contents_mutation = subnet_1_mutation - .mutations - .iter_mut() - .find(|mutation| mutation.key == cup_contents_key) - .expect("the destination subnet should have CUP contents"); - let mut cup_contents = - CatchUpPackageContents::decode(&cup_contents_mutation.value[..]) - .expect("failed to decode the CUP contents"); - cup_contents.ecdsa_initializations = vec![EcdsaInitialization::default()]; - cup_contents.chain_key_initializations = vec![ChainKeyInitialization::default()]; - cup_contents_mutation.value = cup_contents.encode_to_vec(); - subnet_1_mutation - }; let rt_mutation = { fn range(start: u64, end: u64) -> CanisterIdRange { CanisterIdRange { @@ -113,10 +76,6 @@ fn test_merge_subnets() { MergeSubnetsPayload { source_subnet: subnet_id_1, destination_subnet: subnet_id_2, - height: MERGE_HEIGHT, - time_ns: MERGE_TIME_NS, - state_hash: MERGED_STATE_HASH.to_vec(), - initial_dkg_subnet_id: None, }, ) .await as Result<(), String>, @@ -131,10 +90,6 @@ fn test_merge_subnets() { Encode!(&MergeSubnetsPayload { source_subnet: subnet_id_1, destination_subnet: subnet_id_2, - height: MERGE_HEIGHT, - time_ns: MERGE_TIME_NS, - state_hash: MERGED_STATE_HASH.to_vec(), - initial_dkg_subnet_id: None, }) .unwrap(), ) @@ -154,46 +109,6 @@ fn test_merge_subnets() { ) .await; - // Step 5: Verify results: the destination subnet got a recovery CUP at the - // height, time and state hash of the merged state, with fresh DKG - // transcripts, and is no longer halted. - let cup_contents = get_cup_contents(®istry, subnet_id_2).await; - assert_eq!(cup_contents.height, MERGE_HEIGHT); - assert_eq!(cup_contents.time, MERGE_TIME_NS); - assert_eq!(cup_contents.state_hash, MERGED_STATE_HASH); - assert_eq!( - cup_contents.cup_type, - Some(CupType::Recovery(RecoveryArgs { - height: MERGE_HEIGHT, - time: MERGE_TIME_NS, - state_hash: MERGED_STATE_HASH.to_vec(), - })), - ); - assert!( - cup_contents - .initial_ni_dkg_transcript_low_threshold - .is_some(), - "the recovery CUP should hold a low threshold DKG transcript", - ); - assert!( - cup_contents - .initial_ni_dkg_transcript_high_threshold - .is_some(), - "the recovery CUP should hold a high threshold DKG transcript", - ); - // Chain key initializations in a CUP take precedence over the chain key - // configuration of the subnet record, so the stale ones seeded above must be - // gone: merging reshares no chain key. - assert_eq!(cup_contents.ecdsa_initializations, vec![]); - assert_eq!(cup_contents.chain_key_initializations, vec![]); - - let subnet_record = get_subnet_record(®istry, subnet_id_2).await; - assert!( - !subnet_record.is_halted, - "the destination subnet should have been brought back online", - ); - assert!(!subnet_record.halt_at_cup_height); - Ok(()) } }); diff --git a/rs/registry/canister/unreleased_changelog.md b/rs/registry/canister/unreleased_changelog.md index fa8895b855db..68d97b8732e1 100644 --- a/rs/registry/canister/unreleased_changelog.md +++ b/rs/registry/canister/unreleased_changelog.md @@ -27,21 +27,10 @@ on the process that this file is part of, see back out of the Registry. * `merge_subnets` endpoint, callable through a `MergeSubnets` proposal. It takes a source and a - destination subnet ID, plus the height, time and state hash of the merged state, and performs the - whole registry-side part of a subnet merge in a single registry version: - - * the canister ID ranges of the source subnet are merged into the canister ID range set of the - destination subnet, i.e., the canisters hosted by the source subnet are routed to the - destination subnet afterwards; - * a recovery catch-up package is created for the destination subnet, at the given height, time - and state hash, running a fresh DKG for the destination subnet's membership; and - * the destination subnet is brought back online. - - The caller is expected to have taken both subnets offline and to have extended the state of the - destination subnet with the state of the canisters of the source subnet beforehand. The source - subnet record is not modified and the source subnet is not deleted: it merely does not host any - canister ID range anymore. Merging into a subnet holding chain keys is rejected, as the recovery - catch-up package does not reshare them. + destination subnet ID, and merges the canister ID ranges of the source subnet into the canister + ID range set of the destination subnet, i.e., the canisters hosted by the source subnet are + routed to the destination subnet afterwards. Only the routing table is updated: neither subnet + record is modified and the source subnet is not deleted. ## Changed diff --git a/rs/state_machine_tests/src/lib.rs b/rs/state_machine_tests/src/lib.rs index e31216f4983a..0698c52a90d9 100644 --- a/rs/state_machine_tests/src/lib.rs +++ b/rs/state_machine_tests/src/lib.rs @@ -1865,42 +1865,6 @@ impl Default for StateMachineBuilder { } } -/// The responses consensus would produce for the pending `setup_initial_dkg` -/// requests of `state`: a dummy transcript per request, derived from `seed` so -/// that the result stays deterministic. -/// -/// `setup_initial_dkg` can only be called on the NNS subnet, so the seed does -/// not need to depend on the subnet ID. -fn setup_initial_dkg_responses(state: &ReplicatedState, seed: u64) -> Vec { - let mut rng = StdRng::seed_from_u64(seed); - state - .metadata - .subnet_call_context_manager - .setup_initial_dkg_contexts - .keys() - .map(|callback_id| { - let ni_dkg_transcript = dummy_initial_dkg_transcript_with_master_key(&mut rng).0; - let public_key = (&ni_dkg_transcript).try_into().unwrap(); - let public_key_der = threshold_sig_public_key_to_der(public_key).unwrap(); - let subnet_id = PrincipalId::new_self_authenticating(&public_key_der).into(); - let mut high_threshold_transcript_record = ni_dkg_transcript.clone(); - high_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::HighThreshold; - let mut low_threshold_transcript_record = ni_dkg_transcript; - low_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::LowThreshold; - let initial_transcript_records = SetupInitialDKGResponse { - low_threshold_transcript_record: low_threshold_transcript_record.into(), - high_threshold_transcript_record: high_threshold_transcript_record.into(), - fresh_subnet_id: subnet_id, - subnet_threshold_public_key: public_key.into(), - }; - ConsensusResponse::new( - *callback_id, - MsgPayload::Data(initial_transcript_records.encode()), - ) - }) - .collect() -} - impl StateMachine { /// Provides the implicit time increment for a single round of execution /// if time does not advance between consecutive rounds. @@ -2021,7 +1985,34 @@ impl StateMachine { } let self_validating = Some(batch_payload.self_validating); let mut consensus_responses = http_responses; - consensus_responses.extend(setup_initial_dkg_responses(&state, certified_height.get())); + // `setup_initial_dkg` can only be called on the NNS subnet + // and thus the seed does not need to depend on the subnet ID + let mut rng = StdRng::seed_from_u64(certified_height.get()); + for callback_id in state + .metadata + .subnet_call_context_manager + .setup_initial_dkg_contexts + .keys() + { + let ni_dkg_transcript = dummy_initial_dkg_transcript_with_master_key(&mut rng).0; + let public_key = (&ni_dkg_transcript).try_into().unwrap(); + let public_key_der = threshold_sig_public_key_to_der(public_key).unwrap(); + let subnet_id = PrincipalId::new_self_authenticating(&public_key_der).into(); + let mut high_threshold_transcript_record = ni_dkg_transcript.clone(); + high_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::HighThreshold; + let mut low_threshold_transcript_record = ni_dkg_transcript; + low_threshold_transcript_record.dkg_id.dkg_tag = NiDkgTag::LowThreshold; + let initial_transcript_records = SetupInitialDKGResponse { + low_threshold_transcript_record: low_threshold_transcript_record.into(), + high_threshold_transcript_record: high_threshold_transcript_record.into(), + fresh_subnet_id: subnet_id, + subnet_threshold_public_key: public_key.into(), + }; + consensus_responses.push(ConsensusResponse::new( + *callback_id, + MsgPayload::Data(initial_transcript_records.encode()), + )); + } let mut payload = PayloadBuilder::new() .with_ingress_messages(ingress_messages) .with_xnet_payload(xnet_payload) @@ -3081,14 +3072,6 @@ impl StateMachine { self.process_threshold_signing_request(id, context, &mut payload_builder); } - // Process `setup_initial_dkg` requests, which consensus would answer. - payload_builder - .consensus_responses - .extend(setup_initial_dkg_responses( - &state, - self.state_manager.latest_state_height().get(), - )); - self.execute_payload(payload_builder); } From ef42adbf7671ba9b359bcdb35f98ec4ba17432f3 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Tue, 1 Sep 2026 15:39:42 +0000 Subject: [PATCH 15/30] fix: drive the subnet merge test's driver calls on the test's own runtime `await_status_is_healthy` and the SSH helpers without the `_async` suffix run their own future through `futures::executor::block_on`, which polls it on the calling thread instead of letting the runtime park on its I/O. A `reqwest` request that finds no live connection in its pool then busy-polls its connect future forever: the test spun at 100% CPU in `status_is_healthy_async`, having logged the unhalting of `R` and nothing after it. `R` reports `WaitingForRootDelegation` for a few minutes after the recovery, which is what gave the pooled connection time to go away, so the health check of step 16 hit this reliably. Use the `_async` variants -- which is what this file already does for the checkpoint listing of `latest_checkpoint_height` -- for every driver call in the async body of the test. Co-Authored-By: Claude Opus 5 --- .../subnet_cooling_down_test.rs | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs index 954401e5f755..3cd3963f6021 100644 --- a/rs/tests/message_routing/subnet_cooling_down_test.rs +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -808,7 +808,8 @@ async fn run(env: TestEnv) { // manager of a running replica owns its state directory, even while // consensus is halted. for (node, name) in [(&m_node, "M"), (&r_node, "R")] { - node.block_on_bash_script("sudo systemctl stop ic-replica") + node.block_on_bash_script_async("sudo systemctl stop ic-replica") + .await .unwrap_or_else(|e| panic!("failed to stop the replica of subnet {name}: {e}")); info!(logger, "Step 13: stopped the replica of subnet {name}"); } @@ -885,7 +886,8 @@ async fn run(env: TestEnv) { // hold the canisters of `M`. info!(logger, "Step 16: Starting the replica of subnet R"); r_node - .block_on_bash_script("sudo systemctl start ic-replica") + .block_on_bash_script_async("sudo systemctl start ic-replica") + .await .expect("failed to start the replica of subnet R"); // Whether `R` resumes from the merged state or from the checkpoint it halted // at is not something to leave to chance: a replica that started before its @@ -924,8 +926,19 @@ async fn run(env: TestEnv) { logger, "Step 16: R is unhalted as of registry version {unhalt_registry_version}" ); + // The `_async` variant, and not the blocking one: the latter drives its + // request through `futures::executor::block_on`, which busy-polls a `reqwest` + // future that needs the runtime this thread is driving, and livelocks as soon + // as an attempt has to open a new connection -- which is exactly what happens + // here, where `R` reports `WaitingForRootDelegation` for minutes before the + // unhalting takes effect. + // The `_async` variants of the driver's SSH and status helpers are what this + // test uses throughout: the blocking ones drive their own future with + // `futures::executor::block_on`, which busy-polls a `reqwest` request that + // has to open a new connection instead of letting the runtime wait for it. r_node - .await_status_is_healthy() + .await_status_is_healthy_async() + .await .expect("subnet R did not become healthy after the merge"); info!(logger, "Step 16 done: subnet R is healthy"); @@ -1978,7 +1991,8 @@ async fn await_halted_at_checkpoint(node: &IcNodeSnapshot, name: &str, logger: & // `from_now()` makes every poll search all entries since this point, so the // condition, once true, stays true. let journal = JournalStreamer::new( - node.block_on_ssh_session() + node.block_on_ssh_session_async() + .await .unwrap_or_else(|e| panic!("failed to open an SSH session to subnet {name}: {e}")), ) .from_now() From 5213e138b3331495b14e9469ac7b70407ee7c55e Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 7 Sep 2026 09:12:54 +0000 Subject: [PATCH 16/30] feat(state_tool): assemble the merged state of a subnet merge The state side of a subnet merge was a sequence of `chmod`, `cp -al` and `std::fs::write` calls in the subnet merge system test, which hand-encoded the `subnet_merged` marker as the bytes `[0x08, 0x01]`. Move it into a `merge` command of `state-tool`, next to the `split` command that performs the state side of a subnet split. The command assembles the checkpoint at `--output` from the ones at `--base` (the destination subnet) and `--source` (the subnet being merged away): the result holds everything of `base`, with the canisters and canister snapshots of `source` added to those of `base`, and is marked as the product of a subnet merge. Only canisters and snapshots are taken over from `source`; the ingress history in particular is not, as the marker makes the replica re-register the ingress messages of the merged-in canisters that are still in progress. File contents are hard linked rather than copied, so this is cheap no matter how large the two states are, and sound because checkpoints are immutable. Unlike the shell version, the inputs may stay read-only: the linking creates the destination directories itself rather than inheriting their permissions from a `cp -al`. The marker is written through `CheckpointLayout::subnet_merged_marker()` as a `SubnetMerged` protobuf rather than by hand, and colliding canister IDs between the two checkpoints are refused rather than silently resolved. Co-Authored-By: Claude Opus 5 (1M context) --- rs/state_tool/src/commands.rs | 1 + rs/state_tool/src/commands/merge.rs | 323 ++++++++++++++++++++++++++++ rs/state_tool/src/main.rs | 21 ++ 3 files changed, 345 insertions(+) create mode 100644 rs/state_tool/src/commands/merge.rs diff --git a/rs/state_tool/src/commands.rs b/rs/state_tool/src/commands.rs index 20d529f1d2a0..702766fdb9f3 100644 --- a/rs/state_tool/src/commands.rs +++ b/rs/state_tool/src/commands.rs @@ -8,6 +8,7 @@ pub mod decode; pub mod import_state; pub mod list; pub mod manifest; +pub mod merge; pub mod parse_overlay; pub mod split; pub mod split_manifest; diff --git a/rs/state_tool/src/commands/merge.rs b/rs/state_tool/src/commands/merge.rs new file mode 100644 index 000000000000..e16a4b5592a9 --- /dev/null +++ b/rs/state_tool/src/commands/merge.rs @@ -0,0 +1,323 @@ +//! Assembles the merged state of a subnet merge. + +use ic_protobuf::state::system_metadata::v1 as pb_metadata; +use ic_state_layout::{CANISTER_STATES_DIR, CheckpointLayout, SNAPSHOTS_DIR, WriteOnly}; +use ic_types::Height; +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; + +/// The height of a checkpoint is part of the name of its directory, which the +/// caller picks, so the layout below is only ever used to name files within it. +const HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED: Height = Height::new(0); + +/// Assembles the checkpoint at `output` from the checkpoints at `base` and +/// `source`: it holds everything of `base`, with the canisters and canister +/// snapshots of `source` added to those of `base`, and is marked as the product +/// of a subnet merge. +/// +/// Only the canisters and their snapshots are taken over from `source`. +/// Everything else (system metadata, subnet queues, ingress history, ...) is +/// `base`'s. In particular, the ingress history of `source` is deliberately not +/// merged in: the subnet merged marker makes the replica re-register the ingress +/// messages of the merged-in canisters that are still in progress. +/// +/// File contents are hard linked rather than copied, so this is cheap no matter +/// how large the two states are. That makes `output` share the storage of +/// `base` and `source`, which is sound because checkpoints are immutable: the +/// links are only ever read afterwards. +pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), String> { + for (path, name) in [(&base, "base"), (&source, "source")] { + if !path.is_dir() { + return Err(format!( + "the {name} checkpoint {} is not a directory", + path.display() + )); + } + } + if output.exists() { + return Err(format!("{} already exists", output.display())); + } + + link_tree(&base, &output)?; + + for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { + let source_dir = source.join(dir); + if !source_dir.exists() { + continue; + } + let output_dir = output.join(dir); + // The canisters of the two subnets are disjoint, as the source subnet + // hosts the canister ID ranges that the merge reassigns to the + // destination subnet. A collision would mean that the two checkpoints + // do not belong to the same merge, so refuse rather than pick a winner. + if let Some(name) = common_entry(&output_dir, &source_dir)? { + return Err(format!( + "{} holds {name} in both {} and {}", + dir, + base.display(), + source.display() + )); + } + link_tree(&source_dir, &output_dir)?; + } + + let layout = CheckpointLayout::::new_untracked( + output.clone(), + HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED, + ) + .map_err(|err| format!("failed to create the checkpoint layout: {err:?}"))?; + layout + .subnet_merged_marker() + .serialize(pb_metadata::SubnetMerged { merged: true }) + .map_err(|err| format!("failed to write the subnet merged marker: {err:?}"))?; + + // `base` was a checkpoint of a running subnet, so it holds no unverified + // checkpoint marker; but a state that was downloaded and reassembled by + // hand may, and the state manager must not take `output` for unverified. + let unverified_marker = layout.unverified_checkpoint_marker(); + if unverified_marker.exists() { + fs::remove_file(&unverified_marker) + .map_err(|err| format!("failed to remove {}: {err}", unverified_marker.display()))?; + } + + Ok(()) +} + +/// Replicates the directory tree rooted at `from` under `to`, hard linking every +/// file. Directories that already exist under `to` are reused, so a tree can be +/// overlaid onto another one. +/// +/// The directories are created writable, unlike the read-only ones of a +/// checkpoint, so that a subsequent call can overlay onto them. +fn link_tree(from: &Path, to: &Path) -> Result<(), String> { + fs::create_dir_all(to).map_err(|err| format!("failed to create {}: {err}", to.display()))?; + + for entry in read_dir(from)? { + let name = entry.file_name(); + let from = entry.path(); + let to = to.join(&name); + + let file_type = entry + .file_type() + .map_err(|err| format!("failed to stat {}: {err}", from.display()))?; + if file_type.is_dir() { + link_tree(&from, &to)?; + } else { + fs::hard_link(&from, &to).map_err(|err| { + format!( + "failed to link {} to {}: {err}", + from.display(), + to.display() + ) + })?; + } + } + + Ok(()) +} + +/// Returns the name of an entry that both directories hold, if any. A directory +/// that does not exist holds nothing. +fn common_entry(left: &Path, right: &Path) -> Result, String> { + let names = |dir: &Path| -> Result, String> { + if !dir.exists() { + return Ok(BTreeSet::new()); + } + Ok(read_dir(dir)? + .into_iter() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect()) + }; + + let left = names(left)?; + Ok(names(right)?.intersection(&left).next().cloned()) +} + +/// The entries of `dir`, with the I/O errors of both the listing and the entries +/// themselves resolved. +fn read_dir(dir: &Path) -> Result, String> { + fs::read_dir(dir) + .map_err(|err| format!("failed to read {}: {err}", dir.display()))? + .collect::, _>>() + .map_err(|err| format!("failed to read an entry of {}: {err}", dir.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + use ic_state_layout::{CompleteCheckpointLayout, UNVERIFIED_CHECKPOINT_MARKER}; + use std::os::unix::fs::MetadataExt; + use tempfile::TempDir; + + /// Creates a checkpoint-shaped directory under `root`, holding a + /// `system_metadata.pbuf` and a canister directory (with a snapshot + /// directory of the same name) per entry of `canisters`. + fn checkpoint(root: &Path, name: &str, canisters: &[&str]) -> PathBuf { + let checkpoint = root.join(name); + fs::create_dir_all(&checkpoint).unwrap(); + fs::write(checkpoint.join("system_metadata.pbuf"), name).unwrap(); + for canister in canisters { + for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { + let canister_dir = checkpoint.join(dir).join(canister); + fs::create_dir_all(&canister_dir).unwrap(); + fs::write(canister_dir.join("canister.pbuf"), *canister).unwrap(); + } + } + checkpoint + } + + fn entries(dir: &Path) -> Vec { + let mut names: Vec<_> = read_dir(dir) + .unwrap() + .into_iter() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names + } + + #[test] + fn merge_unions_canisters_and_snapshots() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1", "c2"]); + let source = checkpoint(tmp.path(), "source", &["c3"]); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { + assert_eq!(entries(&output.join(dir)), ["c1", "c2", "c3"], "{dir}"); + } + } + + #[test] + fn merge_keeps_everything_else_from_base() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + assert_eq!( + fs::read_to_string(output.join("system_metadata.pbuf")).unwrap(), + "base" + ); + } + + #[test] + fn merge_hard_links_file_contents() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + do_merge(base.clone(), source.clone(), output.clone()).unwrap(); + + let inode = |path: PathBuf| fs::metadata(path).unwrap().ino(); + let canister = |root: &Path, canister: &str| { + root.join(CANISTER_STATES_DIR) + .join(canister) + .join("canister.pbuf") + }; + assert_eq!( + inode(canister(&output, "c1")), + inode(canister(&base, "c1")), + "the canister of the base checkpoint is not hard linked" + ); + assert_eq!( + inode(canister(&output, "c2")), + inode(canister(&source, "c2")), + "the canister of the source checkpoint is not hard linked" + ); + } + + #[test] + fn merge_sets_the_subnet_merged_marker() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + let layout = CompleteCheckpointLayout::new_untracked( + output, + HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED, + ) + .unwrap(); + assert!(layout.subnet_merged_marker().deserialize().unwrap().merged); + } + + #[test] + fn merge_removes_the_unverified_checkpoint_marker() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + fs::write(base.join(UNVERIFIED_CHECKPOINT_MARKER), "").unwrap(); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + assert!(!output.join(UNVERIFIED_CHECKPOINT_MARKER).exists()); + } + + #[test] + fn merge_refuses_colliding_canisters() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1", "c2"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + let err = do_merge(base, source, output.clone()).unwrap_err(); + + assert!(err.contains("holds c2 in both"), "unexpected error: {err}"); + } + + #[test] + fn merge_refuses_an_existing_output() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = checkpoint(tmp.path(), "merged", &[]); + + let err = do_merge(base, source, output).unwrap_err(); + + assert!(err.contains("already exists"), "unexpected error: {err}"); + } + + #[test] + fn merge_refuses_a_missing_input() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let missing = tmp.path().join("missing"); + let output = tmp.path().join("merged"); + + let err = do_merge(base.clone(), missing.clone(), output.clone()).unwrap_err(); + assert!( + err.contains("source checkpoint") && err.contains("not a directory"), + "unexpected error: {err}" + ); + + let err = do_merge(missing, base, output).unwrap_err(); + assert!( + err.contains("base checkpoint") && err.contains("not a directory"), + "unexpected error: {err}" + ); + } + + #[test] + fn merge_tolerates_a_source_without_snapshots() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + fs::remove_dir_all(source.join(SNAPSHOTS_DIR)).unwrap(); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + assert_eq!(entries(&output.join(CANISTER_STATES_DIR)), ["c1", "c2"]); + assert_eq!(entries(&output.join(SNAPSHOTS_DIR)), ["c1"]); + } +} diff --git a/rs/state_tool/src/main.rs b/rs/state_tool/src/main.rs index 06c8dfcbb346..007b9456f104 100644 --- a/rs/state_tool/src/main.rs +++ b/rs/state_tool/src/main.rs @@ -115,6 +115,22 @@ enum Opt { bytes: String, }, + /// Assembles the merged state, as part of a subnet merge. + #[clap(name = "merge")] + Merge { + /// Path to the checkpoint of the destination subnet, which the merged + /// state is based on. + #[clap(long, required = true)] + base: PathBuf, + /// Path to the checkpoint of the source subnet, whose canisters and + /// canister snapshots are added to those of the destination subnet. + #[clap(long, required = true)] + source: PathBuf, + /// Path the merged checkpoint is written to. Must not exist yet. + #[clap(long, required = true)] + output: PathBuf, + }, + /// Prunes a replicated state, as part of a subnet split. #[clap(name = "split")] #[clap(group( @@ -268,6 +284,11 @@ pub(crate) fn main_inner(args: Vec) { Opt::PrincipalFromBytes { bytes } => { commands::convert_ids::do_principal_from_byte_string(bytes) } + Opt::Merge { + base, + source, + output, + } => commands::merge::do_merge(base, source, output), Opt::Split { root, subnet_id, From a0168dca661c5eca51d588ebb74faa495cabfe72 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 7 Sep 2026 09:25:23 +0000 Subject: [PATCH 17/30] test: assemble the merged state with `state-tool merge` The state side of the subnet merge moved into a `merge` command of `state-tool`, so drop the `chmod`/`cp -al`/`std::fs::write` sequence that did it here, along with the `run_local` helper it was the only user of. --- .../subnet_cooling_down_test.rs | 78 ++++--------------- 1 file changed, 14 insertions(+), 64 deletions(-) diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs index 3cd3963f6021..d1397028a961 100644 --- a/rs/tests/message_routing/subnet_cooling_down_test.rs +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -139,10 +139,7 @@ use ic_recovery::steps::Step; use ic_recovery::util::SshUser; use ic_recovery::{IC_STATE_DIR, Recovery, RecoveryArgs}; use ic_registry_subnet_type::SubnetType; -use ic_state_layout::{ - CANISTER_STATES_DIR, SNAPSHOTS_DIR, SUBNET_MERGED_FILE, StateLayout, - UNVERIFIED_CHECKPOINT_MARKER, -}; +use ic_state_layout::StateLayout; use ic_system_test_driver::driver::constants::SSH_USERNAME; use ic_system_test_driver::driver::driver_setup::SSH_AUTHORIZED_PRIV_KEYS_DIR; use ic_system_test_driver::driver::group::SystemTestGroup; @@ -1141,12 +1138,7 @@ impl MergeStateArgs { "M halted at time {m_time}, R at {r_time}; the merged state starts at {merged_time}" ); - assemble_merged_checkpoint( - &r_checkpoint, - &m_checkpoint, - &merged_checkpoint, - &self.logger, - ); + assemble_merged_checkpoint(&r_checkpoint, &m_checkpoint, &merged_checkpoint); let state_hash = manifest_root_hash(&merged_checkpoint); self.upload_merged_checkpoint(&merged_checkpoint); @@ -1233,60 +1225,18 @@ impl MergeStateArgs { } } -/// Copies the checkpoint at `base` to `merged`, replacing its canisters and -/// canister snapshots with the union of those of `base` and of `source`, and -/// marks the result as the product of a subnet merge. -/// -/// Only the canisters and their snapshots are taken over from `source`: its -/// ingress history is not, as the `subnet_merged` marker makes the replica -/// re-register the ingress messages of the merged-in canisters that are still in -/// progress. Everything else (system metadata, subnet queues, ...) is `base`'s. -fn assemble_merged_checkpoint(base: &Path, source: &Path, merged: &Path, logger: &Logger) { - // Checkpoints are read-only and `rsync` preserved that, so the downloaded - // trees have to be made writable before anything can be assembled in them. - for path in [base, source] { - run_local(&format!( - "chmod -R u+w {}", - path.parent().expect("a checkpoint has a parent").display() - )); - } - // `cp -al` hard links the file contents rather than copying them, which - // keeps this cheap. The links are only ever read afterwards, except for the - // marker written below, which is a fresh file. - run_local(&format!("cp -al {} {}", base.display(), merged.display())); - run_local(&format!("chmod -R u+w {}", merged.display())); - for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { - let source_dir = source.join(dir); - if !source_dir.exists() { - info!(logger, "{} holds no {dir}", source.display()); - continue; - } - run_local(&format!( - "mkdir -p {merged_dir} && cp -al {source_dir}/. {merged_dir}/", - merged_dir = merged.join(dir).display(), - source_dir = source_dir.display(), - )); - } - // A `SubnetMerged` message with `merged` (field 1) set to `true`. - std::fs::write(merged.join(SUBNET_MERGED_FILE), [0x08, 0x01]) - .expect("failed to write the subnet merged marker"); - // The uploaded checkpoint must not look unverified to the state manager. - let _ = std::fs::remove_file(merged.join(UNVERIFIED_CHECKPOINT_MARKER)); -} - -/// Runs `script` locally, panicking with its output if it fails. -fn run_local(script: &str) { - let output = Command::new("bash") - .arg("-c") - .arg(script) - .output() - .unwrap_or_else(|e| panic!("failed to run {script:?}: {e}")); - assert!( - output.status.success(), - "{script:?} failed: {}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - ); +/// Assembles the checkpoint at `merged` from the checkpoints at `base` and +/// `source`, i.e. runs the state side of the subnet merge. +fn assemble_merged_checkpoint(base: &Path, source: &Path, merged: &Path) { + state_tool(&[ + "merge", + "--base", + &base.display().to_string(), + "--source", + &source.display().to_string(), + "--output", + &merged.display().to_string(), + ]); } /// Submits (and adopts) the `MergeSubnets` proposal rerouting the canister ID From a1e4ff58d48e0e484fccf7550783bb7219e42121 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 7 Sep 2026 09:12:54 +0000 Subject: [PATCH 18/30] feat(state_tool): assemble the merged state of a subnet merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `merge` command to `state-tool`, performing the state side of a subnet merge: the counterpart of the existing `split` command, which performs the state side of a subnet split. ``` state-tool merge --base \ --source \ --output ``` The command assembles the checkpoint at `--output` from the ones at `--base` (the subnet the canisters are merged into) and `--source` (the subnet being merged away): the result holds everything of `base`, with the canisters and canister snapshots of `source` added to those of `base`, and is marked as the product of a subnet merge. Only the canisters and their snapshots are taken over from `source`. Everything else — system metadata, subnet queues, ingress history — is `base`'s. The ingress history in particular is deliberately not merged in: the marker is what makes the replica re-register the ingress messages of the merged-in canisters that are still in progress. File contents are hard linked rather than copied, so the assembly is cheap no matter how large the two states are, and sound because checkpoints are immutable: the links are only ever read afterwards. The two input checkpoints may stay read-only, as the linking creates the destination directories itself rather than inheriting their permissions from the tree it copies. The marker is written through `CheckpointLayout::subnet_merged_marker()` as a `SubnetMerged` protobuf rather than by hand, and canister IDs that collide between the two checkpoints are refused rather than silently resolved: a collision means the two checkpoints do not belong to the same merge. The change is purely additive — a new command module plus its registration and CLI wiring. No existing behaviour changes, and there are no new dependencies: `ic-protobuf`, `ic-state-layout` and `ic-types` are already dependencies of `state-tool`. The rest of a subnet merge — halting both subnets, downloading the two states, computing the batch time the merged state starts from, and proposing and uploading it — is deliberately out of scope here. A subnet merge system test, not yet on master, drives that workflow and calls this command for the assembly step. That test passes with the assembly performed by this command: the destination subnet adopts the recovery CUP at the merged state, which means the manifest of the assembled checkpoint matches the hash the recovery proposal was made with. The test then checks one canister that the source subnet hosted, now served by the destination subnet: it still holds the blob it had in its stable memory and its canister snapshot, and its cycles balance dropped by no more than an idle canister is expected to burn. Co-Authored-By: Claude Opus 5 (1M context) --- rs/state_tool/src/commands.rs | 1 + rs/state_tool/src/commands/merge.rs | 323 ++++++++++++++++++++++++++++ rs/state_tool/src/main.rs | 21 ++ 3 files changed, 345 insertions(+) create mode 100644 rs/state_tool/src/commands/merge.rs diff --git a/rs/state_tool/src/commands.rs b/rs/state_tool/src/commands.rs index 20d529f1d2a0..702766fdb9f3 100644 --- a/rs/state_tool/src/commands.rs +++ b/rs/state_tool/src/commands.rs @@ -8,6 +8,7 @@ pub mod decode; pub mod import_state; pub mod list; pub mod manifest; +pub mod merge; pub mod parse_overlay; pub mod split; pub mod split_manifest; diff --git a/rs/state_tool/src/commands/merge.rs b/rs/state_tool/src/commands/merge.rs new file mode 100644 index 000000000000..e16a4b5592a9 --- /dev/null +++ b/rs/state_tool/src/commands/merge.rs @@ -0,0 +1,323 @@ +//! Assembles the merged state of a subnet merge. + +use ic_protobuf::state::system_metadata::v1 as pb_metadata; +use ic_state_layout::{CANISTER_STATES_DIR, CheckpointLayout, SNAPSHOTS_DIR, WriteOnly}; +use ic_types::Height; +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; + +/// The height of a checkpoint is part of the name of its directory, which the +/// caller picks, so the layout below is only ever used to name files within it. +const HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED: Height = Height::new(0); + +/// Assembles the checkpoint at `output` from the checkpoints at `base` and +/// `source`: it holds everything of `base`, with the canisters and canister +/// snapshots of `source` added to those of `base`, and is marked as the product +/// of a subnet merge. +/// +/// Only the canisters and their snapshots are taken over from `source`. +/// Everything else (system metadata, subnet queues, ingress history, ...) is +/// `base`'s. In particular, the ingress history of `source` is deliberately not +/// merged in: the subnet merged marker makes the replica re-register the ingress +/// messages of the merged-in canisters that are still in progress. +/// +/// File contents are hard linked rather than copied, so this is cheap no matter +/// how large the two states are. That makes `output` share the storage of +/// `base` and `source`, which is sound because checkpoints are immutable: the +/// links are only ever read afterwards. +pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), String> { + for (path, name) in [(&base, "base"), (&source, "source")] { + if !path.is_dir() { + return Err(format!( + "the {name} checkpoint {} is not a directory", + path.display() + )); + } + } + if output.exists() { + return Err(format!("{} already exists", output.display())); + } + + link_tree(&base, &output)?; + + for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { + let source_dir = source.join(dir); + if !source_dir.exists() { + continue; + } + let output_dir = output.join(dir); + // The canisters of the two subnets are disjoint, as the source subnet + // hosts the canister ID ranges that the merge reassigns to the + // destination subnet. A collision would mean that the two checkpoints + // do not belong to the same merge, so refuse rather than pick a winner. + if let Some(name) = common_entry(&output_dir, &source_dir)? { + return Err(format!( + "{} holds {name} in both {} and {}", + dir, + base.display(), + source.display() + )); + } + link_tree(&source_dir, &output_dir)?; + } + + let layout = CheckpointLayout::::new_untracked( + output.clone(), + HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED, + ) + .map_err(|err| format!("failed to create the checkpoint layout: {err:?}"))?; + layout + .subnet_merged_marker() + .serialize(pb_metadata::SubnetMerged { merged: true }) + .map_err(|err| format!("failed to write the subnet merged marker: {err:?}"))?; + + // `base` was a checkpoint of a running subnet, so it holds no unverified + // checkpoint marker; but a state that was downloaded and reassembled by + // hand may, and the state manager must not take `output` for unverified. + let unverified_marker = layout.unverified_checkpoint_marker(); + if unverified_marker.exists() { + fs::remove_file(&unverified_marker) + .map_err(|err| format!("failed to remove {}: {err}", unverified_marker.display()))?; + } + + Ok(()) +} + +/// Replicates the directory tree rooted at `from` under `to`, hard linking every +/// file. Directories that already exist under `to` are reused, so a tree can be +/// overlaid onto another one. +/// +/// The directories are created writable, unlike the read-only ones of a +/// checkpoint, so that a subsequent call can overlay onto them. +fn link_tree(from: &Path, to: &Path) -> Result<(), String> { + fs::create_dir_all(to).map_err(|err| format!("failed to create {}: {err}", to.display()))?; + + for entry in read_dir(from)? { + let name = entry.file_name(); + let from = entry.path(); + let to = to.join(&name); + + let file_type = entry + .file_type() + .map_err(|err| format!("failed to stat {}: {err}", from.display()))?; + if file_type.is_dir() { + link_tree(&from, &to)?; + } else { + fs::hard_link(&from, &to).map_err(|err| { + format!( + "failed to link {} to {}: {err}", + from.display(), + to.display() + ) + })?; + } + } + + Ok(()) +} + +/// Returns the name of an entry that both directories hold, if any. A directory +/// that does not exist holds nothing. +fn common_entry(left: &Path, right: &Path) -> Result, String> { + let names = |dir: &Path| -> Result, String> { + if !dir.exists() { + return Ok(BTreeSet::new()); + } + Ok(read_dir(dir)? + .into_iter() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect()) + }; + + let left = names(left)?; + Ok(names(right)?.intersection(&left).next().cloned()) +} + +/// The entries of `dir`, with the I/O errors of both the listing and the entries +/// themselves resolved. +fn read_dir(dir: &Path) -> Result, String> { + fs::read_dir(dir) + .map_err(|err| format!("failed to read {}: {err}", dir.display()))? + .collect::, _>>() + .map_err(|err| format!("failed to read an entry of {}: {err}", dir.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + use ic_state_layout::{CompleteCheckpointLayout, UNVERIFIED_CHECKPOINT_MARKER}; + use std::os::unix::fs::MetadataExt; + use tempfile::TempDir; + + /// Creates a checkpoint-shaped directory under `root`, holding a + /// `system_metadata.pbuf` and a canister directory (with a snapshot + /// directory of the same name) per entry of `canisters`. + fn checkpoint(root: &Path, name: &str, canisters: &[&str]) -> PathBuf { + let checkpoint = root.join(name); + fs::create_dir_all(&checkpoint).unwrap(); + fs::write(checkpoint.join("system_metadata.pbuf"), name).unwrap(); + for canister in canisters { + for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { + let canister_dir = checkpoint.join(dir).join(canister); + fs::create_dir_all(&canister_dir).unwrap(); + fs::write(canister_dir.join("canister.pbuf"), *canister).unwrap(); + } + } + checkpoint + } + + fn entries(dir: &Path) -> Vec { + let mut names: Vec<_> = read_dir(dir) + .unwrap() + .into_iter() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names + } + + #[test] + fn merge_unions_canisters_and_snapshots() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1", "c2"]); + let source = checkpoint(tmp.path(), "source", &["c3"]); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { + assert_eq!(entries(&output.join(dir)), ["c1", "c2", "c3"], "{dir}"); + } + } + + #[test] + fn merge_keeps_everything_else_from_base() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + assert_eq!( + fs::read_to_string(output.join("system_metadata.pbuf")).unwrap(), + "base" + ); + } + + #[test] + fn merge_hard_links_file_contents() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + do_merge(base.clone(), source.clone(), output.clone()).unwrap(); + + let inode = |path: PathBuf| fs::metadata(path).unwrap().ino(); + let canister = |root: &Path, canister: &str| { + root.join(CANISTER_STATES_DIR) + .join(canister) + .join("canister.pbuf") + }; + assert_eq!( + inode(canister(&output, "c1")), + inode(canister(&base, "c1")), + "the canister of the base checkpoint is not hard linked" + ); + assert_eq!( + inode(canister(&output, "c2")), + inode(canister(&source, "c2")), + "the canister of the source checkpoint is not hard linked" + ); + } + + #[test] + fn merge_sets_the_subnet_merged_marker() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + let layout = CompleteCheckpointLayout::new_untracked( + output, + HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED, + ) + .unwrap(); + assert!(layout.subnet_merged_marker().deserialize().unwrap().merged); + } + + #[test] + fn merge_removes_the_unverified_checkpoint_marker() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + fs::write(base.join(UNVERIFIED_CHECKPOINT_MARKER), "").unwrap(); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + assert!(!output.join(UNVERIFIED_CHECKPOINT_MARKER).exists()); + } + + #[test] + fn merge_refuses_colliding_canisters() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1", "c2"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + let err = do_merge(base, source, output.clone()).unwrap_err(); + + assert!(err.contains("holds c2 in both"), "unexpected error: {err}"); + } + + #[test] + fn merge_refuses_an_existing_output() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = checkpoint(tmp.path(), "merged", &[]); + + let err = do_merge(base, source, output).unwrap_err(); + + assert!(err.contains("already exists"), "unexpected error: {err}"); + } + + #[test] + fn merge_refuses_a_missing_input() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let missing = tmp.path().join("missing"); + let output = tmp.path().join("merged"); + + let err = do_merge(base.clone(), missing.clone(), output.clone()).unwrap_err(); + assert!( + err.contains("source checkpoint") && err.contains("not a directory"), + "unexpected error: {err}" + ); + + let err = do_merge(missing, base, output).unwrap_err(); + assert!( + err.contains("base checkpoint") && err.contains("not a directory"), + "unexpected error: {err}" + ); + } + + #[test] + fn merge_tolerates_a_source_without_snapshots() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + fs::remove_dir_all(source.join(SNAPSHOTS_DIR)).unwrap(); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + assert_eq!(entries(&output.join(CANISTER_STATES_DIR)), ["c1", "c2"]); + assert_eq!(entries(&output.join(SNAPSHOTS_DIR)), ["c1"]); + } +} diff --git a/rs/state_tool/src/main.rs b/rs/state_tool/src/main.rs index 06c8dfcbb346..007b9456f104 100644 --- a/rs/state_tool/src/main.rs +++ b/rs/state_tool/src/main.rs @@ -115,6 +115,22 @@ enum Opt { bytes: String, }, + /// Assembles the merged state, as part of a subnet merge. + #[clap(name = "merge")] + Merge { + /// Path to the checkpoint of the destination subnet, which the merged + /// state is based on. + #[clap(long, required = true)] + base: PathBuf, + /// Path to the checkpoint of the source subnet, whose canisters and + /// canister snapshots are added to those of the destination subnet. + #[clap(long, required = true)] + source: PathBuf, + /// Path the merged checkpoint is written to. Must not exist yet. + #[clap(long, required = true)] + output: PathBuf, + }, + /// Prunes a replicated state, as part of a subnet split. #[clap(name = "split")] #[clap(group( @@ -268,6 +284,11 @@ pub(crate) fn main_inner(args: Vec) { Opt::PrincipalFromBytes { bytes } => { commands::convert_ids::do_principal_from_byte_string(bytes) } + Opt::Merge { + base, + source, + output, + } => commands::merge::do_merge(base, source, output), Opt::Split { root, subnet_id, From cc647f9d15880507d71ae6ef9d9e732fcb474ad8 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 7 Sep 2026 11:12:59 +0000 Subject: [PATCH 19/30] test: upload the merged state with the recovery upload step Assembling the merged state and adding it to the destination node were one step that ended in a sequence of `sudo` commands over SSH, because `UploadStateAndRestartStep` replaces the whole state directory, which deletes the checkpoint the destination subnet halted at. Deleting that checkpoint is what we want. It does not hold the canisters of the subnet being merged away, so a replica coming up on it serves a state that silently lost them -- a failure mode this test used to guard against by starting the replica only once the recovery CUP existed. With the state directory replaced, the merged state is the only state the destination subnet can resume from, so the failure mode is gone rather than guarded against. The upload step also restarts the replica, so the two proposals of the merge now run before it rather than after, which is the order subnet splitting uses: propose, upload, wait for the CUP, unhalt. Steps 14 and 15 swap accordingly. The merged state is assembled in a directory of its own, as the upload step insists that what it uploads hold a single checkpoint, and `R`'s states metadata is copied next to it, as the step hands rsync every path it transfers as an explicit source. `UploadStateAndRestartStep` rather than `Recovery::get_upload_state_and_restart_step`: the latter hardcodes the check that the uploaded checkpoint matches the height an `ic-replay` run reported, and this merge runs no `ic-replay`. Subnet splitting builds the step directly for the same reason. Co-Authored-By: Claude Opus 5 (1M context) --- .../subnet_cooling_down_test.rs | 229 +++++++++--------- 1 file changed, 117 insertions(+), 112 deletions(-) diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs index d1397028a961..590fff5140a8 100644 --- a/rs/tests/message_routing/subnet_cooling_down_test.rs +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -83,28 +83,29 @@ Runbook:: nodes reports in its journal that it is halted. Record the heights of the checkpoints they halted at. 13. Stop the replicas of both subnets and download the states they halted at. - Assemble the merged state as a new checkpoint of `R`, at the next multiple of - the DKG interval after the height `R` halted at, so that `R`'s own checkpoint - is left untouched: the canisters and canister snapshots of `M` are added to - those of `R`, and the result is marked as the product of a subnet merge. The - ingress history of `M` is deliberately not merged in: the marker makes the - replica re-register the ingress messages of the merged-in canisters that are - still in progress. Compute the block time the merged state starts from, which - must be larger than the times of both checkpoints, and the hash of its - manifest. -14. Add the merged state to the checkpoints of `R`'s node, leaving its replica - stopped. -15. Submit (and adopt) a `MergeSubnets` NNS proposal for `M` and `R`, which + Assemble the merged state locally, as a checkpoint at the next multiple of the + DKG interval after the height `R` halted at: the canisters and canister + snapshots of `M` are added to those of `R`, and the result is marked as the + product of a subnet merge. The ingress history of `M` is deliberately not + merged in: the marker makes the replica re-register the ingress messages of + the merged-in canisters that are still in progress. Compute the block time the + merged state starts from, which must be larger than the times of both + checkpoints, and the hash of its manifest. +14. Submit (and adopt) a `MergeSubnets` NNS proposal for `M` and `R`, which reroutes the canister ID ranges of `M` to `R`, and then a `RecoverSubnet` NNS proposal for `R`, which creates a recovery CUP for `R` at the merged state, running a fresh DKG for `R`'s membership. Recovering a subnet that was instructed to halt at its next CUP replaces that instruction with a plain halt, so `R` stays halted for now. -16. Start `R`'s replica and wait until it adopted the recovery CUP. Only now: a - replica started before the recovery CUP exists resumes from the checkpoint - `R` halted at, which does not hold the canisters of `M`. Then submit (and - adopt) an `UpdateConfigOfSubnet` NNS proposal unhalting `R` and wait until it - is healthy. +15. Upload the merged state to `R`'s node, replacing the state directory holding + the checkpoint it halted at, and restart its replica. Deleting that checkpoint + is what makes the recovery unambiguous: it does not hold the canisters of `M`, + so a replica coming up on it would serve a state that silently lost them, and + the merged state is now the only one `R` can resume from. The recovery CUP of + step 14 has to exist by this point, as the replica is restarted right away. +16. Wait until `R` reports the recovery CUP, i.e. it did come up on the merged + state. Then submit (and adopt) an `UpdateConfigOfSubnet` NNS proposal unhalting + `R` and wait until it is healthy. 17. Check that `U8`, now served by `R`, kept the stable memory, the snapshot and (up to what an idle canister burns) the cycles balance of step 4, and that `UR`, which `R` hosted all along, is undisturbed and can call `U8` now that @@ -134,10 +135,9 @@ use ic_agent::{Agent, RequestId, agent::RequestStatusResponse}; use ic_management_canister_types::{SnapshotId, TakeCanisterSnapshotArgs}; use ic_nns_governance_api::NnsFunction; use ic_recovery::registry_helper::RegistryPollingStrategy; -use ic_recovery::ssh_helper::SshHelper; -use ic_recovery::steps::Step; -use ic_recovery::util::SshUser; -use ic_recovery::{IC_STATE_DIR, Recovery, RecoveryArgs}; +use ic_recovery::steps::{Step, UploadStateAndRestartStep}; +use ic_recovery::util::{DataLocation, SshUser}; +use ic_recovery::{IC_STATE_DIR, Recovery, RecoveryArgs, STATES_METADATA}; use ic_registry_subnet_type::SubnetType; use ic_state_layout::StateLayout; use ic_system_test_driver::driver::constants::SSH_USERNAME; @@ -785,8 +785,8 @@ async fn run(env: TestEnv) { // Step 13: Assemble the merged state: `R`'s state at the checkpoint it // halted at, with the canisters (and canister snapshots) of `M` added to it, - // as a new checkpoint at the next multiple of the DKG interval, so that - // `R`'s own checkpoint is left untouched. + // as a checkpoint at the next multiple of the DKG interval, which is the + // first height a recovery CUP for `R` can be created at. // // Taking `R`'s system metadata and subnet queues wholesale, i.e. dropping // `M`'s, is only sound because `M`'s were empty, which is what the merge @@ -798,7 +798,7 @@ async fn run(env: TestEnv) { let merged_height = r_height + CHECKPOINT_INTERVAL; info!( logger, - "Step 13: Assembling the merged state as checkpoint {merged_height} of R" + "Step 13: Assembling the merged state as checkpoint {merged_height}" ); // The replicas have to be stopped before their states are touched: the state @@ -827,32 +827,39 @@ async fn run(env: TestEnv) { .get_public_url(), m_dir: env.get_path("recovery_m"), r_dir: env.get_path("recovery_r"), + merged_dir: env.get_path("recovery_merged"), m_node_ip: m_node.get_ip_addr(), r_node_ip: r_node.get_ip_addr(), m_height, r_height, merged_height, }; - let (merged_time, state_hash) = tokio::task::spawn_blocking(move || merge.exec()) - .await - .expect("the state merging task panicked"); + let (merged_time, state_hash) = { + let merge = merge.clone(); + tokio::task::spawn_blocking(move || merge.assemble()) + .await + .expect("the state merging task panicked") + }; info!( logger, - "Step 14 done: R holds the merged state, which hashes to {} and starts at {merged_time}", + "Step 13 done: the merged state hashes to {} and starts at {merged_time}", hex::encode(&state_hash), ); - // Step 15: Merge `M` into `R`: reroute `M`'s canister ID ranges to `R`, and - // recover `R` at the merged state. + // Step 14: Merge `M` into `R`: reroute `M`'s canister ID ranges to `R`, and + // recover `R` at the merged state. Both proposals have to be executed before + // the merged state is uploaded in step 15, which restarts `R`'s replica: a + // replica that comes up before the recovery CUP exists has nothing to resume + // from, as the upload replaced the state it halted at. info!( logger, - "Step 15: Submitting the MergeSubnets proposal for M -> R" + "Step 14: Submitting the MergeSubnets proposal for M -> R" ); let merge_registry_version = merge_subnets(&env, m_subnet.subnet_id, r_subnet.subnet_id, &logger).await; info!( logger, - "Step 15: M is merged into R as of registry version {merge_registry_version}" + "Step 14: M is merged into R as of registry version {merge_registry_version}" ); // `merge_subnets` only updates the routing table: making `R` resume from the @@ -861,7 +868,7 @@ async fn run(env: TestEnv) { // merged and stays available throughout. info!( logger, - "Step 15: Submitting the RecoverSubnet proposal for R at height {merged_height}" + "Step 14: Submitting the RecoverSubnet proposal for R at height {merged_height}" ); let recovery_registry_version = recover_subnet( &env, @@ -874,24 +881,24 @@ async fn run(env: TestEnv) { .await; info!( logger, - "Step 15 done: R is recovered at the merged state as of registry version \ + "Step 14 done: R is recovered at the merged state as of registry version \ {recovery_registry_version}" ); - // Step 16: Start `R`'s replica, now that the recovery CUP exists. Starting it - // any earlier would have it resume from its own checkpoint, which does not - // hold the canisters of `M`. - info!(logger, "Step 16: Starting the replica of subnet R"); - r_node - .block_on_bash_script_async("sudo systemctl start ic-replica") + // Step 15: Upload the merged state to `R`, replacing the state it halted at, + // and restart its replica. The recovery CUP of step 14 exists by now, so the + // replica comes up on the merged state. + info!(logger, "Step 15: Uploading the merged state to R"); + tokio::task::spawn_blocking(move || merge.upload_merged_state()) .await - .expect("failed to start the replica of subnet R"); - // Whether `R` resumes from the merged state or from the checkpoint it halted - // at is not something to leave to chance: a replica that started before its - // node had synced the registry version holding the recovery CUP would come - // up on the latter, silently serving a state without the canisters of `M`. - // Wait for the node to report exactly the recovery CUP, so that this fails - // loudly and promptly instead. + .expect("the state uploading task panicked"); + info!(logger, "Step 15 done: R holds the merged state"); + + // Step 16: Wait until `R` came up on the merged state and lift its halt. + // + // That the merged state is the only checkpoint `R` has does not by itself + // mean it resumed from it, so wait for the node to report exactly the + // recovery CUP. { let logger = logger.clone(); let node_ip = r_node.get_ip_addr(); @@ -1080,12 +1087,20 @@ async fn run(env: TestEnv) { /// This is a plain struct of owned data rather than a closure over the test's /// state because it has to be moved onto a blocking thread: `ic-recovery` blocks /// on its own runtime, which a thread driving the test's runtime cannot do. +#[derive(Clone)] struct MergeStateArgs { logger: Logger, admin_key_file: PathBuf, nns_url: Url, m_dir: PathBuf, r_dir: PathBuf, + /// Recovery directory holding nothing but the merged checkpoint, which is + /// what makes it uploadable as a whole: the upload step insists that the + /// directory it uploads hold a single checkpoint. + /// + /// Besides the checkpoint it holds the states metadata, which the upload step + /// transfers alongside it. + merged_dir: PathBuf, m_node_ip: IpAddr, r_node_ip: IpAddr, m_height: u64, @@ -1094,9 +1109,13 @@ struct MergeStateArgs { } impl MergeStateArgs { + /// Downloads the states the two subnets halted at and assembles the merged + /// state from them, as a checkpoint of `merged_dir`. + /// /// Returns the block time the recovered destination subnet should start from - /// and the hash of the manifest of the merged state. - fn exec(self) -> (u64, Vec) { + /// and the hash of the manifest of the merged state, i.e. what the recovery + /// proposal of the destination subnet has to carry. + fn assemble(&self) -> (u64, Vec) { let m_recovery = self.recovery(self.m_dir.clone()); let r_recovery = self.recovery(self.r_dir.clone()); @@ -1124,9 +1143,9 @@ impl MergeStateArgs { m_checkpoints.join(StateLayout::checkpoint_name(Height::from(self.m_height))); let r_checkpoint = r_checkpoints.join(StateLayout::checkpoint_name(Height::from(self.r_height))); - let merged_checkpoint = r_checkpoints.join(StateLayout::checkpoint_name(Height::from( - self.merged_height, - ))); + let merged_checkpoint = self.merged_dir.join(IC_STATE_DIR).join("checkpoints").join( + StateLayout::checkpoint_name(Height::from(self.merged_height)), + ); // The block time the recovered subnet starts from has to be larger than // the times of both checkpoints the merged state is assembled from. @@ -1139,71 +1158,57 @@ impl MergeStateArgs { ); assemble_merged_checkpoint(&r_checkpoint, &m_checkpoint, &merged_checkpoint); - let state_hash = manifest_root_hash(&merged_checkpoint); - self.upload_merged_checkpoint(&merged_checkpoint); + // The upload step of step 15 transfers the states metadata alongside the + // checkpoint, and rsync is given every path it transfers as an explicit + // source, so a missing one fails the whole transfer: take `R`'s along. + // + // It is a manifest cache, which the state manager recomputes for the + // checkpoints it finds whenever it is missing or does not describe them, + // and the heights it names here are the ones `R` held before the merge, + // none of which the merged state directory has. That is the same mismatch + // a plain subnet recovery uploads, where the metadata comes from the state + // that was downloaded and the checkpoint from the replay that followed. + std::fs::copy( + r_recovery.work_dir.join(IC_STATE_DIR).join(STATES_METADATA), + self.merged_dir.join(IC_STATE_DIR).join(STATES_METADATA), + ) + .expect("failed to copy the states metadata of subnet R"); - (merged_time, state_hash) + (merged_time, manifest_root_hash(&merged_checkpoint)) } - /// Adds `merged_checkpoint` to the checkpoints of the destination node, - /// leaving its replica stopped. + /// Uploads the merged state to the destination node, replacing its state + /// directory, and restarts its replica. /// - /// Not `Recovery::get_upload_state_and_restart_step`: that one replaces the - /// whole state directory (and insists that it hold a single checkpoint), - /// which would delete the checkpoint the destination subnet halted at. That - /// checkpoint is meant to survive the merge untouched, so the merged state is - /// added next to it instead. - fn upload_merged_checkpoint(&self, merged_checkpoint: &Path) { - let ssh_helper = SshHelper::new( - self.logger.clone(), - SshUser::Admin, - self.r_node_ip, - /* require_confirmation= */ false, - Some(self.admin_key_file.clone()), - ); - let staging = PathBuf::from("/var/lib/ic/data/merged_state"); - - info!( - self.logger, - "Uploading the merged state to {}", - staging.display() - ); - // `/var/lib/ic/data` is not writable by the SSH user, so the staging - // directory has to be created with `sudo` and then handed over to it, or - // the `rsync` below (which runs as that user) cannot write into it. - ssh_helper - .ssh(format!( - "set -e; - sudo rm -rf {staging}; - sudo mkdir -p {staging}; - sudo chown -R {ssh_user} {staging};", - staging = staging.display(), - ssh_user = SshUser::Admin, - )) - .expect("failed to prepare the staging directory on R"); - ssh_helper - .rsync( - format!("{}/", merged_checkpoint.display()), - ssh_helper.remote_path(staging.join("")), - ) - .expect("failed to rsync the merged state to R"); - - info!(self.logger, "Installing the merged state on R"); - let name = StateLayout::checkpoint_name(Height::from(self.merged_height)); - ssh_helper - .ssh(format!( - "set -e; - CHECKPOINTS={NODE_IC_STATE_DIR}/checkpoints; - OWNER_UID=$(sudo stat -c '%u' $CHECKPOINTS); - GROUP_UID=$(sudo stat -c '%g' $CHECKPOINTS); - sudo mv {staging} $CHECKPOINTS/{name}; - sudo chown -R \"$OWNER_UID:$GROUP_UID\" $CHECKPOINTS/{name}; - sudo chmod -R a-w $CHECKPOINTS/{name}; - sudo systemctl restart setup-permissions;", - staging = staging.display(), - )) - .expect("failed to install the merged state on R"); + /// This deletes the checkpoint the destination subnet halted at, which is + /// what makes the recovery unambiguous: that checkpoint does not hold the + /// canisters of the source subnet, so a replica coming up on it would serve + /// a state that silently lost them. With the state directory replaced, the + /// merged state is the only one the replica can resume from. + /// + /// The recovery CUP has to exist by the time this runs, since the replica is + /// restarted right away. + /// + /// `UploadStateAndRestartStep` rather than + /// `Recovery::get_upload_state_and_restart_step`: the latter hardcodes the + /// check that the uploaded checkpoint matches the height an `ic-replay` run + /// reported, and this merge runs no `ic-replay`. Subnet splitting builds the + /// step directly for the same reason. + fn upload_merged_state(&self) { + info!(self.logger, "Uploading the merged state to R"); + UploadStateAndRestartStep { + logger: self.logger.clone(), + ssh_user: SshUser::Admin, + upload_method: DataLocation::Remote(self.r_node_ip), + work_dir: self.merged_dir.clone(), + data_src: self.merged_dir.join(IC_STATE_DIR), + require_confirmation: false, + key_file: Some(self.admin_key_file.clone()), + check_ic_replay_height: false, + } + .exec() + .expect("failed to upload the merged state to subnet R"); } fn recovery(&self, dir: PathBuf) -> Recovery { From 8333ff93207a7073c9fd38de8237ab26ec0e1566 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 7 Sep 2026 11:23:12 +0000 Subject: [PATCH 20/30] fix(state_tool): address the review of the merge command An output nested under one of the input checkpoints was linked into itself: it is created before the input is listed, so the linking descended into it over and over, leaving a deeply nested partial checkpoint behind. Reject that, resolving the paths first, as either side may reach the same directory through a link or a `..`. Remove the state sync checkpoint marker as well as the unverified one. A state sync marker alone makes `checkpoint_status()` report `UnverifiedStateSync`, so dropping only the unverified marker left an output the state manager would not treat as verified, which was the point of dropping it. Mark the files of the merged checkpoint read-only and sync them, as the state manager does before a directory it assembled becomes a checkpoint. The files linked in are read-only already, being the very files of the inputs, but the marker written by the merge was not, and nothing reached the disk. Directories stay writable, which is what a checkpoint's directories look like too. Co-Authored-By: Claude Opus 5 (1M context) --- rs/state_tool/src/commands/merge.rs | 178 ++++++++++++++++++++++++++-- 1 file changed, 165 insertions(+), 13 deletions(-) diff --git a/rs/state_tool/src/commands/merge.rs b/rs/state_tool/src/commands/merge.rs index e16a4b5592a9..c0aff80728c9 100644 --- a/rs/state_tool/src/commands/merge.rs +++ b/rs/state_tool/src/commands/merge.rs @@ -18,9 +18,7 @@ const HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED: Height = Height::new(0); /// /// Only the canisters and their snapshots are taken over from `source`. /// Everything else (system metadata, subnet queues, ingress history, ...) is -/// `base`'s. In particular, the ingress history of `source` is deliberately not -/// merged in: the subnet merged marker makes the replica re-register the ingress -/// messages of the merged-in canisters that are still in progress. +/// `base`'s. /// /// File contents are hard linked rather than copied, so this is cheap no matter /// how large the two states are. That makes `output` share the storage of @@ -38,6 +36,20 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S if output.exists() { return Err(format!("{} already exists", output.display())); } + // An output under one of the inputs would be linked into itself: it is + // created before the input is listed, so the linking would descend into it + // over and over. Resolve the paths first, as either side may reach the same + // directory through a link or a `..`. + let resolved_output = resolve(&output)?; + for (input, name) in [(&base, "base"), (&source, "source")] { + if resolved_output.starts_with(resolve(input)?) { + return Err(format!( + "the output {} is nested under the {name} checkpoint {}", + output.display(), + input.display() + )); + } + } link_tree(&base, &output)?; @@ -72,15 +84,30 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S .serialize(pb_metadata::SubnetMerged { merged: true }) .map_err(|err| format!("failed to write the subnet merged marker: {err:?}"))?; - // `base` was a checkpoint of a running subnet, so it holds no unverified - // checkpoint marker; but a state that was downloaded and reassembled by - // hand may, and the state manager must not take `output` for unverified. - let unverified_marker = layout.unverified_checkpoint_marker(); - if unverified_marker.exists() { - fs::remove_file(&unverified_marker) - .map_err(|err| format!("failed to remove {}: {err}", unverified_marker.display()))?; + // `base` was a checkpoint of a running subnet, so it holds neither marker; + // but a state that was downloaded and reassembled by hand may, and the state + // manager must not take `output` for unverified. Both markers have to go: a + // state sync marker alone makes `checkpoint_status()` report + // `UnverifiedStateSync`, whether or not the unverified marker is there. + for marker in [ + layout.unverified_checkpoint_marker(), + layout.state_sync_checkpoint_marker(), + ] { + if marker.exists() { + fs::remove_file(&marker) + .map_err(|err| format!("failed to remove {}: {err}", marker.display()))?; + } } + // The files of a checkpoint are read-only, and the ones linked in already + // are, being the very files of `base` and `source`; the marker written above + // is not. Mark and sync as the state manager does before a directory it + // assembled becomes a checkpoint. Directories stay writable, which is what a + // checkpoint's directories look like too. + layout + .mark_files_readonly_and_sync(/* thread_pool= */ None, /* perform_sync= */ true) + .map_err(|err| format!("failed to mark the merged checkpoint read-only: {err:?}"))?; + Ok(()) } @@ -88,8 +115,8 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S /// file. Directories that already exist under `to` are reused, so a tree can be /// overlaid onto another one. /// -/// The directories are created writable, unlike the read-only ones of a -/// checkpoint, so that a subsequent call can overlay onto them. +/// The directories are created writable, so that a subsequent call can overlay +/// onto them. fn link_tree(from: &Path, to: &Path) -> Result<(), String> { fs::create_dir_all(to).map_err(|err| format!("failed to create {}: {err}", to.display()))?; @@ -117,6 +144,35 @@ fn link_tree(from: &Path, to: &Path) -> Result<(), String> { Ok(()) } +/// Resolves `path` to an absolute path with links and `..` components taken out. +/// +/// `canonicalize` needs the path to exist, which the output does not, so the +/// deepest ancestor that does exist is resolved and the rest is appended. That is +/// enough for the nesting check: what an existing directory is nested under does +/// not change by appending to it. +fn resolve(path: &Path) -> Result { + let mut suffix = PathBuf::new(); + let mut existing = path; + loop { + if existing.exists() { + return Ok(existing + .canonicalize() + .map_err(|err| format!("failed to resolve {}: {err}", existing.display()))? + .join(&suffix)); + } + let name = existing.file_name().ok_or_else(|| { + format!( + "{} has no ancestor that exists, so it cannot be created", + path.display() + ) + })?; + suffix = PathBuf::from(name).join(&suffix); + existing = existing + .parent() + .expect("a path with a file name has a parent"); + } +} + /// Returns the name of an entry that both directories hold, if any. A directory /// that does not exist holds nothing. fn common_entry(left: &Path, right: &Path) -> Result, String> { @@ -146,7 +202,10 @@ fn read_dir(dir: &Path) -> Result, String> { #[cfg(test)] mod tests { use super::*; - use ic_state_layout::{CompleteCheckpointLayout, UNVERIFIED_CHECKPOINT_MARKER}; + use ic_state_layout::{ + CheckpointStatus, CompleteCheckpointLayout, STATE_SYNC_CHECKPOINT_MARKER, + SUBNET_MERGED_FILE, UNVERIFIED_CHECKPOINT_MARKER, + }; use std::os::unix::fs::MetadataExt; use tempfile::TempDir; @@ -263,6 +322,99 @@ mod tests { assert!(!output.join(UNVERIFIED_CHECKPOINT_MARKER).exists()); } + #[test] + fn merge_makes_the_files_read_only() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + // The marker is the one file the merge writes itself, so it is the one + // that is not already read-only by virtue of being a link into an input. + let marker = output.join(SUBNET_MERGED_FILE); + assert!( + fs::metadata(&marker).unwrap().permissions().readonly(), + "the subnet merged marker is writable", + ); + for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { + let canister = output.join(dir).join("c1").join("canister.pbuf"); + assert!( + fs::metadata(&canister).unwrap().permissions().readonly(), + "{} is writable", + canister.display(), + ); + } + // Directories stay writable, as they are in a checkpoint the state + // manager wrote: only the files are marked. + assert!( + !fs::metadata(output.join(CANISTER_STATES_DIR)) + .unwrap() + .permissions() + .readonly(), + "the canister states directory is read-only", + ); + } + + #[test] + fn merge_removes_the_state_sync_checkpoint_marker() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + // A state sync marker on its own makes `checkpoint_status()` report + // `UnverifiedStateSync`, so the merge has to drop it as well. + fs::write(base.join(STATE_SYNC_CHECKPOINT_MARKER), "").unwrap(); + fs::write(base.join(UNVERIFIED_CHECKPOINT_MARKER), "").unwrap(); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap(); + + assert!(!output.join(STATE_SYNC_CHECKPOINT_MARKER).exists()); + assert!(!output.join(UNVERIFIED_CHECKPOINT_MARKER).exists()); + let layout = CompleteCheckpointLayout::new_untracked( + output, + HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED, + ) + .unwrap(); + assert!( + matches!(layout.checkpoint_status(), CheckpointStatus::Verified), + "the merged checkpoint is not verified", + ); + } + + #[test] + fn merge_refuses_an_output_nested_under_an_input() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + + // Directly under an input, under a directory of one, and reached through + // a `..` that lands back inside one. + for output in [ + base.join("merged"), + base.join(CANISTER_STATES_DIR).join("merged"), + source.join("merged"), + tmp.path() + .join("base") + .join("..") + .join("base") + .join("merged"), + ] { + let err = do_merge(base.clone(), source.clone(), output.clone()).unwrap_err(); + assert!( + err.contains("is nested under the"), + "unexpected error for {}: {err}", + output.display(), + ); + assert!( + !output.exists(), + "{} was created despite being rejected", + output.display(), + ); + } + } + #[test] fn merge_refuses_colliding_canisters() { let tmp = TempDir::new().unwrap(); From 099542a777bd4d1d11cb8913b9f2def312d3d44e Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 7 Sep 2026 11:40:03 +0000 Subject: [PATCH 21/30] fix(state_tool): assemble the merged checkpoint out of the way A relative output was rejected outright: resolving it walked off the front of the path, since the ancestors of a bare relative path run out before reaching the directory it is relative to. Make the path absolute before walking it. The same oversight made the directory sync open the empty path, so that a merge with a relative output failed after the rename had already gone through. A merge that failed partway left its work at the output path, where a checkpoint is expected: an incomplete one, named as a checkpoint but never marked, which also blocked a retry through the check that the output must not exist. Assemble in a staging directory next to the output instead and rename it into place once the checkpoint is complete, removing it again if any step fails. The staging directory is named so that it cannot be taken for a checkpoint, whose name is a height in hexadecimal, and a leftover one is reported rather than silently reused: a merge that was interrupted outright cannot clean up after itself, and whoever does should know it was there. The collision check now runs before anything is assembled, being the one failure a caller is at all likely to hit, and compares the two inputs directly rather than the output against the source. Co-Authored-By: Claude Opus 5 (1M context) --- rs/state_tool/src/commands/merge.rs | 166 ++++++++++++++++++++++++---- 1 file changed, 143 insertions(+), 23 deletions(-) diff --git a/rs/state_tool/src/commands/merge.rs b/rs/state_tool/src/commands/merge.rs index c0aff80728c9..c70cfdbf5218 100644 --- a/rs/state_tool/src/commands/merge.rs +++ b/rs/state_tool/src/commands/merge.rs @@ -36,10 +36,10 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S if output.exists() { return Err(format!("{} already exists", output.display())); } - // An output under one of the inputs would be linked into itself: it is - // created before the input is listed, so the linking would descend into it - // over and over. Resolve the paths first, as either side may reach the same - // directory through a link or a `..`. + // An output under one of the inputs would be linked into itself, as the + // linking creates it before listing the input it reads. Resolve the paths + // first, as either side may reach the same directory through a link or a + // `..`. let resolved_output = resolve(&output)?; for (input, name) in [(&base, "base"), (&source, "source")] { if resolved_output.starts_with(resolve(input)?) { @@ -50,32 +50,74 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S )); } } - - link_tree(&base, &output)?; - + // The canisters of the two subnets are disjoint, as the source subnet hosts + // the canister ID ranges that the merge reassigns to the destination subnet. + // A collision would mean that the two checkpoints do not belong to the same + // merge, so refuse rather than pick a winner. Before anything is assembled: + // this is the one failure the caller is at all likely to hit. for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { - let source_dir = source.join(dir); - if !source_dir.exists() { - continue; - } - let output_dir = output.join(dir); - // The canisters of the two subnets are disjoint, as the source subnet - // hosts the canister ID ranges that the merge reassigns to the - // destination subnet. A collision would mean that the two checkpoints - // do not belong to the same merge, so refuse rather than pick a winner. - if let Some(name) = common_entry(&output_dir, &source_dir)? { + if let Some(name) = common_entry(&base.join(dir), &source.join(dir))? { return Err(format!( - "{} holds {name} in both {} and {}", - dir, + "{dir} holds {name} in both {} and {}", base.display(), source.display() )); } - link_tree(&source_dir, &output_dir)?; + } + + // Assemble next to the output and rename when done, so that a merge that + // fails halfway leaves no directory where a checkpoint is expected. One that + // is interrupted outright leaves the staging directory behind, which is why + // it is not silently reused: whoever cleans it up should know it is there. + let staging = staging_path(&output)?; + if staging.exists() { + return Err(format!( + "{} exists, presumably left behind by an interrupted merge; remove it to retry", + staging.display() + )); + } + + let result = assemble(&base, &source, &staging).and_then(|()| { + fs::rename(&staging, &output).map_err(|err| { + format!( + "failed to move {} to {}: {err}", + staging.display(), + output.display() + ) + })?; + // The rename itself has to reach the disk, as the state manager syncs the + // directory a checkpoint was renamed into. Through the resolved path: the + // parent of a bare relative one is the empty path, which opens nothing. + let parent = resolved_output + .parent() + .expect("a resolved path is absolute, so it has a parent"); + fs::File::open(parent) + .and_then(|dir| dir.sync_all()) + .map_err(|err| format!("failed to sync {}: {err}", parent.display()))?; + Ok(()) + }); + if result.is_err() { + // A no-op once the rename went through. The original error is what the + // caller needs to see, so a failure to clean up must not replace it. + let _ = fs::remove_dir_all(&staging); + } + + result +} + +/// Assembles the merged checkpoint at `staging`, which must not exist. +fn assemble(base: &Path, source: &Path, staging: &Path) -> Result<(), String> { + link_tree(base, staging)?; + + for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { + let source_dir = source.join(dir); + if source_dir.exists() { + link_tree(&source_dir, &staging.join(dir))?; + } } let layout = CheckpointLayout::::new_untracked( - output.clone(), + staging.to_path_buf(), HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED, ) .map_err(|err| format!("failed to create the checkpoint layout: {err:?}"))?; @@ -86,7 +128,7 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S // `base` was a checkpoint of a running subnet, so it holds neither marker; // but a state that was downloaded and reassembled by hand may, and the state - // manager must not take `output` for unverified. Both markers have to go: a + // manager must not take the result for unverified. Both markers have to go: a // state sync marker alone makes `checkpoint_status()` report // `UnverifiedStateSync`, whether or not the unverified marker is there. for marker in [ @@ -111,6 +153,22 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S Ok(()) } +/// The directory the merged checkpoint is assembled in: a sibling of `output`, +/// so that the two are on the same file system and the hard links and the rename +/// both work. +/// +/// The name is not one a checkpoint can have -- checkpoint directories are named +/// after a height in hexadecimal -- so the staging directory is recognizable as +/// what it is for as long as it exists. +fn staging_path(output: &Path) -> Result { + let name = output + .file_name() + .ok_or_else(|| format!("the output {} has no file name", output.display()))?; + let mut staging = name.to_os_string(); + staging.push(".merging"); + Ok(output.with_file_name(staging)) +} + /// Replicates the directory tree rooted at `from` under `to`, hard linking every /// file. Directories that already exist under `to` are reused, so a tree can be /// overlaid onto another one. @@ -151,8 +209,13 @@ fn link_tree(from: &Path, to: &Path) -> Result<(), String> { /// enough for the nesting check: what an existing directory is nested under does /// not change by appending to it. fn resolve(path: &Path) -> Result { + // Absolute first: the ancestors of a bare relative path run out before + // reaching the directory it is relative to, which is the one that exists. + let absolute = std::path::absolute(path) + .map_err(|err| format!("failed to resolve {}: {err}", path.display()))?; + let mut suffix = PathBuf::new(); - let mut existing = path; + let mut existing = absolute.as_path(); loop { if existing.exists() { return Ok(existing @@ -439,6 +502,63 @@ mod tests { assert!(err.contains("already exists"), "unexpected error: {err}"); } + #[test] + fn resolve_handles_a_relative_path() { + // A bare relative output used to run out of ancestors before reaching the + // directory it is relative to, and was rejected as having none. + let resolved = resolve(Path::new("merged")).unwrap(); + + assert!( + resolved.is_absolute(), + "{} is not absolute", + resolved.display() + ); + assert_eq!(resolved.file_name().unwrap(), "merged"); + assert_eq!( + resolved, + std::env::current_dir().unwrap().join("merged"), + "a relative path should resolve against the working directory", + ); + } + + #[test] + fn merge_refuses_an_existing_staging_directory() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + let output = tmp.path().join("merged"); + // What an interrupted merge would have left behind. + fs::create_dir(staging_path(&output).unwrap()).unwrap(); + + let err = do_merge(base, source, output.clone()).unwrap_err(); + + assert!(err.contains("interrupted merge"), "unexpected error: {err}"); + assert!(!output.exists()); + } + + #[test] + fn merge_leaves_nothing_behind_when_it_fails() { + let tmp = TempDir::new().unwrap(); + let base = checkpoint(tmp.path(), "base", &["c1"]); + let source = checkpoint(tmp.path(), "source", &["c2"]); + // A link that resolves to nothing cannot be hard linked, so the merge + // fails after the base checkpoint has been linked in. + std::os::unix::fs::symlink( + tmp.path().join("nowhere"), + source.join(CANISTER_STATES_DIR).join("c2").join("dangling"), + ) + .unwrap(); + let output = tmp.path().join("merged"); + + do_merge(base, source, output.clone()).unwrap_err(); + + assert!(!output.exists(), "the output was left behind"); + assert!( + !staging_path(&output).unwrap().exists(), + "the staging directory was left behind", + ); + } + #[test] fn merge_refuses_a_missing_input() { let tmp = TempDir::new().unwrap(); From 850ea16f95a5f3b8d8fa0c1202380e1285886586 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 7 Sep 2026 11:55:59 +0000 Subject: [PATCH 22/30] fix(state_tool): remove the merged checkpoint if the merge fails after it moved A merge whose only remaining step was to sync the directory it renamed the checkpoint into reported a failure with the checkpoint left in place, so the guarantee that a failed merge leaves nothing where a checkpoint is expected only held up to the rename. Track the rename and clean up whichever of the two directories the work ended up in. Removing a checkpoint that is complete, and whose durability is the one thing that could not be established, is the point: the caller is told the merge failed, so what it finds afterwards should be what a failed merge leaves, and a retry should not be blocked by the output already existing. Co-Authored-By: Claude Opus 5 (1M context) --- rs/state_tool/src/commands/merge.rs | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/rs/state_tool/src/commands/merge.rs b/rs/state_tool/src/commands/merge.rs index c70cfdbf5218..904eeb5d4c81 100644 --- a/rs/state_tool/src/commands/merge.rs +++ b/rs/state_tool/src/commands/merge.rs @@ -77,7 +77,10 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S )); } - let result = assemble(&base, &source, &staging).and_then(|()| { + let mut renamed = false; + let result = (|| -> Result<(), String> { + assemble(&base, &source, &staging)?; + fs::rename(&staging, &output).map_err(|err| { format!( "failed to move {} to {}: {err}", @@ -85,6 +88,8 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S output.display() ) })?; + renamed = true; + // The rename itself has to reach the disk, as the state manager syncs the // directory a checkpoint was renamed into. Through the resolved path: the // parent of a bare relative one is the empty path, which opens nothing. @@ -94,12 +99,16 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S fs::File::open(parent) .and_then(|dir| dir.sync_all()) .map_err(|err| format!("failed to sync {}: {err}", parent.display()))?; + Ok(()) - }); + })(); if result.is_err() { - // A no-op once the rename went through. The original error is what the - // caller needs to see, so a failure to clean up must not replace it. - let _ = fs::remove_dir_all(&staging); + // Whichever of the two the work is sitting in: a merge that reports a + // failure must not leave a checkpoint behind, not even a complete one + // whose durability is all that could not be established. The original + // error is what the caller needs to see, so a failure to clean up must + // not replace it. + let _ = fs::remove_dir_all(if renamed { &output } else { &staging }); } result @@ -541,13 +550,9 @@ mod tests { let tmp = TempDir::new().unwrap(); let base = checkpoint(tmp.path(), "base", &["c1"]); let source = checkpoint(tmp.path(), "source", &["c2"]); - // A link that resolves to nothing cannot be hard linked, so the merge - // fails after the base checkpoint has been linked in. - std::os::unix::fs::symlink( - tmp.path().join("nowhere"), - source.join(CANISTER_STATES_DIR).join("c2").join("dangling"), - ) - .unwrap(); + // A directory where the subnet merged marker belongs cannot be written as + // a file, so the merge fails with both checkpoints already linked in. + fs::create_dir(base.join(SUBNET_MERGED_FILE)).unwrap(); let output = tmp.path().join("merged"); do_merge(base, source, output.clone()).unwrap_err(); From 91d2fdbf62f1cbaba6272c3391e47edb2cedaa22 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 7 Sep 2026 14:44:36 +0000 Subject: [PATCH 23/30] refactor(state_tool): trim the merge command The markers are no longer removed from the merged checkpoint. Both inputs are checkpoints a node had verified, which hold neither marker, so there was nothing to remove; and removing one would have asserted that the result is a verified checkpoint on the strength of nothing, where inheriting it says what is true. The comments that explained more than the code does are gone with it, and the test fixture writes the files a checkpoint really holds -- `canister.pbuf` in a canister directory, `snapshot.pbuf` in a snapshot one -- through the constants that name them. Co-Authored-By: Claude Opus 5 (1M context) --- rs/state_tool/src/commands/merge.rs | 99 +++++++---------------------- 1 file changed, 24 insertions(+), 75 deletions(-) diff --git a/rs/state_tool/src/commands/merge.rs b/rs/state_tool/src/commands/merge.rs index 904eeb5d4c81..225bafb7fe41 100644 --- a/rs/state_tool/src/commands/merge.rs +++ b/rs/state_tool/src/commands/merge.rs @@ -7,8 +7,8 @@ use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; -/// The height of a checkpoint is part of the name of its directory, which the -/// caller picks, so the layout below is only ever used to name files within it. +/// A `CheckpointLayout` has to be given a height, but the merge only asks it for +/// the paths of files inside the checkpoint, and those do not depend on one. const HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED: Height = Height::new(0); /// Assembles the checkpoint at `output` from the checkpoints at `base` and @@ -22,8 +22,7 @@ const HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED: Height = Height::new(0); /// /// File contents are hard linked rather than copied, so this is cheap no matter /// how large the two states are. That makes `output` share the storage of -/// `base` and `source`, which is sound because checkpoints are immutable: the -/// links are only ever read afterwards. +/// `base` and `source`, which is sound because checkpoints are immutable. pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), String> { for (path, name) in [(&base, "base"), (&source, "source")] { if !path.is_dir() { @@ -52,9 +51,6 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S } // The canisters of the two subnets are disjoint, as the source subnet hosts // the canister ID ranges that the merge reassigns to the destination subnet. - // A collision would mean that the two checkpoints do not belong to the same - // merge, so refuse rather than pick a winner. Before anything is assembled: - // this is the one failure the caller is at all likely to hit. for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { if let Some(name) = common_entry(&base.join(dir), &source.join(dir))? { return Err(format!( @@ -90,9 +86,10 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S })?; renamed = true; - // The rename itself has to reach the disk, as the state manager syncs the - // directory a checkpoint was renamed into. Through the resolved path: the - // parent of a bare relative one is the empty path, which opens nothing. + // A rename is not durable until the directory it happened in is synced, + // so the checkpoint could otherwise be back at the staging path after a + // crash. Through the resolved path: the parent of a bare relative one is + // the empty path, which opens nothing. let parent = resolved_output .parent() .expect("a resolved path is absolute, so it has a parent"); @@ -135,26 +132,11 @@ fn assemble(base: &Path, source: &Path, staging: &Path) -> Result<(), String> { .serialize(pb_metadata::SubnetMerged { merged: true }) .map_err(|err| format!("failed to write the subnet merged marker: {err:?}"))?; - // `base` was a checkpoint of a running subnet, so it holds neither marker; - // but a state that was downloaded and reassembled by hand may, and the state - // manager must not take the result for unverified. Both markers have to go: a - // state sync marker alone makes `checkpoint_status()` report - // `UnverifiedStateSync`, whether or not the unverified marker is there. - for marker in [ - layout.unverified_checkpoint_marker(), - layout.state_sync_checkpoint_marker(), - ] { - if marker.exists() { - fs::remove_file(&marker) - .map_err(|err| format!("failed to remove {}: {err}", marker.display()))?; - } - } - // The files of a checkpoint are read-only, and the ones linked in already // are, being the very files of `base` and `source`; the marker written above // is not. Mark and sync as the state manager does before a directory it - // assembled becomes a checkpoint. Directories stay writable, which is what a - // checkpoint's directories look like too. + // assembled becomes a checkpoint. Directories stay writable, as they are in a + // checkpoint the state manager wrote: only its files are marked. layout .mark_files_readonly_and_sync(/* thread_pool= */ None, /* perform_sync= */ true) .map_err(|err| format!("failed to mark the merged checkpoint read-only: {err:?}"))?; @@ -275,8 +257,8 @@ fn read_dir(dir: &Path) -> Result, String> { mod tests { use super::*; use ic_state_layout::{ - CheckpointStatus, CompleteCheckpointLayout, STATE_SYNC_CHECKPOINT_MARKER, - SUBNET_MERGED_FILE, UNVERIFIED_CHECKPOINT_MARKER, + CANISTER_FILE, CompleteCheckpointLayout, SNAPSHOT_FILE, SUBNET_MERGED_FILE, + SYSTEM_METADATA_FILE, }; use std::os::unix::fs::MetadataExt; use tempfile::TempDir; @@ -287,12 +269,15 @@ mod tests { fn checkpoint(root: &Path, name: &str, canisters: &[&str]) -> PathBuf { let checkpoint = root.join(name); fs::create_dir_all(&checkpoint).unwrap(); - fs::write(checkpoint.join("system_metadata.pbuf"), name).unwrap(); + fs::write(checkpoint.join(SYSTEM_METADATA_FILE), name).unwrap(); for canister in canisters { - for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { + for (dir, file) in [ + (CANISTER_STATES_DIR, CANISTER_FILE), + (SNAPSHOTS_DIR, SNAPSHOT_FILE), + ] { let canister_dir = checkpoint.join(dir).join(canister); fs::create_dir_all(&canister_dir).unwrap(); - fs::write(canister_dir.join("canister.pbuf"), *canister).unwrap(); + fs::write(canister_dir.join(file), *canister).unwrap(); } } checkpoint @@ -332,7 +317,7 @@ mod tests { do_merge(base, source, output.clone()).unwrap(); assert_eq!( - fs::read_to_string(output.join("system_metadata.pbuf")).unwrap(), + fs::read_to_string(output.join(SYSTEM_METADATA_FILE)).unwrap(), "base" ); } @@ -350,7 +335,7 @@ mod tests { let canister = |root: &Path, canister: &str| { root.join(CANISTER_STATES_DIR) .join(canister) - .join("canister.pbuf") + .join(CANISTER_FILE) }; assert_eq!( inode(canister(&output, "c1")), @@ -381,19 +366,6 @@ mod tests { assert!(layout.subnet_merged_marker().deserialize().unwrap().merged); } - #[test] - fn merge_removes_the_unverified_checkpoint_marker() { - let tmp = TempDir::new().unwrap(); - let base = checkpoint(tmp.path(), "base", &["c1"]); - fs::write(base.join(UNVERIFIED_CHECKPOINT_MARKER), "").unwrap(); - let source = checkpoint(tmp.path(), "source", &["c2"]); - let output = tmp.path().join("merged"); - - do_merge(base, source, output.clone()).unwrap(); - - assert!(!output.join(UNVERIFIED_CHECKPOINT_MARKER).exists()); - } - #[test] fn merge_makes_the_files_read_only() { let tmp = TempDir::new().unwrap(); @@ -410,8 +382,11 @@ mod tests { fs::metadata(&marker).unwrap().permissions().readonly(), "the subnet merged marker is writable", ); - for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { - let canister = output.join(dir).join("c1").join("canister.pbuf"); + for (dir, file) in [ + (CANISTER_STATES_DIR, CANISTER_FILE), + (SNAPSHOTS_DIR, SNAPSHOT_FILE), + ] { + let canister = output.join(dir).join("c1").join(file); assert!( fs::metadata(&canister).unwrap().permissions().readonly(), "{} is writable", @@ -429,32 +404,6 @@ mod tests { ); } - #[test] - fn merge_removes_the_state_sync_checkpoint_marker() { - let tmp = TempDir::new().unwrap(); - let base = checkpoint(tmp.path(), "base", &["c1"]); - // A state sync marker on its own makes `checkpoint_status()` report - // `UnverifiedStateSync`, so the merge has to drop it as well. - fs::write(base.join(STATE_SYNC_CHECKPOINT_MARKER), "").unwrap(); - fs::write(base.join(UNVERIFIED_CHECKPOINT_MARKER), "").unwrap(); - let source = checkpoint(tmp.path(), "source", &["c2"]); - let output = tmp.path().join("merged"); - - do_merge(base, source, output.clone()).unwrap(); - - assert!(!output.join(STATE_SYNC_CHECKPOINT_MARKER).exists()); - assert!(!output.join(UNVERIFIED_CHECKPOINT_MARKER).exists()); - let layout = CompleteCheckpointLayout::new_untracked( - output, - HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED, - ) - .unwrap(); - assert!( - matches!(layout.checkpoint_status(), CheckpointStatus::Verified), - "the merged checkpoint is not verified", - ); - } - #[test] fn merge_refuses_an_output_nested_under_an_input() { let tmp = TempDir::new().unwrap(); From 67a25bed58947ab2aeccc6fb81cb426d1ae703ab Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 7 Sep 2026 15:38:40 +0000 Subject: [PATCH 24/30] fix(state_tool): claim the staging directory by creating it Checking that the staging directory is free and then creating it are two steps, so two merges of the same output could both get past the check and assemble into the same directory, where the first to fail would remove what the other was still putting together. Create it instead, which is one step that only one of them can win, and report the directory as left behind by an interrupted merge when the creation says it is already there. Its parents are still created as before, so an output whose directory does not exist yet keeps working. Co-Authored-By: Claude Opus 5 (1M context) --- rs/state_tool/src/commands/merge.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/rs/state_tool/src/commands/merge.rs b/rs/state_tool/src/commands/merge.rs index 225bafb7fe41..54158e057cae 100644 --- a/rs/state_tool/src/commands/merge.rs +++ b/rs/state_tool/src/commands/merge.rs @@ -66,12 +66,25 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S // is interrupted outright leaves the staging directory behind, which is why // it is not silently reused: whoever cleans it up should know it is there. let staging = staging_path(&output)?; - if staging.exists() { - return Err(format!( - "{} exists, presumably left behind by an interrupted merge; remove it to retry", - staging.display() - )); + if let Some(parent) = staging.parent() { + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; } + // Creating the staging directory is what claims it, rather than a check that + // it is free: the check would let two merges of the same output both proceed + // into it, and the one that failed first would clean up while the other was + // still assembling. Creating it is a single step that only one of them can + // win, and the cleanup below is then this merge's to do. + fs::create_dir(&staging).map_err(|err| { + if err.kind() == std::io::ErrorKind::AlreadyExists { + format!( + "{} exists, presumably left behind by an interrupted merge; remove it to retry", + staging.display() + ) + } else { + format!("failed to create {}: {err}", staging.display()) + } + })?; let mut renamed = false; let result = (|| -> Result<(), String> { From fd654f069cd42648bd4dab2a0ddbdca9a3b98e94 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Wed, 9 Sep 2026 15:28:29 +0000 Subject: [PATCH 25/30] test: check the canisters of a cooling down subnet through the subnet it is merged into A subnet that is cooling down now also stops serving query calls (#11495), so the two steps of this test that queried a canister of `M` while it was cooling down can no longer do so: * Step 10 checked that `U1`'s `install_code` calls installed the universal canister module, which a query reveals because a canister without a module rejects every query. That check moves to step 17, where `R` serves the merged-in canisters and answers for them again. Step 10 now checks the rejection itself, as the counterpart to step 8's ingress rejection. * Step 11 read the iteration counters of both call loops to show they are stalled. `UT` is on `T` and is still read directly; for `US` on `M`, the number of rounds `M` skipped canister execution in stands in, which is the stronger statement: while it grows, `M` executes no canister message at all. Verified with `bazel test //rs/tests/message_routing:subnet_cooling_down_test_head_nns`, which passed in 1090s (the plain `subnet_cooling_down_test` target is tagged `manual`: it runs the mainnet NNS, which knows neither the `cooling_down` subnet record field nor the `MergeSubnets` proposal). Co-Authored-By: Claude Opus 5 (1M context) --- .../subnet_cooling_down_test.rs | 143 ++++++++++++------ 1 file changed, 99 insertions(+), 44 deletions(-) diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs index 590fff5140a8..6d26655f93a6 100644 --- a/rs/tests/message_routing/subnet_cooling_down_test.rs +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -71,13 +71,17 @@ Runbook:: entries, `M`'s subnet input and output queues are empty, `M`'s subnet call context manager holds no call context, and the pending anonymous refunds are worth at most `MAX_REFUND_VALUE_CYCLES`. -10. Check that `U2a` .. `U2e` have been installed, i.e. that the `install_code` - calls of step 5 ran to completion rather than being lost or rejected while - `M` was cooling down. -11. Check that the two loops of step 2 are indeed stalled (their iteration - counters, read via queries, no longer advance): while `M` is cooling down, - neither `M` nor `T` routes any message to or from `M`, so the messages of - both loops are retained in their senders' output queues. +10. Check that `M` answers no query call, which is the other half of what a + cooling down subnet stops doing: it neither accepts ingress messages nor + serves queries, and it executes no canister message. Whether the + `install_code` calls of step 5 installed the code is therefore only observable + after the merge, in step 17. +11. Check that the two loops of step 2 are indeed stalled: while `M` is cooling + down, it executes no canister message, and neither `M` nor `T` routes any + message to or from `M`, so the messages of both loops are retained in their + senders' output queues. `UT`'s iteration counter is read via a query to `T`; + `US` sits on `M`, which answers no query, so `M`'s count of the rounds it + skipped canister execution in stands in for it. 12. Submit (and adopt) `UpdateConfigOfSubnet` NNS proposals setting the `halt_at_cup_height` flag of both `M` and `R`, and wait until each of their nodes reports in its journal that it is halted. Record the heights of the @@ -109,7 +113,9 @@ Runbook:: 17. Check that `U8`, now served by `R`, kept the stable memory, the snapshot and (up to what an idle canister burns) the cycles balance of step 4, and that `UR`, which `R` hosted all along, is undisturbed and can call `U8` now that - both are on the same subnet. + both are on the same subnet. Check that `U2a` .. `U2e`, also served by `R` + now, have been installed, i.e. that the `install_code` calls of step 5 ran to + completion while `M` was cooling down rather than being lost or rejected. 18. Set the global data of `U3`, `U5` and `U7` to `LOOP_BREAK_TRIGGER`, ending the three endless loops, and check that every ingress message that was in progress across the merge completed. `U3` and `U5` are reached through `R`, @@ -187,6 +193,8 @@ const METRIC_SUBNET_INPUT_QUEUE_MESSAGES: &str = "execution_subnet_input_queue_m const METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES: &str = "execution_subnet_output_queue_messages"; const METRIC_SUBNET_CALL_CONTEXTS: &str = "replicated_state_subnet_call_contexts"; const METRIC_PENDING_REFUNDS_CYCLES: &str = "replicated_state_pending_refunds_cycles"; +const METRIC_ROUNDS_SKIPPED_CANISTER_EXECUTION: &str = + "round_skipped_canister_execution_due_to_cooling_down"; /// Timeout for a subnet to halt at its next CUP, which is up to a full DKG /// interval away. const HALT_TIMEOUT: Duration = Duration::from_secs(900); @@ -710,56 +718,58 @@ async fn run(env: TestEnv) { .unwrap_or_else(|e| panic!("subnet M did not become \"merge ready\": {e}")); info!(logger, "Step 9 done: subnet M is \"merge ready\""); - // Step 10: Check that `U1`'s `install_code` calls did install the universal - // canister module: a canister that has no module rejects every query. + // Step 10: Check that `M` answers no query call. Step 8 saw it stop accepting + // ingress messages; refusing queries is the other half of what a cooling down + // subnet stops doing, and the reason the state of `M`'s canisters can only be + // inspected once `R` serves them (step 17). info!( logger, - "Step 10: Checking that {} have been installed", - INSTALL_CODE_TARGETS.join(", "), + "Step 10: Checking that subnet M rejects query calls" ); - for (&target, name) in targets.iter().zip(INSTALL_CODE_TARGETS) { - let canister = UniversalCanister::from_canister_id(&m_agent, target); - let reply = canister - .query(wasm().reply_data(name.as_bytes())) - .await - .unwrap_or_else(|e| { - panic!("{name} ({target}) does not answer queries, so it was not installed: {e}") - }); - assert_eq!( - reply, - name.as_bytes(), - "{name} ({target}) answered a query with an unexpected reply", - ); - } - info!( - logger, - "Step 10 done: {} have been installed", - INSTALL_CODE_TARGETS.join(", "), + let err = us + .query(wasm().reply_data(&[])) + .await + .expect_err("query call to US was answered while subnet M was cooling down"); + let err = err.to_string(); + assert!( + err.contains("cooling down"), + "query call to US failed unexpectedly: {err}", ); + info!(logger, "Step 10 done: subnet M rejects query calls"); // Step 11: Check that both call loops are stalled, i.e. that `M` became // "merge ready" because it is cooling down and not because the loops // stopped making calls. + // + // `UT` is on `T`, so its iteration counter can be read directly. `US` is on + // `M`, which answers no query, so the number of rounds `M` skipped canister + // execution in stands in for it: while that keeps growing, `M` executes no + // canister message at all, the next iteration of `US`'s loop included. info!( logger, "Step 11: Checking that both call loops are stalled over {STALL_OBSERVATION_PERIOD:?}" ); - let before = [ - global_counter(&us).await.unwrap(), - global_counter(&ut).await.unwrap(), - ]; + let ut_before = global_counter(&ut).await.unwrap(); + let skipped_before = rounds_with_skipped_canister_execution(&m_subnet).await; tokio::time::sleep(STALL_OBSERVATION_PERIOD).await; - for ((canister, name), before) in [(&us, "US"), (&ut, "UT")].into_iter().zip(before) { - let after = global_counter(canister).await.unwrap(); - assert_eq!( - before, after, - "{name}'s call loop advanced from iteration {before} to {after} while subnet M was \ - cooling down", - ); - } + let ut_after = global_counter(&ut).await.unwrap(); + let skipped_after = rounds_with_skipped_canister_execution(&m_subnet).await; + assert_eq!( + ut_before, ut_after, + "UT's call loop advanced from iteration {ut_before} to {ut_after} while subnet M was \ + cooling down", + ); + assert!( + skipped_after > skipped_before, + "subnet M skipped canister execution in no round over {STALL_OBSERVATION_PERIOD:?} \ + ({skipped_before} rounds before, {skipped_after} after), so it was not cooling down and \ + US's loop was stalled for some other reason", + ); info!( logger, - "Step 11 done: both call loops are stalled at iterations {before:?}" + "Step 11 done: UT's call loop is stalled at iteration {ut_after} and subnet M skipped \ + canister execution in {} rounds while waiting", + skipped_after - skipped_before, ); // Step 12: Halt both `M` and `R` at their next CUP, i.e. at a checkpoint @@ -995,9 +1005,36 @@ async fn run(env: TestEnv) { ur_reply, MERGED_CALL_REPLY, "UR got an unexpected reply from U8", ); + // `U1`'s `install_code` calls ran while `M` was cooling down, and whether they + // installed the universal canister module shows in a query: a canister that + // has no module rejects every query. `R` has to be the one asked, as `M` + // answered no query from the moment it started cooling down until it was + // halted for the merge. + info!( + logger, + "Step 17: Checking that {} have been installed", + INSTALL_CODE_TARGETS.join(", "), + ); + for (&target, name) in targets.iter().zip(INSTALL_CODE_TARGETS) { + let canister = UniversalCanister::from_canister_id(&r_agent, target); + let reply = canister + .query(wasm().reply_data(name.as_bytes())) + .await + .unwrap_or_else(|e| { + panic!("{name} ({target}) does not answer queries, so it was not installed: {e}") + }); + assert_eq!( + reply, + name.as_bytes(), + "{name} ({target}) answered a query with an unexpected reply", + ); + } + info!( logger, - "Step 17 done: U8 kept its stable memory, snapshot and cycles, and UR can call it" + "Step 17 done: U8 kept its stable memory, snapshot and cycles, UR can call it, and {} \ + have been installed", + INSTALL_CODE_TARGETS.join(", "), ); // Step 18: Let the endless loops finish and check that every ingress message @@ -1870,6 +1907,24 @@ fn matching_series<'a>( .collect() } +/// The number of rounds `subnet` executed no canister message in because it is +/// cooling down, as the median across its replicas. +/// +/// A subnet that is not cooling down never touches the counter, and a replica +/// that never touched it does not report it at all, which is a zero here. +async fn rounds_with_skipped_canister_execution(subnet: &SubnetSnapshot) -> f64 { + let metrics = fetch_metrics(subnet, &[METRIC_ROUNDS_SKIPPED_CANISTER_EXECUTION]) + .await + .unwrap_or_else(|e| { + panic!( + "failed to fetch the skipped canister execution metric of subnet {}: {e}", + subnet.subnet_id + ) + }); + median_across_replicas(&metrics, METRIC_ROUNDS_SKIPPED_CANISTER_EXECUTION, |_| true) + .unwrap_or(0.0) +} + /// Prometheus' `quantile(0.5, ...)`: the median of `values`, interpolating /// between the two middle values if there is an even number of them. `None` iff /// `values` is empty. From fa06f5f698671aa30bd94c98732fd2b6fd53b407 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Wed, 9 Sep 2026 16:02:36 +0000 Subject: [PATCH 26/30] revert(state_tool): take master's version of the merge command Drops this branch's follow-ups to `commands/merge.rs`, which were written after #11469 was cut, so that the file is master's again: assembling in a `.merging` sibling and renaming on success (`staging_path()`, `assemble()`), claiming that directory by creating it rather than checking that it is free, and the `resolve()`-based check refusing an output nested under either input, along with the three tests covering those. The `--output` documentation in `main.rs` goes back to master's wording, which states the nesting constraint that is once again the caller's to keep. The subnet merge test is unaffected: it passes an output under its own `recovery_merged` directory, outside both input checkpoints, and never looks for a staging directory. Verified with `bazel test //rs/tests/message_routing:subnet_cooling_down_test_head_nns`, which passed in 1055s, assembling the merged state with the reverted command. Co-Authored-By: Claude Opus 5 (1M context) --- rs/state_tool/src/commands/merge.rs | 266 +++++----------------------- rs/state_tool/src/main.rs | 3 +- 2 files changed, 50 insertions(+), 219 deletions(-) diff --git a/rs/state_tool/src/commands/merge.rs b/rs/state_tool/src/commands/merge.rs index e0dd6d9b1b20..8b84792cd966 100644 --- a/rs/state_tool/src/commands/merge.rs +++ b/rs/state_tool/src/commands/merge.rs @@ -7,10 +7,6 @@ use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; -/// A `CheckpointLayout` has to be given a height, but the merge only asks it for -/// the paths of files inside the checkpoint, and those do not depend on one. -const HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED: Height = Height::new(0); - /// Assembles the checkpoint at `output` from the checkpoints at `base` and /// `source`: it holds everything of `base`, with the canisters and canister /// snapshots of `source` added to those of `base`, and is marked as the product @@ -23,6 +19,12 @@ const HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED: Height = Height::new(0); /// File contents are hard linked rather than copied, so this is cheap no matter /// how large the two states are. That makes `output` share the storage of /// `base` and `source`, which is sound because checkpoints are immutable. +/// +/// `output` has to be outside both inputs, which is left to the caller rather +/// than checked. Under `base`, or under `source`'s canister or snapshot +/// directory, the linking finds the output and links it into itself until the +/// merge fails; anywhere else under `source` it is never reached and the merge +/// succeeds, leaving the merged checkpoint inside the source checkpoint. pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), String> { for (path, name) in [(&base, "base"), (&source, "source")] { if !path.is_dir() { @@ -32,25 +34,15 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S )); } } - if output.exists() { - return Err(format!("{} already exists", output.display())); - } - // An output under one of the inputs would be linked into itself, as the - // linking creates it before listing the input it reads. Resolve the paths - // first, as either side may reach the same directory through a link or a - // `..`. - let resolved_output = resolve(&output)?; - for (input, name) in [(&base, "base"), (&source, "source")] { - if resolved_output.starts_with(resolve(input)?) { - return Err(format!( - "the output {} is nested under the {name} checkpoint {}", - output.display(), - input.display() - )); - } - } - // The canisters of the two subnets are disjoint, as the source subnet hosts - // the canister ID ranges that the merge reassigns to the destination subnet. + // Absolute, so that the parent below is a directory rather than the empty + // path a bare relative output would give. + let absolute_output = std::path::absolute(&output) + .map_err(|err| format!("failed to resolve {}: {err}", output.display()))?; + let parent = absolute_output + .parent() + .expect("an absolute path has a parent"); + // The canisters of the two subnets are disjoint, so a collision means these + // two checkpoints are not from the same merge. for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { if let Some(name) = common_entry(&base.join(dir), &source.join(dir))? { return Err(format!( @@ -61,95 +53,60 @@ pub fn do_merge(base: PathBuf, source: PathBuf, output: PathBuf) -> Result<(), S } } - // Assemble next to the output and rename when done, so that a merge that - // fails halfway leaves no directory where a checkpoint is expected. One that - // is interrupted outright leaves the staging directory behind, which is why - // it is not silently reused: whoever cleans it up should know it is there. - let staging = staging_path(&output)?; - if let Some(parent) = staging.parent() { - fs::create_dir_all(parent) - .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; - } - // Creating the staging directory is what claims it, rather than a check that - // it is free: the check would let two merges of the same output both proceed - // into it, and the one that failed first would clean up while the other was - // still assembling. Creating it is a single step that only one of them can - // win, and the cleanup below is then this merge's to do. - fs::create_dir(&staging).map_err(|err| { + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; + fs::create_dir(&output).map_err(|err| { if err.kind() == std::io::ErrorKind::AlreadyExists { - format!( - "{} exists, presumably left behind by an interrupted merge; remove it to retry", - staging.display() - ) + format!("{} already exists", output.display()) } else { - format!("failed to create {}: {err}", staging.display()) + format!("failed to create {}: {err}", output.display()) } })?; - let mut renamed = false; - let result = (|| -> Result<(), String> { - assemble(&base, &source, &staging)?; - - fs::rename(&staging, &output).map_err(|err| { - format!( - "failed to move {} to {}: {err}", - staging.display(), - output.display() - ) - })?; - renamed = true; - - // A rename is not durable until the directory it happened in is synced, - // so the checkpoint could otherwise be back at the staging path after a - // crash. Through the resolved path: the parent of a bare relative one is - // the empty path, which opens nothing. - let parent = resolved_output - .parent() - .expect("a resolved path is absolute, so it has a parent"); + let result = assemble(&base, &source, &output).and_then(|()| { + // Creating a directory is not durable until the directory it was + // created in is synced. fs::File::open(parent) .and_then(|dir| dir.sync_all()) - .map_err(|err| format!("failed to sync {}: {err}", parent.display()))?; - - Ok(()) - })(); - if result.is_err() { - // Whichever of the two the work is sitting in: a merge that reports a - // failure must not leave a checkpoint behind, not even a complete one - // whose durability is all that could not be established. The original - // error is what the caller needs to see, so a failure to clean up must - // not replace it. - let _ = fs::remove_dir_all(if renamed { &output } else { &staging }); + .map_err(|err| format!("failed to sync {}: {err}", parent.display())) + }); + // A merge that reports a failure must not leave a checkpoint behind. The + // original error is the one the caller needs, so a cleanup failure is + // appended to it rather than replacing it. + if let Err(err) = &result + && let Err(cleanup) = fs::remove_dir_all(&output) + { + return Err(format!( + "{err}, and {} could not be removed: {cleanup}", + output.display() + )); } result } -/// Assembles the merged checkpoint at `staging`, which must not exist. -fn assemble(base: &Path, source: &Path, staging: &Path) -> Result<(), String> { - link_tree(base, staging)?; +/// Assembles the merged checkpoint at `output`, an existing empty directory. +fn assemble(base: &Path, source: &Path, output: &Path) -> Result<(), String> { + link_tree(base, output)?; for dir in [CANISTER_STATES_DIR, SNAPSHOTS_DIR] { let source_dir = source.join(dir); if source_dir.exists() { - link_tree(&source_dir, &staging.join(dir))?; + link_tree(&source_dir, &output.join(dir))?; } } - let layout = CheckpointLayout::::new_untracked( - staging.to_path_buf(), - HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED, - ) - .map_err(|err| format!("failed to create the checkpoint layout: {err:?}"))?; + // Any height will do: the layout is only asked for the paths of files inside + // the checkpoint, which do not depend on one. + let layout = CheckpointLayout::::new_untracked(output.to_path_buf(), Height::new(0)) + .map_err(|err| format!("failed to create the checkpoint layout: {err:?}"))?; layout .subnet_merged_marker() .serialize(pb_metadata::SubnetMerged { merged: true }) .map_err(|err| format!("failed to write the subnet merged marker: {err:?}"))?; - // The files of a checkpoint are read-only, and the ones linked in already - // are, being the very files of `base` and `source`; the marker written above - // is not. Mark and sync as the state manager does before a directory it - // assembled becomes a checkpoint. Directories stay writable, as they are in a - // checkpoint the state manager wrote: only its files are marked. + // The linked-in files are read-only already, being the files of `base` and + // `source`; the marker written above is not. layout .mark_files_readonly_and_sync(/* thread_pool= */ None, /* perform_sync= */ true) .map_err(|err| format!("failed to mark the merged checkpoint read-only: {err:?}"))?; @@ -157,28 +114,9 @@ fn assemble(base: &Path, source: &Path, staging: &Path) -> Result<(), String> { Ok(()) } -/// The directory the merged checkpoint is assembled in: a sibling of `output`, -/// so that the two are on the same file system and the hard links and the rename -/// both work. -/// -/// The name is not one a checkpoint can have -- checkpoint directories are named -/// after a height in hexadecimal -- so the staging directory is recognizable as -/// what it is for as long as it exists. -fn staging_path(output: &Path) -> Result { - let name = output - .file_name() - .ok_or_else(|| format!("the output {} has no file name", output.display()))?; - let mut staging = name.to_os_string(); - staging.push(".merging"); - Ok(output.with_file_name(staging)) -} - /// Replicates the directory tree rooted at `from` under `to`, hard linking every -/// file. Directories that already exist under `to` are reused, so a tree can be -/// overlaid onto another one. -/// -/// The directories are created writable, so that a subsequent call can overlay -/// onto them. +/// file. Directories that already exist under `to` are reused and are created +/// writable, so that trees can be overlaid onto one another. fn link_tree(from: &Path, to: &Path) -> Result<(), String> { fs::create_dir_all(to).map_err(|err| format!("failed to create {}: {err}", to.display()))?; @@ -206,40 +144,6 @@ fn link_tree(from: &Path, to: &Path) -> Result<(), String> { Ok(()) } -/// Resolves `path` to an absolute path with links and `..` components taken out. -/// -/// `canonicalize` needs the path to exist, which the output does not, so the -/// deepest ancestor that does exist is resolved and the rest is appended. That is -/// enough for the nesting check: what an existing directory is nested under does -/// not change by appending to it. -fn resolve(path: &Path) -> Result { - // Absolute first: the ancestors of a bare relative path run out before - // reaching the directory it is relative to, which is the one that exists. - let absolute = std::path::absolute(path) - .map_err(|err| format!("failed to resolve {}: {err}", path.display()))?; - - let mut suffix = PathBuf::new(); - let mut existing = absolute.as_path(); - loop { - if existing.exists() { - return Ok(existing - .canonicalize() - .map_err(|err| format!("failed to resolve {}: {err}", existing.display()))? - .join(&suffix)); - } - let name = existing.file_name().ok_or_else(|| { - format!( - "{} has no ancestor that exists, so it cannot be created", - path.display() - ) - })?; - suffix = PathBuf::from(name).join(&suffix); - existing = existing - .parent() - .expect("a path with a file name has a parent"); - } -} - /// Returns the name of an entry that both directories hold, if any. A directory /// that does not exist holds nothing. fn common_entry(left: &Path, right: &Path) -> Result, String> { @@ -385,11 +289,7 @@ mod tests { do_merge(base, source, output.clone()).unwrap(); - let layout = CompleteCheckpointLayout::new_untracked( - output, - HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED, - ) - .unwrap(); + let layout = CompleteCheckpointLayout::new_untracked(output, Height::new(0)).unwrap(); assert!(layout.subnet_merged_marker().deserialize().unwrap().merged); } @@ -433,38 +333,6 @@ mod tests { ); } - #[test] - fn merge_refuses_an_output_nested_under_an_input() { - let tmp = TempDir::new().unwrap(); - let base = checkpoint(tmp.path(), "base", &["c1"]); - let source = checkpoint(tmp.path(), "source", &["c2"]); - - // Directly under an input, under a directory of one, and reached through - // a `..` that lands back inside one. - for output in [ - base.join("merged"), - base.join(CANISTER_STATES_DIR).join("merged"), - source.join("merged"), - tmp.path() - .join("base") - .join("..") - .join("base") - .join("merged"), - ] { - let err = do_merge(base.clone(), source.clone(), output.clone()).unwrap_err(); - assert!( - err.contains("is nested under the"), - "unexpected error for {}: {err}", - output.display(), - ); - assert!( - !output.exists(), - "{} was created despite being rejected", - output.display(), - ); - } - } - #[test] fn merge_refuses_colliding_canisters() { let tmp = TempDir::new().unwrap(); @@ -489,40 +357,6 @@ mod tests { assert!(err.contains("already exists"), "unexpected error: {err}"); } - #[test] - fn resolve_handles_a_relative_path() { - // A bare relative output used to run out of ancestors before reaching the - // directory it is relative to, and was rejected as having none. - let resolved = resolve(Path::new("merged")).unwrap(); - - assert!( - resolved.is_absolute(), - "{} is not absolute", - resolved.display() - ); - assert_eq!(resolved.file_name().unwrap(), "merged"); - assert_eq!( - resolved, - std::env::current_dir().unwrap().join("merged"), - "a relative path should resolve against the working directory", - ); - } - - #[test] - fn merge_refuses_an_existing_staging_directory() { - let tmp = TempDir::new().unwrap(); - let base = checkpoint(tmp.path(), "base", &["c1"]); - let source = checkpoint(tmp.path(), "source", &["c2"]); - let output = tmp.path().join("merged"); - // What an interrupted merge would have left behind. - fs::create_dir(staging_path(&output).unwrap()).unwrap(); - - let err = do_merge(base, source, output.clone()).unwrap_err(); - - assert!(err.contains("interrupted merge"), "unexpected error: {err}"); - assert!(!output.exists()); - } - #[test] fn merge_leaves_nothing_behind_when_it_fails() { let tmp = TempDir::new().unwrap(); @@ -536,10 +370,6 @@ mod tests { do_merge(base, source, output.clone()).unwrap_err(); assert!(!output.exists(), "the output was left behind"); - assert!( - !staging_path(&output).unwrap().exists(), - "the staging directory was left behind", - ); } #[test] diff --git a/rs/state_tool/src/main.rs b/rs/state_tool/src/main.rs index 414ce6c68dc3..428d1dc8b587 100644 --- a/rs/state_tool/src/main.rs +++ b/rs/state_tool/src/main.rs @@ -134,7 +134,8 @@ enum Opt { /// canister snapshots are added to those of the destination subnet. #[clap(long, required = true)] source: PathBuf, - /// Path the merged checkpoint is written to. Must not exist yet. + /// Path the merged checkpoint is written to. Must not exist yet, and + /// must be outside the base and source checkpoints. #[clap(long, required = true)] output: PathBuf, }, From ef027ecf9fe93e9c81fdfc620b9f9a0da0383e0a Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Wed, 9 Sep 2026 17:15:18 +0000 Subject: [PATCH 27/30] feat(consensus): report the consensus status as a metric Whether a subnet is halted is only observable in the replica's journal today, as the line batch delivery logs every five seconds instead of delivering a batch. Report it as `consensus_status{status=...}`, an `IntGaugeVec` holding 1 for the status that batch delivery last computed and 0 for the other two, so that the question "is this subnet halted right now" can be asked of the metrics that already answer the rest of the subnet merging dashboard's conditions. The gauge is set where the status is computed, which is also the only place that computes it, so it says what that path saw the last time it looked: a subnet that stopped producing blocks altogether keeps reporting the status that made it stop. `deliver_batches` takes the observer as a parameter next to its `result_processor`, and `deliver_batches_for_ic_replay` passes a no-op, leaving its signature unchanged. The three statuses are reported separately rather than as one number because they are not ordered, and the distinction matters: `Halting` still produces empty blocks, so the consensus pool keeps moving, while `Halted` produces none and the latest checkpoint is final. The log line fires for both, so it cannot tell them apart. Step 12 of the subnet merge test now waits for `consensus_status{status="halted"}` on the node whose state it is about to download, rather than opening an SSH session and searching that node's journal for a log message. The height it halted at comes from `state_manager_last_computed_manifest_height` instead of listing the checkpoint directory over SSH, which is the stronger signal: a checkpoint whose manifest is not computed yet has no CUP, and the manifest is what the recovery proposal's state hash is compared against. Verified with `bazel test //rs/tests/message_routing:subnet_cooling_down_test_head_nns`, which passed in 1025s, reporting `M halted at checkpoint 2000, R halted at checkpoint 2500` off the metrics and making no `journalctl` call at all, plus the seven affected unit and integration tests of `//rs/consensus`, `//rs/replay` and `//rs/state_machine_tests`. Co-Authored-By: Claude Opus 5 (1M context) --- rs/consensus/src/consensus/batch_delivery.rs | 10 +- rs/consensus/src/consensus/finalizer.rs | 1 + rs/consensus/src/consensus/metrics.rs | 37 ++++++ .../subnet_cooling_down_test.rs | 120 +++++++++--------- 4 files changed, 109 insertions(+), 59 deletions(-) diff --git a/rs/consensus/src/consensus/batch_delivery.rs b/rs/consensus/src/consensus/batch_delivery.rs index 5555dd945a03..7a753b2b0571 100644 --- a/rs/consensus/src/consensus/batch_delivery.rs +++ b/rs/consensus/src/consensus/batch_delivery.rs @@ -67,6 +67,7 @@ pub fn deliver_batches_for_ic_replay( subnet_id, max_batch_height_to_deliver, /*result_processor=*/ |_, _, _| {}, + /*status_observer=*/ |_| {}, ) } @@ -84,6 +85,7 @@ pub(crate) fn deliver_batches_for_finalizer( node_id: NodeId, subnet_id: SubnetId, result_processor: impl FnMut(&Result<(), MessageRoutingError>, BlockStats, BatchStats), + status_observer: impl Fn(Status), ) -> Result { deliver_batches( message_routing, @@ -95,6 +97,7 @@ pub(crate) fn deliver_batches_for_finalizer( subnet_id, /*max_batch_height_to_deliver=*/ None, result_processor, + status_observer, ) } @@ -111,6 +114,7 @@ fn deliver_batches( subnet_id: SubnetId, max_batch_height_to_deliver: Option, mut result_processor: impl FnMut(&Result<(), MessageRoutingError>, BlockStats, BatchStats), + status_observer: impl Fn(Status), ) -> Result { let finalized_height = pool.get_finalized_height(); // If `max_batch_height_to_deliver` is specified and smaller than @@ -191,7 +195,8 @@ fn deliver_batches( replica_version, log, ) { - Some(Status::Halting | Status::Halted) => { + Some(status @ (Status::Halting | Status::Halted)) => { + status_observer(status); info!( every_n_seconds => 5, log, @@ -200,7 +205,7 @@ fn deliver_batches( ); return Ok(last_delivered_batch_height); } - Some(Status::Running) => {} + Some(Status::Running) => status_observer(Status::Running), None => { warn!( log, @@ -939,6 +944,7 @@ mod tests { replica_config.node_id, replica_config.subnet_id, |_, _, _| {}, + |_| {}, ); assert_eq!(result, Ok(summary_height)); diff --git a/rs/consensus/src/consensus/finalizer.rs b/rs/consensus/src/consensus/finalizer.rs index a76adc827c3d..5ddc3c979f1d 100644 --- a/rs/consensus/src/consensus/finalizer.rs +++ b/rs/consensus/src/consensus/finalizer.rs @@ -106,6 +106,7 @@ impl Finalizer { |result, block_stats, batch_stats| { self.process_batch_delivery_result(result, block_stats, batch_stats) }, + |status| self.metrics.observe_status(status), ); // Try to finalize rounds from finalized_height + 1 up to (and including) diff --git a/rs/consensus/src/consensus/metrics.rs b/rs/consensus/src/consensus/metrics.rs index 495039e71192..c5138678208a 100644 --- a/rs/consensus/src/consensus/metrics.rs +++ b/rs/consensus/src/consensus/metrics.rs @@ -1,3 +1,4 @@ +use crate::consensus::status::Status; use ic_consensus_dkg::metrics::DkgPayloadStats; use ic_consensus_idkg::{ metrics::{CounterPerMasterPublicKeyId, IDkgPayloadStats, KEY_ID_LABEL, key_id_label}, @@ -25,6 +26,13 @@ use std::sync::RwLock; // the range of ranks that are permitted to show up in metrics. const RANKS_TO_RECORD: [&str; 6] = ["0", "1", "2", "3", "4", "5"]; +/// The label of `consensus_status`, whose values are the statuses of +/// [`Status`], lowercased. +const STATUS_LABEL: &str = "status"; +const STATUS_RUNNING: &str = "running"; +const STATUS_HALTING: &str = "halting"; +const STATUS_HALTED: &str = "halted"; + pub(crate) const CRITICAL_ERROR_PAYLOAD_TOO_LARGE: &str = "consensus_payload_too_large"; pub(crate) const CRITICAL_ERROR_VALIDATION_NOT_PASSED: &str = "consensus_validation_not_passed"; pub(crate) const CRITICAL_ERROR_SUBNET_RECORD_ISSUE: &str = "consensus_subnet_record_issue"; @@ -166,6 +174,7 @@ impl BatchStats { pub(crate) struct FinalizerMetrics { pub batches_delivered: IntCounterVec, pub batch_height: IntGauge, + pub consensus_status: IntGaugeVec, pub batch_delivery_interval: Histogram, pub batch_delivery_latency: Histogram, pub ingress_messages_delivered: Histogram, @@ -206,6 +215,13 @@ impl FinalizerMetrics { "consensus_batch_height", "The height of batches sent to Message Routing", ), + consensus_status: metrics_registry.int_gauge_vec( + "consensus_status", + "Whether consensus is running, halting (producing empty blocks but delivering \ + no batches) or halted (producing no blocks either), as of the last time batch \ + delivery looked. 1 for the status that held then, 0 for the other two.", + &[STATUS_LABEL], + ), batch_delivery_interval: metrics_registry.histogram( "consensus_batch_delivery_interval_seconds", "Time elapsed since the delivery of the previous batch, in seconds", @@ -328,6 +344,27 @@ impl FinalizerMetrics { } } + /// Records `status` as the status consensus is in, and the other two as ones + /// it is not. + /// + /// Reported as a gauge per status rather than a single number, so that a + /// dashboard can select the status it asks about by name. Only the batch + /// delivery path computes the status, and only when it has a block to + /// consider, so this says what that path saw the last time it looked: a + /// subnet that stopped producing blocks altogether keeps reporting the + /// status that made it stop. + pub fn observe_status(&self, status: Status) { + for (label, value) in [ + (STATUS_RUNNING, Status::Running), + (STATUS_HALTING, Status::Halting), + (STATUS_HALTED, Status::Halted), + ] { + self.consensus_status + .with_label_values(&[label]) + .set((value == status) as i64); + } + } + pub fn process(&self, block_stats: &BlockStats, batch_stats: &BatchStats) { self.batches_delivered.with_label_values(&["success"]).inc(); self.batch_height.set(batch_stats.batch_height as i64); diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs index 6d26655f93a6..398234ad95b2 100644 --- a/rs/tests/message_routing/subnet_cooling_down_test.rs +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -83,9 +83,11 @@ Runbook:: `US` sits on `M`, which answers no query, so `M`'s count of the rounds it skipped canister execution in stands in for it. 12. Submit (and adopt) `UpdateConfigOfSubnet` NNS proposals setting the - `halt_at_cup_height` flag of both `M` and `R`, and wait until each of their - nodes reports in its journal that it is halted. Record the heights of the - checkpoints they halted at. + `halt_at_cup_height` flag of both `M` and `R`, and wait until the node of each + that the state is taken from reports `consensus_status{status="halted"}`, i.e. + it produces no block and delivers no batch anymore. Record the heights of the + checkpoints they halted at, which are the heights of the manifests they + computed last. 13. Stop the replicas of both subnets and download the states they halted at. Assemble the merged state locally, as a checkpoint at the next multiple of the DKG interval after the height `R` halted at: the canisters and canister @@ -163,8 +165,8 @@ use ic_system_test_driver::nns::{ use ic_system_test_driver::retry_with_msg_async; use ic_system_test_driver::systest; use ic_system_test_driver::util::{ - JournalStreamer, MetricsFetcher, UniversalCanister, assert_create_agent, block_on, - create_canister, runtime_from_url, set_controller, + MetricsFetcher, UniversalCanister, assert_create_agent, block_on, create_canister, + runtime_from_url, set_controller, }; use ic_types::{Height, SubnetId}; use ic_universal_canister::management::InstallMode; @@ -195,16 +197,18 @@ const METRIC_SUBNET_CALL_CONTEXTS: &str = "replicated_state_subnet_call_contexts const METRIC_PENDING_REFUNDS_CYCLES: &str = "replicated_state_pending_refunds_cycles"; const METRIC_ROUNDS_SKIPPED_CANISTER_EXECUTION: &str = "round_skipped_canister_execution_due_to_cooling_down"; +const METRIC_CONSENSUS_STATUS: &str = "consensus_status"; +const METRIC_LAST_COMPUTED_MANIFEST_HEIGHT: &str = "state_manager_last_computed_manifest_height"; + +/// The series of `METRIC_CONSENSUS_STATUS` that is 1 while consensus is halted, +/// i.e. produces no block and delivers no batch. +const LABEL_STATUS_HALTED: &str = "status=\"halted\""; /// Timeout for a subnet to halt at its next CUP, which is up to a full DKG /// interval away. const HALT_TIMEOUT: Duration = Duration::from_secs(900); -/// Backoff between two searches of a node's journal for the halt message. +/// Backoff between two checks of whether a subnet has halted. const HALT_BACKOFF: Duration = Duration::from_secs(10); -/// What a halted replica logs, once every few seconds, instead of delivering the -/// batches it would otherwise deliver (see `rs/consensus/src/consensus/batch_delivery.rs`). -const HALTED_LOG_PATTERN: &str = "is not delivered because replica is halted"; - /// The label selecting the `install_code` call contexts of /// `METRIC_SUBNET_CALL_CONTEXTS`. const LABEL_INSTALL_CODE: &str = "type=\"install_code\""; @@ -1887,6 +1891,24 @@ async fn fetch_metrics( }) } +/// Fetches `metrics` from a single node, rather than from all the nodes of a +/// subnet: the values of a subnet-wide property still differ per replica while +/// they observe it in different rounds, and some questions are about one node, +/// such as whether the very node a state is about to be downloaded from has +/// stopped moving. +async fn fetch_node_metrics( + node: &IcNodeSnapshot, + metrics: &[&str], +) -> Result>> { + MetricsFetcher::new( + std::iter::once(node.clone()), + metrics.iter().map(|metric| metric.to_string()).collect(), + ) + .fetch::() + .await + .map_err(|e| anyhow!("failed to fetch the metrics of node {}: {e}", node.node_id)) +} + /// The per-node values of every series of `metric` whose labels (`{...}`, or the /// empty string for an unlabeled series) match `labels_match`. /// @@ -1970,9 +1992,6 @@ fn median_across_replicas( // Merging subnet M into subnet R. // --------------------------------------------------------------------------- -/// The name of the directory the replica keeps its states in, on a node. -const NODE_IC_STATE_DIR: &str = "/var/lib/ic/data/ic_state"; - /// Waits until `node`'s subnet is halted, and returns the height of the /// checkpoint it halted at, i.e. of the state it stopped in. /// @@ -1994,46 +2013,37 @@ const NODE_IC_STATE_DIR: &str = "/var/lib/ic/data/ic_state"; /// subnet holds precisely the state it stopped in. async fn await_halted_at_checkpoint(node: &IcNodeSnapshot, name: &str, logger: &Logger) -> u64 { info!(logger, "Waiting until subnet {name} is halted"); - // Polling the journal rather than following it: `follow()` blocks in - // `journalctl --follow | grep -m 1` until the line shows up, with no timeout - // of its own, so a line that never comes (because it was reworded, say) - // would hang the test until the whole test times out. The cursor of - // `from_now()` makes every poll search all entries since this point, so the - // condition, once true, stays true. - let journal = JournalStreamer::new( - node.block_on_ssh_session_async() - .await - .unwrap_or_else(|e| panic!("failed to open an SSH session to subnet {name}: {e}")), - ) - .from_now() - .unwrap_or_else(|e| panic!("failed to create a journal streamer for subnet {name}: {e}")); + // `node`'s own metrics, not the subnet's: the state that is downloaded below + // is this node's, so this node is the one that has to have stopped, and the + // replicas of a subnet observe the halt in different rounds. + // + // Halted and not merely halting: a halting subnet still produces (empty) + // blocks, so its consensus pool keeps moving, while a halted one produces + // none and its latest checkpoint is final. retry_with_msg_async!( format!("waiting until subnet {name} reports that it is halted"), logger, HALT_TIMEOUT, HALT_BACKOFF, || async { - // `contains` runs `journalctl | grep`, and `grep` exits non-zero when - // it matches nothing, which the SSH helper in turn reports as an - // error: an error here is indistinguishable from the line not being - // there yet, so both mean "keep waiting". A journal that cannot be - // searched at all therefore surfaces as the timeout below. - match journal.contains(HALTED_LOG_PATTERN) { - Ok(true) => Ok(()), - Ok(false) => bail!("subnet {name} has not reported that it is halted yet"), - Err(e) => bail!( - "subnet {name} has not reported that it is halted yet (or its journal could \ - not be searched: {e})" - ), + let metrics = fetch_node_metrics(node, &[METRIC_CONSENSUS_STATUS]).await?; + match median_across_replicas(&metrics, METRIC_CONSENSUS_STATUS, |labels| { + labels.contains(LABEL_STATUS_HALTED) + }) { + Some(1.0) => Ok(()), + Some(_) => bail!("subnet {name} is not halted yet"), + // Only the batch delivery path reports the status, and only once + // it has looked at a block, so the series is missing until then. + None => bail!("subnet {name} has not reported a consensus status yet"), } } ) .await .unwrap_or_else(|e| panic!("subnet {name} did not report that it is halted: {e}")); - let height = latest_checkpoint_height(node) + let height = halted_checkpoint_height(node) .await - .unwrap_or_else(|e| panic!("failed to read the checkpoint of subnet {name}: {e}")); + .unwrap_or_else(|e| panic!("failed to read the checkpoint height of subnet {name}: {e}")); assert_eq!( height % CHECKPOINT_INTERVAL, 0, @@ -2042,23 +2052,19 @@ async fn await_halted_at_checkpoint(node: &IcNodeSnapshot, name: &str, logger: & height } -/// The height of the highest checkpoint `node` holds. Checkpoint directories are -/// named after their height, in hexadecimal. -async fn latest_checkpoint_height(node: &IcNodeSnapshot) -> Result { - let output = node - .block_on_bash_script_async(&format!("sudo ls -1 {NODE_IC_STATE_DIR}/checkpoints")) - .await - .map_err(|e| anyhow!("failed to list the checkpoints: {e}"))?; - output - .split_whitespace() - .map(|name| { - u64::from_str_radix(name, 16) - .map_err(|e| anyhow!("checkpoint name {name} is not a hex height: {e}")) - }) - .collect::>>()? - .into_iter() - .max() - .ok_or_else(|| anyhow!("no checkpoint yet")) +/// The height of the checkpoint `node` came to rest at, taken from the manifest +/// it computed last. +/// +/// The manifest, rather than the checkpoint directory: the state is downloaded +/// and its manifest recomputed to compare against the one a recovery proposal +/// carries, and a checkpoint whose manifest this node has not finished computing +/// is one whose CUP does not exist yet. Nothing is delivered after the halt, so +/// no later checkpoint follows the one this names. +async fn halted_checkpoint_height(node: &IcNodeSnapshot) -> Result { + let metrics = fetch_node_metrics(node, &[METRIC_LAST_COMPUTED_MANIFEST_HEIGHT]).await?; + let height = median_across_replicas(&metrics, METRIC_LAST_COMPUTED_MANIFEST_HEIGHT, |_| true) + .ok_or_else(|| anyhow!("no manifest has been computed yet"))?; + Ok(height as u64) } /// Submits (and adopts) an `UpdateConfigOfSubnet` proposal setting the From a7ce5706521f65d6232524fbc2fb42734617f5c3 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Fri, 11 Sep 2026 11:43:09 +0000 Subject: [PATCH 28/30] test: wait for the CUP a subnet halts at rather than for the halt itself Step 12 of the subnet merge test waited until the node whose state it downloads reported `consensus_status{status="halted"}` and then read the height of the checkpoint it stopped in off `state_manager_last_computed_manifest_height`. It now waits for the CUP the subnet halts at instead, the way `subnet_splitting_test` does: poll `_/catch_up_package` on that node until the registry version of the CUP's summary is at least the one the `halt_at_cup_height` proposal created. That version is the one `should_halt` reads the flag at, so such a CUP is the one batch delivery stops at, and its height is the height of the last checkpoint the subnet wrote. The CUP is the better signal in both halves: it names the state the subnet came to rest in, and it exists only once that state is certified and its hash agreed upon, which is what the recovery proposal's state hash is compared against. The two metrics together could not say that: a subnet that has just stopped delivering batches may not have hashed the checkpoint it stopped at yet, and the last computed manifest height is then the previous checkpoint, a whole DKG interval before the state the merge is meant to be assembled from. As in the reference, the node's certification height is checked against the CUP height as well, so that a node holding a CUP the rest of the subnet assembled before it got there is waited for, and one that ran past the CUP it should have halted at fails the test. That leaves `consensus_status` without a user, so the gauge, `observe_status` and the `status_observer` parameter threaded through `deliver_batches` in ef027ecf9f are dropped again: `rs/consensus` is master's once more. Verified with `bazel test //rs/tests/message_routing:subnet_cooling_down_test_head_nns`, which passed in 1045s, reporting `M halted at checkpoint 2000, R halted at checkpoint 2500`, the heights of the previous run, after waiting 250s on `M` for the CUP at 2000 to replace the one at 1500, whose registry version still preceded the halting one. Plus the seven affected unit and integration tests of `//rs/consensus`, `//rs/replay` and `//rs/state_machine_tests`. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 1 + rs/consensus/src/consensus/batch_delivery.rs | 10 +- rs/consensus/src/consensus/finalizer.rs | 1 - rs/consensus/src/consensus/metrics.rs | 37 ----- rs/tests/message_routing/BUILD.bazel | 1 + rs/tests/message_routing/Cargo.toml | 1 + .../subnet_cooling_down_test.rs | 156 ++++++++++-------- 7 files changed, 95 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe6c47c59929..c104612354da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18242,6 +18242,7 @@ dependencies = [ "ic-types", "ic-universal-canister", "ic-utils 0.49.1", + "ic_consensus_system_test_utils", "itertools 0.12.1", "rand 0.8.6", "rand_chacha 0.3.1", diff --git a/rs/consensus/src/consensus/batch_delivery.rs b/rs/consensus/src/consensus/batch_delivery.rs index 7a753b2b0571..5555dd945a03 100644 --- a/rs/consensus/src/consensus/batch_delivery.rs +++ b/rs/consensus/src/consensus/batch_delivery.rs @@ -67,7 +67,6 @@ pub fn deliver_batches_for_ic_replay( subnet_id, max_batch_height_to_deliver, /*result_processor=*/ |_, _, _| {}, - /*status_observer=*/ |_| {}, ) } @@ -85,7 +84,6 @@ pub(crate) fn deliver_batches_for_finalizer( node_id: NodeId, subnet_id: SubnetId, result_processor: impl FnMut(&Result<(), MessageRoutingError>, BlockStats, BatchStats), - status_observer: impl Fn(Status), ) -> Result { deliver_batches( message_routing, @@ -97,7 +95,6 @@ pub(crate) fn deliver_batches_for_finalizer( subnet_id, /*max_batch_height_to_deliver=*/ None, result_processor, - status_observer, ) } @@ -114,7 +111,6 @@ fn deliver_batches( subnet_id: SubnetId, max_batch_height_to_deliver: Option, mut result_processor: impl FnMut(&Result<(), MessageRoutingError>, BlockStats, BatchStats), - status_observer: impl Fn(Status), ) -> Result { let finalized_height = pool.get_finalized_height(); // If `max_batch_height_to_deliver` is specified and smaller than @@ -195,8 +191,7 @@ fn deliver_batches( replica_version, log, ) { - Some(status @ (Status::Halting | Status::Halted)) => { - status_observer(status); + Some(Status::Halting | Status::Halted) => { info!( every_n_seconds => 5, log, @@ -205,7 +200,7 @@ fn deliver_batches( ); return Ok(last_delivered_batch_height); } - Some(Status::Running) => status_observer(Status::Running), + Some(Status::Running) => {} None => { warn!( log, @@ -944,7 +939,6 @@ mod tests { replica_config.node_id, replica_config.subnet_id, |_, _, _| {}, - |_| {}, ); assert_eq!(result, Ok(summary_height)); diff --git a/rs/consensus/src/consensus/finalizer.rs b/rs/consensus/src/consensus/finalizer.rs index 5ddc3c979f1d..a76adc827c3d 100644 --- a/rs/consensus/src/consensus/finalizer.rs +++ b/rs/consensus/src/consensus/finalizer.rs @@ -106,7 +106,6 @@ impl Finalizer { |result, block_stats, batch_stats| { self.process_batch_delivery_result(result, block_stats, batch_stats) }, - |status| self.metrics.observe_status(status), ); // Try to finalize rounds from finalized_height + 1 up to (and including) diff --git a/rs/consensus/src/consensus/metrics.rs b/rs/consensus/src/consensus/metrics.rs index c5138678208a..495039e71192 100644 --- a/rs/consensus/src/consensus/metrics.rs +++ b/rs/consensus/src/consensus/metrics.rs @@ -1,4 +1,3 @@ -use crate::consensus::status::Status; use ic_consensus_dkg::metrics::DkgPayloadStats; use ic_consensus_idkg::{ metrics::{CounterPerMasterPublicKeyId, IDkgPayloadStats, KEY_ID_LABEL, key_id_label}, @@ -26,13 +25,6 @@ use std::sync::RwLock; // the range of ranks that are permitted to show up in metrics. const RANKS_TO_RECORD: [&str; 6] = ["0", "1", "2", "3", "4", "5"]; -/// The label of `consensus_status`, whose values are the statuses of -/// [`Status`], lowercased. -const STATUS_LABEL: &str = "status"; -const STATUS_RUNNING: &str = "running"; -const STATUS_HALTING: &str = "halting"; -const STATUS_HALTED: &str = "halted"; - pub(crate) const CRITICAL_ERROR_PAYLOAD_TOO_LARGE: &str = "consensus_payload_too_large"; pub(crate) const CRITICAL_ERROR_VALIDATION_NOT_PASSED: &str = "consensus_validation_not_passed"; pub(crate) const CRITICAL_ERROR_SUBNET_RECORD_ISSUE: &str = "consensus_subnet_record_issue"; @@ -174,7 +166,6 @@ impl BatchStats { pub(crate) struct FinalizerMetrics { pub batches_delivered: IntCounterVec, pub batch_height: IntGauge, - pub consensus_status: IntGaugeVec, pub batch_delivery_interval: Histogram, pub batch_delivery_latency: Histogram, pub ingress_messages_delivered: Histogram, @@ -215,13 +206,6 @@ impl FinalizerMetrics { "consensus_batch_height", "The height of batches sent to Message Routing", ), - consensus_status: metrics_registry.int_gauge_vec( - "consensus_status", - "Whether consensus is running, halting (producing empty blocks but delivering \ - no batches) or halted (producing no blocks either), as of the last time batch \ - delivery looked. 1 for the status that held then, 0 for the other two.", - &[STATUS_LABEL], - ), batch_delivery_interval: metrics_registry.histogram( "consensus_batch_delivery_interval_seconds", "Time elapsed since the delivery of the previous batch, in seconds", @@ -344,27 +328,6 @@ impl FinalizerMetrics { } } - /// Records `status` as the status consensus is in, and the other two as ones - /// it is not. - /// - /// Reported as a gauge per status rather than a single number, so that a - /// dashboard can select the status it asks about by name. Only the batch - /// delivery path computes the status, and only when it has a block to - /// consider, so this says what that path saw the last time it looked: a - /// subnet that stopped producing blocks altogether keeps reporting the - /// status that made it stop. - pub fn observe_status(&self, status: Status) { - for (label, value) in [ - (STATUS_RUNNING, Status::Running), - (STATUS_HALTING, Status::Halting), - (STATUS_HALTED, Status::Halted), - ] { - self.consensus_status - .with_label_values(&[label]) - .set((value == status) as i64); - } - } - pub fn process(&self, block_stats: &BlockStats, batch_stats: &BatchStats) { self.batches_delivered.with_label_values(&["success"]).inc(); self.batch_height.set(batch_stats.batch_height as i64); diff --git a/rs/tests/message_routing/BUILD.bazel b/rs/tests/message_routing/BUILD.bazel index 3bb4b1b534c3..2a6a0539cbb1 100644 --- a/rs/tests/message_routing/BUILD.bazel +++ b/rs/tests/message_routing/BUILD.bazel @@ -157,6 +157,7 @@ system_test_nns( "//rs/registry/canister", "//rs/registry/subnet_type", "//rs/state_layout", + "//rs/tests/consensus/utils", "//rs/tests/driver:ic-system-test-driver", "//rs/types/types", "//rs/universal_canister/lib", diff --git a/rs/tests/message_routing/Cargo.toml b/rs/tests/message_routing/Cargo.toml index 0e5c860aa3af..ea62521d0cc3 100644 --- a/rs/tests/message_routing/Cargo.toml +++ b/rs/tests/message_routing/Cargo.toml @@ -14,6 +14,7 @@ canister-test = { path = "../../rust_canisters/canister_test" } dfn_candid = { path = "../../rust_canisters/dfn_candid" } ic-agent = { workspace = true } ic-base-types = { path = "../../types/base_types" } +ic_consensus_system_test_utils = { path = "../consensus/utils" } ic-management-canister-types = { workspace = true } ic-nns-governance-api = { path = "../../nns/governance/api" } ic-recovery = { path = "../../recovery" } diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs index 398234ad95b2..7da4e44f7e31 100644 --- a/rs/tests/message_routing/subnet_cooling_down_test.rs +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -84,10 +84,10 @@ Runbook:: skipped canister execution in stands in for it. 12. Submit (and adopt) `UpdateConfigOfSubnet` NNS proposals setting the `halt_at_cup_height` flag of both `M` and `R`, and wait until the node of each - that the state is taken from reports `consensus_status{status="halted"}`, i.e. - it produces no block and delivers no batch anymore. Record the heights of the - checkpoints they halted at, which are the heights of the manifests they - computed last. + that the state is taken from holds the CUP its subnet halts at, i.e. the first + one whose summary is created at the registry version carrying that flag. + Record the heights of those CUPs, which are the heights of the checkpoints + holding the states the two subnets stopped in. 13. Stop the replicas of both subnets and download the states they halted at. Assemble the merged state locally, as a checkpoint at the next multiple of the DKG interval after the height `R` halted at: the canisters and canister @@ -140,6 +140,7 @@ end::catalog[] */ use anyhow::{Result, anyhow, bail}; use candid::{CandidType, Principal}; use ic_agent::{Agent, RequestId, agent::RequestStatusResponse}; +use ic_consensus_system_test_utils::get_cup_from_node; use ic_management_canister_types::{SnapshotId, TakeCanisterSnapshotArgs}; use ic_nns_governance_api::NnsFunction; use ic_recovery::registry_helper::RegistryPollingStrategy; @@ -168,6 +169,7 @@ use ic_system_test_driver::util::{ MetricsFetcher, UniversalCanister, assert_create_agent, block_on, create_canister, runtime_from_url, set_controller, }; +use ic_types::consensus::HasHeight; use ic_types::{Height, SubnetId}; use ic_universal_canister::management::InstallMode; use ic_universal_canister::{ @@ -197,13 +199,9 @@ const METRIC_SUBNET_CALL_CONTEXTS: &str = "replicated_state_subnet_call_contexts const METRIC_PENDING_REFUNDS_CYCLES: &str = "replicated_state_pending_refunds_cycles"; const METRIC_ROUNDS_SKIPPED_CANISTER_EXECUTION: &str = "round_skipped_canister_execution_due_to_cooling_down"; -const METRIC_CONSENSUS_STATUS: &str = "consensus_status"; -const METRIC_LAST_COMPUTED_MANIFEST_HEIGHT: &str = "state_manager_last_computed_manifest_height"; +const METRIC_CERTIFICATION_HEIGHT: &str = r#"artifact_pool_certification_height_stat{pool_type="validated",stat="max",type="certification"}"#; -/// The series of `METRIC_CONSENSUS_STATUS` that is 1 while consensus is halted, -/// i.e. produces no block and delivers no batch. -const LABEL_STATUS_HALTED: &str = "status=\"halted\""; -/// Timeout for a subnet to halt at its next CUP, which is up to a full DKG +/// Timeout for a subnet to reach the CUP it halts at, which is up to a full DKG /// interval away. const HALT_TIMEOUT: Duration = Duration::from_secs(900); /// Backoff between two checks of whether a subnet has halted. @@ -783,15 +781,17 @@ async fn run(env: TestEnv) { logger, "Step 12: Halting subnets M and R at their next checkpoint" ); + let mut halt_versions = BTreeMap::new(); for (subnet, name) in [(&m_subnet, "M"), (&r_subnet, "R")] { let version = halt_subnet_at_cup_height(&env, subnet.subnet_id, &logger).await; info!( logger, "Step 12: subnet {name} is set to halt at its next CUP as of registry version {version}" ); + halt_versions.insert(name, version); } - let m_height = await_halted_at_checkpoint(&m_node, "M", &logger).await; - let r_height = await_halted_at_checkpoint(&r_node, "R", &logger).await; + let m_height = await_halting_cup(&m_node, "M", halt_versions["M"], &logger).await; + let r_height = await_halting_cup(&r_node, "R", halt_versions["R"], &logger).await; info!( logger, "Step 12 done: M halted at checkpoint {m_height}, R halted at checkpoint {r_height}" @@ -1992,78 +1992,102 @@ fn median_across_replicas( // Merging subnet M into subnet R. // --------------------------------------------------------------------------- -/// Waits until `node`'s subnet is halted, and returns the height of the -/// checkpoint it halted at, i.e. of the state it stopped in. +/// Waits until `node` holds the CUP its subnet halts at, and returns its +/// height, i.e. the height of the checkpoint holding the state the subnet +/// stopped in. /// -/// Whether the subnet halted is read off the node's journal, which is where a -/// halted replica says that it stops delivering batches. Waiting for the -/// *checkpoint* height to stop advancing instead would not do: while the subnet -/// is running, its state runs ahead of its latest checkpoint by up to a whole DKG -/// interval, which is minutes of wall clock time, so the checkpoint height looks -/// stable long before the subnet halts. The state of that checkpoint is then -/// hundreds of rounds behind the state the merge readiness of step 9 was -/// established on, and may hold, say, an `install_code` that was aborted at the -/// checkpoint and only completed afterwards. +/// A CUP whose summary block was created at `halt_registry_version` or later is +/// one the subnet halts at: the `halt_at_cup_height` flag is read at the +/// registry version of the summary block active at a height, and that version +/// only changes at a summary, so batch delivery stops exactly when the summary +/// carrying the flag becomes active. As checkpoints are written at CUP heights, +/// the height of that CUP is the height of the last checkpoint the subnet +/// wrote. /// -/// The batch heights of a subnet halting because of `halt_at_cup_height` stop at -/// a CUP height: the flag is read at the registry version of the summary block -/// active at a height, and that version only changes at a summary, so batch -/// delivery stops exactly when the summary carrying it becomes active. As -/// checkpoints are written at CUP heights, the latest checkpoint of a halted -/// subnet holds precisely the state it stopped in. -async fn await_halted_at_checkpoint(node: &IcNodeSnapshot, name: &str, logger: &Logger) -> u64 { - info!(logger, "Waiting until subnet {name} is halted"); - // `node`'s own metrics, not the subnet's: the state that is downloaded below - // is this node's, so this node is the one that has to have stopped, and the - // replicas of a subnet observe the halt in different rounds. - // - // Halted and not merely halting: a halting subnet still produces (empty) - // blocks, so its consensus pool keeps moving, while a halted one produces - // none and its latest checkpoint is final. - retry_with_msg_async!( - format!("waiting until subnet {name} reports that it is halted"), +/// Waiting for the CUP rather than for the subnet to report that it is halted: +/// the CUP is what names the state the subnet came to rest in, and it exists +/// only once that state has been certified and its hash agreed upon, which is +/// what the recovery proposal of step 14 compares its state hash against. A +/// subnet that has just stopped delivering batches, on the other hand, may not +/// have finished hashing the checkpoint it stopped at, and reading its latest +/// checkpoint height then yields the previous one, a whole DKG interval before +/// the state the merge is supposed to be assembled from. +/// +/// `node`'s own CUP and metrics, not the subnet's: the state that is downloaded +/// below is this node's, so this node is the one that has to have reached the +/// CUP. +async fn await_halting_cup( + node: &IcNodeSnapshot, + name: &str, + halt_registry_version: u64, + logger: &Logger, +) -> u64 { + info!( + logger, + "Waiting until subnet {name} reaches the CUP it halts at" + ); + let height = retry_with_msg_async!( + format!("waiting until subnet {name} reaches the CUP it halts at"), logger, HALT_TIMEOUT, HALT_BACKOFF, || async { - let metrics = fetch_node_metrics(node, &[METRIC_CONSENSUS_STATUS]).await?; - match median_across_replicas(&metrics, METRIC_CONSENSUS_STATUS, |labels| { - labels.contains(LABEL_STATUS_HALTED) - }) { - Some(1.0) => Ok(()), - Some(_) => bail!("subnet {name} is not halted yet"), - // Only the batch delivery path reports the status, and only once - // it has looked at a block, so the series is missing until then. - None => bail!("subnet {name} has not reported a consensus status yet"), + let cup = get_cup_from_node(node, logger).await?; + let cup_height = cup.height().get(); + let cup_registry_version = cup + .content + .block + .get_value() + .payload + .as_ref() + .as_summary() + .dkg + .registry_version + .get(); + if cup_registry_version < halt_registry_version { + bail!( + "subnet {name} is at the CUP at height {cup_height}, whose registry \ + version {cup_registry_version} precedes the version \ + {halt_registry_version} it is instructed to halt at" + ); } + + // The node has to have caught up with the CUP itself: it is its + // state that is downloaded below, and a node can hold a CUP that + // the rest of the subnet assembled before it got there. + let certification_height = certification_height(node).await?; + assert!( + certification_height <= cup_height, + "subnet {name} certified height {certification_height}, past the CUP at \ + height {cup_height} it should have halted at", + ); + if certification_height < cup_height { + bail!( + "subnet {name} holds the CUP at height {cup_height} but has only \ + certified up to height {certification_height}" + ); + } + + Ok(cup_height) } ) .await - .unwrap_or_else(|e| panic!("subnet {name} did not report that it is halted: {e}")); + .unwrap_or_else(|e| panic!("subnet {name} did not reach the CUP it halts at: {e}")); - let height = halted_checkpoint_height(node) - .await - .unwrap_or_else(|e| panic!("failed to read the checkpoint height of subnet {name}: {e}")); assert_eq!( height % CHECKPOINT_INTERVAL, 0, - "subnet {name} halted at checkpoint {height}, which is not a CUP height", + "subnet {name} halted at height {height}, which is not a checkpoint height", ); height } -/// The height of the checkpoint `node` came to rest at, taken from the manifest -/// it computed last. -/// -/// The manifest, rather than the checkpoint directory: the state is downloaded -/// and its manifest recomputed to compare against the one a recovery proposal -/// carries, and a checkpoint whose manifest this node has not finished computing -/// is one whose CUP does not exist yet. Nothing is delivered after the halt, so -/// no later checkpoint follows the one this names. -async fn halted_checkpoint_height(node: &IcNodeSnapshot) -> Result { - let metrics = fetch_node_metrics(node, &[METRIC_LAST_COMPUTED_MANIFEST_HEIGHT]).await?; - let height = median_across_replicas(&metrics, METRIC_LAST_COMPUTED_MANIFEST_HEIGHT, |_| true) - .ok_or_else(|| anyhow!("no manifest has been computed yet"))?; +/// The height of the highest certification `node` holds, i.e. how far its state +/// is certified. +async fn certification_height(node: &IcNodeSnapshot) -> Result { + let metrics = fetch_node_metrics(node, &[METRIC_CERTIFICATION_HEIGHT]).await?; + let height = median_across_replicas(&metrics, METRIC_CERTIFICATION_HEIGHT, |_| true) + .ok_or_else(|| anyhow!("no certification height has been reported yet"))?; Ok(height as u64) } From fdd53d0589aff78905f847b08769af2401a80942 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Fri, 11 Sep 2026 15:55:30 +0000 Subject: [PATCH 29/30] feat(recovery): a subnet merging tool, and drive the system test through it Adds `rs/recovery/subnet_merging`, the operator tool that performs a subnet merge, built like `rs/recovery/subnet_splitting`: a resumable sequence of steps, each of which either submits a proposal through `ic-admin`, waits for a condition, or works on the states of the two subnets. The steps: label the source subnet as "cooling down" and read back the registry version `V` this created; wait until the subnet is ready to be merged, i.e. until every term of the readiness condition of the `Subnet merging` dashboard holds for `V` (all subnets at `V`, no message in any stream in either direction, nothing but `processing` entries in the ingress history, empty subnet queues, no subnet call context, and an empty refund pool); halt both subnets at their next CUP and wait until the node each state is taken from holds that CUP -- served at its public endpoint, written to its disk, and with the state it names certified; stop both replicas, download both states and validate each against its CUP and the subnet's public key in the NNS signed state tree; assemble the merged state and compute the height, block time and state hash the recovery needs; reroute the canister id ranges to the destination subnet, recover it at the merged state, upload that state and wait for the recovery CUP; unhalt it; and, once every other subnet routes the merged canisters to it, delete the subnet that was merged away. The readiness condition is evaluated by the tool itself, by scraping the metrics of every node of every subnet, in addition to printing the dashboard link for the operator to confirm. Shared code this needs: * `ic-admin propose-to-merge-subnets`, as the `MergeSubnets` NNS function had no subcommand yet and every step of these tools proposes through `ic-admin`; * an explicit block time for a recovery CUP proposal, as the merged state starts later than the checkpoints it was assembled from; * the batch time of a checkpoint as a `state_tool` library function, rather than only as something the binary prints; * a versioned subnet record getter on `RegistryHelper`, which polls the local store like every other getter: the halting CUP names a registry version that was created moments ago; * `ic_cup_explorer::get_cup` and `ic_recovery::{get_member_node_ids_and_ips, steps::DownloadIcDataStep}` made public: a merge downloads two states, so it cannot use the helper that downloads into the single working directory of a recovery. `subnet_cooling_down_test` now drives the tool instead of carrying its own implementation of the merge: it sets the scenario up, hands over to the tool, and makes its checks in between the tool's steps, at the points the runbook names. Like the subnet splitting test, the part that drives the tool is plain synchronous code -- the tool blocks on runtimes of its own -- with the asynchronous checks driven through a runtime that lives for the whole test. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 35 + Cargo.toml | 1 + rs/cup_explorer/src/lib.rs | 2 +- rs/recovery/src/app_subnet_recovery.rs | 1 + rs/recovery/src/lib.rs | 9 +- .../src/nns_recovery_failover_nodes.rs | 1 + rs/recovery/src/registry_helper.rs | 23 + rs/recovery/src/steps.rs | 6 +- rs/recovery/subnet_merging/BUILD.bazel | 64 + rs/recovery/subnet_merging/Cargo.toml | 44 + .../subnet_merging/src/admin_helper.rs | 236 +++ .../subnet_merging/src/agent_helper.rs | 166 ++ rs/recovery/subnet_merging/src/layout.rs | 193 +++ rs/recovery/subnet_merging/src/lib.rs | 12 + rs/recovery/subnet_merging/src/main.rs | 180 ++ .../subnet_merging/src/metrics_helper.rs | 219 +++ rs/recovery/subnet_merging/src/readiness.rs | 362 ++++ .../subnet_merging/src/state_tool_helper.rs | 53 + rs/recovery/subnet_merging/src/steps.rs | 669 ++++++++ .../subnet_merging/src/subnet_merging.rs | 754 ++++++++ .../subnet_merging/src/target_subnet.rs | 25 + rs/recovery/subnet_merging/src/utils.rs | 172 ++ rs/recovery/subnet_merging/src/validation.rs | 177 ++ .../subnet_splitting/src/subnet_splitting.rs | 1 + rs/registry/admin/bin/main.rs | 60 + rs/state_tool/BUILD.bazel | 5 +- rs/state_tool/src/commands/checkpoint_time.rs | 12 +- rs/tests/message_routing/BUILD.bazel | 12 +- rs/tests/message_routing/Cargo.toml | 1 + .../subnet_cooling_down_test.rs | 1515 +++++------------ 30 files changed, 3873 insertions(+), 1137 deletions(-) create mode 100644 rs/recovery/subnet_merging/BUILD.bazel create mode 100644 rs/recovery/subnet_merging/Cargo.toml create mode 100644 rs/recovery/subnet_merging/src/admin_helper.rs create mode 100644 rs/recovery/subnet_merging/src/agent_helper.rs create mode 100644 rs/recovery/subnet_merging/src/layout.rs create mode 100644 rs/recovery/subnet_merging/src/lib.rs create mode 100644 rs/recovery/subnet_merging/src/main.rs create mode 100644 rs/recovery/subnet_merging/src/metrics_helper.rs create mode 100644 rs/recovery/subnet_merging/src/readiness.rs create mode 100644 rs/recovery/subnet_merging/src/state_tool_helper.rs create mode 100644 rs/recovery/subnet_merging/src/steps.rs create mode 100644 rs/recovery/subnet_merging/src/subnet_merging.rs create mode 100644 rs/recovery/subnet_merging/src/target_subnet.rs create mode 100644 rs/recovery/subnet_merging/src/utils.rs create mode 100644 rs/recovery/subnet_merging/src/validation.rs diff --git a/Cargo.lock b/Cargo.lock index c104612354da..fe417ecb0ac0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14973,6 +14973,40 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ic-subnet-merging" +version = "0.9.0" +dependencies = [ + "anyhow", + "clap", + "futures", + "hex", + "ic-agent", + "ic-base-types", + "ic-crypto-utils-threshold-sig", + "ic-crypto-utils-threshold-sig-der", + "ic-cup-explorer", + "ic-protobuf", + "ic-recovery", + "ic-registry-client-helpers", + "ic-registry-routing-table", + "ic-registry-subnet-type", + "ic-state-layout", + "ic-state-manager", + "ic-state-tool", + "ic-test-utilities-tmpdir", + "ic-types", + "reqwest", + "serde", + "serde_cbor", + "serde_json", + "slog", + "strum 0.26.3", + "strum_macros 0.26.4", + "tokio", + "url", +] + [[package]] name = "ic-subnet-splitting" version = "0.9.0" @@ -18238,6 +18272,7 @@ dependencies = [ "ic-recovery", "ic-registry-subnet-type", "ic-state-layout", + "ic-subnet-merging", "ic-system-test-driver", "ic-types", "ic-universal-canister", diff --git a/Cargo.toml b/Cargo.toml index ba21cb9ee93e..558f2776b243 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -341,6 +341,7 @@ members = [ "rs/protobuf/generator", "rs/query_stats", "rs/recovery", + "rs/recovery/subnet_merging", "rs/recovery/subnet_splitting", "rs/registry/admin", "rs/registry/admin-derive", diff --git a/rs/cup_explorer/src/lib.rs b/rs/cup_explorer/src/lib.rs index 295e5c133dee..dd7b8367f985 100644 --- a/rs/cup_explorer/src/lib.rs +++ b/rs/cup_explorer/src/lib.rs @@ -43,7 +43,7 @@ pub async fn get_catchup_content(url: &Url) -> Result } /// Fetches the CatchUp package, if it's present. -async fn get_cup(url: &Url) -> Result, String> { +pub async fn get_cup(url: &Url) -> Result, String> { let agent = Agent::new(url.clone(), Sender::Anonymous); agent .query_cup_endpoint(None) diff --git a/rs/recovery/src/app_subnet_recovery.rs b/rs/recovery/src/app_subnet_recovery.rs index a942bd4d1d55..3651f539d7a7 100644 --- a/rs/recovery/src/app_subnet_recovery.rs +++ b/rs/recovery/src/app_subnet_recovery.rs @@ -608,6 +608,7 @@ impl RecoveryIterator for AppSubnetRecovery { None, self.params.initial_dkg_subnet_id, self.params.chain_key_subnet_id, + /*time=*/ None, )?)) } diff --git a/rs/recovery/src/lib.rs b/rs/recovery/src/lib.rs index d0c54a619d02..95a247cf305b 100644 --- a/rs/recovery/src/lib.rs +++ b/rs/recovery/src/lib.rs @@ -817,6 +817,7 @@ impl Recovery { registry_params: Option, initial_dkg_subnet_id: Option, chain_key_subnet_id: Option, + time: Option, ) -> RecoveryResult> { let chain_key_config = chain_key_subnet_id .map(|id| match self.registry_helper.get_chain_key_config(id) { @@ -847,7 +848,11 @@ impl Recovery { chain_key_config, replacement_nodes, registry_params, - SystemTime::now(), + // The block time the recovered subnet starts from. Defaults + // to now, which is what a recovery replaying up to the + // present wants; a subnet merge passes the time it computed + // from the states it merged instead. + time.unwrap_or_else(SystemTime::now), ), }) } @@ -1209,7 +1214,7 @@ pub fn get_available_nodes_heights_from_metrics( } /// Lookup node IDs and corresponding IP addresses of all members of the given subnet -fn get_member_node_ids_and_ips( +pub fn get_member_node_ids_and_ips( registry_helper: &RegistryHelper, subnet_id: SubnetId, ) -> RecoveryResult> { diff --git a/rs/recovery/src/nns_recovery_failover_nodes.rs b/rs/recovery/src/nns_recovery_failover_nodes.rs index 11e5f0420340..b2f41bed9e2d 100644 --- a/rs/recovery/src/nns_recovery_failover_nodes.rs +++ b/rs/recovery/src/nns_recovery_failover_nodes.rs @@ -431,6 +431,7 @@ impl RecoveryIterator for NNSRecoveryFailoverNodes { Some(registry_params), None, None, + /*time=*/ None, )?)) } else { Err(RecoveryError::StepSkipped) diff --git a/rs/recovery/src/registry_helper.rs b/rs/recovery/src/registry_helper.rs index 20eee4a4f6b7..7ec5db028dce 100644 --- a/rs/recovery/src/registry_helper.rs +++ b/rs/recovery/src/registry_helper.rs @@ -102,6 +102,29 @@ impl RegistryHelper { }) } + /// Returns the subnet record of the given subnet as of `registry_version`, + /// rather than as of the latest version [Self::get_subnet_record] reads. + /// + /// Polls the [RegistryReplicator] first, as every other getter does: a + /// version that the local store has not caught up with yet is not readable, + /// and the caller may well be asking about one that was just created. + pub fn get_subnet_record_at_version( + &self, + subnet_id: SubnetId, + registry_version: RegistryVersion, + ) -> RecoveryResult> { + let _ = self.latest_registry_version()?; + + self.registry_client() + .get_subnet_record(subnet_id, registry_version) + .map_err(|err| { + RecoveryError::RegistryError(format!( + "Failed to get the record of subnet {subnet_id} at registry version \ + {registry_version}: {err}" + )) + }) + } + /// Returns the [SubnetRecord] of the given subnet. pub fn get_subnet_record(&self, subnet_id: SubnetId) -> VersionedRecoveryResult { self.get(|registry_version, registry_client| { diff --git a/rs/recovery/src/steps.rs b/rs/recovery/src/steps.rs index d582eafa9af7..8c5f288fbd93 100644 --- a/rs/recovery/src/steps.rs +++ b/rs/recovery/src/steps.rs @@ -274,7 +274,11 @@ impl Step for MergeCertificationPoolsStep { } } -pub(crate) struct DownloadIcDataStep { +/// Downloads data (the state, the consensus pool, ...) of a node into a +/// working directory. Public so that a tool that works with more than one +/// working directory, such as subnet merging, can direct the download at the +/// right one. +pub struct DownloadIcDataStep { pub logger: Logger, pub ssh_helper: SshHelper, pub backup_dir: PathBuf, diff --git a/rs/recovery/subnet_merging/BUILD.bazel b/rs/recovery/subnet_merging/BUILD.bazel new file mode 100644 index 000000000000..37c0af6dd351 --- /dev/null +++ b/rs/recovery/subnet_merging/BUILD.bazel @@ -0,0 +1,64 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") + +DEPENDENCIES = [ + # Keep sorted. + "//rs/crypto/utils/threshold_sig", + "//rs/crypto/utils/threshold_sig_der", + "//rs/cup_explorer", + "//rs/protobuf", + "//rs/recovery", + "//rs/registry/helpers", + "//rs/registry/routing_table", + "//rs/registry/subnet_type", + "//rs/state_layout", + "//rs/state_manager", + "//rs/state_tool:state_tool_lib", + "//rs/types/base_types", + "//rs/types/types", + "@crate_index//:anyhow", + "@crate_index//:clap", + "@crate_index//:futures", + "@crate_index//:hex", + "@crate_index//:ic-agent", + "@crate_index//:reqwest", + "@crate_index//:serde", + "@crate_index//:serde_cbor", + "@crate_index//:serde_json", + "@crate_index//:slog", + "@crate_index//:strum", + "@crate_index//:tokio", + "@crate_index//:url", +] + +rust_library( + name = "subnet_merging", + srcs = glob( + ["src/**/*.rs"], + exclude = ["src/main.rs"], + ), + crate_name = "ic_subnet_merging", + proc_macro_deps = ["@crate_index//:strum_macros"], + version = "0.1.0", + visibility = ["//rs:system-tests-pkg"], + deps = DEPENDENCIES, +) + +rust_binary( + name = "subnet-merging-tool", + srcs = ["src/main.rs"], + visibility = ["//rs:release-pkg"], + deps = DEPENDENCIES + [ + # Keep sorted. + ":subnet_merging", + "//rs/canister_sandbox:backend_lib", + ], +) + +rust_test( + name = "subnet_merging_tool_test", + crate = "subnet_merging", + deps = DEPENDENCIES + [ + # Keep sorted. + "//rs/test_utilities/tmpdir", + ], +) diff --git a/rs/recovery/subnet_merging/Cargo.toml b/rs/recovery/subnet_merging/Cargo.toml new file mode 100644 index 000000000000..ba50f3ca82e3 --- /dev/null +++ b/rs/recovery/subnet_merging/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "ic-subnet-merging" +version.workspace = true +authors.workspace = true +edition.workspace = true +description.workspace = true +documentation.workspace = true + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true } +futures = { workspace = true } +hex = { workspace = true } +ic-agent = { workspace = true } +ic-base-types = { path = "../../types/base_types/" } +ic-crypto-utils-threshold-sig = { path = "../../crypto/utils/threshold_sig" } +ic-crypto-utils-threshold-sig-der = { path = "../../crypto/utils/threshold_sig_der" } +ic-cup-explorer = { path = "../../cup_explorer" } +ic-protobuf = { path = "../../protobuf" } +ic-recovery = { path = "../" } +ic-registry-client-helpers = { path = "../../registry/helpers" } +ic-registry-routing-table = { path = "../../registry/routing_table" } +ic-registry-subnet-type = { path = "../../registry/subnet_type" } +ic-state-layout = { path = "../../state_layout" } +ic-state-manager = { path = "../../state_manager" } +ic-state-tool = { path = "../../state_tool" } +ic-types = { path = "../../types/types" } +reqwest = { workspace = true } +serde = { workspace = true } +serde_cbor = { workspace = true } +serde_json = { workspace = true } +slog = { workspace = true } + +strum = { workspace = true } +strum_macros = { workspace = true } +tokio = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +ic-test-utilities-tmpdir = { path = "../../test_utilities/tmpdir" } + +[[bin]] +name = "subnet-merging-tool" +path = "src/main.rs" diff --git a/rs/recovery/subnet_merging/src/admin_helper.rs b/rs/recovery/subnet_merging/src/admin_helper.rs new file mode 100644 index 000000000000..29ec7b74c7f6 --- /dev/null +++ b/rs/recovery/subnet_merging/src/admin_helper.rs @@ -0,0 +1,236 @@ +use ic_base_types::SubnetId; +use ic_recovery::admin_helper::{ + AdminHelper, CommandHelper, IcAdmin, SSH_READONLY_ACCESS_ARG, SUMMARY_ARG, quote, +}; + +const SOURCE_SUBNET_ARG: &str = "source-subnet"; +const DESTINATION_SUBNET_ARG: &str = "destination-subnet"; +const SUBNET_ARG: &str = "subnet"; +const SUBNET_ID_ARG: &str = "subnet-id"; + +/// Propose to label the subnet as "cooling down", i.e. to have it stop +/// accepting ingress messages, answering queries and executing canister +/// messages, so that it quiesces and can be merged into another subnet. +/// +/// Optionally adds a ssh-readonly-access key to the subnet, which is needed to +/// download its state later on. +pub(crate) fn get_propose_to_cool_down_subnet_command( + admin_helper: &AdminHelper, + subnet_id: SubnetId, + key: &Option, +) -> IcAdmin { + let mut ic_admin = admin_helper.get_ic_admin_cmd_base(); + + ic_admin + .add_positional_argument("propose-to-update-subnet") + .add_argument(SUBNET_ARG, subnet_id) + .add_argument( + SUMMARY_ARG, + quote(format!( + "Label subnet {subnet_id} as cooling down and optionally update ssh readonly access", + )), + ) + .add_argument("cooling-down", true); + + if let Some(key) = key { + ic_admin.add_argument(SSH_READONLY_ACCESS_ARG, quote(key)); + } + + admin_helper.add_proposer_args(&mut ic_admin); + + ic_admin +} + +/// Propose to make the subnet halt after reaching the next CUP height. +/// +/// Optionally adds a ssh-readonly-access key to the subnet. +pub(crate) fn get_halt_subnet_at_cup_height_command( + admin_helper: &AdminHelper, + subnet_id: SubnetId, + key: &Option, +) -> IcAdmin { + let mut ic_admin = admin_helper.get_ic_admin_cmd_base(); + + ic_admin + .add_positional_argument("propose-to-update-subnet") + .add_argument(SUBNET_ARG, subnet_id) + .add_argument( + SUMMARY_ARG, + quote(format!( + "Halt subnet {subnet_id} at cup height and optionally update ssh readonly access", + )), + ) + .add_argument("halt-at-cup-height", true); + + if let Some(key) = key { + ic_admin.add_argument(SSH_READONLY_ACCESS_ARG, quote(key)); + } + + admin_helper.add_proposer_args(&mut ic_admin); + + ic_admin +} + +/// Propose to reroute the canister ID ranges of the source subnet to the +/// destination subnet, i.e. to merge the former into the latter. +pub(crate) fn get_propose_to_merge_subnets_command( + admin_helper: &AdminHelper, + source_subnet_id: SubnetId, + destination_subnet_id: SubnetId, +) -> IcAdmin { + let mut ic_admin = admin_helper.get_ic_admin_cmd_base(); + + ic_admin + .add_positional_argument("propose-to-merge-subnets") + .add_argument( + SUMMARY_ARG, + quote(format!( + "Merge subnet {source_subnet_id} into subnet {destination_subnet_id}", + )), + ) + .add_argument(SOURCE_SUBNET_ARG, source_subnet_id) + .add_argument(DESTINATION_SUBNET_ARG, destination_subnet_id); + + admin_helper.add_proposer_args(&mut ic_admin); + + ic_admin +} + +/// Propose to delete the subnet that was merged away and that hosts no canister +/// ID range anymore. +pub(crate) fn get_propose_to_delete_subnet_command( + admin_helper: &AdminHelper, + subnet_id: SubnetId, +) -> IcAdmin { + let mut ic_admin = admin_helper.get_ic_admin_cmd_base(); + + ic_admin + .add_positional_argument("propose-to-delete-subnet") + .add_argument( + SUMMARY_ARG, + quote(format!( + "Delete subnet {subnet_id}, which was merged into another subnet and hosts no \ + canister id range anymore", + )), + ) + .add_argument(SUBNET_ID_ARG, subnet_id); + + admin_helper.add_proposer_args(&mut ic_admin); + + ic_admin +} + +#[cfg(test)] +mod tests { + use super::*; + + use ic_base_types::PrincipalId; + use url::Url; + + use std::{path::PathBuf, str::FromStr}; + + const FAKE_IC_ADMIN: &str = "/fake/ic/admin/dir/ic-admin"; + const FAKE_NNS_URL: &str = "https://fake_nns_url.com:8080"; + const FAKE_SUBNET_ID_1: &str = + "gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe"; + const FAKE_SUBNET_ID_2: &str = + "mklno-zzmhy-zutel-oujwg-dzcli-h6nfy-2serg-gnwru-vuwck-hcxit-wqe"; + const SSH_KEY: &str = "fake ssh key"; + + #[test] + fn get_propose_to_cool_down_subnet_command_test() { + let result = get_propose_to_cool_down_subnet_command( + &fake_admin_helper(), + subnet_id_from_str(FAKE_SUBNET_ID_1), + &Some(SSH_KEY.to_string()), + ) + .join(" "); + + assert_eq!( + result, + "/fake/ic/admin/dir/ic-admin \ + --nns-url \"https://fake_nns_url.com:8080/\" \ + propose-to-update-subnet \ + --subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe \ + --summary \"Label subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe as cooling down and optionally update ssh readonly access\" \ + --cooling-down true \ + --ssh-readonly-access \"fake ssh key\" \ + --test-neuron-proposer" + ); + } + + #[test] + fn get_halt_subnet_at_cup_height_command_test() { + let result = get_halt_subnet_at_cup_height_command( + &fake_admin_helper(), + subnet_id_from_str(FAKE_SUBNET_ID_1), + &None, + ) + .join(" "); + + assert_eq!( + result, + "/fake/ic/admin/dir/ic-admin \ + --nns-url \"https://fake_nns_url.com:8080/\" \ + propose-to-update-subnet \ + --subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe \ + --summary \"Halt subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe at cup height and optionally update ssh readonly access\" \ + --halt-at-cup-height true \ + --test-neuron-proposer" + ); + } + + #[test] + fn get_propose_to_merge_subnets_command_test() { + let result = get_propose_to_merge_subnets_command( + &fake_admin_helper(), + subnet_id_from_str(FAKE_SUBNET_ID_1), + subnet_id_from_str(FAKE_SUBNET_ID_2), + ) + .join(" "); + + assert_eq!( + result, + "/fake/ic/admin/dir/ic-admin \ + --nns-url \"https://fake_nns_url.com:8080/\" \ + propose-to-merge-subnets \ + --summary \"Merge subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe into subnet mklno-zzmhy-zutel-oujwg-dzcli-h6nfy-2serg-gnwru-vuwck-hcxit-wqe\" \ + --source-subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe \ + --destination-subnet mklno-zzmhy-zutel-oujwg-dzcli-h6nfy-2serg-gnwru-vuwck-hcxit-wqe \ + --test-neuron-proposer" + ); + } + + #[test] + fn get_propose_to_delete_subnet_command_test() { + let result = get_propose_to_delete_subnet_command( + &fake_admin_helper(), + subnet_id_from_str(FAKE_SUBNET_ID_1), + ) + .join(" "); + + assert_eq!( + result, + "/fake/ic/admin/dir/ic-admin \ + --nns-url \"https://fake_nns_url.com:8080/\" \ + propose-to-delete-subnet \ + --summary \"Delete subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe, which was merged into another subnet and hosts no canister id range anymore\" \ + --subnet-id gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe \ + --test-neuron-proposer" + ); + } + + fn fake_admin_helper() -> AdminHelper { + AdminHelper::new( + PathBuf::from(FAKE_IC_ADMIN), + Url::try_from(FAKE_NNS_URL).unwrap(), + /*neuron_args=*/ None, + ) + } + + fn subnet_id_from_str(subnet_id: &str) -> SubnetId { + PrincipalId::from_str(subnet_id) + .map(SubnetId::from) + .unwrap() + } +} diff --git a/rs/recovery/subnet_merging/src/agent_helper.rs b/rs/recovery/subnet_merging/src/agent_helper.rs new file mode 100644 index 000000000000..dba0c19e4e9c --- /dev/null +++ b/rs/recovery/subnet_merging/src/agent_helper.rs @@ -0,0 +1,166 @@ +use ic_agent::{Agent, Certificate, export::Principal, hash_tree::Label, lookup_value}; +use ic_base_types::SubnetId; +use ic_crypto_utils_threshold_sig_der::{parse_threshold_sig_key_from_pem_file, public_key_to_der}; +use ic_recovery::{ + error::{RecoveryError, RecoveryResult}, + file_sync_helper::{read_bytes, write_bytes}, + util::{block_on, write_public_key_to_file}, +}; +use slog::{Logger, debug, info}; +use url::Url; + +use std::{fmt::Display, path::Path}; + +const NNS_REGISTRY_CANISTER_ID: &str = "rwlgt-iiaaa-aaaaa-aaaaa-cai"; + +const SUBNET_LABEL: &[u8] = b"subnet"; +const PUBLIC_KEY_LABEL: &[u8] = b"public_key"; +const CANISTER_RANGES_LABEL: &[u8] = b"canister_ranges"; + +type StorageType = Vec; + +/// Wrapper around the raw state tree with some utility functions. +/// +/// Note: the state tree is pruned to include only the information (public key and canister ranges) +/// of a single subnet. +pub(crate) struct StateTree { + certificate: Certificate, + subnet_id: SubnetId, +} + +impl StateTree { + /// Saves the raw state tree to the disk, in CBOR format. + pub(crate) fn save_to_file(&self, path: &Path) -> RecoveryResult<()> { + serde_cbor::to_vec(&self.certificate) + .map_err(|err| agent_error("Failed to serialize the state tree", err)) + .and_then(|bytes| write_bytes(path, bytes)) + .map_err(|err| agent_error("Failed to write the state tree to disk", err)) + } + + /// Reads the raw state tree from the disk. + pub(crate) fn read_from_file(path: &Path, subnet_id: SubnetId) -> RecoveryResult { + let serialized_state_tree = read_bytes(path) + .map_err(|err| agent_error("Failed to read the state tree from the disk", err))?; + + let certificate = serde_cbor::from_slice(serialized_state_tree.as_slice()) + .map_err(|err| agent_error("Failed to deserialize the state tree", err))?; + + Ok(Self { + subnet_id, + certificate, + }) + } + + /// Extracts the public key from the raw state tree and saves it to the disk. + pub(crate) fn save_public_key_to_file(&self, path: &Path) -> RecoveryResult<()> { + self.lookup_public_key() + .and_then(|public_key| write_public_key_to_file(public_key, path)) + .map_err(|err| agent_error("Failed to write the public key to disk", err)) + } + + pub(crate) fn lookup_public_key(&self) -> RecoveryResult<&[u8]> { + lookup_value( + &self.certificate, + create_path(self.subnet_id, PUBLIC_KEY_LABEL), + ) + .map_err(|err| agent_error("Failed to retrieve the public key", err)) + } +} + +/// Wrapper around [Agent] with some utility functions. +pub(crate) struct AgentHelper { + agent: Agent, + nns_registry: Principal, + logger: Logger, +} + +impl AgentHelper { + /// Creates a new instance of [AgentHelper]. + /// + /// When the `nns_public_key_path` argument is not specified, the mainnet root key will be + /// used. + /// + /// Returns an error when the underlying [Agent] fails to build or when there is something + /// wrong with the provided NNS public key. + pub(crate) fn new( + nns_url: &Url, + nns_public_key_path: Option<&Path>, + logger: Logger, + ) -> RecoveryResult { + let agent = Agent::builder() + .with_url(nns_url.to_string()) + .build() + .map_err(|err| agent_error("Failed to build an Agent", err))?; + + // If we don't set a root key, the [Agent] will use the mainnet root key. + if let Some(nns_public_key_path) = nns_public_key_path { + info!( + logger, + "Reading the NNS public key from {}", + nns_public_key_path.display() + ); + + let nns_public_key = parse_threshold_sig_key_from_pem_file(nns_public_key_path) + .map_err(|err| agent_error("Failed to parse NNS public key", err))?; + let der_bytes = public_key_to_der(&nns_public_key.into_bytes()) + .map_err(|err| agent_error("Failed to convert the NNS public key to DER", err))?; + + agent.set_root_key(der_bytes); + } + + let nns_registry = Principal::from_text(NNS_REGISTRY_CANISTER_ID) + .map_err(|err| agent_error("Failed to parse NNS registry canister id", err))?; + + Ok(Self { + agent, + nns_registry, + logger, + }) + } + + /// Reads the state tree and prunes it to contain only the following paths: + /// * /subnet/$subnet_id/public_key + /// * /canister_ranges/$subnet_id + /// + /// See: https://internetcomputer.org/docs/current/references/ic-interface-spec#state-tree-subnet + /// for more information + pub(crate) fn read_subnet_data(&self, subnet_id: SubnetId) -> RecoveryResult { + let certificate = block_on(self.agent.read_subnet_state_raw( + vec![ + create_path(subnet_id, PUBLIC_KEY_LABEL), + vec![ + CANISTER_RANGES_LABEL.into(), + subnet_id.get().as_slice().into(), + ], + ], + subnet_id.get().into(), + )) + .map_err(|err| agent_error("Failed to read the state tree", err))?; + + debug!(self.logger, "State tree: {:#?}", certificate.tree); + + Ok(StateTree { + certificate, + subnet_id, + }) + } + + /// Validates the state tree. + pub(crate) fn validate_state_tree(&self, state_tree: &StateTree) -> RecoveryResult<()> { + self.agent + .verify(&state_tree.certificate, self.nns_registry) + .map_err(|err| agent_error("Failed to verify the state tree", err)) + } +} + +fn agent_error(message: impl Display, error: impl Display) -> RecoveryError { + RecoveryError::AgentError(format!("{message}: {error}")) +} + +fn create_path(subnet_id: SubnetId, label: &[u8]) -> Vec> { + vec![ + SUBNET_LABEL.into(), + subnet_id.get().as_slice().into(), + label.into(), + ] +} diff --git a/rs/recovery/subnet_merging/src/layout.rs b/rs/recovery/subnet_merging/src/layout.rs new file mode 100644 index 000000000000..42fd5fcd5065 --- /dev/null +++ b/rs/recovery/subnet_merging/src/layout.rs @@ -0,0 +1,193 @@ +use crate::{ + target_subnet::TargetSubnet, + utils::{ + COOLING_DOWN_REGISTRY_VERSION_FILE, MERGE_REGISTRY_VERSION_FILE, read_registry_version, + write_registry_version, + }, +}; + +use ic_base_types::SubnetId; +use ic_recovery::{ + CHECKPOINTS, CUPS_DIR, IC_STATE, Recovery, error::RecoveryResult, file_sync_helper::create_dir, +}; +use ic_state_layout::StateLayout; +use ic_types::Height; + +use std::path::{Path, PathBuf}; + +/// The name the orchestrator stores the latest CUP of a subnet under. +pub(crate) const CUP_FILE_NAME: &str = "cup.types.v1.CatchUpPackage.pb"; + +#[derive(Clone)] +/// Describes the layout of the working directory of subnet merging: +/// +/// |-- root/ +/// | |-- ${destination_subnet_id}.manifest +/// | |-- ${source_subnet_id}.manifest +/// | |-- ${source_subnet_id}.pem +/// | |-- ${destination_subnet_id}.pem +/// | |-- merged.manifest +/// | |-- merged_state_params.json +/// | |-- cooling_down_registry_version +/// | |-- merge_registry_version +/// | |-- nns.pem +/// | |-- ${source_subnet_id}.pruned_state_tree.cbor +/// | |-- ${destination_subnet_id}.pruned_state_tree.cbor +/// | |-- ${source_subnet_id}.halting_cup.pb +/// | |-- ${destination_subnet_id}.halting_cup.pb +/// | |-- (destination_|merged_)work_dir/ +/// | | |-- data/ +/// | | | |-- cups/cup.types.v1.CatchUpPackage.pb +/// | | | |-- ic_state/states_metadata.pbuf +/// | | | |-- ic_state/checkpoints/ +/// | | | | |-- 1/ +pub(crate) struct Layout { + root: PathBuf, + source_working_dir: PathBuf, + + nns_public_key: PathBuf, + merged_state_manifest: PathBuf, + merged_state_params: PathBuf, + cooling_down_registry_version: PathBuf, + merge_registry_version: PathBuf, +} + +impl Layout { + pub(crate) fn new(recovery: &Recovery) -> Self { + Self { + root: recovery.recovery_dir.clone(), + source_working_dir: recovery.work_dir.clone(), + nns_public_key: recovery.recovery_dir.join("nns.pem"), + merged_state_manifest: recovery.recovery_dir.join("merged.manifest"), + merged_state_params: recovery.recovery_dir.join("merged_state_params.json"), + cooling_down_registry_version: recovery + .recovery_dir + .join(COOLING_DOWN_REGISTRY_VERSION_FILE), + merge_registry_version: recovery.recovery_dir.join(MERGE_REGISTRY_VERSION_FILE), + } + } + + /// Creates the working directories of the two subnets being merged and of + /// the merged state. `Recovery` only creates the one working directory a + /// recovery has, which is the source subnet's here. + pub(crate) fn create_dirs(&self) -> RecoveryResult<()> { + for target_subnet in [ + TargetSubnet::Source, + TargetSubnet::Destination, + TargetSubnet::Merged, + ] { + create_dir(&self.data_dir(target_subnet))?; + } + + Ok(()) + } + + pub(crate) fn nns_public_key_file(&self) -> &Path { + &self.nns_public_key + } + + pub(crate) fn merged_state_manifest_file(&self) -> &Path { + &self.merged_state_manifest + } + + pub(crate) fn merged_state_params_file(&self) -> &Path { + &self.merged_state_params + } + + /// The registry version at which the subnet that is merged away was + /// labeled "cooling down", i.e. the `V` of the merge readiness condition. + pub(crate) fn write_cooling_down_registry_version(&self, version: u64) -> RecoveryResult<()> { + write_registry_version(&self.cooling_down_registry_version, version) + } + + pub(crate) fn read_cooling_down_registry_version(&self) -> RecoveryResult { + read_registry_version(&self.cooling_down_registry_version) + } + + /// The registry version the merge was applied at, i.e. the one every other + /// subnet has to have reached before the merged subnet may be deleted. + pub(crate) fn write_merge_registry_version(&self, version: u64) -> RecoveryResult<()> { + write_registry_version(&self.merge_registry_version, version) + } + + pub(crate) fn read_merge_registry_version(&self) -> RecoveryResult { + read_registry_version(&self.merge_registry_version) + } + + pub(crate) fn actual_manifest_file(&self, subnet_id: SubnetId) -> PathBuf { + self.root.join(format!("{subnet_id}.manifest")) + } + + pub(crate) fn subnet_public_key_file(&self, subnet_id: SubnetId) -> PathBuf { + self.root.join(format!("{subnet_id}.pem")) + } + + pub(crate) fn pruned_state_tree_file(&self, subnet_id: SubnetId) -> PathBuf { + self.root + .join(format!("{subnet_id}.pruned_state_tree.cbor")) + } + + /// The copy of the CUP a subnet halted at that is pulled off its node while + /// waiting for the halt, before the state is downloaded. + pub(crate) fn halting_cup_file(&self, subnet_id: SubnetId) -> PathBuf { + self.root.join(format!("{subnet_id}.halting_cup.pb")) + } + + /// The CUP that came with the downloaded state, i.e. the one the validation + /// of the downloaded state checks against. + pub(crate) fn downloaded_cup_file(&self, target_subnet: TargetSubnet) -> PathBuf { + self.data_dir(target_subnet) + .join(CUPS_DIR) + .join(CUP_FILE_NAME) + } + + pub(crate) fn work_dir(&self, target_subnet: TargetSubnet) -> PathBuf { + match target_subnet { + TargetSubnet::Source => self.source_working_dir.clone(), + TargetSubnet::Destination => self.root.join("destination_working_dir"), + TargetSubnet::Merged => self.root.join("merged_working_dir"), + } + } + + /// Where the state of a subnet is downloaded to when the downloaded state + /// is to be kept, before it is copied into the working directory. + pub(crate) fn original_data_dir(&self, target_subnet: TargetSubnet) -> PathBuf { + self.work_dir(target_subnet).join("original_data") + } + + pub(crate) fn data_dir(&self, target_subnet: TargetSubnet) -> PathBuf { + self.work_dir(target_subnet).join("data") + } + + pub(crate) fn ic_state_dir(&self, target_subnet: TargetSubnet) -> PathBuf { + self.data_dir(target_subnet).join(IC_STATE) + } + + pub(crate) fn checkpoints_dir(&self, target_subnet: TargetSubnet) -> PathBuf { + self.ic_state_dir(target_subnet).join(CHECKPOINTS) + } + + pub(crate) fn checkpoint_dir(&self, target_subnet: TargetSubnet, height: Height) -> PathBuf { + self.checkpoints_dir(target_subnet) + .join(StateLayout::checkpoint_name(height)) + } + + pub(crate) fn latest_checkpoint_dir( + &self, + target_subnet: TargetSubnet, + ) -> RecoveryResult { + let checkpoints_dir = self.checkpoints_dir(target_subnet); + + let (max_name, _) = Recovery::get_latest_checkpoint_name_and_height(&checkpoints_dir)?; + + Ok(checkpoints_dir.join(max_name)) + } + + pub(crate) fn latest_checkpoint_height( + &self, + target_subnet: TargetSubnet, + ) -> RecoveryResult { + Recovery::get_latest_checkpoint_name_and_height(&self.checkpoints_dir(target_subnet)) + .map(|(_, height)| height) + } +} diff --git a/rs/recovery/subnet_merging/src/lib.rs b/rs/recovery/subnet_merging/src/lib.rs new file mode 100644 index 000000000000..00da8137e793 --- /dev/null +++ b/rs/recovery/subnet_merging/src/lib.rs @@ -0,0 +1,12 @@ +pub mod readiness; +pub mod subnet_merging; +pub mod utils; +pub mod validation; + +mod admin_helper; +mod agent_helper; +mod layout; +mod metrics_helper; +mod state_tool_helper; +mod steps; +mod target_subnet; diff --git a/rs/recovery/subnet_merging/src/main.rs b/rs/recovery/subnet_merging/src/main.rs new file mode 100644 index 000000000000..539bd95b0a17 --- /dev/null +++ b/rs/recovery/subnet_merging/src/main.rs @@ -0,0 +1,180 @@ +use anyhow::Context; +use clap::Parser; +use ic_base_types::SubnetId; +use ic_recovery::{NeuronArgs, RecoveryArgs, cli, util}; +use ic_subnet_merging::{ + subnet_merging::{SubnetMerging, SubnetMergingArgs}, + validation::validate_artifacts, +}; +use ic_types::ReplicaVersion; +use slog::{Logger, info, warn}; +use url::Url; + +use std::path::PathBuf; + +const FORUM_ANNOUNCEMENT_TEMPLATE_URL: &str = + "https://wiki.internetcomputer.org/wiki/Subnet_merging_forum_announcement_template"; + +#[derive(Parser)] +struct MergeArgs { + #[clap( + short = 'r', + long, + alias = "registry-url", + default_value = "https://ic0.app" + )] + /// The URL of an NNS entry point. That is, the URL of any replica on the + /// NNS subnet. + nns_url: Url, + + /// replica version of ic-admin binary + #[clap(long)] + replica_version: Option, + + /// The directory to do the subnet merging in + #[clap(long)] + dir: PathBuf, + + /// The path to a private key to be considered for admin SSH connections + #[clap(long)] + admin_key_file: Option, + + /// Flag to enter test mode + #[clap(long)] + test: bool, + + /// Flag to make the tool non interactive. No input from the user is requested. + #[clap(long)] + pub skip_prompts: bool, + + #[clap(flatten)] + subnet_merging_args: SubnetMergingArgs, +} + +#[derive(Parser)] +struct ValidateArgs { + /// Path to the State Tree signed by the NNS + #[clap(long)] + state_tree_path: PathBuf, + + /// (Optional) path to the NNS public key. If not set, the built-in public key is used. + #[clap(long)] + nns_public_key_path: Option, + + /// Path to the CUP the subnet halted at, retrieved from one of its nodes. + #[clap(long)] + cup_path: PathBuf, + + /// Path to the manifest computed from the state the subnet halted at. + #[clap(long)] + state_manifest_path: PathBuf, + + /// SubnetId of the subnet the artifacts belong to. + #[clap(long, value_parser=ic_recovery::util::subnet_id_from_str)] + subnet_id: SubnetId, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Parser)] +enum Subcommand { + /// Perform Subnet Merging + Merge(MergeArgs), + + /// Validate artifacts produced during subnet merging + Validate(ValidateArgs), +} + +#[derive(Parser)] +#[clap(version = "1.0")] +struct SubnetMergingToolArgs { + #[clap(subcommand)] + subcommand: Subcommand, +} + +fn subnet_merging( + logger: Logger, + recovery_args: RecoveryArgs, + subnet_merging_args: SubnetMergingArgs, + mut neuron_args: Option, +) { + cli::print_step(&logger, "Subnet Merging"); + + info!( + logger, + "Merging subnet with id {} into subnet with id {}", + subnet_merging_args.source_subnet_id, + subnet_merging_args.destination_subnet_id, + ); + warn!( + logger, + "Don't forget to announce at the forum the upcoming series of proposals to merge the \ + subnet" + ); + warn!( + logger, + "See the template at: {}", FORUM_ANNOUNCEMENT_TEMPLATE_URL + ); + + if !recovery_args.skip_prompts { + cli::wait_for_confirmation(&logger); + } + + if neuron_args.is_none() && !recovery_args.test_mode { + neuron_args = Some(cli::read_neuron_args(&logger)); + } + + let subnet_merging = SubnetMerging::new( + logger.clone(), + recovery_args.clone(), + neuron_args, + subnet_merging_args, + ); + + cli::execute_steps(&logger, recovery_args.skip_prompts, subnet_merging); +} + +fn do_merge(args: MergeArgs, logger: Logger) -> anyhow::Result<()> { + let recovery_args = RecoveryArgs { + dir: args.dir, + nns_url: args.nns_url, + replica_version: args.replica_version, + admin_key_file: args.admin_key_file, + test_mode: args.test, + skip_prompts: args.skip_prompts, + }; + + let subnet_merging_state = + cli::read_and_maybe_update_state(&logger, recovery_args, Some(args.subnet_merging_args)); + + subnet_merging( + logger, + subnet_merging_state.recovery_args, + subnet_merging_state.subcommand_args, + subnet_merging_state.neuron_args, + ); + + Ok(()) +} + +fn do_validate(args: ValidateArgs, logger: Logger) -> anyhow::Result<()> { + validate_artifacts( + args.state_tree_path, + args.nns_public_key_path.as_deref(), + args.cup_path, + args.state_manifest_path, + args.subnet_id, + &logger, + ) + .context("Failed to validate the artifacts") +} + +fn main() -> anyhow::Result<()> { + let args = SubnetMergingToolArgs::parse(); + + let logger = util::make_logger(); + + match args.subcommand { + Subcommand::Merge(merge_args) => do_merge(merge_args, logger), + Subcommand::Validate(validate_args) => do_validate(validate_args, logger), + } +} diff --git a/rs/recovery/subnet_merging/src/metrics_helper.rs b/rs/recovery/subnet_merging/src/metrics_helper.rs new file mode 100644 index 000000000000..5c9aa1dc882d --- /dev/null +++ b/rs/recovery/subnet_merging/src/metrics_helper.rs @@ -0,0 +1,219 @@ +//! Scraping of replica metrics, as the `Subnet merging` dashboard reads them. +//! +//! The dashboard evaluates its conditions on Prometheus queries over the +//! metrics of all replicas of a subnet; this module provides the same data to +//! the tool, by scraping the metrics endpoints of the nodes directly. + +use futures::future::join_all; +use ic_recovery::util::block_on; +use slog::{Logger, warn}; + +use std::{collections::BTreeMap, net::IpAddr, time::Duration}; + +/// Timeout of a single metrics request, as in `ic_recovery::get_node_metrics`. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); + +/// The metrics of a set of nodes, keyed by series (i.e. metric name plus +/// labels), with one value per node reporting the series. +pub type Metrics = BTreeMap>; + +/// Fetches all series of the given metrics from the given nodes. +/// +/// Best effort, exactly like the dashboard: a node that cannot be scraped +/// contributes no value, and a series that no node reports is absent (which +/// every helper below reads as zero). A condition that has to hold on every +/// node of every subnet therefore has to be written so that missing data keeps +/// it unsatisfied -- see `readiness::evaluate_merge_readiness`. +pub fn fetch_metrics(logger: &Logger, node_ips: &[IpAddr], metrics: &[&str]) -> Metrics { + let responses = block_on(join_all( + node_ips.iter().map(|ip| fetch_node_metrics(logger, ip)), + )); + + let mut result = Metrics::new(); + for (ip, body) in node_ips.iter().zip(responses) { + let Some(body) = body else { + warn!(logger, "Failed to scrape the metrics of node {ip}"); + continue; + }; + for (series, value) in parse_metrics(&body, metrics) { + result.entry(series).or_default().push(value); + } + } + result +} + +async fn fetch_node_metrics(logger: &Logger, ip: &IpAddr) -> Option { + let response = + tokio::time::timeout(REQUEST_TIMEOUT, reqwest::get(format!("http://[{ip}]:9090"))).await; + match response { + Ok(Ok(response)) => match response.text().await { + Ok(body) => Some(body), + Err(err) => { + warn!(logger, "Failed to decode the metrics of node {ip}: {err}"); + None + } + }, + Ok(Err(err)) => { + warn!(logger, "Failed to request the metrics of node {ip}: {err}"); + None + } + Err(_) => { + warn!(logger, "Timed out requesting the metrics of node {ip}"); + None + } + } +} + +/// Picks the series of the requested metrics out of a Prometheus text exposition. +fn parse_metrics(body: &str, metrics: &[&str]) -> Vec<(String, f64)> { + body.lines() + .filter(|line| !line.starts_with('#')) + .filter_map(|line| { + let (series, value) = line.rsplit_once(' ')?; + let series = series.trim(); + if !metrics.iter().any(|metric| is_series_of(series, metric)) { + return None; + } + let value = value.trim().parse::().ok()?; + (!value.is_nan()).then(|| (series.to_string(), value)) + }) + .collect() +} + +/// Whether `series` is a series of `metric`, i.e. the metric name followed by +/// its labels (if any). Metric names are prefixes of one another (e.g. +/// `..._messages` and `..._messages_total`), so a plain prefix check would mix +/// up their series. +fn is_series_of(series: &str, metric: &str) -> bool { + match series.strip_prefix(metric) { + Some(labels) => labels.is_empty() || labels.starts_with('{'), + None => false, + } +} + +/// The per-node values of every series of `metric` whose labels (`{...}`, or +/// the empty string for an unlabeled series) match `labels_match`. +pub fn matching_series<'a>( + metrics: &'a Metrics, + metric: &str, + labels_match: impl Fn(&str) -> bool, +) -> Vec<&'a Vec> { + metrics + .iter() + .filter(|(series, _)| match series.strip_prefix(metric) { + Some(labels) if labels.is_empty() || labels.starts_with('{') => labels_match(labels), + _ => false, + }) + .map(|(_, values)| values) + .collect() +} + +/// Prometheus' `quantile(0.5, ...)`: the median of `values`, interpolating +/// between the two middle values if there is an even number of them. `None` iff +/// `values` is empty. +pub fn median(values: &[f64]) -> Option { + if values.is_empty() { + return None; + } + let mut values = values.to_vec(); + values.sort_by(|a, b| a.partial_cmp(b).expect("metric value should not be NaN")); + let middle = (values.len() - 1) as f64 / 2.0; + Some((values[middle.floor() as usize] + values[middle.ceil() as usize]) / 2.0) +} + +/// `sum(quantile by () (0.5, {}))`: the median across +/// the replicas reporting each matching series, summed over those series. +pub fn sum_of_medians(metrics: &Metrics, metric: &str, labels_match: impl Fn(&str) -> bool) -> f64 { + matching_series(metrics, metric, labels_match) + .into_iter() + .filter_map(|values| median(values)) + .sum() +} + +/// `quantile(0.5, {})`: the median across all replicas +/// reporting any matching series. `None` if there is no such series. +pub fn median_across_replicas( + metrics: &Metrics, + metric: &str, + labels_match: impl Fn(&str) -> bool, +) -> Option { + let values: Vec = matching_series(metrics, metric, labels_match) + .into_iter() + .flatten() + .copied() + .collect(); + median(&values) +} + +#[cfg(test)] +mod tests { + use super::*; + + const BODY: &str = "\ +# HELP mr_stream_messages Messages in streams. +# TYPE mr_stream_messages gauge +mr_stream_messages{remote=\"subnet_1\"} 3 +mr_stream_messages{remote=\"subnet_2\"} 4 +mr_stream_messages_total 7 +replicated_state_pending_refunds 0 +some_other_metric 12 +"; + + #[test] + fn parse_metrics_test() { + let parsed = parse_metrics( + BODY, + &["mr_stream_messages", "replicated_state_pending_refunds"], + ); + + assert_eq!( + parsed, + vec![ + ("mr_stream_messages{remote=\"subnet_1\"}".to_string(), 3.0), + ("mr_stream_messages{remote=\"subnet_2\"}".to_string(), 4.0), + ("replicated_state_pending_refunds".to_string(), 0.0), + ], + "the series of `mr_stream_messages_total`, which `mr_stream_messages` is a prefix \ + of, must not be picked up", + ); + } + + #[test] + fn median_test() { + assert_eq!(median(&[]), None); + assert_eq!(median(&[3.0]), Some(3.0)); + assert_eq!(median(&[3.0, 1.0]), Some(2.0)); + assert_eq!(median(&[3.0, 1.0, 2.0]), Some(2.0)); + assert_eq!(median(&[4.0, 1.0, 3.0, 2.0]), Some(2.5)); + } + + #[test] + fn medians_across_series_test() { + let metrics = Metrics::from([ + ( + "mr_stream_messages{remote=\"a\"}".to_string(), + vec![1.0, 3.0], + ), + ("mr_stream_messages{remote=\"b\"}".to_string(), vec![5.0]), + ("mr_stream_messages_total".to_string(), vec![100.0]), + ]); + + assert_eq!( + sum_of_medians(&metrics, "mr_stream_messages", |_| true), + 7.0 + ); + assert_eq!( + sum_of_medians(&metrics, "mr_stream_messages", |labels| labels + .contains("remote=\"b\"")), + 5.0 + ); + assert_eq!( + median_across_replicas(&metrics, "mr_stream_messages", |_| true), + Some(3.0) + ); + assert_eq!( + median_across_replicas(&metrics, "mr_registry_version", |_| true), + None + ); + } +} diff --git a/rs/recovery/subnet_merging/src/readiness.rs b/rs/recovery/subnet_merging/src/readiness.rs new file mode 100644 index 000000000000..86556dad52b7 --- /dev/null +++ b/rs/recovery/subnet_merging/src/readiness.rs @@ -0,0 +1,362 @@ +//! The "merge readiness" condition of the `Subnet merging` dashboard, evaluated +//! from the metrics of the replicas rather than from a Grafana panel. +//! +//! A subnet that is cooling down is ready to be merged once it has come to a +//! complete rest: every subnet has observed that it is cooling down, nothing is +//! in flight to or from it, and it holds no state that only it could act upon. +//! The terms below spell that out, in the order the dashboard states them. + +use crate::metrics_helper::{self, Metrics}; + +use ic_base_types::SubnetId; +use ic_recovery::{ + error::{RecoveryError, RecoveryResult}, + get_member_node_ids_and_ips, + registry_helper::RegistryHelper, +}; +use ic_registry_client_helpers::subnet::SubnetListRegistry; +use slog::{Logger, info, warn}; + +use std::{ + collections::BTreeMap, + net::IpAddr, + thread::sleep, + time::{Duration, Instant}, +}; + +/// The nodes of every subnet the conditions below range over. +pub type SubnetNodeIps = BTreeMap>; + +/// The registry version every subnet has to have observed. +const METRIC_REGISTRY_VERSION: &str = "mr_registry_version"; +/// The messages held in the streams of a subnet, by remote subnet (the subnet +/// itself included, i.e. its loopback stream). +const METRIC_STREAM_MESSAGES: &str = "mr_stream_messages"; +/// The entries of the ingress history, by status. +const METRIC_INGRESS_HISTORY_BY_STATE: &str = "replicated_state_ingress_history_length_by_state"; +/// The messages held in the input and output queues of the subnet itself, i.e. +/// the management canister's. +const METRIC_SUBNET_INPUT_QUEUE_MESSAGES: &str = "execution_subnet_input_queue_messages"; +const METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES: &str = "execution_subnet_output_queue_messages"; +/// The call contexts of the subnet call context manager, e.g. the `install_code` +/// calls that are still running. +const METRIC_SUBNET_CALL_CONTEXTS: &str = "replicated_state_subnet_call_contexts"; +/// The number of pending anonymous refunds, i.e. the size of the refund pool. +const METRIC_PENDING_REFUNDS: &str = "replicated_state_pending_refunds"; +/// The total value of those refunds, which is logged next to the term above +/// when the refund pool is not empty, but is not itself a condition. +const METRIC_PENDING_REFUNDS_CYCLES: &str = "replicated_state_pending_refunds_cycles"; + +/// A term of the readiness condition: what has to hold, and whether it does. +pub struct Term { + pub description: String, + pub satisfied: bool, +} + +/// Evaluates the terms of the "merge readiness" condition of the `Subnet +/// merging` dashboard for the subnet that is cooling down and the registry +/// version `V` at which it was labeled as such. +/// +/// As in the dashboard, every term is evaluated on the median across the +/// replicas reporting the respective series, and missing data reads as zero +/// (the dashboard's `or vector(0)` fallback). The first term is the one that +/// keeps an unreachable subnet from reading as ready: a subnet whose metrics +/// cannot be scraped reports no registry version, i.e. zero, which is below +/// `V`. +pub fn evaluate_merge_readiness( + subnets: &SubnetNodeIps, + source_subnet_id: SubnetId, + registry_version: u64, + logger: &Logger, +) -> Vec { + let own_metrics = subnet_metrics( + subnets, + source_subnet_id, + &[ + METRIC_REGISTRY_VERSION, + METRIC_STREAM_MESSAGES, + METRIC_INGRESS_HISTORY_BY_STATE, + METRIC_SUBNET_INPUT_QUEUE_MESSAGES, + METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES, + METRIC_SUBNET_CALL_CONTEXTS, + METRIC_PENDING_REFUNDS, + METRIC_PENDING_REFUNDS_CYCLES, + ], + logger, + ); + + // Terms 1 and 2 range over all subnets: the registry version of every + // subnet and the streams of all remote subnets towards this one. + let remote_label = format!("remote=\"{source_subnet_id}\""); + let mut min_registry_version = None; + let mut incoming_stream_messages = 0.0; + for &subnet_id in subnets.keys() { + let metrics = if subnet_id == source_subnet_id { + own_metrics.clone() + } else { + subnet_metrics( + subnets, + subnet_id, + &[METRIC_REGISTRY_VERSION, METRIC_STREAM_MESSAGES], + logger, + ) + }; + let version = + metrics_helper::median_across_replicas(&metrics, METRIC_REGISTRY_VERSION, |_| true) + .unwrap_or(0.0); + min_registry_version = Some(min_registry_version.map_or(version, |v: f64| v.min(version))); + if subnet_id != source_subnet_id { + incoming_stream_messages += + metrics_helper::sum_of_medians(&metrics, METRIC_STREAM_MESSAGES, |labels| { + labels.contains(&remote_label) + }); + } + } + let min_registry_version = min_registry_version.unwrap_or(0.0); + + let outgoing_stream_messages = + metrics_helper::sum_of_medians(&own_metrics, METRIC_STREAM_MESSAGES, |_| true); + let ingress_history_messages = + metrics_helper::sum_of_medians(&own_metrics, METRIC_INGRESS_HISTORY_BY_STATE, |labels| { + !labels.contains("state=\"processing\"") + }); + let subnet_input_queue_messages = + metrics_helper::sum_of_medians(&own_metrics, METRIC_SUBNET_INPUT_QUEUE_MESSAGES, |_| true); + let subnet_output_queue_messages = metrics_helper::median_across_replicas( + &own_metrics, + METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES, + |_| true, + ) + .unwrap_or(0.0); + let subnet_call_contexts = + metrics_helper::sum_of_medians(&own_metrics, METRIC_SUBNET_CALL_CONTEXTS, |_| true); + let pending_refunds = + metrics_helper::median_across_replicas(&own_metrics, METRIC_PENDING_REFUNDS, |_| true) + .unwrap_or(0.0); + let pending_refunds_cycles = + metrics_helper::median_across_replicas(&own_metrics, METRIC_PENDING_REFUNDS_CYCLES, |_| { + true + }) + .unwrap_or(0.0); + + let term = |description: String, satisfied: bool| Term { + description, + satisfied, + }; + vec![ + term( + format!( + "every subnet has reached registry version {registry_version} (the lowest one is \ + at {min_registry_version})" + ), + min_registry_version >= registry_version as f64, + ), + term( + format!( + "no remote subnet holds a message in its stream to subnet {source_subnet_id} \ + ({incoming_stream_messages} messages)" + ), + incoming_stream_messages == 0.0, + ), + term( + format!( + "subnet {source_subnet_id} holds no message in any of its streams, loopback \ + included ({outgoing_stream_messages} messages)" + ), + outgoing_stream_messages == 0.0, + ), + term( + format!( + "the ingress history holds nothing but `processing` entries \ + ({ingress_history_messages} other entries)" + ), + ingress_history_messages == 0.0, + ), + term( + format!("the subnet input queues are empty ({subnet_input_queue_messages} messages)"), + subnet_input_queue_messages == 0.0, + ), + term( + format!("the subnet output queues are empty ({subnet_output_queue_messages} messages)"), + subnet_output_queue_messages == 0.0, + ), + term( + format!( + "the subnet call context manager holds no call context ({subnet_call_contexts} \ + call contexts)" + ), + subnet_call_contexts == 0.0, + ), + term( + // A cooling down subnet routes no refunds either (see `route_refunds` + // in `rs/messaging/src/routing/stream_builder.rs`), so a refund that + // is in the pool stays pending until the subnet is merged, and is + // then lost: the merged state takes the refunds of the destination + // subnet, not those of the subnet that is merged away. + format!( + "the refund pool holds no pending anonymous refund ({pending_refunds} refunds, \ + worth {pending_refunds_cycles} cycles)" + ), + pending_refunds == 0.0, + ), + ] +} + +/// Waits until every term of the readiness condition holds, logging the ones +/// that do not after every round. +pub fn await_merge_readiness( + registry_helper: &RegistryHelper, + source_subnet_id: SubnetId, + registry_version: u64, + timeout: Duration, + poll_interval: Duration, + logger: &Logger, +) -> RecoveryResult<()> { + let deadline = Instant::now() + timeout; + loop { + let terms = evaluate_merge_readiness( + &subnet_node_ips(registry_helper)?, + source_subnet_id, + registry_version, + logger, + ); + let unsatisfied: Vec<&Term> = terms.iter().filter(|term| !term.satisfied).collect(); + + if unsatisfied.is_empty() { + info!( + logger, + "Subnet {source_subnet_id} is ready to be merged; every term holds:" + ); + for term in &terms { + info!(logger, " [x] {}", term.description); + } + return Ok(()); + } + + info!( + logger, + "Subnet {source_subnet_id} is not ready to be merged yet, {} of {} terms do not hold:", + unsatisfied.len(), + terms.len(), + ); + for term in &terms { + let mark = if term.satisfied { "x" } else { " " }; + info!(logger, " [{mark}] {}", term.description); + } + + if Instant::now() >= deadline { + return Err(RecoveryError::UnexpectedError(format!( + "Subnet {source_subnet_id} did not become ready to be merged within {timeout:?}; \ + the terms that do not hold: {}", + unsatisfied + .iter() + .map(|term| term.description.clone()) + .collect::>() + .join("; "), + ))); + } + sleep(poll_interval); + } +} + +/// Waits until every subnet other than `skipped` has reached `registry_version`. +/// +/// `skipped` is the subnet that was merged away: its replicas are stopped by +/// the time this runs, so it reports no metrics anymore, and it is about to be +/// deleted. What matters is that every *other* subnet already routes its +/// canisters to the destination subnet. +pub fn await_registry_version_on_all_subnets( + registry_helper: &RegistryHelper, + skipped: SubnetId, + registry_version: u64, + timeout: Duration, + poll_interval: Duration, + logger: &Logger, +) -> RecoveryResult<()> { + let deadline = Instant::now() + timeout; + loop { + let mut behind = Vec::new(); + let subnets = subnet_node_ips(registry_helper)?; + for &subnet_id in subnets.keys() { + if subnet_id == skipped { + continue; + } + let metrics = subnet_metrics(&subnets, subnet_id, &[METRIC_REGISTRY_VERSION], logger); + let version = + metrics_helper::median_across_replicas(&metrics, METRIC_REGISTRY_VERSION, |_| true) + .unwrap_or(0.0); + if version < registry_version as f64 { + behind.push(format!("{subnet_id} is at registry version {version}")); + } + } + + if behind.is_empty() { + info!( + logger, + "All subnets other than {skipped} have reached registry version {registry_version}" + ); + return Ok(()); + } + + info!( + logger, + "Waiting until all subnets reached registry version {registry_version}: {}", + behind.join("; "), + ); + + if Instant::now() >= deadline { + return Err(RecoveryError::UnexpectedError(format!( + "Not all subnets reached registry version {registry_version} within {timeout:?}: \ + {}", + behind.join("; "), + ))); + } + sleep(poll_interval); + } +} + +/// The nodes of every subnet in the registry. +pub fn subnet_node_ips(registry_helper: &RegistryHelper) -> RecoveryResult { + let registry_version = registry_helper.latest_registry_version()?; + let subnet_ids = registry_helper + .registry_client() + .get_subnet_ids(registry_version) + .map_err(|err| { + RecoveryError::RegistryError(format!( + "Failed to get the subnet ids at registry version {registry_version}: {err}" + )) + })? + .ok_or_else(|| { + RecoveryError::RegistryError(format!( + "No subnet ids at registry version {registry_version}" + )) + })?; + + subnet_ids + .into_iter() + .map(|subnet_id| { + let node_ips = get_member_node_ids_and_ips(registry_helper, subnet_id)? + .into_values() + .collect(); + Ok((subnet_id, node_ips)) + }) + .collect() +} + +/// Fetches the given metrics from all nodes of `subnet_id`. +fn subnet_metrics( + subnets: &SubnetNodeIps, + subnet_id: SubnetId, + metrics: &[&str], + logger: &Logger, +) -> Metrics { + let node_ips = match subnets.get(&subnet_id) { + Some(node_ips) if !node_ips.is_empty() => node_ips.as_slice(), + _ => { + warn!(logger, "Subnet {subnet_id} has no node to scrape"); + &[] + } + }; + + metrics_helper::fetch_metrics(logger, node_ips, metrics) +} diff --git a/rs/recovery/subnet_merging/src/state_tool_helper.rs b/rs/recovery/subnet_merging/src/state_tool_helper.rs new file mode 100644 index 000000000000..d91bbe9e5764 --- /dev/null +++ b/rs/recovery/subnet_merging/src/state_tool_helper.rs @@ -0,0 +1,53 @@ +use ic_recovery::{ + error::{RecoveryError, RecoveryResult}, + file_sync_helper::write_file, +}; + +use std::{fs::File, path::Path}; + +/// Computes manifest of a checkpoint at `dir` and writes it to `output_path`. +pub(crate) fn compute_manifest(dir: &Path, output_path: &Path) -> RecoveryResult<()> { + ic_state_tool::commands::manifest::compute_manifest(dir) + .map_err(|err| { + RecoveryError::StateToolError(format!("Failed to compute the state manifest: {err}")) + }) + .and_then(|manifest| write_file(output_path, manifest)) +} + +/// Verifies whether the textual representation of a manifest matches its root hash, and +/// returns the root hash. +pub(crate) fn verify_manifest(manifest_path: &Path) -> RecoveryResult { + let manifest_file = + File::open(manifest_path).map_err(|err| RecoveryError::file_error(manifest_path, err))?; + + ic_state_tool::commands::verify_manifest::verify_manifest(manifest_file) + .map_err(|err| { + RecoveryError::StateToolError(format!("Failed to verify the state manifest: {err}")) + }) + .map(hex::encode) +} + +/// Assembles the checkpoint at `output` from the checkpoints at `base` (the +/// state of the destination subnet) and `source` (the state of the subnet that +/// is merged away): it holds everything of `base`, with the canisters and +/// canister snapshots of `source` added to those of `base`, and is marked as +/// the product of a subnet merge. +pub(crate) fn merge_checkpoints(base: &Path, source: &Path, output: &Path) -> RecoveryResult<()> { + ic_state_tool::commands::merge::do_merge( + base.to_path_buf(), + source.to_path_buf(), + output.to_path_buf(), + ) + .map_err(|err| RecoveryError::StateToolError(format!("Failed to merge the states: {err}"))) +} + +/// The batch time of the checkpoint at `path`, in nanoseconds since the Epoch, +/// i.e. the IC time the subnet had reached when it wrote the checkpoint. +pub(crate) fn checkpoint_time_nanos(path: &Path) -> RecoveryResult { + ic_state_tool::commands::checkpoint_time::batch_time_nanos(path.to_path_buf()).map_err(|err| { + RecoveryError::StateToolError(format!( + "Failed to read the batch time of the checkpoint {}: {err}", + path.display() + )) + }) +} diff --git a/rs/recovery/subnet_merging/src/steps.rs b/rs/recovery/subnet_merging/src/steps.rs new file mode 100644 index 000000000000..e925a80d0474 --- /dev/null +++ b/rs/recovery/subnet_merging/src/steps.rs @@ -0,0 +1,669 @@ +use crate::{ + agent_helper::AgentHelper, + layout::{CUP_FILE_NAME, Layout}, + readiness, state_tool_helper, + target_subnet::TargetSubnet, + utils::{MergedStateParams, first_registry_version_where, get_cup, get_state_hash}, + validation::validate_artifacts, +}; + +use ic_base_types::SubnetId; +use ic_recovery::{ + IC_DATA_PATH, Recovery, STATES_METADATA, + error::{RecoveryError, RecoveryResult}, + get_node_metrics, + registry_helper::RegistryHelper, + ssh_helper::SshHelper, + steps::Step, + util::block_on, +}; +use ic_registry_client_helpers::routing_table::RoutingTableRegistry; +use ic_registry_routing_table::RoutingTable; +use ic_types::{Height, consensus::CatchUpPackage, consensus::HasHeight}; +use slog::{Logger, info, warn}; +use url::Url; + +use std::{ + net::IpAddr, + path::PathBuf, + thread::sleep, + time::{Duration, Instant}, +}; + +/// Retries `check` until it reports success, `timeout` elapses, or it fails +/// with an error that is not worth retrying. +/// +/// What the steps below use it for is the registry catching up with a proposal +/// that `ic-admin` just reported as executed: the local store is polled on +/// every read, but the mutation may not have made it into the registry canister +/// the poll reads from yet. +fn wait_for( + what: &str, + timeout: Duration, + poll_interval: Duration, + logger: &Logger, + check: impl Fn() -> RecoveryResult, +) -> RecoveryResult<()> { + let deadline = Instant::now() + timeout; + loop { + match check() { + Ok(true) => return Ok(()), + Ok(false) => info!(logger, "Waiting until {what}"), + Err(err) => warn!(logger, "Waiting until {what}, last attempt failed: {err}"), + } + + if Instant::now() >= deadline { + return Err(RecoveryError::UnexpectedError(format!( + "Gave up waiting until {what} after {timeout:?}" + ))); + } + sleep(poll_interval); + } +} + +/// Reads the record of the subnet that is being merged away, checks that it is +/// labeled "cooling down" and determines (and persists) the registry version +/// `V` at which it was labeled so, which is what the merge readiness condition +/// is evaluated against. +pub(crate) struct CheckCoolingDownStep { + pub(crate) source_subnet_id: SubnetId, + pub(crate) registry_helper: RegistryHelper, + pub(crate) layout: Layout, + pub(crate) timeout: Duration, + pub(crate) poll_interval: Duration, + pub(crate) logger: Logger, +} + +impl Step for CheckCoolingDownStep { + fn descr(&self) -> String { + format!( + "Read the registry to check that subnet {} is labeled \"cooling down\", and determine \ + the registry version it was labeled so at.", + self.source_subnet_id, + ) + } + + fn exec(&self) -> RecoveryResult<()> { + let source_subnet_id = self.source_subnet_id; + wait_for( + &format!("subnet {source_subnet_id} is labeled \"cooling down\""), + self.timeout, + self.poll_interval, + &self.logger, + || { + let (registry_version, subnet_record) = + self.registry_helper.get_subnet_record(source_subnet_id)?; + let Some(subnet_record) = subnet_record else { + return Err(RecoveryError::RegistryError(format!( + "No record of subnet {source_subnet_id} at registry version \ + {registry_version}" + ))); + }; + Ok(subnet_record.cooling_down) + }, + )?; + + let registry_helper = self.registry_helper.clone(); + let cooling_down_version = + first_registry_version_where(&self.registry_helper, |version| { + Ok(registry_helper + .get_subnet_record_at_version(source_subnet_id, version)? + .is_some_and(|record| record.cooling_down)) + })?; + + info!( + self.logger, + "Subnet {source_subnet_id} has been labeled \"cooling down\" since registry version \ + {cooling_down_version}", + ); + + self.layout + .write_cooling_down_registry_version(cooling_down_version.get()) + } +} + +/// Waits until the merge readiness condition of the `Subnet merging` dashboard +/// holds for the subnet that is cooling down. +pub(crate) struct CheckMergeReadinessStep { + pub(crate) source_subnet_id: SubnetId, + pub(crate) registry_helper: RegistryHelper, + pub(crate) layout: Layout, + pub(crate) timeout: Duration, + pub(crate) poll_interval: Duration, + pub(crate) logger: Logger, +} + +impl Step for CheckMergeReadinessStep { + fn descr(&self) -> String { + format!( + "Wait until subnet {} is ready to be merged, i.e. until it has come to a complete \ + rest: every subnet has observed that it is cooling down, no message is in flight to \ + or from it, its ingress history holds nothing but `processing` entries, its subnet \ + queues and call context manager are empty and its refund pool holds no pending \ + anonymous refund.", + self.source_subnet_id, + ) + } + + fn exec(&self) -> RecoveryResult<()> { + let registry_version = self.layout.read_cooling_down_registry_version()?; + + readiness::await_merge_readiness( + &self.registry_helper, + self.source_subnet_id, + registry_version, + self.timeout, + self.poll_interval, + &self.logger, + ) + } +} + +/// Waits until the node the state is downloaded from holds the CUP its subnet +/// halts at, i.e. the first one whose summary is created at a registry version +/// carrying the `halt_at_cup_height` flag. +/// +/// Waiting for the CUP rather than for the subnet to report that it is halted: +/// the CUP is what names the state the subnet came to rest in, and it exists +/// only once that state has been certified and its hash agreed upon. A subnet +/// that has just stopped delivering batches, on the other hand, may not have +/// finished writing and hashing the checkpoint it stopped at, and downloading +/// its latest checkpoint then yields the previous one, a whole DKG interval +/// before the state the merge is supposed to be assembled from. +/// +/// The CUP the node serves over HTTP is not enough: the orchestrator writes the +/// CUP to disk asynchronously, and it is the on-disk copy that is downloaded +/// with the state and validated afterwards. So this waits for both, and for +/// them to agree. +pub(crate) struct WaitForHaltingCupStep { + pub(crate) subnet_id: SubnetId, + pub(crate) node_ip: IpAddr, + pub(crate) registry_helper: RegistryHelper, + pub(crate) layout: Layout, + pub(crate) ssh_helper: SshHelper, + pub(crate) timeout: Duration, + pub(crate) poll_interval: Duration, + pub(crate) logger: Logger, +} + +impl Step for WaitForHaltingCupStep { + fn descr(&self) -> String { + format!( + "Wait until node {} holds the CUP subnet {} halts at, both at its public endpoint and \ + on disk, and until it has certified the state that CUP names.", + self.node_ip, self.subnet_id, + ) + } + + fn exec(&self) -> RecoveryResult<()> { + let deadline = Instant::now() + self.timeout; + loop { + match self.check() { + Ok(Some(height)) => { + info!( + self.logger, + "Subnet {} halted at height {height}", self.subnet_id + ); + return Ok(()); + } + Ok(None) => {} + // A node that is not answering yet, or a CUP that cannot be + // pulled off it yet, is the normal state of affairs while the + // subnet is still running towards its halting CUP. + Err(err) => warn!( + self.logger, + "Subnet {} has not halted yet: {err}", self.subnet_id + ), + } + + if Instant::now() >= deadline { + return Err(RecoveryError::UnexpectedError(format!( + "Subnet {} did not reach the CUP it halts at within {:?}", + self.subnet_id, self.timeout, + ))); + } + sleep(self.poll_interval); + } + } +} + +impl WaitForHaltingCupStep { + /// Returns the height of the halting CUP if the node has reached it, `None` + /// if it has not yet, and an error if the node could not be asked. + fn check(&self) -> RecoveryResult> { + let cup = self.served_cup()?; + let cup_height = cup.height(); + let cup_registry_version = cup + .content + .block + .get_value() + .payload + .as_ref() + .as_summary() + .dkg + .registry_version; + + // The `halt_at_cup_height` flag is read at the registry version of the + // summary block active at a height, and that version only changes at a + // summary, so batch delivery stops exactly when the summary carrying the + // flag becomes active. + let halting = self + .registry_helper + .get_subnet_record_at_version(self.subnet_id, cup_registry_version)? + .is_some_and(|record| record.halt_at_cup_height); + if !halting { + info!( + self.logger, + "Subnet {} is at the CUP at height {cup_height}, whose registry version \ + {cup_registry_version} does not carry the `halt_at_cup_height` flag yet", + self.subnet_id, + ); + return Ok(None); + } + + // The node has to have caught up with the CUP itself: it is its state + // that is downloaded, and a node can hold a CUP that the rest of the + // subnet assembled before it got there. + let metrics = block_on(get_node_metrics(&self.logger, &self.node_ip)).ok_or_else(|| { + RecoveryError::UnexpectedError(format!( + "Failed to get the metrics of node {}", + self.node_ip + )) + })?; + if metrics.certification_height > cup_height { + return Err(RecoveryError::ValidationFailed(format!( + "Subnet {} certified height {}, past the CUP at height {cup_height} it should \ + have halted at", + self.subnet_id, metrics.certification_height, + ))); + } + if metrics.certification_height < cup_height { + info!( + self.logger, + "Subnet {} holds the CUP at height {cup_height} but node {} has only certified up \ + to height {}", + self.subnet_id, + self.node_ip, + metrics.certification_height, + ); + return Ok(None); + } + + // The CUP the orchestrator has written to disk is the one that is + // downloaded with the state and validated afterwards, so it has to be + // the halting CUP, too. + let on_disk_cup = self.on_disk_cup()?; + if on_disk_cup.height() != cup_height + || on_disk_cup.content.state_hash != cup.content.state_hash + { + info!( + self.logger, + "Node {} still holds the CUP at height {} on disk, not the one at height \ + {cup_height} it serves", + self.node_ip, + on_disk_cup.height(), + ); + return Ok(None); + } + + Ok(Some(cup_height)) + } + + /// The CUP the node serves at its public endpoint. + fn served_cup(&self) -> RecoveryResult { + let url = Url::parse(&format!("http://[{}]:8080/", self.node_ip)).map_err(|err| { + RecoveryError::UnexpectedError(format!( + "Could not parse the URL of node {}: {err}", + self.node_ip + )) + })?; + + let cup_proto = block_on(ic_cup_explorer::get_cup(&url)) + .map_err(|err| { + RecoveryError::UnexpectedError(format!( + "Failed to get the CUP of node {}: {err}", + self.node_ip + )) + })? + .ok_or_else(|| { + RecoveryError::UnexpectedError(format!("Node {} serves no CUP", self.node_ip)) + })?; + + CatchUpPackage::try_from(&cup_proto).map_err(|err| { + RecoveryError::UnexpectedError(format!( + "Failed to deserialize the CUP of node {}: {err}", + self.node_ip + )) + }) + } + + /// The CUP the orchestrator of the node has written to disk, pulled off the + /// node and kept next to the other artifacts of the merge. + fn on_disk_cup(&self) -> RecoveryResult { + let remote_cup = self.ssh_helper.remote_path( + PathBuf::from(IC_DATA_PATH) + .join(ic_recovery::CUPS_DIR) + .join(CUP_FILE_NAME), + ); + let local_cup = self.layout.halting_cup_file(self.subnet_id); + + self.ssh_helper.rsync(remote_cup, &local_cup)?; + + get_cup(&local_cup) + } +} + +/// Validates the CUP a subnet halted at and the state that was downloaded with +/// it: the subnet's public key is taken from the NNS signed state tree, the CUP +/// signature is verified against it, and the manifest recomputed from the +/// downloaded state has to match the state hash the CUP names. +pub(crate) struct ValidateCupStep { + pub(crate) subnet_id: SubnetId, + pub(crate) target_subnet: TargetSubnet, + pub(crate) nns_url: Url, + pub(crate) layout: Layout, + pub(crate) logger: Logger, +} + +impl Step for ValidateCupStep { + fn descr(&self) -> String { + format!( + "Validate the CUP and the state downloaded from the {} subnet {}, and preserve the \ + subnet's public key and the state tree (with only the relevant paths) so that they \ + can be verified independently.", + self.target_subnet, self.subnet_id, + ) + } + + fn exec(&self) -> RecoveryResult<()> { + // 1. Get the subnet's public key using `ic-agent` and persist it on disk. + info!(self.logger, "Getting the NNS signed State Tree"); + let agent_helper = AgentHelper::new( + &self.nns_url, + Some(self.layout.nns_public_key_file()), + self.logger.clone(), + )?; + + let pruned_state_tree = agent_helper.read_subnet_data(self.subnet_id)?; + pruned_state_tree.save_to_file(&self.layout.pruned_state_tree_file(self.subnet_id))?; + pruned_state_tree + .save_public_key_to_file(&self.layout.subnet_public_key_file(self.subnet_id))?; + + // 2. Compute the manifest of the downloaded state. + info!(self.logger, "Computing the state manifest"); + let checkpoint_dir = self.layout.latest_checkpoint_dir(self.target_subnet)?; + let manifest_path = self.layout.actual_manifest_file(self.subnet_id); + + state_tool_helper::compute_manifest(&checkpoint_dir, &manifest_path)?; + + // 3. Validate all the artifacts (state tree, CUP, state manifest). + validate_artifacts( + self.layout.pruned_state_tree_file(self.subnet_id), + Some(self.layout.nns_public_key_file()), + self.layout.downloaded_cup_file(self.target_subnet), + manifest_path, + self.subnet_id, + &self.logger, + ) + } +} + +/// Assembles the merged state from the states the two subnets halted at: the +/// state of the destination subnet, with the canisters and canister snapshots +/// of the source subnet added to it, as a new checkpoint that the destination +/// subnet is then recovered at. +pub(crate) struct MergeStatesStep { + pub(crate) layout: Layout, + pub(crate) time_margin: Duration, + pub(crate) logger: Logger, +} + +impl Step for MergeStatesStep { + fn descr(&self) -> String { + format!( + "Assemble the merged state from the states downloaded to {} and {}, as a new \ + checkpoint in {}, and compute the height, the batch time and the state hash the \ + recovery of the destination subnet needs.", + self.layout.work_dir(TargetSubnet::Source).display(), + self.layout.work_dir(TargetSubnet::Destination).display(), + self.layout.work_dir(TargetSubnet::Merged).display(), + ) + } + + fn exec(&self) -> RecoveryResult<()> { + let source_height = self.layout.latest_checkpoint_height(TargetSubnet::Source)?; + let destination_height = self + .layout + .latest_checkpoint_height(TargetSubnet::Destination)?; + let source_checkpoint = self + .layout + .checkpoint_dir(TargetSubnet::Source, source_height); + let destination_checkpoint = self + .layout + .checkpoint_dir(TargetSubnet::Destination, destination_height); + + // The recovery CUP is created at the next recovery height above the one + // the destination subnet halted at, exactly as in any other recovery. + let merged_height = Recovery::get_recovery_height(destination_height); + let merged_checkpoint = self + .layout + .checkpoint_dir(TargetSubnet::Merged, merged_height); + + // The block time the recovered subnet starts from has to be larger than + // the times of both checkpoints the merged state is assembled from. + let source_time = state_tool_helper::checkpoint_time_nanos(&source_checkpoint)?; + let destination_time = state_tool_helper::checkpoint_time_nanos(&destination_checkpoint)?; + let merged_time = source_time.max(destination_time) + self.time_margin.as_nanos() as u64; + info!( + self.logger, + "The source subnet halted at height {source_height} and time {source_time}, the \ + destination subnet at height {destination_height} and time {destination_time}; the \ + merged state is the checkpoint {merged_height} and starts at time {merged_time}", + ); + + info!(self.logger, "Merging the states"); + state_tool_helper::merge_checkpoints( + &destination_checkpoint, + &source_checkpoint, + &merged_checkpoint, + )?; + + info!(self.logger, "Computing the manifest of the merged state"); + let manifest_path = self.layout.merged_state_manifest_file(); + state_tool_helper::compute_manifest(&merged_checkpoint, manifest_path)?; + + info!(self.logger, "Validating the manifest of the merged state"); + let manifest_hash = state_tool_helper::verify_manifest(manifest_path) + .map_err(|err| RecoveryError::validation_failed("Manifest verification failed", err))?; + let state_hash = get_state_hash(&merged_checkpoint)?; + if manifest_hash != state_hash { + return Err(RecoveryError::ValidationFailed(format!( + "The root hash {manifest_hash} of the manifest of the merged state differs from \ + the hash {state_hash} recomputed from the merged checkpoint", + ))); + } + + // The upload step transfers the states metadata alongside the + // checkpoint, and rsync is given every path it transfers as an explicit + // source, so a missing one fails the whole transfer: take the + // destination subnet's along. + // + // It is a manifest cache, which the state manager recomputes for the + // checkpoints it finds whenever it is missing or does not describe them, + // and the heights it names here are the ones the destination subnet held + // before the merge, none of which the merged state directory has. That is + // the same mismatch a plain subnet recovery uploads, where the metadata + // comes from the state that was downloaded and the checkpoint from the + // replay that followed. + std::fs::copy( + self.layout + .ic_state_dir(TargetSubnet::Destination) + .join(STATES_METADATA), + self.layout + .ic_state_dir(TargetSubnet::Merged) + .join(STATES_METADATA), + ) + .map_err(|err| { + RecoveryError::UnexpectedError(format!("Failed to copy the states metadata: {err}")) + })?; + + let params = MergedStateParams { + height: merged_height.get(), + time_nanos: merged_time, + state_hash, + }; + info!(self.logger, "The merged state: {params:?}"); + + params.write(self.layout.merged_state_params_file()) + } +} + +/// Reads the routing table, checks that the canister ID ranges of the subnet +/// that was merged away are hosted by the destination subnet now, and +/// determines (and persists) the registry version the merge was applied at. +pub(crate) struct CheckRoutingTableStep { + pub(crate) source_subnet_id: SubnetId, + pub(crate) destination_subnet_id: SubnetId, + pub(crate) registry_helper: RegistryHelper, + pub(crate) layout: Layout, + pub(crate) timeout: Duration, + pub(crate) poll_interval: Duration, + pub(crate) logger: Logger, +} + +impl Step for CheckRoutingTableStep { + fn descr(&self) -> String { + format!( + "Read the routing table to check that subnet {} hosts no canister id range anymore \ + and that subnet {} hosts them now.", + self.source_subnet_id, self.destination_subnet_id, + ) + } + + fn exec(&self) -> RecoveryResult<()> { + let source_subnet_id = self.source_subnet_id; + wait_for( + &format!("subnet {source_subnet_id} hosts no canister id range anymore"), + self.timeout, + self.poll_interval, + &self.logger, + || { + let (registry_version, routing_table) = self.registry_helper.get_routing_table()?; + let Some(routing_table) = routing_table else { + return Err(RecoveryError::RegistryError(format!( + "No routing table at registry version {registry_version}" + ))); + }; + Ok(source_subnet_is_empty(&routing_table, source_subnet_id)) + }, + )?; + + let (registry_version, routing_table) = self.registry_helper.get_routing_table()?; + let Some(routing_table) = routing_table else { + return Err(RecoveryError::RegistryError(format!( + "No routing table at registry version {registry_version}" + ))); + }; + info!( + self.logger, + "Canister id ranges of subnet {} at registry version {registry_version}: {:#?}", + self.destination_subnet_id, + routing_table.ranges(self.destination_subnet_id), + ); + + let registry_client = self.registry_helper.registry_client(); + let merge_registry_version = + first_registry_version_where(&self.registry_helper, |version| { + let routing_table = registry_client + .get_routing_table(version) + .map_err(|err| { + RecoveryError::RegistryError(format!( + "Failed to get the routing table at registry version {version}: {err}" + )) + })? + .ok_or_else(|| { + RecoveryError::RegistryError(format!( + "No routing table at registry version {version}" + )) + })?; + Ok(source_subnet_is_empty(&routing_table, source_subnet_id)) + })?; + + info!( + self.logger, + "Subnet {source_subnet_id} has been merged away as of registry version \ + {merge_registry_version}", + ); + + self.layout + .write_merge_registry_version(merge_registry_version.get()) + } +} + +fn source_subnet_is_empty(routing_table: &RoutingTable, source_subnet_id: SubnetId) -> bool { + routing_table.ranges(source_subnet_id).is_empty() +} + +/// Waits until every subnet other than the one that was merged away routes the +/// canisters of the merged subnet to the destination subnet, i.e. until it is +/// safe to delete the merged subnet. +pub(crate) struct CheckRegistryVersionOnAllSubnetsStep { + pub(crate) source_subnet_id: SubnetId, + pub(crate) registry_helper: RegistryHelper, + pub(crate) layout: Layout, + pub(crate) timeout: Duration, + pub(crate) poll_interval: Duration, + pub(crate) logger: Logger, +} + +impl Step for CheckRegistryVersionOnAllSubnetsStep { + fn descr(&self) -> String { + format!( + "Wait until every subnet other than {} has reached the registry version the merge \ + created, i.e. routes the canisters that used to be hosted by it to the destination \ + subnet.", + self.source_subnet_id, + ) + } + + fn exec(&self) -> RecoveryResult<()> { + let registry_version = self.layout.read_merge_registry_version()?; + + readiness::await_registry_version_on_all_subnets( + &self.registry_helper, + self.source_subnet_id, + registry_version, + self.timeout, + self.poll_interval, + &self.logger, + ) + } +} + +/// Waits until the destination subnet came up on the merged state, i.e. until +/// the node the merged state was uploaded to reports the recovery CUP. +pub(crate) struct WaitForRecoveryCupStep { + pub(crate) node_ip: IpAddr, + pub(crate) layout: Layout, + pub(crate) logger: Logger, +} + +impl Step for WaitForRecoveryCupStep { + fn descr(&self) -> String { + format!( + "Wait until node {} reports the recovery CUP holding the merged state.", + self.node_ip, + ) + } + + fn exec(&self) -> RecoveryResult<()> { + let params = MergedStateParams::read(self.layout.merged_state_params_file())?; + + Recovery::wait_for_recovery_cup( + &self.logger, + self.node_ip, + Height::from(params.height), + params.state_hash, + ) + } +} diff --git a/rs/recovery/subnet_merging/src/subnet_merging.rs b/rs/recovery/subnet_merging/src/subnet_merging.rs new file mode 100644 index 000000000000..53b8c72cf782 --- /dev/null +++ b/rs/recovery/subnet_merging/src/subnet_merging.rs @@ -0,0 +1,754 @@ +use crate::{ + admin_helper::{ + get_halt_subnet_at_cup_height_command, get_propose_to_cool_down_subnet_command, + get_propose_to_delete_subnet_command, get_propose_to_merge_subnets_command, + }, + layout::Layout, + steps::{ + CheckCoolingDownStep, CheckMergeReadinessStep, CheckRegistryVersionOnAllSubnetsStep, + CheckRoutingTableStep, MergeStatesStep, ValidateCupStep, WaitForHaltingCupStep, + WaitForRecoveryCupStep, + }, + target_subnet::TargetSubnet, + utils::MergedStateParams, +}; + +use clap::Parser; +use ic_base_types::SubnetId; +use ic_protobuf::registry::subnet::v1::SubnetRecord; +use ic_recovery::{ + CUPS_DIR, NeuronArgs, Recovery, RecoveryArgs, + cli::{consent_given, read_optional, wait_for_confirmation}, + error::{RecoveryError, RecoveryResult}, + recovery_iterator::RecoveryIterator, + recovery_state::{HasRecoveryState, RecoveryState}, + registry_helper::RegistryPollingStrategy, + ssh_helper::SshHelper, + steps::{AdminStep, DownloadIcDataStep, Step, UploadStateAndRestartStep}, + util::{CheckpointHeight, DataLocation, ExecutionMode, SshUser}, +}; +use ic_registry_subnet_type::SubnetType; +use ic_types::Height; +use serde::{Deserialize, Serialize}; +use slog::{Logger, error, info, warn}; +use strum::{EnumMessage, IntoEnumIterator}; +use strum_macros::{EnumIter, EnumString}; +use url::Url; + +use std::{ + iter::Peekable, + net::IpAddr, + path::PathBuf, + time::{Duration, UNIX_EPOCH}, +}; + +const SUBNET_TYPE_ALLOW_LIST: [SubnetType; 2] = + [SubnetType::Application, SubnetType::VerifiedApplication]; + +#[derive( + Copy, + Clone, + PartialEq, + Debug, + Deserialize, + EnumIter, + EnumMessage, + EnumString, + Serialize, + clap::ValueEnum, +)] +pub enum StepType { + CoolDownSourceSubnet, + CheckRegistryForCoolingDownFlag, + CheckMergeReadiness, + HaltSourceSubnetAtCupHeight, + HaltDestinationSubnetAtCupHeight, + WaitForHaltingCupOnSourceSubnet, + WaitForHaltingCupOnDestinationSubnet, + StopSourceReplica, + StopDestinationReplica, + DownloadStateFromSourceSubnet, + DownloadStateFromDestinationSubnet, + ValidateSourceSubnetCup, + ValidateDestinationSubnetCup, + MergeStates, + MergeSubnets, + CheckRegistryForRoutingTableEntry, + ProposeCupForDestinationSubnet, + UploadStateToDestinationSubnet, + WaitForCUPOnDestinationSubnet, + UnhaltDestinationSubnet, + CheckRegistryVersionOnAllSubnets, + DeleteSourceSubnet, + Cleanup, +} + +#[derive(Clone, PartialEq, Debug, Deserialize, Parser, Serialize)] +#[clap(version = "1.0")] +pub struct SubnetMergingArgs { + /// Id of the subnet that is cooling down, whose canisters are merged into + /// the destination subnet and which is deleted afterwards. + #[clap(long, value_parser=ic_recovery::util::subnet_id_from_str)] + pub source_subnet_id: SubnetId, + + /// Id of the subnet that hosts the canisters of the source subnet after the + /// merge, and that is recovered at the merged state. + #[clap(long, value_parser=ic_recovery::util::subnet_id_from_str)] + pub destination_subnet_id: SubnetId, + + /// Public ssh key to be deployed to both subnets for read only access. + #[clap(long)] + pub readonly_pub_key: Option, + + /// The path to a file containing the private key associated with `readonly_pub_key`. + #[clap(long)] + pub readonly_key_file: Option, + + /// If the downloaded states should be backed up locally. + #[clap(long)] + pub keep_downloaded_state: Option, + + /// IP address of the node of the source subnet to download the state from. + #[clap(long)] + pub download_node_source: Option, + + /// IP address of the node of the destination subnet to download the state from. + #[clap(long)] + pub download_node_destination: Option, + + /// IP address of the node of the destination subnet to upload the merged state to. + #[clap(long)] + pub upload_node_destination: Option, + + /// How much later than the checkpoints it is assembled from the merged + /// state starts, in seconds. + #[clap(long, default_value = "60")] + pub time_margin_secs: u64, + + /// How long to wait for the subnet that is cooling down to become ready to + /// be merged, in seconds. + #[clap(long, default_value = "2400")] + pub merge_ready_timeout_secs: u64, + + /// How long to wait for a subnet to reach the CUP it halts at, in seconds. + #[clap(long, default_value = "900")] + pub halt_timeout_secs: u64, + + /// How long to wait for the registry to reflect a proposal that `ic-admin` + /// reported as executed, in seconds. + #[clap(long, default_value = "300")] + pub registry_timeout_secs: u64, + + /// How long to wait between two evaluations of a condition this tool waits + /// for, in seconds. + #[clap(long, default_value = "10")] + pub poll_interval_secs: u64, + + /// If present the tool will start execution for the provided step, skipping the initial ones. + #[clap(long = "resume")] + #[clap(value_enum)] + pub next_step: Option, +} + +pub struct SubnetMerging { + step_iterator: Peekable, + params: SubnetMergingArgs, + recovery_args: RecoveryArgs, + neuron_args: Option, + recovery: Recovery, + layout: Layout, + logger: Logger, +} + +impl SubnetMerging { + pub fn new( + logger: Logger, + recovery_args: RecoveryArgs, + neuron_args: Option, + subnet_merging_args: SubnetMergingArgs, + ) -> Self { + let recovery = Recovery::new( + logger.clone(), + recovery_args.clone(), + neuron_args.clone(), + recovery_args.nns_url.clone(), + RegistryPollingStrategy::WithEveryRead, + ) + .expect("Failed to initialize recovery"); + + Self::check_subnets_preconditions( + &recovery, + subnet_merging_args.source_subnet_id, + subnet_merging_args.destination_subnet_id, + ) + .expect("Subnets should satisfy all the preconditions"); + + let layout = Layout::new(&recovery); + layout + .create_dirs() + .expect("Failed to create the working directories"); + + Self { + step_iterator: StepType::iter().peekable(), + params: subnet_merging_args, + recovery_args, + neuron_args, + layout, + recovery, + logger, + } + } + + /// Checks whether the subnets satisfy the following preconditions: + /// + /// Both subnets: + /// 1) Are `Application` (or `VerifiedApplication`) subnets; + /// 2) Are not Chain key subnets; + /// 3) Are not halted: this tool halts them itself, at their next CUP; + /// 4) Have the same subnet type, as the merged state inherits the + /// destination subnet's. + /// + /// And they are two different subnets. Unlike subnet splitting, the + /// destination subnet is a subnet in operation: it keeps serving its own + /// canisters across the merge, and it may or may not be empty. + fn check_subnets_preconditions( + recovery: &Recovery, + source_subnet_id: SubnetId, + destination_subnet_id: SubnetId, + ) -> RecoveryResult<()> { + if source_subnet_id == destination_subnet_id { + return Err(RecoveryError::ValidationFailed(format!( + "A subnet cannot be merged into itself ({source_subnet_id})" + ))); + } + + let source_subnet_record = + Self::get_and_pre_validate_subnet_record(recovery, source_subnet_id, None)?; + + let _ = Self::get_and_pre_validate_subnet_record( + recovery, + destination_subnet_id, + Some(source_subnet_record), + )?; + + Ok(()) + } + + fn get_and_pre_validate_subnet_record( + recovery: &Recovery, + subnet_id: SubnetId, + other_subnet_record: Option, + ) -> RecoveryResult { + let validation_error = |error_message| { + Err(RecoveryError::ValidationFailed(format!( + "Subnet {subnet_id}: {error_message}" + ))) + }; + + let (_, Some(subnet_record)) = recovery.registry_helper.get_subnet_record(subnet_id)? + else { + return validation_error("Subnet Record should not be empty".to_string()); + }; + + if subnet_record + .chain_key_config + .as_ref() + .is_some_and(|chain_key_config| !chain_key_config.key_configs.is_empty()) + { + return validation_error("Subnet should not be a Chain key subnet".to_string()); + } + + let subnet_type = subnet_record + .subnet_type() + .try_into() + .expect("Unexpected subnet type"); + + if !SUBNET_TYPE_ALLOW_LIST.contains(&subnet_type) { + return validation_error(format!( + "Subnet's type ({subnet_type:?}) is not allowed for subnet merging. Allowlist: {SUBNET_TYPE_ALLOW_LIST:?}", + )); + } + + if subnet_record.is_halted { + return validation_error(String::from( + "Subnet should not be halted: subnet merging halts both subnets itself, at the \ + CUP whose state it merges", + )); + } + + if let Some(other_subnet_record) = other_subnet_record + && subnet_record.subnet_type() != other_subnet_record.subnet_type() + { + return validation_error(format!( + "Both subnets should have the same subnet type. \ + Expected subnet type = {:?}, actual subnet type = {:?}", + other_subnet_record.subnet_type(), + subnet_record.subnet_type(), + )); + } + + Ok(subnet_record) + } + + fn wait_for_halting_cup_step( + &self, + target_subnet: TargetSubnet, + ) -> RecoveryResult> { + let Some(node_ip) = self.download_node(target_subnet) else { + return Err(RecoveryError::StepSkipped); + }; + + Ok(WaitForHaltingCupStep { + subnet_id: self.subnet_id(target_subnet), + node_ip, + registry_helper: self.recovery.registry_helper.clone(), + layout: self.layout.clone(), + ssh_helper: self.ssh_helper(node_ip), + timeout: Duration::from_secs(self.params.halt_timeout_secs), + poll_interval: Duration::from_secs(self.params.poll_interval_secs), + logger: self.recovery.logger.clone(), + }) + } + + fn stop_replica_step(&self, target_subnet: TargetSubnet) -> RecoveryResult> { + // The state manager of a running replica owns its state directory, even + // while consensus is halted, so the replica has to be stopped before the + // state is downloaded. The destination subnet's replica stays stopped + // until the merged state is uploaded to it; the source subnet's stays + // stopped for good, as that subnet is deleted at the end of the merge. + match self.download_node(target_subnet) { + Some(node_ip) => Ok(self.recovery.get_stop_replica_step(node_ip)), + None => Err(RecoveryError::StepSkipped), + } + } + + /// Downloads the state a subnet halted at into that subnet's working + /// directory. + /// + /// `DownloadIcDataStep` rather than `Recovery::get_download_data_step`: a + /// merge downloads two states, and the latter always downloads into the + /// single working directory a `Recovery` has. + fn download_state_step( + &self, + target_subnet: TargetSubnet, + ) -> RecoveryResult> { + let Some(node_ip) = self.download_node(target_subnet) else { + return Err(RecoveryError::StepSkipped); + }; + + let mut ssh_helper = self.ssh_helper(node_ip); + if ssh_helper.wait_for_access().is_err() { + ssh_helper.ssh_user = SshUser::Admin; + ssh_helper.key_file = self.recovery.admin_key_file.clone(); + if !ssh_helper.can_connect() { + return Err(RecoveryError::UnexpectedError(format!( + "SSH access to node {node_ip} denied" + ))); + } + } + info!( + self.recovery.logger, + "Continuing with account: {}", ssh_helper.ssh_user + ); + + let mut includes = Recovery::get_ic_state_includes( + &self.recovery.logger, + ExecutionMode::Remote(&ssh_helper), + CheckpointHeight::Latest, + )?; + // The CUP the subnet halted at, which the validation step checks the + // downloaded state against. + includes.push(PathBuf::from(CUPS_DIR)); + + Ok(DownloadIcDataStep { + logger: self.recovery.logger.clone(), + ssh_helper, + backup_dir: self.layout.original_data_dir(target_subnet), + working_dir: self.layout.work_dir(target_subnet), + keep_downloaded_data: self.params.keep_downloaded_state == Some(true), + data_includes: includes, + include_config: true, + }) + } + + fn validate_cup_step(&self, target_subnet: TargetSubnet) -> impl Step + use<> { + ValidateCupStep { + subnet_id: self.subnet_id(target_subnet), + target_subnet, + nns_url: self.recovery_args.nns_url.clone(), + layout: self.layout.clone(), + logger: self.recovery.logger.clone(), + } + } + + /// The recovery CUP of the destination subnet, at the merged state. + fn propose_cup(&self) -> RecoveryResult> { + let params = MergedStateParams::read(self.layout.merged_state_params_file())?; + + self.recovery.update_recovery_cup( + self.params.destination_subnet_id, + Height::from(params.height), + params.state_hash, + /*replacement_nodes=*/ &[], + /*registry_params=*/ None, + // The merged subnets are both unavailable while the recovery CUP is + // created, so its DKG is handled by whichever subnet the NNS picks + // by default, which is neither of them. + /*initial_dkg_subnet_id=*/ + None, + /*chain_key_subnet_id=*/ None, + Some(UNIX_EPOCH + Duration::from_nanos(params.time_nanos)), + ) + } + + fn upload_state_and_restart_step(&self) -> RecoveryResult> { + match self.params.upload_node_destination { + // The replica of the destination subnet has been stopped before its + // state was downloaded, so this step replaces its state directory + // and starts it back up on the merged state. + Some(node_ip) => Ok(UploadStateAndRestartStep { + logger: self.recovery.logger.clone(), + ssh_user: SshUser::Admin, + upload_method: DataLocation::Remote(node_ip), + work_dir: self.layout.work_dir(TargetSubnet::Merged), + data_src: self.layout.ic_state_dir(TargetSubnet::Merged), + require_confirmation: !self.recovery_args.skip_prompts, + key_file: self.recovery.admin_key_file.clone(), + check_ic_replay_height: false, + }), + None => Err(RecoveryError::StepSkipped), + } + } + + fn wait_for_recovery_cup_step(&self) -> RecoveryResult> { + match self.params.upload_node_destination { + Some(node_ip) => Ok(WaitForRecoveryCupStep { + node_ip, + layout: self.layout.clone(), + logger: self.recovery.logger.clone(), + }), + None => Err(RecoveryError::StepSkipped), + } + } + + fn ssh_helper(&self, node_ip: IpAddr) -> SshHelper { + let (ssh_user, key_file) = if self.params.readonly_pub_key.is_some() { + (SshUser::Readonly, self.params.readonly_key_file.clone()) + } else { + (SshUser::Admin, self.recovery.admin_key_file.clone()) + }; + + SshHelper::new( + self.recovery.logger.clone(), + ssh_user, + node_ip, + self.recovery.ssh_confirmation, + key_file, + ) + } + + fn download_node(&self, target_subnet: TargetSubnet) -> Option { + match target_subnet { + TargetSubnet::Source => self.params.download_node_source, + TargetSubnet::Destination => self.params.download_node_destination, + TargetSubnet::Merged => None, + } + } + + fn subnet_id(&self, target_subnet: TargetSubnet) -> SubnetId { + match target_subnet { + TargetSubnet::Source => self.params.source_subnet_id, + TargetSubnet::Destination | TargetSubnet::Merged => self.params.destination_subnet_id, + } + } + + fn dashboard_url(&self, dashboard: &str) -> String { + match self.recovery.registry_helper.latest_registry_version() { + Ok(registry_version) => format!( + "https://grafana.mainnet.dfinity.network/d/{dashboard}?var-datasource=IC+Metrics&var-ic=mercury&var-ic_subnet={}&var-registry_version={}", + self.params.source_subnet_id, registry_version, + ), + Err(err) => { + warn!( + self.logger, + "Failed to get the latest registry version: {}", err + ); + format!( + "https://grafana.mainnet.dfinity.network/d/{dashboard}?var-datasource=IC+Metrics&var-ic=mercury&var-ic_subnet={}", + self.params.source_subnet_id, + ) + } + } + } +} + +impl RecoveryIterator for SubnetMerging { + fn get_step_iterator(&mut self) -> &mut Peekable { + &mut self.step_iterator + } + + fn store_next_step(&mut self, step_type: Option) { + self.params.next_step = step_type; + } + + fn get_logger(&self) -> &Logger { + &self.logger + } + + fn interactive(&self) -> bool { + !self.recovery_args.skip_prompts + } + + fn read_step_params(&mut self, step_type: StepType) { + match step_type { + StepType::CoolDownSourceSubnet => { + if self.params.readonly_pub_key.is_none() { + self.params.readonly_pub_key = read_optional( + &self.logger, + "Enter public key to add readonly SSH access to both subnets. Ensure the right format.\n\ + Format: ssh-ed25519 \n\ + Example: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPwS/0S6xH0g/xLDV0Tz7VeMZE9AKPeSbLmCsq9bY3F1 foo@dfinity.org\n\ + Enter your key: ", + ) + } + } + + StepType::CheckMergeReadiness => { + print_url_and_ask_for_confirmation( + &self.logger, + self.dashboard_url("subnet-merging"), + "The tool evaluates the merge readiness condition itself, and waits until it \ + holds. Please check the dashboard as well, to see whether it is safe to \ + merge the subnet", + ); + } + + StepType::DownloadStateFromSourceSubnet => { + if self.params.download_node_source.is_none() { + self.params.download_node_source = read_optional( + &self.logger, + "Enter the IP of the node of the Source Subnet to download the state from:", + ); + } + + self.params.keep_downloaded_state = Some(consent_given( + &self.logger, + "Preserve original downloaded state locally?", + )); + } + + StepType::DownloadStateFromDestinationSubnet => { + if self.params.download_node_destination.is_none() { + self.params.download_node_destination = read_optional( + &self.logger, + "Enter the IP of the node of the Destination Subnet to download the state from:", + ); + } + } + + #[allow(clippy::collapsible_match)] + StepType::UploadStateToDestinationSubnet => { + if self.params.upload_node_destination.is_none() { + self.params.upload_node_destination = read_optional( + &self.logger, + "Enter IP of node in the Destination Subnet with admin access: ", + ); + } + } + + StepType::DeleteSourceSubnet => { + print_url_and_ask_for_confirmation( + &self.logger, + self.dashboard_url("subnet-merging"), + "Please check the dashboard to see if it is safe to delete the subnet that \ + was merged away", + ); + } + + _ => (), + } + } + + fn get_step_impl(&self, step_type: StepType) -> RecoveryResult> { + let step: Box = match step_type { + StepType::CoolDownSourceSubnet => AdminStep { + logger: self.recovery.logger.clone(), + ic_admin_cmd: get_propose_to_cool_down_subnet_command( + &self.recovery.admin_helper, + self.params.source_subnet_id, + &self.params.readonly_pub_key, + ), + } + .into(), + + StepType::CheckRegistryForCoolingDownFlag => CheckCoolingDownStep { + source_subnet_id: self.params.source_subnet_id, + registry_helper: self.recovery.registry_helper.clone(), + layout: self.layout.clone(), + timeout: Duration::from_secs(self.params.registry_timeout_secs), + poll_interval: Duration::from_secs(self.params.poll_interval_secs), + logger: self.recovery.logger.clone(), + } + .into(), + + StepType::CheckMergeReadiness => CheckMergeReadinessStep { + source_subnet_id: self.params.source_subnet_id, + registry_helper: self.recovery.registry_helper.clone(), + layout: self.layout.clone(), + timeout: Duration::from_secs(self.params.merge_ready_timeout_secs), + poll_interval: Duration::from_secs(self.params.poll_interval_secs), + logger: self.recovery.logger.clone(), + } + .into(), + + StepType::HaltSourceSubnetAtCupHeight => AdminStep { + logger: self.recovery.logger.clone(), + ic_admin_cmd: get_halt_subnet_at_cup_height_command( + &self.recovery.admin_helper, + self.params.source_subnet_id, + &self.params.readonly_pub_key, + ), + } + .into(), + + StepType::HaltDestinationSubnetAtCupHeight => AdminStep { + logger: self.recovery.logger.clone(), + ic_admin_cmd: get_halt_subnet_at_cup_height_command( + &self.recovery.admin_helper, + self.params.destination_subnet_id, + &self.params.readonly_pub_key, + ), + } + .into(), + + StepType::WaitForHaltingCupOnSourceSubnet => { + self.wait_for_halting_cup_step(TargetSubnet::Source)?.into() + } + StepType::WaitForHaltingCupOnDestinationSubnet => self + .wait_for_halting_cup_step(TargetSubnet::Destination)? + .into(), + + StepType::StopSourceReplica => self.stop_replica_step(TargetSubnet::Source)?.into(), + StepType::StopDestinationReplica => { + self.stop_replica_step(TargetSubnet::Destination)?.into() + } + + StepType::DownloadStateFromSourceSubnet => { + self.download_state_step(TargetSubnet::Source)?.into() + } + StepType::DownloadStateFromDestinationSubnet => { + self.download_state_step(TargetSubnet::Destination)?.into() + } + + StepType::ValidateSourceSubnetCup => { + self.validate_cup_step(TargetSubnet::Source).into() + } + StepType::ValidateDestinationSubnetCup => { + self.validate_cup_step(TargetSubnet::Destination).into() + } + + StepType::MergeStates => MergeStatesStep { + layout: self.layout.clone(), + time_margin: Duration::from_secs(self.params.time_margin_secs), + logger: self.recovery.logger.clone(), + } + .into(), + + StepType::MergeSubnets => AdminStep { + logger: self.recovery.logger.clone(), + ic_admin_cmd: get_propose_to_merge_subnets_command( + &self.recovery.admin_helper, + self.params.source_subnet_id, + self.params.destination_subnet_id, + ), + } + .into(), + + StepType::CheckRegistryForRoutingTableEntry => CheckRoutingTableStep { + source_subnet_id: self.params.source_subnet_id, + destination_subnet_id: self.params.destination_subnet_id, + registry_helper: self.recovery.registry_helper.clone(), + layout: self.layout.clone(), + timeout: Duration::from_secs(self.params.registry_timeout_secs), + poll_interval: Duration::from_secs(self.params.poll_interval_secs), + logger: self.recovery.logger.clone(), + } + .into(), + + StepType::ProposeCupForDestinationSubnet => self.propose_cup()?.into(), + StepType::UploadStateToDestinationSubnet => { + self.upload_state_and_restart_step()?.into() + } + StepType::WaitForCUPOnDestinationSubnet => self.wait_for_recovery_cup_step()?.into(), + + StepType::UnhaltDestinationSubnet => self + .recovery + .bring_subnet_back_online_after_repairs(self.params.destination_subnet_id) + .into(), + + StepType::CheckRegistryVersionOnAllSubnets => CheckRegistryVersionOnAllSubnetsStep { + source_subnet_id: self.params.source_subnet_id, + registry_helper: self.recovery.registry_helper.clone(), + layout: self.layout.clone(), + timeout: Duration::from_secs(self.params.merge_ready_timeout_secs), + poll_interval: Duration::from_secs(self.params.poll_interval_secs), + logger: self.recovery.logger.clone(), + } + .into(), + + StepType::DeleteSourceSubnet => AdminStep { + logger: self.recovery.logger.clone(), + ic_admin_cmd: get_propose_to_delete_subnet_command( + &self.recovery.admin_helper, + self.params.source_subnet_id, + ), + } + .into(), + + StepType::Cleanup => self.recovery.get_cleanup_step().into(), + }; + + Ok(step) + } +} + +impl Iterator for SubnetMerging { + type Item = (StepType, Box); + fn next(&mut self) -> Option { + self.next_step() + } +} + +impl HasRecoveryState for SubnetMerging { + type StepType = StepType; + type SubcommandArgsType = SubnetMergingArgs; + + fn get_next_step(&self) -> Option { + self.params.next_step + } + + fn get_state(&self) -> RecoveryResult> { + Ok(RecoveryState { + recovery_args: self.recovery_args.clone(), + neuron_args: self.neuron_args.clone(), + subcommand_args: self.params.clone(), + }) + } +} + +fn print_url_and_ask_for_confirmation( + logger: &Logger, + url: String, + text_to_display: impl std::fmt::Display, +) { + match Url::parse(&url) { + Ok(url) => { + warn!(logger, "{}", text_to_display); + warn!(logger, "{}", url); + wait_for_confirmation(logger); + } + Err(err) => { + error!(logger, "Failed to parse url {}: {}", url, err); + } + } +} diff --git a/rs/recovery/subnet_merging/src/target_subnet.rs b/rs/recovery/subnet_merging/src/target_subnet.rs new file mode 100644 index 000000000000..b4fc210963b5 --- /dev/null +++ b/rs/recovery/subnet_merging/src/target_subnet.rs @@ -0,0 +1,25 @@ +/// The three states a subnet merge works with: the two states that are +/// downloaded and the merged one that is assembled from them. +#[derive(Copy, Clone, PartialEq, Debug)] +pub(crate) enum TargetSubnet { + /// The subnet that is cooling down, whose canisters are merged away and + /// which is deleted afterwards. + Source, + /// The subnet that hosts the canisters of the source subnet after the merge + /// and that is recovered at the merged state. + Destination, + /// Not a subnet of its own: the state assembled from the states of the two + /// subnets above, which the destination subnet is recovered at. + Merged, +} + +impl std::fmt::Display for TargetSubnet { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let name = match self { + TargetSubnet::Source => "source", + TargetSubnet::Destination => "destination", + TargetSubnet::Merged => "merged", + }; + write!(f, "{name}") + } +} diff --git a/rs/recovery/subnet_merging/src/utils.rs b/rs/recovery/subnet_merging/src/utils.rs new file mode 100644 index 000000000000..9d5a83a49ab6 --- /dev/null +++ b/rs/recovery/subnet_merging/src/utils.rs @@ -0,0 +1,172 @@ +use ic_base_types::RegistryVersion; +use ic_protobuf::types::v1 as pb; +use ic_recovery::{ + RECOVERY_DIRECTORY_NAME, + error::{RecoveryError, RecoveryResult}, + file_sync_helper::{read_file, write_file}, + registry_helper::RegistryHelper, +}; +use ic_state_manager::manifest::{manifest_from_path, manifest_hash}; +use ic_types::consensus::CatchUpPackage; +use serde::{Deserialize, Serialize}; + +use std::{fmt::Display, path::Path}; + +/// Everything the recovery of the destination subnet needs to know about the +/// merged state, as computed by the step that assembles it. +/// +/// Persisted next to the merged state so that the steps proposing the recovery +/// CUP and waiting for it agree on the same values, and so that a resumed run +/// neither re-merges the states nor re-hashes the merged checkpoint. +#[derive(Clone, PartialEq, Debug, Deserialize, Serialize)] +pub struct MergedStateParams { + /// The height of the merged checkpoint, i.e. the height the destination + /// subnet is recovered at. + pub height: u64, + /// The block time the recovered destination subnet starts from, in + /// nanoseconds since the Epoch. Larger than the batch times of both + /// checkpoints the merged state was assembled from. + pub time_nanos: u64, + /// The root hash of the manifest of the merged state. + pub state_hash: String, +} + +impl MergedStateParams { + pub fn read(path: &Path) -> RecoveryResult { + let contents = read_file(path)?; + serde_json::from_str(&contents).map_err(|err| { + RecoveryError::UnexpectedError(format!( + "Failed to parse the merged state params at {}: {err}", + path.display() + )) + }) + } + + pub fn write(&self, path: &Path) -> RecoveryResult<()> { + let contents = serde_json::to_string_pretty(self).map_err(|err| { + RecoveryError::UnexpectedError(format!( + "Failed to serialize the merged state params: {err}" + )) + })?; + write_file(path, contents) + } +} + +/// The lowest registry version at which `predicate` holds, in a registry whose +/// history has it hold from some version onwards -- e.g. "the subnet is labeled +/// cooling down" or "the subnet hosts no canister id range anymore", both of +/// which a single proposal of this tool brings about and nothing here undoes. +/// +/// Found by binary search, i.e. in a logarithmic number of (local) registry +/// reads rather than by scanning the whole history. Fails if the predicate does +/// not hold at the latest registry version; if the property was turned on and +/// off repeatedly, the version returned is the start of one of the stretches it +/// held in, which is why the callers log it for the operator to confirm. +pub(crate) fn first_registry_version_where( + registry_helper: &RegistryHelper, + predicate: impl Fn(RegistryVersion) -> RecoveryResult, +) -> RecoveryResult { + let latest = registry_helper.latest_registry_version()?; + if !predicate(latest)? { + return Err(RecoveryError::ValidationFailed(format!( + "The expected registry state has not been reached at the latest registry version \ + {latest}" + ))); + } + + // Invariant: the predicate does not hold at `low` and holds at `high`. The + // registry starts at version 1, whose predecessor 0 is the empty registry, + // where nothing holds. + let mut low = RegistryVersion::from(0); + let mut high = latest; + while high > low + RegistryVersion::from(1) { + let middle = RegistryVersion::from((low.get() + high.get()) / 2); + if predicate(middle)? { + high = middle; + } else { + low = middle; + } + } + + Ok(high) +} + +/// The file the tool records the registry version at which the subnet that is +/// merged away was labeled "cooling down" in, i.e. the `V` of the merge +/// readiness condition. +pub const COOLING_DOWN_REGISTRY_VERSION_FILE: &str = "cooling_down_registry_version"; +/// The file the tool records the registry version the merge was applied at in. +pub const MERGE_REGISTRY_VERSION_FILE: &str = "merge_registry_version"; + +pub fn write_registry_version(path: &Path, version: u64) -> RecoveryResult<()> { + write_file(path, version.to_string()) +} + +pub fn read_registry_version(path: &Path) -> RecoveryResult { + let contents = read_file(path)?; + contents.trim().parse().map_err(|err| { + RecoveryError::UnexpectedError(format!( + "Failed to parse the registry version {contents:?} at {}: {err}", + path.display() + )) + }) +} + +/// The registry version at which the subnet that is merged away was labeled +/// "cooling down", as recorded by the tool working in `dir`, i.e. in the +/// directory that was passed to it as `RecoveryArgs::dir`. +pub fn read_cooling_down_registry_version(dir: &Path) -> RecoveryResult { + read_registry_version( + &dir.join(RECOVERY_DIRECTORY_NAME) + .join(COOLING_DOWN_REGISTRY_VERSION_FILE), + ) +} + +pub(crate) fn get_cup(cup_path: &Path) -> RecoveryResult { + let cup_proto = pb::CatchUpPackage::read_from_file(cup_path) + .map_err(|err| cup_error("Failed to decode the CUP file", cup_path, err))?; + + CatchUpPackage::try_from(&cup_proto) + .map_err(|err| cup_error("Failed to deserialize the CUP file", cup_path, err)) +} + +fn cup_error(message: impl Display, cup_path: &Path, error: impl Display) -> RecoveryError { + RecoveryError::UnexpectedError(format!("{} ({}): {}", message, cup_path.display(), error)) +} + +/// Computes the state hash of the given checkpoint. +pub(crate) fn get_state_hash(checkpoint_dir: impl AsRef) -> RecoveryResult { + let manifest = manifest_from_path(checkpoint_dir.as_ref()).map_err(|e| { + RecoveryError::CheckpointError( + format!( + "Failed to read the manifest from path {}", + checkpoint_dir.as_ref().display() + ), + e, + ) + })?; + + Ok(hex::encode(manifest_hash(&manifest))) +} + +#[cfg(test)] +mod tests { + use super::*; + + use ic_test_utilities_tmpdir::tmpdir; + + #[test] + fn merged_state_params_round_trip_test() { + let dir = tmpdir("test_dir"); + let path = dir.as_ref().join("merged_state_params.json"); + + let params = MergedStateParams { + height: 42_000, + time_nanos: 1_700_000_000_000_000_000, + state_hash: "deadbeef".to_string(), + }; + params.write(&path).unwrap(); + + assert_eq!(MergedStateParams::read(&path).unwrap(), params); + } +} diff --git a/rs/recovery/subnet_merging/src/validation.rs b/rs/recovery/subnet_merging/src/validation.rs new file mode 100644 index 000000000000..31dd259ba71b --- /dev/null +++ b/rs/recovery/subnet_merging/src/validation.rs @@ -0,0 +1,177 @@ +use crate::{ + agent_helper::{AgentHelper, StateTree}, + state_tool_helper, + utils::get_cup, +}; + +use ic_base_types::SubnetId; +use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; +use ic_recovery::error::{RecoveryError, RecoveryResult}; +use ic_types::{consensus::HasHeight, crypto::threshold_sig::ThresholdSigPublicKey}; +use slog::{Logger, error, info}; +use url::Url; + +use std::{fmt::Display, path::Path}; + +/// Validates the following artifacts of a subnet that was halted at a CUP: +/// 1. NNS signed State Tree; +/// 2. the CUP the subnet halted at; +/// 3. the manifest of the state the subnet halted at. +pub fn validate_artifacts( + state_tree_path: impl AsRef, + nns_public_key_path: Option<&Path>, + cup_path: impl AsRef, + state_manifest_path: impl AsRef, + subnet_id: SubnetId, + logger: &Logger, +) -> RecoveryResult<()> { + let validated_subnet_public_key = validation_helper( + "State Tree signed by the NNS", + "extracted authentic subnet key from the NNS state tree", + logger, + || { + validate_state_tree_and_extract_subnet_public_key( + state_tree_path.as_ref(), + nns_public_key_path, + subnet_id, + logger, + ) + }, + )?; + + let state_hash = validation_helper( + "the CUP the subnet halted at", + "the CUP signature is valid", + logger, + || { + validate_cup_and_extract_state_hash( + cup_path.as_ref(), + &validated_subnet_public_key, + logger, + ) + }, + )?; + + validation_helper( + "the manifest of the state the subnet halted at", + "recomputed manifest root hash matches the one in the CUP", + logger, + || validate_manifest(state_manifest_path.as_ref(), &state_hash, logger), + )?; + + Ok(()) +} + +fn validate_cup_and_extract_state_hash( + cup_path: &Path, + subnet_public_key: &ThresholdSigPublicKey, + logger: &Logger, +) -> RecoveryResult { + let cup = get_cup(cup_path)?; + + if let Some((_, transcript)) = cup + .content + .block + .as_ref() + .payload + .as_ref() + .as_summary() + .dkg + .current_transcripts() + .iter() + .next() + { + info!( + logger, + "Dealer subnet from the CUP: {}", transcript.dkg_id.dealer_subnet + ); + } + info!(logger, "CUP height: {}", &cup.content.height()); + + let block_time = cup.content.block.as_ref().context.time; + + info!( + logger, + "Block time from the CUP: {} (nanos since unix epoch: {})", + block_time, + block_time.as_nanos_since_unix_epoch() + ); + + let state_hash = hex::encode(&cup.content.state_hash.get_ref().0); + info!(logger, "State hash from the CUP: {}", state_hash); + + ic_crypto_utils_threshold_sig::verify_combined( + &cup.content, + &cup.signature.signature, + subnet_public_key, + ) + .map_err(|err| { + RecoveryError::ValidationFailed(format!("Failed to validate the CUP signature: {err}")) + })?; + + Ok(state_hash) +} + +fn validate_state_tree_and_extract_subnet_public_key( + state_tree_path: &Path, + nns_public_key_path: Option<&Path>, + subnet_id: SubnetId, + logger: &Logger, +) -> RecoveryResult { + let agent_helper = AgentHelper::new( + &Url::parse("https://ic0.app").unwrap(), + nns_public_key_path, + logger.clone(), + )?; + + let state_tree = StateTree::read_from_file(state_tree_path, subnet_id)?; + + agent_helper.validate_state_tree(&state_tree)?; + + let bytes = state_tree.lookup_public_key()?; + + parse_threshold_sig_key_from_der(bytes).map_err(|err| { + RecoveryError::UnexpectedError(format!("Failed to parse the public key bytes: {err}")) + }) +} + +fn validate_manifest( + state_manifest_path: &Path, + state_hash_from_cup: &String, + logger: &Logger, +) -> RecoveryResult<()> { + let state_hash = state_tool_helper::verify_manifest(state_manifest_path).map_err(|err| { + RecoveryError::validation_failed("Failed to validate the state manifest", err) + })?; + + info!(logger, "state hash from the CUP: {}", state_hash_from_cup); + info!(logger, "state hash from the State Manifest: {}", state_hash); + + if state_hash != *state_hash_from_cup { + return Err(RecoveryError::validation_failed( + "Failed to validate the state manifest", + "hash from the state manifest differs from the state hash from the CUP", + )); + } + + Ok(()) +} + +fn validation_helper( + label: impl Display, + on_success_message: impl Display, + logger: &Logger, + validator: impl FnOnce() -> RecoveryResult, +) -> RecoveryResult { + info!(logger, "Validating {}", label); + + let result = (validator)(); + match &result { + Ok(_) => info!(logger, "Validation succeeded: {}.", on_success_message), + Err(err) => error!(logger, "Validation failed: {}.", err), + } + + info!(logger, ""); + + result +} diff --git a/rs/recovery/subnet_splitting/src/subnet_splitting.rs b/rs/recovery/subnet_splitting/src/subnet_splitting.rs index ef042f375b1a..868d1f3631b8 100644 --- a/rs/recovery/subnet_splitting/src/subnet_splitting.rs +++ b/rs/recovery/subnet_splitting/src/subnet_splitting.rs @@ -328,6 +328,7 @@ impl SubnetSplitting { /*registry_params=*/ None, /*initial_dkg_subnet_id=*/ None, /*chain_key_subnet_id=*/ None, + /*time=*/ None, ) } diff --git a/rs/registry/admin/bin/main.rs b/rs/registry/admin/bin/main.rs index 2bdfdb930aa7..43ac9d0681d0 100644 --- a/rs/registry/admin/bin/main.rs +++ b/rs/registry/admin/bin/main.rs @@ -160,6 +160,7 @@ use registry_canister::mutations::{ add_firewall_rules_compute_entries, compute_firewall_ruleset_hash, remove_firewall_rules_compute_entries, update_firewall_rules_compute_entries, }, + merge_subnets::MergeSubnetsPayload, node_management::do_remove_nodes::RemoveNodesPayload, prepare_canister_migration::PrepareCanisterMigrationPayload, reroute_canister_ranges::RerouteCanisterRangesPayload, @@ -478,6 +479,10 @@ enum SubCommand { // Submits a proposal to add custom upgrade path entries ProposeToInsertSnsWasmUpgradePathEntries(ProposeToInsertSnsWasmUpgradePathEntriesCmd), + /// Submits a proposal to merge a subnet into another one, i.e. to reroute + /// the canister ID ranges of the source subnet to the destination subnet. + ProposeToMergeSubnets(ProposeToMergeSubnetsCmd), + /// Propose additions or updates to `canister_migrations`. Step 1 of canister migration. ProposeToPrepareCanisterMigration(ProposeToPrepareCanisterMigrationCmd), @@ -1124,6 +1129,45 @@ impl ProposalPayload for ProposeToDeleteSubnetCmd { } } +/// Sub-command to submit a proposal to merge a subnet into another one. +#[derive_common_proposal_fields] +#[derive(Parser, ProposalMetadata)] +struct ProposeToMergeSubnetsCmd { + /// The subnet whose canister ID ranges are merged into those of the + /// destination subnet. It hosts no canister ID range after the merge and is + /// expected to be deleted afterwards. + #[clap(long)] + pub source_subnet: PrincipalId, + + /// The subnet that hosts the canister ID ranges of the source subnet after + /// the merge. + #[clap(long)] + pub destination_subnet: PrincipalId, +} + +impl ProposalTitle for ProposeToMergeSubnetsCmd { + fn title(&self) -> String { + match &self.proposal_title { + Some(title) => title.clone(), + None => format!( + "Merge subnet {} into subnet {}", + shortened_pid_string(&self.source_subnet), + shortened_pid_string(&self.destination_subnet), + ), + } + } +} + +#[async_trait] +impl ProposalPayload for ProposeToMergeSubnetsCmd { + async fn payload(&self, _: &Agent) -> MergeSubnetsPayload { + MergeSubnetsPayload { + source_subnet: SubnetId::from(self.source_subnet), + destination_subnet: SubnetId::from(self.destination_subnet), + } + } +} + /// Sub-command to submit a proposal to change the public keys with "readonly" /// access privileges. There is no easy way to set a privilege to an empty list. #[derive_common_proposal_fields] @@ -4750,6 +4794,7 @@ async fn main() { SubCommand::ProposeToDeployHostosToSomeNodes(_) => (), SubCommand::ProposeToHardResetNnsRootToVersion(_) => (), SubCommand::ProposeToInsertSnsWasmUpgradePathEntries(_) => (), + SubCommand::ProposeToMergeSubnets(_) => (), SubCommand::ProposeToPrepareCanisterMigration(_) => (), SubCommand::ProposeToRemoveApiBoundaryNodes(_) => (), SubCommand::ProposeToRemoveFirewallRules(_) => (), @@ -5200,6 +5245,21 @@ async fn main() { ) .await; } + SubCommand::ProposeToMergeSubnets(cmd) => { + let (proposer, sender) = cmd.proposer_and_sender(sender); + propose_external_proposal_from_command( + cmd, + NnsFunction::MergeSubnets, + make_canister_client( + reachable_nns_urls, + opts.verify_nns_responses, + opts.nns_public_key_pem_file, + sender, + ), + proposer, + ) + .await; + } SubCommand::ProposeToCreateServiceNervousSystem(cmd) => { let (proposer, sender) = cmd.proposer_and_sender(sender); propose_to_create_service_nervous_system( diff --git a/rs/state_tool/BUILD.bazel b/rs/state_tool/BUILD.bazel index 4e9d61cfb5c3..161ae2ebf30b 100644 --- a/rs/state_tool/BUILD.bazel +++ b/rs/state_tool/BUILD.bazel @@ -9,7 +9,10 @@ rust_library( ), crate_name = "ic_state_tool", version = "0.1.0", - visibility = ["//rs/recovery/subnet_splitting:__subpackages__"], + visibility = [ + "//rs/recovery/subnet_merging:__subpackages__", + "//rs/recovery/subnet_splitting:__subpackages__", + ], deps = [ # Keep sorted. "//rs/config", diff --git a/rs/state_tool/src/commands/checkpoint_time.rs b/rs/state_tool/src/commands/checkpoint_time.rs index e32c3d206b14..0d0cf657cb7c 100644 --- a/rs/state_tool/src/commands/checkpoint_time.rs +++ b/rs/state_tool/src/commands/checkpoint_time.rs @@ -6,10 +6,10 @@ use std::path::PathBuf; const HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED: Height = Height::new(0); -/// Prints the batch time of the checkpoint rooted at `path`, in nanoseconds +/// Returns the batch time of the checkpoint rooted at `path`, in nanoseconds /// since the Epoch, i.e. the IC time the subnet had reached when it wrote the /// checkpoint. -pub fn do_print_checkpoint_time(path: PathBuf) -> Result<(), String> { +pub fn batch_time_nanos(path: PathBuf) -> Result { let checkpoint_layout = CompleteCheckpointLayout::new_untracked(path, HEIGHT_IS_IRRELEVANT_BECAUSE_ITS_UNUSED) .map_err(|err| format!("Failed to create CheckpointLayout: {err:?}"))?; @@ -18,7 +18,13 @@ pub fn do_print_checkpoint_time(path: PathBuf) -> Result<(), String> { .deserialize() .map_err(|err| format!("Failed to read the system metadata: {err:?}"))?; - println!("{}", system_metadata.batch_time_nanos); + Ok(system_metadata.batch_time_nanos) +} + +/// Prints the batch time of the checkpoint rooted at `path`, in nanoseconds +/// since the Epoch. +pub fn do_print_checkpoint_time(path: PathBuf) -> Result<(), String> { + println!("{}", batch_time_nanos(path)?); Ok(()) } diff --git a/rs/tests/message_routing/BUILD.bazel b/rs/tests/message_routing/BUILD.bazel index 2a6a0539cbb1..43ee214e9c40 100644 --- a/rs/tests/message_routing/BUILD.bazel +++ b/rs/tests/message_routing/BUILD.bazel @@ -1,4 +1,4 @@ -load("//rs/tests:common.bzl", "DEFAULT_VCPUS_PER_VM", "MAINNET_ENV", "MAINNET_QUEUES_COMPATIBILITY_RUNTIME_DEPS", "MIN_LOCAL_CPUS", "UNIVERSAL_CANISTER_RUNTIME_DEPS") +load("//rs/tests:common.bzl", "CANISTER_SANDBOX_RUNTIME_DEPS", "DEFAULT_VCPUS_PER_VM", "MAINNET_ENV", "MAINNET_QUEUES_COMPATIBILITY_RUNTIME_DEPS", "MIN_LOCAL_CPUS", "UNIVERSAL_CANISTER_RUNTIME_DEPS") load("//rs/tests:system_tests.bzl", "system_test", "system_test_nns") package(default_visibility = ["//rs:system-tests-pkg"]) @@ -147,17 +147,14 @@ system_test_nns( "long_test", # since the subnet only quiesces once its long-running `install_code` calls are done and the ingress history is pruned. ], test_timeout = "eternal", - runtime_deps = UNIVERSAL_CANISTER_RUNTIME_DEPS | { - "ENV_DEPS__STATE_TOOL": "//rs/state_tool:state-tool", + runtime_deps = UNIVERSAL_CANISTER_RUNTIME_DEPS | CANISTER_SANDBOX_RUNTIME_DEPS | { + "IC_ADMIN_PATH": "//rs/registry/admin:ic-admin", }, deps = [ # Keep sorted. - "//rs/nns/governance/api", "//rs/recovery", - "//rs/registry/canister", + "//rs/recovery/subnet_merging", "//rs/registry/subnet_type", - "//rs/state_layout", - "//rs/tests/consensus/utils", "//rs/tests/driver:ic-system-test-driver", "//rs/types/types", "//rs/universal_canister/lib", @@ -169,7 +166,6 @@ system_test_nns( "@crate_index//:ic-utils", "@crate_index//:slog", "@crate_index//:tokio", - "@crate_index//:url", ], ) diff --git a/rs/tests/message_routing/Cargo.toml b/rs/tests/message_routing/Cargo.toml index ea62521d0cc3..43ae83a26bb5 100644 --- a/rs/tests/message_routing/Cargo.toml +++ b/rs/tests/message_routing/Cargo.toml @@ -19,6 +19,7 @@ ic-management-canister-types = { workspace = true } ic-nns-governance-api = { path = "../../nns/governance/api" } ic-recovery = { path = "../../recovery" } ic-registry-subnet-type = { path = "../../registry/subnet_type" } +ic-subnet-merging = { path = "../../recovery/subnet_merging" } ic-state-layout = { path = "../../state_layout" } ic-system-test-driver = { path = "../driver" } ic-types = { path = "../../types/types" } diff --git a/rs/tests/message_routing/subnet_cooling_down_test.rs b/rs/tests/message_routing/subnet_cooling_down_test.rs index 7da4e44f7e31..b244cbd3ac16 100644 --- a/rs/tests/message_routing/subnet_cooling_down_test.rs +++ b/rs/tests/message_routing/subnet_cooling_down_test.rs @@ -1,14 +1,14 @@ /* tag::catalog[] -Title:: Draining a subnet that is "cooling down". +Title:: Draining and merging a subnet that is "cooling down". Goal:: Verify that a subnet labeled "cooling down" in its subnet record quiesces while its canisters are busy making cross-subnet calls in a loop, installing code on one another and waiting for responses that never arrive, i.e. that it reaches the "merge readiness" condition of the `Subnet merging` dashboard (see `bases/apps/ic-dashboards/core/subnet-merging.json` on branch -`mraszyk/subnet-merging-dashboard` of `dfinity/k8s`) for `V` = the registry -version at which the subnet was labeled "cooling down" and a pending refund -budget (the dashboard's `R`) of 0 cycles. +`mraszyk/subnet-merging-dashboard` of `dfinity/k8s`), and that the subnet +merging tool (`rs/recovery/subnet_merging`) then merges it into another subnet +without losing any of that state. The subnet that is cooling down, i.e. the one that is merged away, is called `M` and the Application subnet it is merged into is called `R`. A third Application @@ -17,6 +17,9 @@ NNS subnet is none of these: it has to stay available throughout, as it is where the proposals of this test are executed, including the one recovering `R` at the merged state. +Every proposal of the merge itself is submitted by the tool, through `ic-admin` +with the test neuron; this test submits none. + "Executing" an update call below always means submitting it as an ingress message without waiting for it to complete: most of the calls of this test are never meant to complete. @@ -58,76 +61,86 @@ Runbook:: loop as its payload, so that `M` holds a canister waiting for a response from another subnet that never arrives. Wait until all three loops are running. -7. Submit (and adopt) an `UpdateConfigOfSubnet` NNS proposal labeling `M` as - "cooling down" in its subnet record, and record the registry version `V` it - creates. -8. Wait until `M` rejects ingress messages, i.e. the replicas of `M` observed - the "cooling down" label. -9. Check that `M` is not "merge ready" yet, so that the wait below is known to - be waiting for something, and then wait until it is, according to the - dashboard's condition for `V` and `MAX_REFUND_VALUE_CYCLES`: all subnets have - reached registry version `V`, no stream in either direction holds a message - (loopback included), the ingress history holds nothing but `processing` - entries, `M`'s subnet input and output queues are empty, `M`'s subnet call - context manager holds no call context, and the pending anonymous refunds are - worth at most `MAX_REFUND_VALUE_CYCLES`. -10. Check that `M` answers no query call, which is the other half of what a - cooling down subnet stops doing: it neither accepts ingress messages nor - serves queries, and it executes no canister message. Whether the - `install_code` calls of step 5 installed the code is therefore only observable - after the merge, in step 17. -11. Check that the two loops of step 2 are indeed stalled: while `M` is cooling - down, it executes no canister message, and neither `M` nor `T` routes any - message to or from `M`, so the messages of both loops are retained in their - senders' output queues. `UT`'s iteration counter is read via a query to `T`; - `US` sits on `M`, which answers no query, so `M`'s count of the rounds it - skipped canister execution in stands in for it. -12. Submit (and adopt) `UpdateConfigOfSubnet` NNS proposals setting the - `halt_at_cup_height` flag of both `M` and `R`, and wait until the node of each - that the state is taken from holds the CUP its subnet halts at, i.e. the first - one whose summary is created at the registry version carrying that flag. - Record the heights of those CUPs, which are the heights of the checkpoints - holding the states the two subnets stopped in. -13. Stop the replicas of both subnets and download the states they halted at. - Assemble the merged state locally, as a checkpoint at the next multiple of the - DKG interval after the height `R` halted at: the canisters and canister - snapshots of `M` are added to those of `R`, and the result is marked as the - product of a subnet merge. The ingress history of `M` is deliberately not - merged in: the marker makes the replica re-register the ingress messages of - the merged-in canisters that are still in progress. Compute the block time the - merged state starts from, which must be larger than the times of both - checkpoints, and the hash of its manifest. -14. Submit (and adopt) a `MergeSubnets` NNS proposal for `M` and `R`, which - reroutes the canister ID ranges of `M` to `R`, and then a `RecoverSubnet` NNS - proposal for `R`, which creates a recovery CUP for `R` at the merged state, - running a fresh DKG for `R`'s membership. Recovering a subnet that was - instructed to halt at its next CUP replaces that instruction with a plain - halt, so `R` stays halted for now. -15. Upload the merged state to `R`'s node, replacing the state directory holding - the checkpoint it halted at, and restart its replica. Deleting that checkpoint - is what makes the recovery unambiguous: it does not hold the canisters of `M`, - so a replica coming up on it would serve a state that silently lost them, and - the merged state is now the only one `R` can resume from. The recovery CUP of - step 14 has to exist by this point, as the replica is restarted right away. -16. Wait until `R` reports the recovery CUP, i.e. it did come up on the merged - state. Then submit (and adopt) an `UpdateConfigOfSubnet` NNS proposal unhalting - `R` and wait until it is healthy. -17. Check that `U8`, now served by `R`, kept the stable memory, the snapshot and - (up to what an idle canister burns) the cycles balance of step 4, and that - `UR`, which `R` hosted all along, is undisturbed and can call `U8` now that - both are on the same subnet. Check that `U2a` .. `U2e`, also served by `R` - now, have been installed, i.e. that the `install_code` calls of step 5 ran to - completion while `M` was cooling down rather than being lost or rejected. -18. Set the global data of `U3`, `U5` and `U7` to `LOOP_BREAK_TRIGGER`, ending - the three endless loops, and check that every ingress message that was in - progress across the merge completed. `U3` and `U5` are reached through `R`, - which serves the canisters of `M` after the merge. -19. Wait until every subnet other than `M` has reached the registry version the - merge created, i.e. routes the canisters that used to be hosted by `M` to - `R`. `M` itself is excluded: its replica was stopped for the merge and it is - about to be deleted. -20. Submit (and adopt) a `DeleteSubnet` NNS proposal deleting `M`, which hosts no - canister ID range anymore, and check that it is gone from the registry. +7. Hand over to the subnet merging tool, which runs one step at a time; the + checks of the steps below are made in between its steps, at the points the + step names give. The tool labels `M` as "cooling down" and reads back the + registry version `V` this created (`CoolDownSourceSubnet`, + `CheckRegistryForCoolingDownFlag`). +8. After `CheckRegistryForCoolingDownFlag`: wait until `M` rejects ingress + messages, i.e. the replicas of `M` observed the "cooling down" label, and + check that `M` is not "merge ready" yet, so that the tool's wait below is + known to be waiting for something. Both use what the tool recorded: the + readiness condition, evaluated by the tool's own code, and `V`, read from the + tool's working directory. +9. The tool waits until `M` is "merge ready" according to the dashboard's + condition for `V`: all subnets have reached registry version `V`, no stream + in either direction holds a message (loopback included), the ingress history + holds nothing but `processing` entries, `M`'s subnet input and output queues + are empty, `M`'s subnet call context manager holds no call context, and its + refund pool holds no pending anonymous refund (`CheckMergeReadiness`). +10. After `CheckMergeReadiness`: check that `M` answers no query call, which is + the other half of what a cooling down subnet stops doing: it neither accepts + ingress messages nor serves queries, and it executes no canister message. + Whether the `install_code` calls of step 5 installed the code is therefore + only observable after the merge, in step 17. +11. Also after `CheckMergeReadiness`: check that the two loops of step 2 are + indeed stalled: while `M` is cooling down, it executes no canister message, + and neither `M` nor `T` routes any message to or from `M`, so the messages of + both loops are retained in their senders' output queues. `UT`'s iteration + counter is read via a query to `T`; `US` sits on `M`, which answers no query, + so `M`'s count of the rounds it skipped canister execution in stands in for + it. +12. The tool sets the `halt_at_cup_height` flag of both `M` and `R` and waits + until the node of each that the state is taken from holds the CUP its subnet + halts at -- served at its public endpoint, written to its disk, and with the + state it names certified (`Halt*SubnetAtCupHeight`, + `WaitForHaltingCupOn*Subnet`). +13. The tool stops the replicas of both subnets, downloads the states they + halted at, validates each against the CUP it was taken at and the subnet's + public key in the NNS signed state tree, and assembles the merged state + locally: the canisters and canister snapshots of `M` are added to those of + `R`, and the result is marked as the product of a subnet merge. The ingress + history of `M` is deliberately not merged in: the marker makes the replica + re-register the ingress messages of the merged-in canisters that are still in + progress. The tool also computes the block time the merged state starts from, + which must be larger than the times of both checkpoints, and the hash of its + manifest (`Stop*Replica`, `DownloadStateFrom*Subnet`, `Validate*SubnetCup`, + `MergeStates`). +14. The tool submits a `MergeSubnets` proposal for `M` and `R`, which reroutes + the canister ID ranges of `M` to `R`, and then a `RecoverSubnet` proposal for + `R`, which creates a recovery CUP for `R` at the merged state, running a + fresh DKG for `R`'s membership. Recovering a subnet that was instructed to + halt at its next CUP replaces that instruction with a plain halt, so `R` + stays halted for now (`MergeSubnets`, + `CheckRegistryForRoutingTableEntry`, `ProposeCupForDestinationSubnet`). +15. The tool uploads the merged state to `R`'s node, replacing the state + directory holding the checkpoint it halted at, and starts its replica back + up. Deleting that checkpoint is what makes the recovery unambiguous: it does + not hold the canisters of `M`, so a replica coming up on it would serve a + state that silently lost them, and the merged state is now the only one `R` + can resume from (`UploadStateToDestinationSubnet`). +16. The tool waits until `R` reports the recovery CUP, i.e. it did come up on + the merged state, and then unhalts it + (`WaitForCUPOnDestinationSubnet`, `UnhaltDestinationSubnet`). +17. After `UnhaltDestinationSubnet`: wait until `R` is healthy, then check that + `U8`, now served by `R`, kept the stable memory, the snapshot and (up to what + an idle canister burns) the cycles balance of step 4, and that `UR`, which + `R` hosted all along, is undisturbed and can call `U8` now that both are on + the same subnet. Check that `U2a` .. `U2e`, also served by `R` now, have been + installed, i.e. that the `install_code` calls of step 5 ran to completion + while `M` was cooling down rather than being lost or rejected. +18. Also after `UnhaltDestinationSubnet`: set the global data of `U3`, `U5` and + `U7` to `LOOP_BREAK_TRIGGER`, ending the three endless loops, and check that + every ingress message that was in progress across the merge completed. `U3` + and `U5` are reached through `R`, which serves the canisters of `M` after the + merge. +19. The tool waits until every subnet other than `M` has reached the registry + version the merge created, i.e. routes the canisters that used to be hosted + by `M` to `R`. `M` itself is excluded: its replica was stopped for the merge + and it is about to be deleted (`CheckRegistryVersionOnAllSubnets`). +20. The tool deletes `M`, which hosts no canister ID range anymore + (`DeleteSourceSubnet`); after that step, check that it is gone from the + registry. Success:: `M` becomes "merge ready", with `U2a` .. `U2e` installed, while both loops of @@ -138,38 +151,33 @@ deleted. end::catalog[] */ use anyhow::{Result, anyhow, bail}; -use candid::{CandidType, Principal}; +use candid::Principal; use ic_agent::{Agent, RequestId, agent::RequestStatusResponse}; -use ic_consensus_system_test_utils::get_cup_from_node; use ic_management_canister_types::{SnapshotId, TakeCanisterSnapshotArgs}; -use ic_nns_governance_api::NnsFunction; -use ic_recovery::registry_helper::RegistryPollingStrategy; -use ic_recovery::steps::{Step, UploadStateAndRestartStep}; -use ic_recovery::util::{DataLocation, SshUser}; -use ic_recovery::{IC_STATE_DIR, Recovery, RecoveryArgs, STATES_METADATA}; +use ic_recovery::{RecoveryArgs, file_sync_helper}; use ic_registry_subnet_type::SubnetType; -use ic_state_layout::StateLayout; +use ic_subnet_merging::{ + readiness::{SubnetNodeIps, evaluate_merge_readiness}, + subnet_merging::{StepType, SubnetMerging, SubnetMergingArgs}, + utils::read_cooling_down_registry_version, +}; use ic_system_test_driver::driver::constants::SSH_USERNAME; -use ic_system_test_driver::driver::driver_setup::SSH_AUTHORIZED_PRIV_KEYS_DIR; +use ic_system_test_driver::driver::driver_setup::{ + SSH_AUTHORIZED_PRIV_KEYS_DIR, SSH_AUTHORIZED_PUB_KEYS_DIR, +}; use ic_system_test_driver::driver::group::SystemTestGroup; use ic_system_test_driver::driver::ic::{InternetComputer, Subnet}; use ic_system_test_driver::driver::test_env::TestEnv; use ic_system_test_driver::driver::test_env_api::{ HasPublicApiUrl, HasRegistryVersion, HasTopologySnapshot, IcNodeContainer, IcNodeSnapshot, - NnsInstallationBuilder, READY_WAIT_TIMEOUT, RETRY_BACKOFF, SshSession, SubnetSnapshot, - TopologySnapshot, get_dependency_path_from_env, -}; -use ic_system_test_driver::nns::{ - get_governance_canister, submit_external_proposal_with_test_id, - vote_execute_proposal_assert_executed, + NnsInstallationBuilder, READY_WAIT_TIMEOUT, RETRY_BACKOFF, SubnetSnapshot, TopologySnapshot, + get_guestos_img_version, }; use ic_system_test_driver::retry_with_msg_async; use ic_system_test_driver::systest; use ic_system_test_driver::util::{ - MetricsFetcher, UniversalCanister, assert_create_agent, block_on, create_canister, - runtime_from_url, set_controller, + MetricsFetcher, UniversalCanister, assert_create_agent, create_canister, set_controller, }; -use ic_types::consensus::HasHeight; use ic_types::{Height, SubnetId}; use ic_universal_canister::management::InstallMode; use ic_universal_canister::{ @@ -177,56 +185,23 @@ use ic_universal_canister::{ }; use ic_utils::call::AsyncCall; use ic_utils::interfaces::ManagementCanister; -use registry_canister::mutations::do_delete_subnet::DeleteSubnetPayload; -use registry_canister::mutations::do_recover_subnet::RecoverSubnetPayload; -use registry_canister::mutations::do_update_subnet::UpdateSubnetPayload; -use registry_canister::mutations::merge_subnets::MergeSubnetsPayload; use slog::{Logger, info}; use std::collections::BTreeMap; -use std::net::IpAddr; -use std::path::{Path, PathBuf}; -use std::process::Command; +use std::path::Path; use std::time::Duration; -use url::Url; +use tokio::runtime::Runtime; -/// Metrics making up the "merge readiness" condition. -const METRIC_REGISTRY_VERSION: &str = "mr_registry_version"; -const METRIC_STREAM_MESSAGES: &str = "mr_stream_messages"; -const METRIC_INGRESS_HISTORY_BY_STATE: &str = "replicated_state_ingress_history_length_by_state"; +/// Metrics this test reads itself; the ones making up the "merge readiness" +/// condition are read by the merging tool. const METRIC_SUBNET_INPUT_QUEUE_MESSAGES: &str = "execution_subnet_input_queue_messages"; -const METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES: &str = "execution_subnet_output_queue_messages"; const METRIC_SUBNET_CALL_CONTEXTS: &str = "replicated_state_subnet_call_contexts"; -const METRIC_PENDING_REFUNDS_CYCLES: &str = "replicated_state_pending_refunds_cycles"; const METRIC_ROUNDS_SKIPPED_CANISTER_EXECUTION: &str = "round_skipped_canister_execution_due_to_cooling_down"; -const METRIC_CERTIFICATION_HEIGHT: &str = r#"artifact_pool_certification_height_stat{pool_type="validated",stat="max",type="certification"}"#; - -/// Timeout for a subnet to reach the CUP it halts at, which is up to a full DKG -/// interval away. -const HALT_TIMEOUT: Duration = Duration::from_secs(900); -/// Backoff between two checks of whether a subnet has halted. -const HALT_BACKOFF: Duration = Duration::from_secs(10); /// The label selecting the `install_code` call contexts of /// `METRIC_SUBNET_CALL_CONTEXTS`. const LABEL_INSTALL_CODE: &str = "type=\"install_code\""; -/// The dashboard's `R` in the readiness condition: the maximum total value in -/// cycles of the pending anonymous refunds of the cooling down subnet. (Not to -/// be confused with the subnet `R` of the runbook above.) -/// -/// This test leaves no pending refunds behind, so it requires them to be worth -/// nothing at all. Making it hold non-zero ones would take a cycle bearing -/// message that is dropped from one of the subnet's queues while owed to a -/// canister of another subnet: the cycles of the best effort call of step 6 are -/// not it, as that call is picked up and its cycles are held by the open call -/// context of its callee rather than by a queued message. Which is just as well: -/// a cooling down subnet routes no refunds either (see `route_refunds` in -/// `rs/messaging/src/routing/stream_builder.rs`), so any refund it does hold -/// stays pending until it is merged, and is then lost -- the merged state takes -/// the refunds of the destination subnet, not those of the merged one. -const MAX_REFUND_VALUE_CYCLES: f64 = 0.0; - /// Number of loop iterations each universal canister must have completed before /// the subnet is labeled "cooling down", so that the loops are known to be /// making cross-subnet calls when the label takes effect. @@ -300,21 +275,15 @@ const SUBNET_SIZE: usize = 4; /// one interval, which matters because a paused `install_code` is aborted at /// every checkpoint and has to start over afterwards. const DKG_INTERVAL_LENGTH: u64 = 499; -/// The distance between two consecutive checkpoint (and CUP) heights. -const CHECKPOINT_INTERVAL: u64 = DKG_INTERVAL_LENGTH + 1; + +/// The directory the merging tool works in, relative to the test environment. +const MERGING_DIR: &str = "subnet_merging"; /// How much later than the checkpoints it is assembled from the merged state /// starts, i.e. the block time of the recovery CUP of `R` minus the larger of /// the two checkpoint times. const MERGED_STATE_TIME_MARGIN: Duration = Duration::from_secs(60); -/// Timeout for an ingress message that was in progress across the merge to -/// complete once the loop it is waiting for is broken. Generous because the -/// destination subnet has just resumed from the merged state and is busy -/// recomputing its manifest and draining the message loops of step 2 at the same -/// time: this has been observed to take up to four minutes. -const INGRESS_COMPLETION_TIMEOUT: Duration = Duration::from_secs(900); - /// Timeout for the subnet to become "merge ready". The binding terms are the /// `install_code` calls of step 5, which take a couple of hundred rounds each /// (and are executed one at a time, as at most one long-running `install_code` @@ -323,9 +292,23 @@ const INGRESS_COMPLETION_TIMEOUT: Duration = Duration::from_secs(900); /// before the subnet started cooling down are pruned, i.e. at their (up to /// `MAX_INGRESS_TTL` = 5 minutes away) expiry times. const MERGE_READY_TIMEOUT: Duration = Duration::from_secs(2400); -/// Backoff between two evaluations of the readiness condition. Longer than the -/// default because every evaluation scrapes the metrics of all subnets. -const MERGE_READY_BACKOFF: Duration = Duration::from_secs(10); +/// Timeout for a subnet to reach the CUP it halts at, which is up to a full DKG +/// interval away. +const HALT_TIMEOUT: Duration = Duration::from_secs(900); +/// Timeout for the registry to reflect a proposal that `ic-admin` reported as +/// executed. +const REGISTRY_TIMEOUT: Duration = Duration::from_secs(300); +/// How long the merging tool waits between two evaluations of a condition it +/// waits for. Longer than a driver retry because every evaluation scrapes the +/// metrics of all subnets. +const POLL_INTERVAL: Duration = Duration::from_secs(10); + +/// Timeout for an ingress message that was in progress across the merge to +/// complete once the loop it is waiting for is broken. Generous because the +/// destination subnet has just resumed from the merged state and is busy +/// recomputing its manifest and draining the message loops of step 2 at the same +/// time: this has been observed to take up to four minutes. +const INGRESS_COMPLETION_TIMEOUT: Duration = Duration::from_secs(900); /// How long the loops are observed to be stalled (step 11). const STALL_OBSERVATION_PERIOD: Duration = Duration::from_secs(15); @@ -351,6 +334,74 @@ fn main() -> Result<()> { Ok(()) } +pub fn test(env: TestEnv) { + // One runtime for the whole test, kept alive across all of its phases: the + // agents of the phase that sets the scenario up are used again by the phase + // that checks the outcome. + // + // The merging tool runs in between, on this thread, which is deliberately + // not inside that runtime: every registry read, ssh command and rsync of + // `ic-recovery` blocks on a runtime of its own, which a thread that is + // driving one cannot do. + let runtime = Runtime::new().expect("failed to create a tokio runtime"); + + let context = runtime.block_on(prepare(&env)); + merge(&env, &context, &runtime); +} + +/// Everything the phases that run in between the steps of the merging tool need +/// from the phase that set the scenario up. +/// +/// Canister ids and agents rather than `UniversalCanister`s, which borrow the +/// agent they were created with: the canisters of the subnet that is merged +/// away are reached through a different agent after the merge than before it. +struct Context { + logger: Logger, + m_subnet: SubnetSnapshot, + r_subnet: SubnetSnapshot, + m_node: IcNodeSnapshot, + r_node: IcNodeSnapshot, + m_agent: Agent, + t_agent: Agent, + r_agent: Agent, + /// The two canisters calling each other in a loop across subnets, `US` on + /// `M` and `UT` on `T`. + us: Principal, + ut: Principal, + /// The canisters holding the endless loops of step 6, `U3` and `U5` on `M` + /// and `U7` on `T`. + u3: Principal, + u5: Principal, + u7: Principal, + /// The canister on `M` whose state has to survive the merge, and that state. + u8: Principal, + u8_snapshot: SnapshotId, + u8_cycles_before: u128, + /// The canister `R` hosted all along, which the merge must leave alone. + ur: Principal, + /// The canisters `U1` installs code on while `M` is cooling down. + targets: Vec, + /// The ingress messages that are in progress when the merge happens, with + /// the agent each of them has to be read through afterwards. + pending_ingress_messages: Vec<(String, Agent, Principal, RequestId)>, +} + +impl Context { + /// The nodes of every subnet, which is what the merge readiness condition + /// is evaluated on. + fn subnet_node_ips(topology: &TopologySnapshot) -> SubnetNodeIps { + topology + .subnets() + .map(|subnet| { + ( + subnet.subnet_id, + subnet.nodes().map(|node| node.get_ip_addr()).collect(), + ) + }) + .collect::>() + } +} + pub fn setup(env: TestEnv) { let subnet = |subnet_type| { Subnet::fast(subnet_type, SUBNET_SIZE) @@ -379,11 +430,8 @@ pub fn setup(env: TestEnv) { .expect("failed to install NNS canisters"); } -pub fn test(env: TestEnv) { - block_on(run(env)); -} - -async fn run(env: TestEnv) { +/// Steps 0 to 6: set up the scenario the merge has to survive. +async fn prepare(env: &TestEnv) -> Context { let logger = env.logger(); let topology = env.topology_snapshot(); @@ -412,7 +460,7 @@ async fn run(env: TestEnv) { let nns_node = topology.root_subnet().nodes().next().unwrap(); info!( logger, - "Subnets under test, with their (single) nodes:\n \ + "Subnets under test, with the nodes the merge works with:\n \ M={} on {}\n \ R={} on {}\n \ T={} on {}\n \ @@ -551,15 +599,13 @@ async fn run(env: TestEnv) { ); // Step 6: Start the three endless loops, and the best effort call whose - // cycles end up as a pending anonymous refund of `M`. + // cycles are held by the callee's call context across the merge. info!( logger, "Step 6: Starting the three endless loops and U9's best effort call to U10" ); // `U10` never responds, so the call is still in flight when `M` starts - // cooling down, and its deadline passes while it is. Dropping it leaves `M` - // owing its cycles to `U9`, which is on `T`: a refund `M` cannot route while - // it is cooling down, and hence one that is still pending when it is merged. + // cooling down, and its deadline passes while it is. u9.submit_update(wasm().call_simple_with_cycles_and_best_effort_response( u10.canister_id(), "update", @@ -570,7 +616,7 @@ async fn run(env: TestEnv) { .await .expect("submitting U9's best effort call should succeed"); // The IDs of the ingress messages that stay in progress across the merge, so - // that step 16 can check that all of them eventually completed. `U3` and + // that step 18 can check that all of them eventually completed. `U3` and // `U6` are on `M` and thus served by `R` after the merge; `U4` stays on `T`. let mut pending_ingress_messages: Vec<(String, Agent, Principal, RequestId)> = Vec::new(); for (canister, name, agent) in [ @@ -610,27 +656,127 @@ async fn run(env: TestEnv) { from U6" ); - // Step 7: Label `M` as "cooling down" in its subnet record. - info!( + Context { logger, - "Step 7: Labeling subnet M ({}) as \"cooling down\"", m_subnet.subnet_id, - ); - let registry_version = set_subnet_cooling_down(&env, m_subnet.subnet_id, &logger).await; + m_subnet, + r_subnet, + m_node, + r_node, + us: us.canister_id(), + ut: ut.canister_id(), + u3: u3.canister_id(), + u5: u5.canister_id(), + u7: u7.canister_id(), + u8: u8.canister_id(), + u8_snapshot, + u8_cycles_before, + ur: ur.canister_id(), + targets, + pending_ingress_messages, + m_agent, + t_agent, + r_agent, + } +} + +/// Steps 7 to 20: hand the scenario over to the subnet merging tool, and make +/// the checks of the steps in between its steps. +/// +/// Plain synchronous code, like `rs/tests/consensus/subnet_splitting_test.rs`: +/// the tool's steps block on runtimes of their own, and the asynchronous checks +/// of this test are driven through `runtime`, which this thread is not inside. +fn merge(env: &TestEnv, context: &Context, runtime: &Runtime) { + let logger = &context.logger; + let topology = env.topology_snapshot(); + + let ssh_priv_key_path = env + .get_path(SSH_AUTHORIZED_PRIV_KEYS_DIR) + .join(SSH_USERNAME); + let readonly_pub_key = + file_sync_helper::read_file(&env.get_path(SSH_AUTHORIZED_PUB_KEYS_DIR).join(SSH_USERNAME)) + .expect("Couldn't read public key"); + let merging_dir = env.get_path(MERGING_DIR); + + let recovery_args = RecoveryArgs { + dir: merging_dir.clone(), + nns_url: topology + .root_subnet() + .nodes() + .next() + .unwrap() + .get_public_url(), + replica_version: Some(get_guestos_img_version()), + admin_key_file: Some(ssh_priv_key_path.clone()), + test_mode: true, + skip_prompts: true, + }; + let merging_args = SubnetMergingArgs { + source_subnet_id: context.m_subnet.subnet_id, + destination_subnet_id: context.r_subnet.subnet_id, + readonly_pub_key: Some(readonly_pub_key), + readonly_key_file: Some(ssh_priv_key_path), + keep_downloaded_state: Some(false), + download_node_source: Some(context.m_node.get_ip_addr()), + download_node_destination: Some(context.r_node.get_ip_addr()), + upload_node_destination: Some(context.r_node.get_ip_addr()), + time_margin_secs: MERGED_STATE_TIME_MARGIN.as_secs(), + merge_ready_timeout_secs: MERGE_READY_TIMEOUT.as_secs(), + halt_timeout_secs: HALT_TIMEOUT.as_secs(), + registry_timeout_secs: REGISTRY_TIMEOUT.as_secs(), + poll_interval_secs: POLL_INTERVAL.as_secs(), + next_step: None, + }; + info!( logger, - "Step 7 done: subnet M is labeled \"cooling down\" as of registry version \ - {registry_version} (V)", + "Step 7: Merging subnet M ({}) into subnet R ({})", + context.m_subnet.subnet_id, + context.r_subnet.subnet_id, + ); + let merging = SubnetMerging::new( + logger.clone(), + recovery_args, + /*neuron_args=*/ None, + merging_args, ); - // Step 8: Wait until the replicas of `M` observed the "cooling down" label, - // i.e. until `M` rejects ingress messages. + for (step_type, step) in merging { + info!(logger, "Next step: {step_type:?}"); + info!(logger, "{}", step.descr()); + step.exec() + .unwrap_or_else(|e| panic!("Execution of step {step_type:?} failed: {e}")); + + match step_type { + StepType::CheckRegistryForCoolingDownFlag => { + runtime.block_on(check_ingress_rejected(context)); + check_not_merge_ready_yet(&topology, context, &merging_dir); + } + StepType::CheckMergeReadiness => { + runtime.block_on(check_no_queries_and_stalled_loops(context)) + } + StepType::UnhaltDestinationSubnet => runtime.block_on(verify_after_merge(context)), + StepType::DeleteSourceSubnet => { + runtime.block_on(check_source_subnet_deleted(&topology, context)) + } + _ => {} + } + } + + info!(logger, "Subnet M has been merged into subnet R and deleted"); +} + +/// Step 8: wait until the replicas of `M` observed the "cooling down" label, +/// i.e. until `M` rejects ingress messages. +async fn check_ingress_rejected(context: &Context) { + let logger = &context.logger; info!( logger, "Step 8: Waiting until subnet M rejects ingress messages" ); + let us = UniversalCanister::from_canister_id(&context.m_agent, context.us); retry_with_msg_async!( "waiting until subnet M rejects ingress messages", - &logger, + logger, READY_WAIT_TIMEOUT, RETRY_BACKOFF, || async { @@ -652,82 +798,58 @@ async fn run(env: TestEnv) { logger, "Step 8 done: subnet M rejects ingress messages, so it is cooling down" ); +} + +/// Step 9: check that `M` is *not* "merge ready" yet, so that the tool's wait +/// for the condition is known to be waiting for something: a readiness +/// condition that held from the start would be satisfied by a subnet that never +/// had anything to drain. +/// +/// The very condition the tool waits for, evaluated by the tool's own code, at +/// the very registry version `V` the tool recorded when it labeled `M` as +/// cooling down. +fn check_not_merge_ready_yet(topology: &TopologySnapshot, context: &Context, merging_dir: &Path) { + let logger = &context.logger; + let registry_version = read_cooling_down_registry_version(merging_dir) + .expect("the merging tool should have recorded the cooling down registry version"); - // Step 9: Check that `M` is *not* "merge ready" yet, so that the wait below - // is known to be waiting for something: a readiness condition that held from - // the start would be satisfied by a subnet that never had anything to drain. let terms = evaluate_merge_readiness( - &topology, - &m_subnet, + &Context::subnet_node_ips(topology), + context.m_subnet.subnet_id, registry_version, - MAX_REFUND_VALUE_CYCLES, - ) - .await - .expect("failed to evaluate the merge readiness of subnet M"); - let unsatisfied: Vec<_> = terms + logger, + ); + let unsatisfied: Vec<&str> = terms .iter() - .filter(|(_, satisfied)| !satisfied) - .map(|(term, _)| term.as_str()) + .filter(|term| !term.satisfied) + .map(|term| term.description.as_str()) .collect(); assert!( !unsatisfied.is_empty(), - "subnet M was already \"merge ready\" right after it started cooling down, so the wait \ - below would prove nothing", + "subnet M was already \"merge ready\" right after it started cooling down, so the tool's \ + wait would prove nothing", ); info!( logger, - "Step 9: subnet M is not \"merge ready\" yet: {}", + "Step 9: subnet M is not \"merge ready\" yet for V={registry_version}: {}", unsatisfied.join("; "), ); +} - // Step 9 (continued): Wait until `M` is "merge ready". - info!( - logger, - "Step 9: Waiting until subnet M is \"merge ready\" for V={registry_version} and at most \ - {MAX_REFUND_VALUE_CYCLES} cycles of pending refunds", - ); - retry_with_msg_async!( - format!( - "waiting until subnet {} is \"merge ready\"", - m_subnet.subnet_id - ), - &logger, - MERGE_READY_TIMEOUT, - MERGE_READY_BACKOFF, - || async { - let terms = evaluate_merge_readiness( - &topology, - &m_subnet, - registry_version, - MAX_REFUND_VALUE_CYCLES, - ) - .await?; - let unsatisfied: Vec<_> = terms - .iter() - .filter(|(_, satisfied)| !satisfied) - .map(|(term, _)| term.as_str()) - .collect(); - if !unsatisfied.is_empty() { - bail!("not merge ready: {}", unsatisfied.join("; ")); - } - for (term, _) in &terms { - info!(logger, "Step 9: merge readiness term holds: {term}"); - } - Ok(()) - } - ) - .await - .unwrap_or_else(|e| panic!("subnet M did not become \"merge ready\": {e}")); - info!(logger, "Step 9 done: subnet M is \"merge ready\""); +/// Steps 10 and 11: check that `M` answers no query call and that both call +/// loops of step 2 are stalled. +async fn check_no_queries_and_stalled_loops(context: &Context) { + let logger = &context.logger; - // Step 10: Check that `M` answers no query call. Step 8 saw it stop accepting - // ingress messages; refusing queries is the other half of what a cooling down - // subnet stops doing, and the reason the state of `M`'s canisters can only be - // inspected once `R` serves them (step 17). + // Step 10: Step 8 saw `M` stop accepting ingress messages; refusing queries + // is the other half of what a cooling down subnet stops doing, and the + // reason the state of `M`'s canisters can only be inspected once `R` serves + // them (step 17). info!( logger, "Step 10: Checking that subnet M rejects query calls" ); + let us = UniversalCanister::from_canister_id(&context.m_agent, context.us); let err = us .query(wasm().reply_data(&[])) .await @@ -739,9 +861,8 @@ async fn run(env: TestEnv) { ); info!(logger, "Step 10 done: subnet M rejects query calls"); - // Step 11: Check that both call loops are stalled, i.e. that `M` became - // "merge ready" because it is cooling down and not because the loops - // stopped making calls. + // Step 11: `M` became "merge ready" because it is cooling down and not + // because the loops stopped making calls. // // `UT` is on `T`, so its iteration counter can be read directly. `US` is on // `M`, which answers no query, so the number of rounds `M` skipped canister @@ -751,11 +872,12 @@ async fn run(env: TestEnv) { logger, "Step 11: Checking that both call loops are stalled over {STALL_OBSERVATION_PERIOD:?}" ); + let ut = UniversalCanister::from_canister_id(&context.t_agent, context.ut); let ut_before = global_counter(&ut).await.unwrap(); - let skipped_before = rounds_with_skipped_canister_execution(&m_subnet).await; + let skipped_before = rounds_with_skipped_canister_execution(&context.m_subnet).await; tokio::time::sleep(STALL_OBSERVATION_PERIOD).await; let ut_after = global_counter(&ut).await.unwrap(); - let skipped_after = rounds_with_skipped_canister_execution(&m_subnet).await; + let skipped_after = rounds_with_skipped_canister_execution(&context.m_subnet).await; assert_eq!( ut_before, ut_after, "UT's call loop advanced from iteration {ut_before} to {ut_after} while subnet M was \ @@ -773,219 +895,48 @@ async fn run(env: TestEnv) { canister execution in {} rounds while waiting", skipped_after - skipped_before, ); +} - // Step 12: Halt both `M` and `R` at their next CUP, i.e. at a checkpoint - // whose state is certified, so that the merged state can be assembled from - // states both subnets agree on. - info!( - logger, - "Step 12: Halting subnets M and R at their next checkpoint" - ); - let mut halt_versions = BTreeMap::new(); - for (subnet, name) in [(&m_subnet, "M"), (&r_subnet, "R")] { - let version = halt_subnet_at_cup_height(&env, subnet.subnet_id, &logger).await; - info!( - logger, - "Step 12: subnet {name} is set to halt at its next CUP as of registry version {version}" - ); - halt_versions.insert(name, version); - } - let m_height = await_halting_cup(&m_node, "M", halt_versions["M"], &logger).await; - let r_height = await_halting_cup(&r_node, "R", halt_versions["R"], &logger).await; - info!( - logger, - "Step 12 done: M halted at checkpoint {m_height}, R halted at checkpoint {r_height}" - ); - - // Step 13: Assemble the merged state: `R`'s state at the checkpoint it - // halted at, with the canisters (and canister snapshots) of `M` added to it, - // as a checkpoint at the next multiple of the DKG interval, which is the - // first height a recovery CUP for `R` can be created at. - // - // Taking `R`'s system metadata and subnet queues wholesale, i.e. dropping - // `M`'s, is only sound because `M`'s were empty, which is what the merge - // readiness of step 9 established. That they are *still* empty at the - // checkpoint `M` halted at, minutes later, is due to `M` cooling down: no - // message is routed out of any of its canisters' output queues, not even - // into the loopback stream, so no management call can be inducted, no subnet - // call context can be created and no `install_code` can start in between. - let merged_height = r_height + CHECKPOINT_INTERVAL; - info!( - logger, - "Step 13: Assembling the merged state as checkpoint {merged_height}" - ); - - // The replicas have to be stopped before their states are touched: the state - // manager of a running replica owns its state directory, even while - // consensus is halted. - for (node, name) in [(&m_node, "M"), (&r_node, "R")] { - node.block_on_bash_script_async("sudo systemctl stop ic-replica") - .await - .unwrap_or_else(|e| panic!("failed to stop the replica of subnet {name}: {e}")); - info!(logger, "Step 13: stopped the replica of subnet {name}"); - } - - // `ic-recovery` is a synchronous library that blocks on its own runtime - // internally (registry polling, rsync steps), which cannot be done from a - // thread that is driving this runtime, so all of it runs on a blocking one. - let merge = MergeStateArgs { - logger: logger.clone(), - admin_key_file: env - .get_path(SSH_AUTHORIZED_PRIV_KEYS_DIR) - .join(SSH_USERNAME), - nns_url: topology - .root_subnet() - .nodes() - .next() - .unwrap() - .get_public_url(), - m_dir: env.get_path("recovery_m"), - r_dir: env.get_path("recovery_r"), - merged_dir: env.get_path("recovery_merged"), - m_node_ip: m_node.get_ip_addr(), - r_node_ip: r_node.get_ip_addr(), - m_height, - r_height, - merged_height, - }; - let (merged_time, state_hash) = { - let merge = merge.clone(); - tokio::task::spawn_blocking(move || merge.assemble()) - .await - .expect("the state merging task panicked") - }; - info!( - logger, - "Step 13 done: the merged state hashes to {} and starts at {merged_time}", - hex::encode(&state_hash), - ); - - // Step 14: Merge `M` into `R`: reroute `M`'s canister ID ranges to `R`, and - // recover `R` at the merged state. Both proposals have to be executed before - // the merged state is uploaded in step 15, which restarts `R`'s replica: a - // replica that comes up before the recovery CUP exists has nothing to resume - // from, as the upload replaced the state it halted at. - info!( - logger, - "Step 14: Submitting the MergeSubnets proposal for M -> R" - ); - let merge_registry_version = - merge_subnets(&env, m_subnet.subnet_id, r_subnet.subnet_id, &logger).await; - info!( - logger, - "Step 14: M is merged into R as of registry version {merge_registry_version}" - ); - - // `merge_subnets` only updates the routing table: making `R` resume from the - // merged state is a subnet recovery like any other. The DKG of the recovery - // CUP is handled by the NNS subnet, which is neither of the two subnets being - // merged and stays available throughout. - info!( - logger, - "Step 14: Submitting the RecoverSubnet proposal for R at height {merged_height}" - ); - let recovery_registry_version = recover_subnet( - &env, - r_subnet.subnet_id, - merged_height, - merged_time, - state_hash.clone(), - &logger, - ) - .await; - info!( - logger, - "Step 14 done: R is recovered at the merged state as of registry version \ - {recovery_registry_version}" - ); - - // Step 15: Upload the merged state to `R`, replacing the state it halted at, - // and restart its replica. The recovery CUP of step 14 exists by now, so the - // replica comes up on the merged state. - info!(logger, "Step 15: Uploading the merged state to R"); - tokio::task::spawn_blocking(move || merge.upload_merged_state()) - .await - .expect("the state uploading task panicked"); - info!(logger, "Step 15 done: R holds the merged state"); - - // Step 16: Wait until `R` came up on the merged state and lift its halt. - // - // That the merged state is the only checkpoint `R` has does not by itself - // mean it resumed from it, so wait for the node to report exactly the - // recovery CUP. - { - let logger = logger.clone(); - let node_ip = r_node.get_ip_addr(); - let state_hash = hex::encode(&state_hash); - tokio::task::spawn_blocking(move || { - Recovery::wait_for_recovery_cup( - &logger, - node_ip, - Height::from(merged_height), - state_hash, - ) - }) - .await - .expect("the recovery CUP waiting task panicked") - .expect("subnet R did not adopt the recovery CUP holding the merged state"); - } - info!( - logger, - "Step 16: subnet R adopted the recovery CUP at height {merged_height}" - ); +/// Steps 17 and 18: check that the merge carried the state of `M`'s canisters +/// over, left `R`'s own canister alone, and let every ingress message that was +/// in progress across it complete. +async fn verify_after_merge(context: &Context) { + let logger = &context.logger; - // `recover_subnet` turned the "halt at the next CUP" instruction of step 12 - // into a plain halt, so that a recovered subnet does not resume before its - // recovery has been checked. Lift it now that `R` came up on the merged - // state: a halted subnet delivers no batches, so none of the ingress - // messages of step 18 would complete. - let unhalt_registry_version = unhalt_subnet(&env, r_subnet.subnet_id, &logger).await; - info!( - logger, - "Step 16: R is unhalted as of registry version {unhalt_registry_version}" - ); - // The `_async` variant, and not the blocking one: the latter drives its - // request through `futures::executor::block_on`, which busy-polls a `reqwest` - // future that needs the runtime this thread is driving, and livelocks as soon - // as an attempt has to open a new connection -- which is exactly what happens - // here, where `R` reports `WaitingForRootDelegation` for minutes before the - // unhalting takes effect. - // The `_async` variants of the driver's SSH and status helpers are what this - // test uses throughout: the blocking ones drive their own future with - // `futures::executor::block_on`, which busy-polls a `reqwest` request that - // has to open a new connection instead of letting the runtime wait for it. - r_node + // The tool has unhalted `R`; a halted subnet delivers no batches, so none of + // the calls below would be answered before it is healthy again. + context + .r_node .await_status_is_healthy_async() .await .expect("subnet R did not become healthy after the merge"); info!(logger, "Step 16 done: subnet R is healthy"); - // Step 17: Check that the merge carried the state of `M`'s canisters over and - // left `R`'s own canister alone. + // Step 17: `U8` was hosted by `M` and is served by `R` now. info!( logger, "Step 17: Checking the state of U8 and UR after the merge" ); - let u8_on_r = UniversalCanister::from_canister_id(&r_agent, u8.canister_id()); + let u8 = UniversalCanister::from_canister_id(&context.r_agent, context.u8); assert_eq!( - u8_on_r - .try_read_stable( - STABLE_MEMORY_OFFSET, - STABLE_MEMORY_BLOB.len().try_into().unwrap() - ) - .await, + u8.try_read_stable( + STABLE_MEMORY_OFFSET, + STABLE_MEMORY_BLOB.len().try_into().unwrap() + ) + .await, STABLE_MEMORY_BLOB, "the stable memory of U8 did not survive the merge", ); assert!( - canister_snapshot_ids(&r_agent, u8.canister_id()) + canister_snapshot_ids(&context.r_agent, context.u8) .await - .contains(&u8_snapshot), + .contains(&context.u8_snapshot), "the snapshot of U8 did not survive the merge", ); - let u8_cycles_after = cycles_balance(&u8_on_r) + let u8_cycles_after = cycles_balance(&u8) .await .expect("failed to read the cycles balance of U8 after the merge"); + let u8_cycles_before = context.u8_cycles_before; assert!( u8_cycles_after <= u8_cycles_before && u8_cycles_before - u8_cycles_after <= u8_cycles_before / MAX_BURNED_CYCLES_FRACTION, @@ -997,9 +948,10 @@ async fn run(env: TestEnv) { // `UR` was hosted by `R` all along: adding the canisters of `M` to `R`'s // state must not have disturbed it. And now that both are on `R`, they must // be able to call each other. + let ur = UniversalCanister::from_canister_id(&context.r_agent, context.ur); let ur_reply = ur .update(wasm().call_simple( - u8.canister_id(), + context.u8, "update", call_args().other_side(wasm().push_bytes(MERGED_CALL_REPLY).append_and_reply()), )) @@ -1019,8 +971,8 @@ async fn run(env: TestEnv) { "Step 17: Checking that {} have been installed", INSTALL_CODE_TARGETS.join(", "), ); - for (&target, name) in targets.iter().zip(INSTALL_CODE_TARGETS) { - let canister = UniversalCanister::from_canister_id(&r_agent, target); + for (&target, name) in context.targets.iter().zip(INSTALL_CODE_TARGETS) { + let canister = UniversalCanister::from_canister_id(&context.r_agent, target); let reply = canister .query(wasm().reply_data(name.as_bytes())) .await @@ -1033,7 +985,6 @@ async fn run(env: TestEnv) { "{name} ({target}) answered a query with an unexpected reply", ); } - info!( logger, "Step 17 done: U8 kept its stable memory, snapshot and cycles, UR can call it, and {} \ @@ -1045,18 +996,21 @@ async fn run(env: TestEnv) { // that was still in progress when the merge happened completed. // // The canisters of `M` now live on `R`, which serves them under the same - // canister IDs, so the agent for `R` is what reaches them. `U4` and `U7` did - // not move: they are on `T`. + // canister IDs, so the agent for `R` is what reaches them. `U7` did not + // move: it is on `T`. info!( logger, "Step 18: Breaking the endless loops and waiting for the pending ingress messages" ); - let u3 = UniversalCanister::from_canister_id(&r_agent, u3.canister_id()); - let u5 = UniversalCanister::from_canister_id(&r_agent, u5.canister_id()); - for (canister, name) in [(&u3, "U3"), (&u5, "U5"), (&u7, "U7")] { + for (agent, canister_id, name) in [ + (&context.r_agent, context.u3, "U3"), + (&context.r_agent, context.u5, "U5"), + (&context.t_agent, context.u7, "U7"), + ] { + let canister = UniversalCanister::from_canister_id(agent, canister_id); retry_with_msg_async!( format!("setting the global data of {name} to {LOOP_BREAK_TRIGGER:?}"), - &logger, + logger, READY_WAIT_TIMEOUT, RETRY_BACKOFF, || async { @@ -1072,45 +1026,29 @@ async fn run(env: TestEnv) { info!(logger, "Step 18: broke {name}'s endless loop"); } - for (name, agent, canister_id, request_id) in pending_ingress_messages { - await_ingress_message_replied(&agent, canister_id, &request_id, &name, &logger).await; + for (name, agent, canister_id, request_id) in &context.pending_ingress_messages { + await_ingress_message_replied(agent, *canister_id, request_id, name, logger).await; info!(logger, "Step 18: {name}'s ingress message completed"); } info!( logger, "Step 18 done: all the ingress messages that were pending across the merge completed" ); +} - // Step 18: Wait until every subnet observed the merge, i.e. routes the - // canisters that used to be hosted by `M` to `R`. Only then may `M` be - // deleted: a subnet still on an older registry version would keep routing - // messages to a subnet that no longer exists. - info!( - logger, - "Step 19: Waiting until all subnets reached registry version \ - {merge_registry_version}, which holds the merge" - ); - await_registry_version_on_all_subnets( - &topology, - m_subnet.subnet_id, - merge_registry_version, - &logger, - ) - .await; - info!(logger, "Step 19 done: all subnets observed the merge"); +/// Step 20: check that the subnet the tool deleted is gone from the registry. +async fn check_source_subnet_deleted(topology: &TopologySnapshot, context: &Context) { + let logger = &context.logger; + let m_subnet_id = context.m_subnet.subnet_id; - // Step 19: Delete the merged subnet, which hosts no canister ID range - // anymore, and check that it is gone from the registry. - info!( - logger, - "Step 20: Deleting subnet M ({})", m_subnet.subnet_id - ); - let topology = delete_subnet(&env, m_subnet.subnet_id, &logger).await; + let topology = topology + .block_for_newer_registry_version() + .await + .expect("the registry should have a newer version after the subnet was deleted"); let remaining: Vec<_> = topology.subnets().map(|subnet| subnet.subnet_id).collect(); assert!( - !remaining.contains(&m_subnet.subnet_id), - "subnet M ({}) is still in the registry at version {}: {remaining:?}", - m_subnet.subnet_id, + !remaining.contains(&m_subnet_id), + "subnet M ({m_subnet_id}) is still in the registry at version {}: {remaining:?}", topology.get_registry_version(), ); info!( @@ -1121,273 +1059,6 @@ async fn run(env: TestEnv) { ); } -/// Everything the synchronous, `ic-recovery` driven part of the merge needs: it -/// downloads the states of both subnets, assembles the merged state as a new -/// checkpoint of the destination subnet, and puts it on the destination node. -/// -/// This is a plain struct of owned data rather than a closure over the test's -/// state because it has to be moved onto a blocking thread: `ic-recovery` blocks -/// on its own runtime, which a thread driving the test's runtime cannot do. -#[derive(Clone)] -struct MergeStateArgs { - logger: Logger, - admin_key_file: PathBuf, - nns_url: Url, - m_dir: PathBuf, - r_dir: PathBuf, - /// Recovery directory holding nothing but the merged checkpoint, which is - /// what makes it uploadable as a whole: the upload step insists that the - /// directory it uploads hold a single checkpoint. - /// - /// Besides the checkpoint it holds the states metadata, which the upload step - /// transfers alongside it. - merged_dir: PathBuf, - m_node_ip: IpAddr, - r_node_ip: IpAddr, - m_height: u64, - r_height: u64, - merged_height: u64, -} - -impl MergeStateArgs { - /// Downloads the states the two subnets halted at and assembles the merged - /// state from them, as a checkpoint of `merged_dir`. - /// - /// Returns the block time the recovered destination subnet should start from - /// and the hash of the manifest of the merged state, i.e. what the recovery - /// proposal of the destination subnet has to carry. - fn assemble(&self) -> (u64, Vec) { - let m_recovery = self.recovery(self.m_dir.clone()); - let r_recovery = self.recovery(self.r_dir.clone()); - - for (recovery, node_ip, height, name) in [ - (&m_recovery, self.m_node_ip, self.m_height, "M"), - (&r_recovery, self.r_node_ip, self.r_height, "R"), - ] { - info!(self.logger, "Downloading the state of subnet {name}"); - recovery - .get_download_state_step( - node_ip, - SshUser::Admin, - Some(self.admin_key_file.clone()), - /* keep_downloaded_state= */ false, - Some(height), - ) - .expect("failed to build the download step") - .exec() - .unwrap_or_else(|e| panic!("failed to download the state of subnet {name}: {e}")); - } - - let m_checkpoints = m_recovery.work_dir.join(IC_STATE_DIR).join("checkpoints"); - let r_checkpoints = r_recovery.work_dir.join(IC_STATE_DIR).join("checkpoints"); - let m_checkpoint = - m_checkpoints.join(StateLayout::checkpoint_name(Height::from(self.m_height))); - let r_checkpoint = - r_checkpoints.join(StateLayout::checkpoint_name(Height::from(self.r_height))); - let merged_checkpoint = self.merged_dir.join(IC_STATE_DIR).join("checkpoints").join( - StateLayout::checkpoint_name(Height::from(self.merged_height)), - ); - - // The block time the recovered subnet starts from has to be larger than - // the times of both checkpoints the merged state is assembled from. - let m_time = checkpoint_time_nanos(&m_checkpoint); - let r_time = checkpoint_time_nanos(&r_checkpoint); - let merged_time = m_time.max(r_time) + MERGED_STATE_TIME_MARGIN.as_nanos() as u64; - info!( - self.logger, - "M halted at time {m_time}, R at {r_time}; the merged state starts at {merged_time}" - ); - - assemble_merged_checkpoint(&r_checkpoint, &m_checkpoint, &merged_checkpoint); - - // The upload step of step 15 transfers the states metadata alongside the - // checkpoint, and rsync is given every path it transfers as an explicit - // source, so a missing one fails the whole transfer: take `R`'s along. - // - // It is a manifest cache, which the state manager recomputes for the - // checkpoints it finds whenever it is missing or does not describe them, - // and the heights it names here are the ones `R` held before the merge, - // none of which the merged state directory has. That is the same mismatch - // a plain subnet recovery uploads, where the metadata comes from the state - // that was downloaded and the checkpoint from the replay that followed. - std::fs::copy( - r_recovery.work_dir.join(IC_STATE_DIR).join(STATES_METADATA), - self.merged_dir.join(IC_STATE_DIR).join(STATES_METADATA), - ) - .expect("failed to copy the states metadata of subnet R"); - - (merged_time, manifest_root_hash(&merged_checkpoint)) - } - - /// Uploads the merged state to the destination node, replacing its state - /// directory, and restarts its replica. - /// - /// This deletes the checkpoint the destination subnet halted at, which is - /// what makes the recovery unambiguous: that checkpoint does not hold the - /// canisters of the source subnet, so a replica coming up on it would serve - /// a state that silently lost them. With the state directory replaced, the - /// merged state is the only one the replica can resume from. - /// - /// The recovery CUP has to exist by the time this runs, since the replica is - /// restarted right away. - /// - /// `UploadStateAndRestartStep` rather than - /// `Recovery::get_upload_state_and_restart_step`: the latter hardcodes the - /// check that the uploaded checkpoint matches the height an `ic-replay` run - /// reported, and this merge runs no `ic-replay`. Subnet splitting builds the - /// step directly for the same reason. - fn upload_merged_state(&self) { - info!(self.logger, "Uploading the merged state to R"); - UploadStateAndRestartStep { - logger: self.logger.clone(), - ssh_user: SshUser::Admin, - upload_method: DataLocation::Remote(self.r_node_ip), - work_dir: self.merged_dir.clone(), - data_src: self.merged_dir.join(IC_STATE_DIR), - require_confirmation: false, - key_file: Some(self.admin_key_file.clone()), - check_ic_replay_height: false, - } - .exec() - .expect("failed to upload the merged state to subnet R"); - } - - fn recovery(&self, dir: PathBuf) -> Recovery { - Recovery::new( - self.logger.clone(), - RecoveryArgs { - dir, - nns_url: self.nns_url.clone(), - replica_version: None, - admin_key_file: Some(self.admin_key_file.clone()), - test_mode: true, - skip_prompts: true, - }, - /* neuron_args= */ None, - self.nns_url.clone(), - RegistryPollingStrategy::OnlyOnInit, - ) - .expect("failed to init recovery") - } -} - -/// Assembles the checkpoint at `merged` from the checkpoints at `base` and -/// `source`, i.e. runs the state side of the subnet merge. -fn assemble_merged_checkpoint(base: &Path, source: &Path, merged: &Path) { - state_tool(&[ - "merge", - "--base", - &base.display().to_string(), - "--source", - &source.display().to_string(), - "--output", - &merged.display().to_string(), - ]); -} - -/// Submits (and adopts) the `MergeSubnets` proposal rerouting the canister ID -/// ranges of `source_subnet` to `destination_subnet`, and returns the registry -/// version it created. -async fn merge_subnets( - env: &TestEnv, - source_subnet: SubnetId, - destination_subnet: SubnetId, - logger: &Logger, -) -> u64 { - let payload = MergeSubnetsPayload { - source_subnet, - destination_subnet, - }; - submit_and_adopt_proposal(env, NnsFunction::MergeSubnets, payload, logger).await -} - -/// Submits (and adopts) the `RecoverSubnet` proposal creating a recovery CUP for -/// `subnet_id` at the given height, time and state hash, and returns the registry -/// version it created. -async fn recover_subnet( - env: &TestEnv, - subnet_id: SubnetId, - height: u64, - time_ns: u64, - state_hash: Vec, - logger: &Logger, -) -> u64 { - let payload = RecoverSubnetPayload { - subnet_id: subnet_id.get(), - // The NNS subnet, which is neither of the two subnets being merged and - // stays available throughout, handles the DKG of the recovery CUP. - initial_dkg_subnet_id: None, - height, - time_ns, - state_hash, - // The subnet keeps its membership, holds no chain key and is not becoming - // the NNS subnet, so there is nothing else to recover. - replacement_nodes: None, - registry_store_uri: None, - chain_key_config: None, - }; - submit_and_adopt_proposal(env, NnsFunction::RecoverSubnet, payload, logger).await -} - -/// Submits (and adopts) the `DeleteSubnet` proposal deleting `subnet_id`, and -/// returns a topology snapshot taken after its mutations were applied. -async fn delete_subnet(env: &TestEnv, subnet_id: SubnetId, logger: &Logger) -> TopologySnapshot { - let topology = env.topology_snapshot(); - let nns_node = topology.root_subnet().nodes().next().unwrap(); - let nns_runtime = runtime_from_url(nns_node.get_public_url(), nns_node.effective_canister_id()); - let governance = get_governance_canister(&nns_runtime); - - let payload = DeleteSubnetPayload { - subnet_id: subnet_id.get().into(), - }; - let proposal_id = - submit_external_proposal_with_test_id(&governance, NnsFunction::DeleteSubnet, payload) - .await; - info!(logger, "Submitted {proposal_id}"); - vote_execute_proposal_assert_executed(&governance, proposal_id).await; - - topology - .block_for_newer_registry_version() - .await - .expect("the registry should have a newer version after the proposal executed") -} - -/// Waits until every subnet other than `stopped` has reached `registry_version`. -/// -/// `stopped` is the merged subnet, whose replica this test stopped for the merge -/// and which therefore does not report metrics anymore. It is also the subnet -/// about to be deleted, so what matters is that every *other* subnet already -/// routes its canisters to the destination subnet. -async fn await_registry_version_on_all_subnets( - topology: &TopologySnapshot, - stopped: SubnetId, - registry_version: u64, - logger: &Logger, -) { - retry_with_msg_async!( - format!("waiting until all subnets reached registry version {registry_version}"), - logger, - READY_WAIT_TIMEOUT, - RETRY_BACKOFF, - || async { - for subnet in topology.subnets().filter(|s| s.subnet_id != stopped) { - let metrics = fetch_metrics(&subnet, &[METRIC_REGISTRY_VERSION]).await?; - let version = median_across_replicas(&metrics, METRIC_REGISTRY_VERSION, |_| true) - .unwrap_or(0.0); - if version < registry_version as f64 { - bail!( - "subnet {} is at registry version {version}", - subnet.subnet_id - ); - } - } - Ok(()) - } - ) - .await - .unwrap_or_else(|e| panic!("not all subnets reached registry version {registry_version}: {e}")); -} - /// Waits until the ingress message `request_id` sent to `canister_id` is /// replied, i.e. until the update call it carries completed successfully. async fn await_ingress_message_replied( @@ -1658,219 +1329,6 @@ async fn global_counter(canister: &UniversalCanister<'_>) -> Result { Ok(u64::from_le_bytes(reply)) } -/// Submits and adopts an `UpdateConfigOfSubnet` proposal labeling `subnet_id` as -/// "cooling down" in its subnet record. Returns the registry version created by -/// the proposal, i.e. `V` in the dashboard's readiness condition. -async fn set_subnet_cooling_down(env: &TestEnv, subnet_id: SubnetId, logger: &Logger) -> u64 { - let payload = UpdateSubnetPayload { - cooling_down: Some(true), - ..empty_update_subnet_payload(subnet_id) - }; - submit_and_adopt_update_subnet_proposal(env, payload, logger).await -} - -/// An `UpdateSubnetPayload` for `subnet_id` that changes nothing, to be used as -/// the base of a payload changing a single field. -fn empty_update_subnet_payload(subnet_id: SubnetId) -> UpdateSubnetPayload { - UpdateSubnetPayload { - subnet_id, - cooling_down: None, - max_ingress_bytes_per_message: None, - max_ingress_messages_per_block: None, - max_ingress_bytes_per_block: None, - max_block_payload_size: None, - unit_delay_millis: None, - initial_notary_delay_millis: None, - dkg_interval_length: None, - dkg_dealings_per_block: None, - start_as_nns: None, - subnet_type: None, - is_halted: None, - halt_at_cup_height: None, - features: None, - resource_limits: None, - chain_key_config: None, - chain_key_signing_enable: None, - chain_key_signing_disable: None, - max_number_of_canisters: None, - ssh_readonly_access: None, - ssh_backup_access: None, - subnet_admins: None, - // Deprecated/unused values follow - max_artifact_streams_per_peer: None, - max_chunk_wait_ms: None, - max_duplicity: None, - max_chunk_size: None, - receive_check_cache_size: None, - pfn_evaluation_period_ms: None, - registry_poll_period_ms: None, - retransmission_request_ms: None, - set_gossip_config_to_default: false, - } -} - -/// Submits (and adopts) `payload` as an `UpdateConfigOfSubnet` proposal, and -/// returns the registry version its mutation created. -async fn submit_and_adopt_update_subnet_proposal( - env: &TestEnv, - payload: UpdateSubnetPayload, - logger: &Logger, -) -> u64 { - submit_and_adopt_proposal(env, NnsFunction::UpdateConfigOfSubnet, payload, logger).await -} - -/// Submits (and adopts) `payload` as a proposal calling `nns_function`, and -/// returns the registry version its mutations created. -async fn submit_and_adopt_proposal( - env: &TestEnv, - nns_function: NnsFunction, - payload: T, - logger: &Logger, -) -> u64 { - let topology = env.topology_snapshot(); - let nns_node = topology.root_subnet().nodes().next().unwrap(); - let nns_runtime = runtime_from_url(nns_node.get_public_url(), nns_node.effective_canister_id()); - let governance = get_governance_canister(&nns_runtime); - - let proposal_id = - submit_external_proposal_with_test_id(&governance, nns_function, payload).await; - info!(logger, "Submitted proposal {proposal_id}"); - vote_execute_proposal_assert_executed(&governance, proposal_id).await; - - // The proposal's registry mutations are applied in a single registry version, - // which is the newest one. The snapshot above was taken before the proposal - // was submitted, so this cannot miss the version the mutations created. - topology - .block_for_newer_registry_version() - .await - .expect("the registry should have a newer version after the proposal executed") - .get_registry_version() - .get() -} - -/// Evaluates the terms of the "merge readiness" condition of the `Subnet -/// merging` dashboard for `subnet` (the subnet that is cooling down), -/// `registry_version` (`V`) and `max_refund_value_cycles` (the dashboard's -/// `R`, not to be confused with the subnet `R`). Returns one -/// (description, satisfied) pair per term, in the order the terms appear in the -/// dashboard's readiness expression. -/// -/// As in the dashboard, every term is evaluated on the median across the -/// replicas reporting the respective series, and missing data reads as zero -/// (the dashboard's `or vector(0)` fallback). -async fn evaluate_merge_readiness( - topology: &TopologySnapshot, - subnet: &SubnetSnapshot, - registry_version: u64, - max_refund_value_cycles: f64, -) -> Result> { - let subnet_id = subnet.subnet_id; - let own_metrics = fetch_metrics( - subnet, - &[ - METRIC_REGISTRY_VERSION, - METRIC_STREAM_MESSAGES, - METRIC_INGRESS_HISTORY_BY_STATE, - METRIC_SUBNET_INPUT_QUEUE_MESSAGES, - METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES, - METRIC_SUBNET_CALL_CONTEXTS, - METRIC_PENDING_REFUNDS_CYCLES, - ], - ) - .await?; - - // Terms 1 and 2 range over all subnets: the registry version of every - // subnet and the streams of all remote subnets towards this one. - let remote_label = format!("remote=\"{subnet_id}\""); - let mut min_registry_version = None; - let mut incoming_stream_messages = 0.0; - for other in topology.subnets() { - let metrics = if other.subnet_id == subnet_id { - own_metrics.clone() - } else { - fetch_metrics(&other, &[METRIC_REGISTRY_VERSION, METRIC_STREAM_MESSAGES]).await? - }; - let version = - median_across_replicas(&metrics, METRIC_REGISTRY_VERSION, |_| true).unwrap_or(0.0); - min_registry_version = Some(min_registry_version.map_or(version, |v: f64| v.min(version))); - if other.subnet_id != subnet_id { - incoming_stream_messages += - sum_of_medians(&metrics, METRIC_STREAM_MESSAGES, |labels| { - labels.contains(&remote_label) - }); - } - } - let min_registry_version = min_registry_version.unwrap_or(0.0); - - let outgoing_stream_messages = sum_of_medians(&own_metrics, METRIC_STREAM_MESSAGES, |_| true); - let ingress_history_messages = - sum_of_medians(&own_metrics, METRIC_INGRESS_HISTORY_BY_STATE, |labels| { - !labels.contains("state=\"processing\"") - }); - let subnet_input_queue_messages = - sum_of_medians(&own_metrics, METRIC_SUBNET_INPUT_QUEUE_MESSAGES, |_| true); - let subnet_output_queue_messages = - median_across_replicas(&own_metrics, METRIC_SUBNET_OUTPUT_QUEUE_MESSAGES, |_| true) - .unwrap_or(0.0); - let subnet_call_contexts = sum_of_medians(&own_metrics, METRIC_SUBNET_CALL_CONTEXTS, |_| true); - let pending_refunds_cycles = - median_across_replicas(&own_metrics, METRIC_PENDING_REFUNDS_CYCLES, |_| true) - .unwrap_or(0.0); - - Ok(vec![ - ( - format!( - "every subnet has reached registry version {registry_version} (the lowest one is \ - at {min_registry_version})" - ), - min_registry_version >= registry_version as f64, - ), - ( - format!( - "no remote subnet holds a message in its stream to subnet {subnet_id} \ - ({incoming_stream_messages} messages)" - ), - incoming_stream_messages == 0.0, - ), - ( - format!( - "subnet {subnet_id} holds no message in any of its streams, loopback included \ - ({outgoing_stream_messages} messages)" - ), - outgoing_stream_messages == 0.0, - ), - ( - format!( - "the ingress history holds nothing but `processing` entries \ - ({ingress_history_messages} other entries)" - ), - ingress_history_messages == 0.0, - ), - ( - format!("the subnet input queues are empty ({subnet_input_queue_messages} messages)"), - subnet_input_queue_messages == 0.0, - ), - ( - format!("the subnet output queues are empty ({subnet_output_queue_messages} messages)"), - subnet_output_queue_messages == 0.0, - ), - ( - format!( - "the subnet call context manager holds no call context ({subnet_call_contexts} \ - call contexts)" - ), - subnet_call_contexts == 0.0, - ), - ( - format!( - "the pending anonymous refunds are worth at most {max_refund_value_cycles} cycles \ - ({pending_refunds_cycles} cycles)" - ), - pending_refunds_cycles <= max_refund_value_cycles, - ), - ]) -} - /// Fetches the given metrics from all nodes of `subnet`, keyed by series (i.e. /// metric name plus labels), with one value per node reporting the series. async fn fetch_metrics( @@ -1891,24 +1349,6 @@ async fn fetch_metrics( }) } -/// Fetches `metrics` from a single node, rather than from all the nodes of a -/// subnet: the values of a subnet-wide property still differ per replica while -/// they observe it in different rounds, and some questions are about one node, -/// such as whether the very node a state is about to be downloaded from has -/// stopped moving. -async fn fetch_node_metrics( - node: &IcNodeSnapshot, - metrics: &[&str], -) -> Result>> { - MetricsFetcher::new( - std::iter::once(node.clone()), - metrics.iter().map(|metric| metric.to_string()).collect(), - ) - .fetch::() - .await - .map_err(|e| anyhow!("failed to fetch the metrics of node {}: {e}", node.node_id)) -} - /// The per-node values of every series of `metric` whose labels (`{...}`, or the /// empty string for an unlabeled series) match `labels_match`. /// @@ -1987,170 +1427,3 @@ fn median_across_replicas( .collect(); median(&values) } - -// --------------------------------------------------------------------------- -// Merging subnet M into subnet R. -// --------------------------------------------------------------------------- - -/// Waits until `node` holds the CUP its subnet halts at, and returns its -/// height, i.e. the height of the checkpoint holding the state the subnet -/// stopped in. -/// -/// A CUP whose summary block was created at `halt_registry_version` or later is -/// one the subnet halts at: the `halt_at_cup_height` flag is read at the -/// registry version of the summary block active at a height, and that version -/// only changes at a summary, so batch delivery stops exactly when the summary -/// carrying the flag becomes active. As checkpoints are written at CUP heights, -/// the height of that CUP is the height of the last checkpoint the subnet -/// wrote. -/// -/// Waiting for the CUP rather than for the subnet to report that it is halted: -/// the CUP is what names the state the subnet came to rest in, and it exists -/// only once that state has been certified and its hash agreed upon, which is -/// what the recovery proposal of step 14 compares its state hash against. A -/// subnet that has just stopped delivering batches, on the other hand, may not -/// have finished hashing the checkpoint it stopped at, and reading its latest -/// checkpoint height then yields the previous one, a whole DKG interval before -/// the state the merge is supposed to be assembled from. -/// -/// `node`'s own CUP and metrics, not the subnet's: the state that is downloaded -/// below is this node's, so this node is the one that has to have reached the -/// CUP. -async fn await_halting_cup( - node: &IcNodeSnapshot, - name: &str, - halt_registry_version: u64, - logger: &Logger, -) -> u64 { - info!( - logger, - "Waiting until subnet {name} reaches the CUP it halts at" - ); - let height = retry_with_msg_async!( - format!("waiting until subnet {name} reaches the CUP it halts at"), - logger, - HALT_TIMEOUT, - HALT_BACKOFF, - || async { - let cup = get_cup_from_node(node, logger).await?; - let cup_height = cup.height().get(); - let cup_registry_version = cup - .content - .block - .get_value() - .payload - .as_ref() - .as_summary() - .dkg - .registry_version - .get(); - if cup_registry_version < halt_registry_version { - bail!( - "subnet {name} is at the CUP at height {cup_height}, whose registry \ - version {cup_registry_version} precedes the version \ - {halt_registry_version} it is instructed to halt at" - ); - } - - // The node has to have caught up with the CUP itself: it is its - // state that is downloaded below, and a node can hold a CUP that - // the rest of the subnet assembled before it got there. - let certification_height = certification_height(node).await?; - assert!( - certification_height <= cup_height, - "subnet {name} certified height {certification_height}, past the CUP at \ - height {cup_height} it should have halted at", - ); - if certification_height < cup_height { - bail!( - "subnet {name} holds the CUP at height {cup_height} but has only \ - certified up to height {certification_height}" - ); - } - - Ok(cup_height) - } - ) - .await - .unwrap_or_else(|e| panic!("subnet {name} did not reach the CUP it halts at: {e}")); - - assert_eq!( - height % CHECKPOINT_INTERVAL, - 0, - "subnet {name} halted at height {height}, which is not a checkpoint height", - ); - height -} - -/// The height of the highest certification `node` holds, i.e. how far its state -/// is certified. -async fn certification_height(node: &IcNodeSnapshot) -> Result { - let metrics = fetch_node_metrics(node, &[METRIC_CERTIFICATION_HEIGHT]).await?; - let height = median_across_replicas(&metrics, METRIC_CERTIFICATION_HEIGHT, |_| true) - .ok_or_else(|| anyhow!("no certification height has been reported yet"))?; - Ok(height as u64) -} - -/// Submits (and adopts) an `UpdateConfigOfSubnet` proposal setting the -/// `halt_at_cup_height` flag of `subnet_id`, so that the subnet halts once it -/// reaches its next CUP, i.e. at a checkpoint whose state is certified. -/// Returns the registry version the proposal created, which is the version at -/// which the subnet is instructed to halt. -async fn halt_subnet_at_cup_height(env: &TestEnv, subnet_id: SubnetId, logger: &Logger) -> u64 { - let payload = UpdateSubnetPayload { - halt_at_cup_height: Some(true), - ..empty_update_subnet_payload(subnet_id) - }; - submit_and_adopt_update_subnet_proposal(env, payload, logger).await -} - -/// Submits (and adopts) an `UpdateConfigOfSubnet` proposal clearing the -/// `is_halted` flag of `subnet_id`, so that the subnet resumes delivering -/// batches. Returns the registry version the proposal created. -/// -/// This is the flag `recover_subnet` sets in place of the `halt_at_cup_height` -/// flag it clears, so that a recovered subnet stays halted until its recovery -/// has been checked. -async fn unhalt_subnet(env: &TestEnv, subnet_id: SubnetId, logger: &Logger) -> u64 { - let payload = UpdateSubnetPayload { - is_halted: Some(false), - ..empty_update_subnet_payload(subnet_id) - }; - submit_and_adopt_update_subnet_proposal(env, payload, logger).await -} - -/// Runs `state-tool` with the given arguments and returns its standard output. -fn state_tool(args: &[&str]) -> String { - let binary = get_dependency_path_from_env("ENV_DEPS__STATE_TOOL"); - let output = Command::new(&binary) - .args(args) - .output() - .unwrap_or_else(|e| panic!("failed to run {}: {e}", binary.display())); - assert!( - output.status.success(), - "{} {args:?} failed: {}", - binary.display(), - String::from_utf8_lossy(&output.stderr), - ); - String::from_utf8(output.stdout).expect("state-tool output should be UTF-8") -} - -/// The batch time of the checkpoint at `path`, in nanoseconds since the Epoch. -fn checkpoint_time_nanos(path: &Path) -> u64 { - let output = state_tool(&["checkpoint_time", "--state", &path.display().to_string()]); - output - .trim() - .parse() - .unwrap_or_else(|e| panic!("failed to parse the checkpoint time {output:?}: {e}")) -} - -/// The root hash of the manifest of the checkpoint at `path`. -fn manifest_root_hash(path: &Path) -> Vec { - let output = state_tool(&["manifest", "--state", &path.display().to_string()]); - let hash = output - .lines() - .find_map(|line| line.strip_prefix("ROOT HASH: ")) - .unwrap_or_else(|| panic!("no root hash in the manifest of {}", path.display())) - .trim(); - hex::decode(hash).unwrap_or_else(|e| panic!("root hash {hash} is not hex: {e}")) -} From 641bcf63ecc8dd5911cd33164d3a1cd13e5f1d18 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Fri, 11 Sep 2026 16:37:24 +0000 Subject: [PATCH 30/30] refactor(recovery): share the common helpers of the subnet reshaping tools `subnet_merging` was created as a sibling of `subnet_splitting` and took a copy of the parts of it that are not specific to splitting a subnet. Both tools halt a subnet at a CUP, work on the state it halted at, validate what they produced against that CUP and the NNS signed state tree, and let an operator confirm a dashboard before the next proposal -- so those parts now live in one place, `rs/recovery/subnet_tools`, which both tools depend on: * `agent_helper`: the NNS signed state tree and the subnet public key in it; * `validation`: state tree -> subnet key -> CUP signature -> manifest root hash; * `state_tool_helper`: the `state_tool` commands these tools run, i.e. computing and verifying a manifest, splitting a manifest, merging two checkpoints and reading the batch time of one; * `utils`: reading a CUP, its block time, and the state hash of a checkpoint; * `admin_helper`: halting a subnet at its next CUP height, and the arguments naming the subnets an operation moves canister id ranges between; * `cli`: printing a dashboard URL and waiting for the operator's confirmation; * `steps`: `ReadRegistryStep`, which is generic over what it reads. What is specific to either operation stays where it was: the canister migration proposals, the state splitting step and the expected manifest cross-check in `subnet_splitting`, and the cooling down, merge and delete proposals, the merge readiness evaluation, the halting CUP wait and the state merging step in `subnet_merging`. The dependencies the moved code brought with it leave the two tools, and `state_tool_lib` is now visible to `//rs/recovery:__subpackages__` rather than to two packages by name. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 35 ++-- Cargo.toml | 1 + rs/recovery/subnet_merging/BUILD.bazel | 8 +- rs/recovery/subnet_merging/Cargo.toml | 8 +- .../subnet_merging/src/admin_helper.rs | 55 +----- rs/recovery/subnet_merging/src/lib.rs | 3 - rs/recovery/subnet_merging/src/main.rs | 6 +- .../subnet_merging/src/state_tool_helper.rs | 53 ------ rs/recovery/subnet_merging/src/steps.rs | 12 +- .../subnet_merging/src/subnet_merging.rs | 29 +-- rs/recovery/subnet_merging/src/utils.rs | 32 +--- rs/recovery/subnet_splitting/BUILD.bazel | 24 +-- rs/recovery/subnet_splitting/Cargo.toml | 6 +- .../subnet_splitting/src/admin_helper.rs | 61 +----- .../subnet_splitting/src/agent_helper.rs | 166 ---------------- rs/recovery/subnet_splitting/src/lib.rs | 3 - rs/recovery/subnet_splitting/src/main.rs | 2 +- rs/recovery/subnet_splitting/src/steps.rs | 48 +---- .../subnet_splitting/src/subnet_splitting.rs | 33 +--- rs/recovery/subnet_splitting/src/utils.rs | 36 +--- .../subnet_splitting/src/validation.rs | 177 ------------------ rs/recovery/subnet_tools/BUILD.bazel | 35 ++++ rs/recovery/subnet_tools/Cargo.toml | 24 +++ rs/recovery/subnet_tools/src/admin_helper.rs | 88 +++++++++ .../src/agent_helper.rs | 18 +- rs/recovery/subnet_tools/src/cli.rs | 22 +++ rs/recovery/subnet_tools/src/lib.rs | 16 ++ .../src/state_tool_helper.rs | 52 +++-- rs/recovery/subnet_tools/src/steps.rs | 42 +++++ rs/recovery/subnet_tools/src/utils.rs | 40 ++++ .../src/validation.rs | 0 rs/state_tool/BUILD.bazel | 5 +- 32 files changed, 388 insertions(+), 752 deletions(-) delete mode 100644 rs/recovery/subnet_merging/src/state_tool_helper.rs delete mode 100644 rs/recovery/subnet_splitting/src/agent_helper.rs delete mode 100644 rs/recovery/subnet_splitting/src/validation.rs create mode 100644 rs/recovery/subnet_tools/BUILD.bazel create mode 100644 rs/recovery/subnet_tools/Cargo.toml create mode 100644 rs/recovery/subnet_tools/src/admin_helper.rs rename rs/recovery/{subnet_merging => subnet_tools}/src/agent_helper.rs (90%) create mode 100644 rs/recovery/subnet_tools/src/cli.rs create mode 100644 rs/recovery/subnet_tools/src/lib.rs rename rs/recovery/{subnet_splitting => subnet_tools}/src/state_tool_helper.rs (56%) create mode 100644 rs/recovery/subnet_tools/src/steps.rs create mode 100644 rs/recovery/subnet_tools/src/utils.rs rename rs/recovery/{subnet_merging => subnet_tools}/src/validation.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index fe417ecb0ac0..af76d864c61d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14980,11 +14980,7 @@ dependencies = [ "anyhow", "clap", "futures", - "hex", - "ic-agent", "ic-base-types", - "ic-crypto-utils-threshold-sig", - "ic-crypto-utils-threshold-sig-der", "ic-cup-explorer", "ic-protobuf", "ic-recovery", @@ -14992,13 +14988,11 @@ dependencies = [ "ic-registry-routing-table", "ic-registry-subnet-type", "ic-state-layout", - "ic-state-manager", - "ic-state-tool", + "ic-subnet-tools", "ic-test-utilities-tmpdir", "ic-types", "reqwest", "serde", - "serde_cbor", "serde_json", "slog", "strum 0.26.3", @@ -15015,11 +15009,7 @@ dependencies = [ "candid", "clap", "csv", - "hex", - "ic-agent", "ic-base-types", - "ic-crypto-utils-threshold-sig", - "ic-crypto-utils-threshold-sig-der", "ic-management-canister-types-private", "ic-metrics", "ic-protobuf", @@ -15030,6 +15020,7 @@ dependencies = [ "ic-state-machine-tests", "ic-state-manager", "ic-state-tool", + "ic-subnet-tools", "ic-test-utilities-logger", "ic-test-utilities-tmpdir", "ic-test-utilities-types", @@ -15038,13 +15029,33 @@ dependencies = [ "ic-universal-canister", "proxy_canister", "serde", - "serde_cbor", "slog", "strum 0.26.3", "strum_macros 0.26.4", "url", ] +[[package]] +name = "ic-subnet-tools" +version = "0.9.0" +dependencies = [ + "hex", + "ic-agent", + "ic-base-types", + "ic-crypto-utils-threshold-sig", + "ic-crypto-utils-threshold-sig-der", + "ic-protobuf", + "ic-recovery", + "ic-registry-routing-table", + "ic-registry-subnet-type", + "ic-state-manager", + "ic-state-tool", + "ic-types", + "serde_cbor", + "slog", + "url", +] + [[package]] name = "ic-sys" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 558f2776b243..f497c666c00a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -343,6 +343,7 @@ members = [ "rs/recovery", "rs/recovery/subnet_merging", "rs/recovery/subnet_splitting", + "rs/recovery/subnet_tools", "rs/registry/admin", "rs/registry/admin-derive", "rs/registry/canister", diff --git a/rs/recovery/subnet_merging/BUILD.bazel b/rs/recovery/subnet_merging/BUILD.bazel index 37c0af6dd351..da493020eb70 100644 --- a/rs/recovery/subnet_merging/BUILD.bazel +++ b/rs/recovery/subnet_merging/BUILD.bazel @@ -2,27 +2,21 @@ load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") DEPENDENCIES = [ # Keep sorted. - "//rs/crypto/utils/threshold_sig", - "//rs/crypto/utils/threshold_sig_der", "//rs/cup_explorer", "//rs/protobuf", "//rs/recovery", + "//rs/recovery/subnet_tools", "//rs/registry/helpers", "//rs/registry/routing_table", "//rs/registry/subnet_type", "//rs/state_layout", - "//rs/state_manager", - "//rs/state_tool:state_tool_lib", "//rs/types/base_types", "//rs/types/types", "@crate_index//:anyhow", "@crate_index//:clap", "@crate_index//:futures", - "@crate_index//:hex", - "@crate_index//:ic-agent", "@crate_index//:reqwest", "@crate_index//:serde", - "@crate_index//:serde_cbor", "@crate_index//:serde_json", "@crate_index//:slog", "@crate_index//:strum", diff --git a/rs/recovery/subnet_merging/Cargo.toml b/rs/recovery/subnet_merging/Cargo.toml index ba50f3ca82e3..21d7650501d5 100644 --- a/rs/recovery/subnet_merging/Cargo.toml +++ b/rs/recovery/subnet_merging/Cargo.toml @@ -10,11 +10,7 @@ documentation.workspace = true anyhow = { workspace = true } clap = { workspace = true } futures = { workspace = true } -hex = { workspace = true } -ic-agent = { workspace = true } ic-base-types = { path = "../../types/base_types/" } -ic-crypto-utils-threshold-sig = { path = "../../crypto/utils/threshold_sig" } -ic-crypto-utils-threshold-sig-der = { path = "../../crypto/utils/threshold_sig_der" } ic-cup-explorer = { path = "../../cup_explorer" } ic-protobuf = { path = "../../protobuf" } ic-recovery = { path = "../" } @@ -22,12 +18,10 @@ ic-registry-client-helpers = { path = "../../registry/helpers" } ic-registry-routing-table = { path = "../../registry/routing_table" } ic-registry-subnet-type = { path = "../../registry/subnet_type" } ic-state-layout = { path = "../../state_layout" } -ic-state-manager = { path = "../../state_manager" } -ic-state-tool = { path = "../../state_tool" } +ic-subnet-tools = { path = "../subnet_tools" } ic-types = { path = "../../types/types" } reqwest = { workspace = true } serde = { workspace = true } -serde_cbor = { workspace = true } serde_json = { workspace = true } slog = { workspace = true } diff --git a/rs/recovery/subnet_merging/src/admin_helper.rs b/rs/recovery/subnet_merging/src/admin_helper.rs index 29ec7b74c7f6..1a592000b35f 100644 --- a/rs/recovery/subnet_merging/src/admin_helper.rs +++ b/rs/recovery/subnet_merging/src/admin_helper.rs @@ -2,10 +2,8 @@ use ic_base_types::SubnetId; use ic_recovery::admin_helper::{ AdminHelper, CommandHelper, IcAdmin, SSH_READONLY_ACCESS_ARG, SUMMARY_ARG, quote, }; +use ic_subnet_tools::admin_helper::{DESTINATION_SUBNET_ARG, SOURCE_SUBNET_ARG, SUBNET_ARG}; -const SOURCE_SUBNET_ARG: &str = "source-subnet"; -const DESTINATION_SUBNET_ARG: &str = "destination-subnet"; -const SUBNET_ARG: &str = "subnet"; const SUBNET_ID_ARG: &str = "subnet-id"; /// Propose to label the subnet as "cooling down", i.e. to have it stop @@ -41,36 +39,6 @@ pub(crate) fn get_propose_to_cool_down_subnet_command( ic_admin } -/// Propose to make the subnet halt after reaching the next CUP height. -/// -/// Optionally adds a ssh-readonly-access key to the subnet. -pub(crate) fn get_halt_subnet_at_cup_height_command( - admin_helper: &AdminHelper, - subnet_id: SubnetId, - key: &Option, -) -> IcAdmin { - let mut ic_admin = admin_helper.get_ic_admin_cmd_base(); - - ic_admin - .add_positional_argument("propose-to-update-subnet") - .add_argument(SUBNET_ARG, subnet_id) - .add_argument( - SUMMARY_ARG, - quote(format!( - "Halt subnet {subnet_id} at cup height and optionally update ssh readonly access", - )), - ) - .add_argument("halt-at-cup-height", true); - - if let Some(key) = key { - ic_admin.add_argument(SSH_READONLY_ACCESS_ARG, quote(key)); - } - - admin_helper.add_proposer_args(&mut ic_admin); - - ic_admin -} - /// Propose to reroute the canister ID ranges of the source subnet to the /// destination subnet, i.e. to merge the former into the latter. pub(crate) fn get_propose_to_merge_subnets_command( @@ -159,27 +127,6 @@ mod tests { ); } - #[test] - fn get_halt_subnet_at_cup_height_command_test() { - let result = get_halt_subnet_at_cup_height_command( - &fake_admin_helper(), - subnet_id_from_str(FAKE_SUBNET_ID_1), - &None, - ) - .join(" "); - - assert_eq!( - result, - "/fake/ic/admin/dir/ic-admin \ - --nns-url \"https://fake_nns_url.com:8080/\" \ - propose-to-update-subnet \ - --subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe \ - --summary \"Halt subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe at cup height and optionally update ssh readonly access\" \ - --halt-at-cup-height true \ - --test-neuron-proposer" - ); - } - #[test] fn get_propose_to_merge_subnets_command_test() { let result = get_propose_to_merge_subnets_command( diff --git a/rs/recovery/subnet_merging/src/lib.rs b/rs/recovery/subnet_merging/src/lib.rs index 00da8137e793..ff5a666ff2cc 100644 --- a/rs/recovery/subnet_merging/src/lib.rs +++ b/rs/recovery/subnet_merging/src/lib.rs @@ -1,12 +1,9 @@ pub mod readiness; pub mod subnet_merging; pub mod utils; -pub mod validation; mod admin_helper; -mod agent_helper; mod layout; mod metrics_helper; -mod state_tool_helper; mod steps; mod target_subnet; diff --git a/rs/recovery/subnet_merging/src/main.rs b/rs/recovery/subnet_merging/src/main.rs index 539bd95b0a17..233f4395357f 100644 --- a/rs/recovery/subnet_merging/src/main.rs +++ b/rs/recovery/subnet_merging/src/main.rs @@ -2,10 +2,8 @@ use anyhow::Context; use clap::Parser; use ic_base_types::SubnetId; use ic_recovery::{NeuronArgs, RecoveryArgs, cli, util}; -use ic_subnet_merging::{ - subnet_merging::{SubnetMerging, SubnetMergingArgs}, - validation::validate_artifacts, -}; +use ic_subnet_merging::subnet_merging::{SubnetMerging, SubnetMergingArgs}; +use ic_subnet_tools::validation::validate_artifacts; use ic_types::ReplicaVersion; use slog::{Logger, info, warn}; use url::Url; diff --git a/rs/recovery/subnet_merging/src/state_tool_helper.rs b/rs/recovery/subnet_merging/src/state_tool_helper.rs deleted file mode 100644 index d91bbe9e5764..000000000000 --- a/rs/recovery/subnet_merging/src/state_tool_helper.rs +++ /dev/null @@ -1,53 +0,0 @@ -use ic_recovery::{ - error::{RecoveryError, RecoveryResult}, - file_sync_helper::write_file, -}; - -use std::{fs::File, path::Path}; - -/// Computes manifest of a checkpoint at `dir` and writes it to `output_path`. -pub(crate) fn compute_manifest(dir: &Path, output_path: &Path) -> RecoveryResult<()> { - ic_state_tool::commands::manifest::compute_manifest(dir) - .map_err(|err| { - RecoveryError::StateToolError(format!("Failed to compute the state manifest: {err}")) - }) - .and_then(|manifest| write_file(output_path, manifest)) -} - -/// Verifies whether the textual representation of a manifest matches its root hash, and -/// returns the root hash. -pub(crate) fn verify_manifest(manifest_path: &Path) -> RecoveryResult { - let manifest_file = - File::open(manifest_path).map_err(|err| RecoveryError::file_error(manifest_path, err))?; - - ic_state_tool::commands::verify_manifest::verify_manifest(manifest_file) - .map_err(|err| { - RecoveryError::StateToolError(format!("Failed to verify the state manifest: {err}")) - }) - .map(hex::encode) -} - -/// Assembles the checkpoint at `output` from the checkpoints at `base` (the -/// state of the destination subnet) and `source` (the state of the subnet that -/// is merged away): it holds everything of `base`, with the canisters and -/// canister snapshots of `source` added to those of `base`, and is marked as -/// the product of a subnet merge. -pub(crate) fn merge_checkpoints(base: &Path, source: &Path, output: &Path) -> RecoveryResult<()> { - ic_state_tool::commands::merge::do_merge( - base.to_path_buf(), - source.to_path_buf(), - output.to_path_buf(), - ) - .map_err(|err| RecoveryError::StateToolError(format!("Failed to merge the states: {err}"))) -} - -/// The batch time of the checkpoint at `path`, in nanoseconds since the Epoch, -/// i.e. the IC time the subnet had reached when it wrote the checkpoint. -pub(crate) fn checkpoint_time_nanos(path: &Path) -> RecoveryResult { - ic_state_tool::commands::checkpoint_time::batch_time_nanos(path.to_path_buf()).map_err(|err| { - RecoveryError::StateToolError(format!( - "Failed to read the batch time of the checkpoint {}: {err}", - path.display() - )) - }) -} diff --git a/rs/recovery/subnet_merging/src/steps.rs b/rs/recovery/subnet_merging/src/steps.rs index e925a80d0474..b7d06764d916 100644 --- a/rs/recovery/subnet_merging/src/steps.rs +++ b/rs/recovery/subnet_merging/src/steps.rs @@ -1,10 +1,8 @@ use crate::{ - agent_helper::AgentHelper, layout::{CUP_FILE_NAME, Layout}, - readiness, state_tool_helper, + readiness, target_subnet::TargetSubnet, - utils::{MergedStateParams, first_registry_version_where, get_cup, get_state_hash}, - validation::validate_artifacts, + utils::{MergedStateParams, first_registry_version_where}, }; use ic_base_types::SubnetId; @@ -19,6 +17,12 @@ use ic_recovery::{ }; use ic_registry_client_helpers::routing_table::RoutingTableRegistry; use ic_registry_routing_table::RoutingTable; +use ic_subnet_tools::{ + agent_helper::AgentHelper, + state_tool_helper, + utils::{get_cup, get_state_hash}, + validation::validate_artifacts, +}; use ic_types::{Height, consensus::CatchUpPackage, consensus::HasHeight}; use slog::{Logger, info, warn}; use url::Url; diff --git a/rs/recovery/subnet_merging/src/subnet_merging.rs b/rs/recovery/subnet_merging/src/subnet_merging.rs index 53b8c72cf782..34f22e7ee366 100644 --- a/rs/recovery/subnet_merging/src/subnet_merging.rs +++ b/rs/recovery/subnet_merging/src/subnet_merging.rs @@ -1,7 +1,7 @@ use crate::{ admin_helper::{ - get_halt_subnet_at_cup_height_command, get_propose_to_cool_down_subnet_command, - get_propose_to_delete_subnet_command, get_propose_to_merge_subnets_command, + get_propose_to_cool_down_subnet_command, get_propose_to_delete_subnet_command, + get_propose_to_merge_subnets_command, }, layout::Layout, steps::{ @@ -18,7 +18,7 @@ use ic_base_types::SubnetId; use ic_protobuf::registry::subnet::v1::SubnetRecord; use ic_recovery::{ CUPS_DIR, NeuronArgs, Recovery, RecoveryArgs, - cli::{consent_given, read_optional, wait_for_confirmation}, + cli::{consent_given, read_optional}, error::{RecoveryError, RecoveryResult}, recovery_iterator::RecoveryIterator, recovery_state::{HasRecoveryState, RecoveryState}, @@ -28,12 +28,14 @@ use ic_recovery::{ util::{CheckpointHeight, DataLocation, ExecutionMode, SshUser}, }; use ic_registry_subnet_type::SubnetType; +use ic_subnet_tools::{ + admin_helper::get_halt_subnet_at_cup_height_command, cli::print_url_and_ask_for_confirmation, +}; use ic_types::Height; use serde::{Deserialize, Serialize}; -use slog::{Logger, error, info, warn}; +use slog::{Logger, info, warn}; use strum::{EnumMessage, IntoEnumIterator}; use strum_macros::{EnumIter, EnumString}; -use url::Url; use std::{ iter::Peekable, @@ -735,20 +737,3 @@ impl HasRecoveryState for SubnetMerging { }) } } - -fn print_url_and_ask_for_confirmation( - logger: &Logger, - url: String, - text_to_display: impl std::fmt::Display, -) { - match Url::parse(&url) { - Ok(url) => { - warn!(logger, "{}", text_to_display); - warn!(logger, "{}", url); - wait_for_confirmation(logger); - } - Err(err) => { - error!(logger, "Failed to parse url {}: {}", url, err); - } - } -} diff --git a/rs/recovery/subnet_merging/src/utils.rs b/rs/recovery/subnet_merging/src/utils.rs index 9d5a83a49ab6..fc43ad40b9a0 100644 --- a/rs/recovery/subnet_merging/src/utils.rs +++ b/rs/recovery/subnet_merging/src/utils.rs @@ -1,16 +1,13 @@ use ic_base_types::RegistryVersion; -use ic_protobuf::types::v1 as pb; use ic_recovery::{ RECOVERY_DIRECTORY_NAME, error::{RecoveryError, RecoveryResult}, file_sync_helper::{read_file, write_file}, registry_helper::RegistryHelper, }; -use ic_state_manager::manifest::{manifest_from_path, manifest_hash}; -use ic_types::consensus::CatchUpPackage; use serde::{Deserialize, Serialize}; -use std::{fmt::Display, path::Path}; +use std::path::Path; /// Everything the recovery of the destination subnet needs to know about the /// merged state, as computed by the step that assembles it. @@ -122,33 +119,6 @@ pub fn read_cooling_down_registry_version(dir: &Path) -> RecoveryResult { ) } -pub(crate) fn get_cup(cup_path: &Path) -> RecoveryResult { - let cup_proto = pb::CatchUpPackage::read_from_file(cup_path) - .map_err(|err| cup_error("Failed to decode the CUP file", cup_path, err))?; - - CatchUpPackage::try_from(&cup_proto) - .map_err(|err| cup_error("Failed to deserialize the CUP file", cup_path, err)) -} - -fn cup_error(message: impl Display, cup_path: &Path, error: impl Display) -> RecoveryError { - RecoveryError::UnexpectedError(format!("{} ({}): {}", message, cup_path.display(), error)) -} - -/// Computes the state hash of the given checkpoint. -pub(crate) fn get_state_hash(checkpoint_dir: impl AsRef) -> RecoveryResult { - let manifest = manifest_from_path(checkpoint_dir.as_ref()).map_err(|e| { - RecoveryError::CheckpointError( - format!( - "Failed to read the manifest from path {}", - checkpoint_dir.as_ref().display() - ), - e, - ) - })?; - - Ok(hex::encode(manifest_hash(&manifest))) -} - #[cfg(test)] mod tests { use super::*; diff --git a/rs/recovery/subnet_splitting/BUILD.bazel b/rs/recovery/subnet_splitting/BUILD.bazel index 70ee217a13cf..29ece9b3c6ea 100644 --- a/rs/recovery/subnet_splitting/BUILD.bazel +++ b/rs/recovery/subnet_splitting/BUILD.bazel @@ -13,11 +13,10 @@ rust_library( visibility = ["//rs:system-tests-pkg"], deps = [ # Keep sorted. - "//rs/crypto/utils/threshold_sig", - "//rs/crypto/utils/threshold_sig_der", "//rs/monitoring/metrics", "//rs/protobuf", "//rs/recovery", + "//rs/recovery/subnet_tools", "//rs/registry/routing_table", "//rs/registry/subnet_type", "//rs/state_layout", @@ -29,10 +28,7 @@ rust_library( "@crate_index//:anyhow", "@crate_index//:clap", "@crate_index//:csv", - "@crate_index//:hex", - "@crate_index//:ic-agent", "@crate_index//:serde", - "@crate_index//:serde_cbor", "@crate_index//:slog", "@crate_index//:strum", "@crate_index//:url", @@ -47,11 +43,10 @@ rust_binary( # Keep sorted. ":subnet_splitting", "//rs/canister_sandbox:backend_lib", - "//rs/crypto/utils/threshold_sig", - "//rs/crypto/utils/threshold_sig_der", "//rs/monitoring/metrics", "//rs/protobuf", "//rs/recovery", + "//rs/recovery/subnet_tools", "//rs/registry/routing_table", "//rs/registry/subnet_type", "//rs/state_layout", @@ -63,10 +58,7 @@ rust_binary( "@crate_index//:anyhow", "@crate_index//:clap", "@crate_index//:csv", - "@crate_index//:hex", - "@crate_index//:ic-agent", "@crate_index//:serde", - "@crate_index//:serde_cbor", "@crate_index//:slog", "@crate_index//:strum", "@crate_index//:url", @@ -79,11 +71,10 @@ rust_test( crate = "subnet_splitting", deps = [ # Keep sorted. - "//rs/crypto/utils/threshold_sig", - "//rs/crypto/utils/threshold_sig_der", "//rs/monitoring/metrics", "//rs/protobuf", "//rs/recovery", + "//rs/recovery/subnet_tools", "//rs/registry/routing_table", "//rs/registry/subnet_type", "//rs/state_layout", @@ -96,10 +87,7 @@ rust_test( "@crate_index//:anyhow", "@crate_index//:clap", "@crate_index//:csv", - "@crate_index//:hex", - "@crate_index//:ic-agent", "@crate_index//:serde", - "@crate_index//:serde_cbor", "@crate_index//:slog", "@crate_index//:strum", "@crate_index//:url", @@ -126,11 +114,10 @@ rust_ic_test( deps = [ # Keep sorted. ":subnet_splitting", - "//rs/crypto/utils/threshold_sig", - "//rs/crypto/utils/threshold_sig_der", "//rs/monitoring/metrics", "//rs/protobuf", "//rs/recovery", + "//rs/recovery/subnet_tools", "//rs/registry/routing_table", "//rs/registry/subnet_type", "//rs/rust_canisters/proxy_canister:lib", @@ -150,10 +137,7 @@ rust_ic_test( "@crate_index//:candid", "@crate_index//:clap", "@crate_index//:csv", - "@crate_index//:hex", - "@crate_index//:ic-agent", "@crate_index//:serde", - "@crate_index//:serde_cbor", "@crate_index//:slog", "@crate_index//:strum", "@crate_index//:url", diff --git a/rs/recovery/subnet_splitting/Cargo.toml b/rs/recovery/subnet_splitting/Cargo.toml index d1b30e6dc680..888a0e975ba4 100644 --- a/rs/recovery/subnet_splitting/Cargo.toml +++ b/rs/recovery/subnet_splitting/Cargo.toml @@ -10,11 +10,7 @@ documentation.workspace = true anyhow = { workspace = true } clap = { workspace = true } csv = { workspace = true } -hex = { workspace = true } -ic-agent = { workspace = true } ic-base-types = { path = "../../types/base_types/" } -ic-crypto-utils-threshold-sig = { path = "../../crypto/utils/threshold_sig" } -ic-crypto-utils-threshold-sig-der = { path = "../../crypto/utils/threshold_sig_der" } ic-metrics = { path = "../../monitoring/metrics" } ic-protobuf = { path = "../../protobuf" } ic-recovery = { path = "../" } @@ -23,10 +19,10 @@ ic-registry-subnet-type = { path = "../../registry/subnet_type" } ic-state-layout = { path = "../../state_layout" } ic-state-manager = { path = "../../state_manager" } ic-state-tool = { path = "../../state_tool" } +ic-subnet-tools = { path = "../subnet_tools" } ic-types-cycles = { path = "../../types/cycles" } ic-types = { path = "../../types/types" } serde = { workspace = true } -serde_cbor = { workspace = true } slog = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } diff --git a/rs/recovery/subnet_splitting/src/admin_helper.rs b/rs/recovery/subnet_splitting/src/admin_helper.rs index ceb676503d77..7a5d5c7bba5b 100644 --- a/rs/recovery/subnet_splitting/src/admin_helper.rs +++ b/rs/recovery/subnet_splitting/src/admin_helper.rs @@ -1,16 +1,12 @@ use crate::utils::canister_id_range_to_string; use ic_base_types::SubnetId; -use ic_recovery::admin_helper::{ - AdminHelper, CommandHelper, IcAdmin, SSH_READONLY_ACCESS_ARG, SUMMARY_ARG, quote, -}; +use ic_recovery::admin_helper::{AdminHelper, CommandHelper, IcAdmin, SUMMARY_ARG, quote}; use ic_registry_routing_table::CanisterIdRange; +use ic_subnet_tools::admin_helper::{DESTINATION_SUBNET_ARG, SOURCE_SUBNET_ARG}; -const SOURCE_SUBNET_ARG: &str = "source-subnet"; -const DESTINATION_SUBNET_ARG: &str = "destination-subnet"; const CANISTER_ID_RANGES_ARG: &str = "canister-id-ranges"; const MIGRATION_TRACE_ARG: &str = "migration-trace"; -const SUBNET_ARG: &str = "subnet"; /// Propose additions or updates to `canister_migrations`. /// @@ -92,36 +88,6 @@ pub(crate) fn get_propose_to_complete_canister_migration_command( ic_admin } -/// Propose to make the Subnet halt after reaching the next CUP height. -/// -/// Optionally adds a ssh-readonly-access key to the Subnet. -pub(crate) fn get_halt_subnet_at_cup_height_command( - admin_helper: &AdminHelper, - subnet_id: SubnetId, - key: &Option, -) -> IcAdmin { - let mut ic_admin = admin_helper.get_ic_admin_cmd_base(); - - ic_admin - .add_positional_argument("propose-to-update-subnet") - .add_argument(SUBNET_ARG, subnet_id) - .add_argument( - SUMMARY_ARG, - quote(format!( - "Halt subnet {subnet_id} at cup height and optionally update ssh readonly access", - )), - ) - .add_argument("halt-at-cup-height", true); - - if let Some(key) = key { - ic_admin.add_argument(SSH_READONLY_ACCESS_ARG, quote(key)); - } - - admin_helper.add_proposer_args(&mut ic_admin); - - ic_admin -} - #[cfg(test)] mod tests { use super::*; @@ -141,29 +107,6 @@ mod tests { "53zcu-tiaaa-aaaaa-qaaba-cai:54yea-6qaaa-aaaaa-qaabq-cai", "5h5yf-eiaaa-aaaaa-qaada-cai:5a46r-jqaaa-aaaaa-qaadq-cai", ]; - const SSH_KEY: &str = "fake ssh key"; - - #[test] - fn get_halt_subnet_at_cup_height_command_test() { - let result = get_halt_subnet_at_cup_height_command( - &fake_admin_helper(), - subnet_id_from_str(FAKE_SUBNET_ID_1), - &Some(SSH_KEY.to_string()), - ) - .join(" "); - - assert_eq!( - result, - "/fake/ic/admin/dir/ic-admin \ - --nns-url \"https://fake_nns_url.com:8080/\" \ - propose-to-update-subnet \ - --subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe \ - --summary \"Halt subnet gpvux-2ejnk-3hgmh-cegwf-iekfc-b7rzs-hrvep-5euo2-3ywz3-k3hcb-cqe at cup height and optionally update ssh readonly access\" \ - --halt-at-cup-height true \ - --ssh-readonly-access \"fake ssh key\" \ - --test-neuron-proposer" - ); - } #[test] fn get_propose_to_prepare_canister_migration_command_test() { diff --git a/rs/recovery/subnet_splitting/src/agent_helper.rs b/rs/recovery/subnet_splitting/src/agent_helper.rs deleted file mode 100644 index dba0c19e4e9c..000000000000 --- a/rs/recovery/subnet_splitting/src/agent_helper.rs +++ /dev/null @@ -1,166 +0,0 @@ -use ic_agent::{Agent, Certificate, export::Principal, hash_tree::Label, lookup_value}; -use ic_base_types::SubnetId; -use ic_crypto_utils_threshold_sig_der::{parse_threshold_sig_key_from_pem_file, public_key_to_der}; -use ic_recovery::{ - error::{RecoveryError, RecoveryResult}, - file_sync_helper::{read_bytes, write_bytes}, - util::{block_on, write_public_key_to_file}, -}; -use slog::{Logger, debug, info}; -use url::Url; - -use std::{fmt::Display, path::Path}; - -const NNS_REGISTRY_CANISTER_ID: &str = "rwlgt-iiaaa-aaaaa-aaaaa-cai"; - -const SUBNET_LABEL: &[u8] = b"subnet"; -const PUBLIC_KEY_LABEL: &[u8] = b"public_key"; -const CANISTER_RANGES_LABEL: &[u8] = b"canister_ranges"; - -type StorageType = Vec; - -/// Wrapper around the raw state tree with some utility functions. -/// -/// Note: the state tree is pruned to include only the information (public key and canister ranges) -/// of a single subnet. -pub(crate) struct StateTree { - certificate: Certificate, - subnet_id: SubnetId, -} - -impl StateTree { - /// Saves the raw state tree to the disk, in CBOR format. - pub(crate) fn save_to_file(&self, path: &Path) -> RecoveryResult<()> { - serde_cbor::to_vec(&self.certificate) - .map_err(|err| agent_error("Failed to serialize the state tree", err)) - .and_then(|bytes| write_bytes(path, bytes)) - .map_err(|err| agent_error("Failed to write the state tree to disk", err)) - } - - /// Reads the raw state tree from the disk. - pub(crate) fn read_from_file(path: &Path, subnet_id: SubnetId) -> RecoveryResult { - let serialized_state_tree = read_bytes(path) - .map_err(|err| agent_error("Failed to read the state tree from the disk", err))?; - - let certificate = serde_cbor::from_slice(serialized_state_tree.as_slice()) - .map_err(|err| agent_error("Failed to deserialize the state tree", err))?; - - Ok(Self { - subnet_id, - certificate, - }) - } - - /// Extracts the public key from the raw state tree and saves it to the disk. - pub(crate) fn save_public_key_to_file(&self, path: &Path) -> RecoveryResult<()> { - self.lookup_public_key() - .and_then(|public_key| write_public_key_to_file(public_key, path)) - .map_err(|err| agent_error("Failed to write the public key to disk", err)) - } - - pub(crate) fn lookup_public_key(&self) -> RecoveryResult<&[u8]> { - lookup_value( - &self.certificate, - create_path(self.subnet_id, PUBLIC_KEY_LABEL), - ) - .map_err(|err| agent_error("Failed to retrieve the public key", err)) - } -} - -/// Wrapper around [Agent] with some utility functions. -pub(crate) struct AgentHelper { - agent: Agent, - nns_registry: Principal, - logger: Logger, -} - -impl AgentHelper { - /// Creates a new instance of [AgentHelper]. - /// - /// When the `nns_public_key_path` argument is not specified, the mainnet root key will be - /// used. - /// - /// Returns an error when the underlying [Agent] fails to build or when there is something - /// wrong with the provided NNS public key. - pub(crate) fn new( - nns_url: &Url, - nns_public_key_path: Option<&Path>, - logger: Logger, - ) -> RecoveryResult { - let agent = Agent::builder() - .with_url(nns_url.to_string()) - .build() - .map_err(|err| agent_error("Failed to build an Agent", err))?; - - // If we don't set a root key, the [Agent] will use the mainnet root key. - if let Some(nns_public_key_path) = nns_public_key_path { - info!( - logger, - "Reading the NNS public key from {}", - nns_public_key_path.display() - ); - - let nns_public_key = parse_threshold_sig_key_from_pem_file(nns_public_key_path) - .map_err(|err| agent_error("Failed to parse NNS public key", err))?; - let der_bytes = public_key_to_der(&nns_public_key.into_bytes()) - .map_err(|err| agent_error("Failed to convert the NNS public key to DER", err))?; - - agent.set_root_key(der_bytes); - } - - let nns_registry = Principal::from_text(NNS_REGISTRY_CANISTER_ID) - .map_err(|err| agent_error("Failed to parse NNS registry canister id", err))?; - - Ok(Self { - agent, - nns_registry, - logger, - }) - } - - /// Reads the state tree and prunes it to contain only the following paths: - /// * /subnet/$subnet_id/public_key - /// * /canister_ranges/$subnet_id - /// - /// See: https://internetcomputer.org/docs/current/references/ic-interface-spec#state-tree-subnet - /// for more information - pub(crate) fn read_subnet_data(&self, subnet_id: SubnetId) -> RecoveryResult { - let certificate = block_on(self.agent.read_subnet_state_raw( - vec![ - create_path(subnet_id, PUBLIC_KEY_LABEL), - vec![ - CANISTER_RANGES_LABEL.into(), - subnet_id.get().as_slice().into(), - ], - ], - subnet_id.get().into(), - )) - .map_err(|err| agent_error("Failed to read the state tree", err))?; - - debug!(self.logger, "State tree: {:#?}", certificate.tree); - - Ok(StateTree { - certificate, - subnet_id, - }) - } - - /// Validates the state tree. - pub(crate) fn validate_state_tree(&self, state_tree: &StateTree) -> RecoveryResult<()> { - self.agent - .verify(&state_tree.certificate, self.nns_registry) - .map_err(|err| agent_error("Failed to verify the state tree", err)) - } -} - -fn agent_error(message: impl Display, error: impl Display) -> RecoveryError { - RecoveryError::AgentError(format!("{message}: {error}")) -} - -fn create_path(subnet_id: SubnetId, label: &[u8]) -> Vec> { - vec![ - SUBNET_LABEL.into(), - subnet_id.get().as_slice().into(), - label.into(), - ] -} diff --git a/rs/recovery/subnet_splitting/src/lib.rs b/rs/recovery/subnet_splitting/src/lib.rs index 1ebc3f829f89..d05c60727b7b 100644 --- a/rs/recovery/subnet_splitting/src/lib.rs +++ b/rs/recovery/subnet_splitting/src/lib.rs @@ -1,11 +1,8 @@ pub mod post_split_estimations; pub mod subnet_splitting; pub mod utils; -pub mod validation; mod admin_helper; -mod agent_helper; mod layout; -mod state_tool_helper; mod steps; mod target_subnet; diff --git a/rs/recovery/subnet_splitting/src/main.rs b/rs/recovery/subnet_splitting/src/main.rs index 45356a693fe7..c018241c199e 100644 --- a/rs/recovery/subnet_splitting/src/main.rs +++ b/rs/recovery/subnet_splitting/src/main.rs @@ -7,8 +7,8 @@ use ic_subnet_splitting::{ post_split_estimations, subnet_splitting::{SubnetSplitting, SubnetSplittingArgs}, utils::canister_id_ranges_to_strings, - validation::validate_artifacts, }; +use ic_subnet_tools::validation::validate_artifacts; use ic_types::ReplicaVersion; use slog::{Logger, info, warn}; use url::Url; diff --git a/rs/recovery/subnet_splitting/src/steps.rs b/rs/recovery/subnet_splitting/src/steps.rs index f8e51a41c5cb..a36258dd5f8e 100644 --- a/rs/recovery/subnet_splitting/src/steps.rs +++ b/rs/recovery/subnet_splitting/src/steps.rs @@ -1,28 +1,27 @@ use crate::{ - agent_helper::AgentHelper, - layout::Layout, - state_tool_helper, - target_subnet::TargetSubnet, - utils::{find_expected_state_hash_for_subnet_id, get_batch_time_from_cup, get_state_hash}, - validation::validate_artifacts, + layout::Layout, target_subnet::TargetSubnet, utils::find_expected_state_hash_for_subnet_id, }; use ic_base_types::SubnetId; use ic_metrics::MetricsRegistry; use ic_recovery::{ Recovery, - cli::consent_given, error::{RecoveryError, RecoveryResult}, file_sync_helper::rsync_includes, - registry_helper::VersionedRecoveryResult, steps::Step, util::parse_hex_str, }; use ic_registry_routing_table::CanisterIdRange; use ic_registry_subnet_type::SubnetType; use ic_state_manager::split::resolve_ranges_and_split; +use ic_subnet_tools::{ + agent_helper::AgentHelper, + state_tool_helper, + utils::{get_batch_time_from_cup, get_state_hash}, + validation::validate_artifacts, +}; use ic_types::Height; -use slog::{Logger, error, info}; +use slog::{Logger, info}; use url::Url; use std::{net::IpAddr, path::PathBuf}; @@ -283,34 +282,3 @@ impl Step for WaitForCUPStep { Recovery::wait_for_recovery_cup(&self.logger, self.node_ip, new_cup_height, state_hash) } } - -pub(crate) struct ReadRegistryStep VersionedRecoveryResult> { - pub(crate) logger: Logger, - pub(crate) label: String, - pub(crate) interactive: bool, - pub(crate) querier: F, -} - -impl VersionedRecoveryResult> Step for ReadRegistryStep { - fn descr(&self) -> String { - format!("Read Registry to get the most recent {}", self.label) - } - - fn exec(&self) -> RecoveryResult<()> { - loop { - match (self.querier)() { - Ok((registry_version, value)) => info!( - self.logger, - "{} at registry version {}: {:#?}", self.label, registry_version, value, - ), - Err(err) => error!(self.logger, "Failed getting {}, error: {}", self.label, err), - } - - if !self.interactive || !consent_given(&self.logger, "Read registry again?") { - break; - } - } - - Ok(()) - } -} diff --git a/rs/recovery/subnet_splitting/src/subnet_splitting.rs b/rs/recovery/subnet_splitting/src/subnet_splitting.rs index 868d1f3631b8..f9a0d63fbc02 100644 --- a/rs/recovery/subnet_splitting/src/subnet_splitting.rs +++ b/rs/recovery/subnet_splitting/src/subnet_splitting.rs @@ -1,16 +1,15 @@ use crate::{ admin_helper::{ - get_halt_subnet_at_cup_height_command, get_propose_to_complete_canister_migration_command, + get_propose_to_complete_canister_migration_command, get_propose_to_prepare_canister_migration_command, get_propose_to_reroute_canister_ranges_command, }, layout::Layout, steps::{ - ComputeExpectedManifestsStep, CopyWorkDirStep, ReadRegistryStep, SplitStateStep, - StateSplitStrategy, ValidateCUPStep, WaitForCUPStep, + ComputeExpectedManifestsStep, CopyWorkDirStep, SplitStateStep, StateSplitStrategy, + ValidateCUPStep, WaitForCUPStep, }, target_subnet::TargetSubnet, - utils::get_state_hash, }; use clap::Parser; @@ -18,7 +17,7 @@ use ic_base_types::SubnetId; use ic_protobuf::registry::subnet::v1::SubnetRecord; use ic_recovery::{ CUPS_DIR, IC_STATE_DIR, NeuronArgs, Recovery, RecoveryArgs, - cli::{consent_given, read_optional, wait_for_confirmation}, + cli::{consent_given, read_optional}, error::{RecoveryError, RecoveryResult}, get_available_nodes_heights_from_metrics, recovery_iterator::RecoveryIterator, @@ -30,12 +29,15 @@ use ic_recovery::{ }; use ic_registry_routing_table::{CanisterIdRange, RoutingTable}; use ic_registry_subnet_type::SubnetType; +use ic_subnet_tools::{ + admin_helper::get_halt_subnet_at_cup_height_command, cli::print_url_and_ask_for_confirmation, + steps::ReadRegistryStep, utils::get_state_hash, +}; use ic_types::Height; use serde::{Deserialize, Serialize}; -use slog::{Logger, error, warn}; +use slog::{Logger, warn}; use strum::{EnumMessage, IntoEnumIterator}; use strum_macros::{EnumIter, EnumString}; -use url::Url; use std::{collections::HashMap, iter::Peekable, net::IpAddr, path::PathBuf}; @@ -691,20 +693,3 @@ impl HasRecoveryState for SubnetSplitting { }) } } - -fn print_url_and_ask_for_confirmation( - logger: &Logger, - url: String, - text_to_display: impl std::fmt::Display, -) { - match Url::parse(&url) { - Ok(url) => { - warn!(logger, "{}", text_to_display); - warn!(logger, "{}", url); - wait_for_confirmation(logger); - } - Err(err) => { - error!(logger, "Failed to parse url {}: {}", url, err); - } - } -} diff --git a/rs/recovery/subnet_splitting/src/utils.rs b/rs/recovery/subnet_splitting/src/utils.rs index f73b707647f0..c416b86b18b5 100644 --- a/rs/recovery/subnet_splitting/src/utils.rs +++ b/rs/recovery/subnet_splitting/src/utils.rs @@ -1,31 +1,12 @@ use ic_base_types::SubnetId; -use ic_protobuf::types::v1 as pb; use ic_recovery::{ error::{RecoveryError, RecoveryResult}, file_sync_helper::read_file, util::subnet_id_from_str, }; use ic_registry_routing_table::CanisterIdRange; -use ic_state_manager::manifest::{manifest_from_path, manifest_hash}; -use ic_types::{Time, consensus::CatchUpPackage}; -use std::{fmt::Display, path::Path}; - -pub(crate) fn get_batch_time_from_cup(cup_path: &Path) -> RecoveryResult