test: drain a subnet that is "cooling down"

Cargo.toml

@@ -341,7 +341,9 @@ members = [
     "rs/protobuf/generator",
     "rs/query_stats",
     "rs/recovery",
+    "rs/recovery/subnet_merging",
     "rs/recovery/subnet_splitting",
+    "rs/recovery/subnet_tools",
     "rs/registry/admin",
     "rs/registry/admin-derive",
     "rs/registry/canister",

rs/cup_explorer/src/lib.rs

@@ -43,7 +43,7 @@ pub async fn get_catchup_content(url: &Url) -> Result<Option<pb::CatchUpContent>
 }
 
 /// Fetches the CatchUp package, if it's present.
-async fn get_cup(url: &Url) -> Result<Option<pb::CatchUpPackage>, String> {
+pub async fn get_cup(url: &Url) -> Result<Option<pb::CatchUpPackage>, String> {
     let agent = Agent::new(url.clone(), Sender::Anonymous);
     agent
         .query_cup_endpoint(None)

rs/recovery/src/app_subnet_recovery.rs

@@ -608,6 +608,7 @@ impl RecoveryIterator<StepType, StepTypeIter> for AppSubnetRecovery {
                     None,
                     self.params.initial_dkg_subnet_id,
                     self.params.chain_key_subnet_id,
+                    /*time=*/ None,
                 )?))
             }
 

rs/recovery/src/lib.rs

@@ -817,6 +817,7 @@ impl Recovery {
         registry_params: Option<RegistryParams>,
         initial_dkg_subnet_id: Option<SubnetId>,
         chain_key_subnet_id: Option<SubnetId>,
+        time: Option<SystemTime>,
     ) -> RecoveryResult<impl Step + use<>> {
         let chain_key_config = chain_key_subnet_id
             .map(|id| match self.registry_helper.get_chain_key_config(id) {
@@ -847,7 +848,11 @@ impl Recovery {
                     chain_key_config,
                     replacement_nodes,
                     registry_params,
-                    SystemTime::now(),
+                    // The block time the recovered subnet starts from. Defaults
+                    // to now, which is what a recovery replaying up to the
+                    // present wants; a subnet merge passes the time it computed
+                    // from the states it merged instead.
+                    time.unwrap_or_else(SystemTime::now),
                 ),
         })
     }
@@ -1209,7 +1214,7 @@ pub fn get_available_nodes_heights_from_metrics(
 }
 
 /// Lookup node IDs and corresponding IP addresses of all members of the given subnet
-fn get_member_node_ids_and_ips(
+pub fn get_member_node_ids_and_ips(
     registry_helper: &RegistryHelper,
     subnet_id: SubnetId,
 ) -> RecoveryResult<BTreeMap<NodeId, IpAddr>> {

rs/recovery/src/nns_recovery_failover_nodes.rs

@@ -431,6 +431,7 @@ impl RecoveryIterator<StepType, StepTypeIter> for NNSRecoveryFailoverNodes {
                         Some(registry_params),
                         None,
                         None,
+                        /*time=*/ None,
                     )?))
                 } else {
                     Err(RecoveryError::StepSkipped)

rs/recovery/src/registry_helper.rs

@@ -102,6 +102,29 @@ impl RegistryHelper {
         })
     }
 
+    /// Returns the subnet record of the given subnet as of `registry_version`,
+    /// rather than as of the latest version [Self::get_subnet_record] reads.
+    ///
+    /// Polls the [RegistryReplicator] first, as every other getter does: a
+    /// version that the local store has not caught up with yet is not readable,
+    /// and the caller may well be asking about one that was just created.
+    pub fn get_subnet_record_at_version(
+        &self,
+        subnet_id: SubnetId,
+        registry_version: RegistryVersion,
+    ) -> RecoveryResult<Option<SubnetRecord>> {
+        let _ = self.latest_registry_version()?;
+
+        self.registry_client()
+            .get_subnet_record(subnet_id, registry_version)
+            .map_err(|err| {
+                RecoveryError::RegistryError(format!(
+                    "Failed to get the record of subnet {subnet_id} at registry version \
+                     {registry_version}: {err}"
+                ))
+            })
+    }
+
     /// Returns the [SubnetRecord] of the given subnet.
     pub fn get_subnet_record(&self, subnet_id: SubnetId) -> VersionedRecoveryResult<SubnetRecord> {
         self.get(|registry_version, registry_client| {

rs/recovery/src/steps.rs

@@ -274,7 +274,11 @@ impl Step for MergeCertificationPoolsStep {
     }
 }
 
-pub(crate) struct DownloadIcDataStep {
+/// Downloads data (the state, the consensus pool, ...) of a node into a
+/// working directory. Public so that a tool that works with more than one
+/// working directory, such as subnet merging, can direct the download at the
+/// right one.
+pub struct DownloadIcDataStep {
     pub logger: Logger,
     pub ssh_helper: SshHelper,
     pub backup_dir: PathBuf,

rs/recovery/subnet_merging/BUILD.bazel

@@ -0,0 +1,58 @@
+load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test")
+
+DEPENDENCIES = [
+    # Keep sorted.
+    "//rs/cup_explorer",
+    "//rs/protobuf",
+    "//rs/recovery",
+    "//rs/recovery/subnet_tools",
+    "//rs/registry/helpers",
+    "//rs/registry/routing_table",
+    "//rs/registry/subnet_type",
+    "//rs/state_layout",
+    "//rs/types/base_types",
+    "//rs/types/types",
+    "@crate_index//:anyhow",
+    "@crate_index//:clap",
+    "@crate_index//:futures",
+    "@crate_index//:reqwest",
+    "@crate_index//:serde",
+    "@crate_index//:serde_json",
+    "@crate_index//:slog",
+    "@crate_index//:strum",
+    "@crate_index//:tokio",
+    "@crate_index//:url",
+]
+
+rust_library(
+    name = "subnet_merging",
+    srcs = glob(
+        ["src/**/*.rs"],
+        exclude = ["src/main.rs"],
+    ),
+    crate_name = "ic_subnet_merging",
+    proc_macro_deps = ["@crate_index//:strum_macros"],
+    version = "0.1.0",
+    visibility = ["//rs:system-tests-pkg"],
+    deps = DEPENDENCIES,
+)
+
+rust_binary(
+    name = "subnet-merging-tool",
+    srcs = ["src/main.rs"],
+    visibility = ["//rs:release-pkg"],
+    deps = DEPENDENCIES + [
+        # Keep sorted.
+        ":subnet_merging",
+        "//rs/canister_sandbox:backend_lib",
+    ],
+)
+
+rust_test(
+    name = "subnet_merging_tool_test",
+    crate = "subnet_merging",
+    deps = DEPENDENCIES + [
+        # Keep sorted.
+        "//rs/test_utilities/tmpdir",
+    ],
+)

rs/recovery/subnet_merging/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "ic-subnet-merging"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+description.workspace = true
+documentation.workspace = true
+
+[dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true }
+futures = { workspace = true }
+ic-base-types = { path = "../../types/base_types/" }
+ic-cup-explorer = { path = "../../cup_explorer" }
+ic-protobuf = { path = "../../protobuf" }
+ic-recovery = { path = "../" }
+ic-registry-client-helpers = { path = "../../registry/helpers" }
+ic-registry-routing-table = { path = "../../registry/routing_table" }
+ic-registry-subnet-type = { path = "../../registry/subnet_type" }
+ic-state-layout = { path = "../../state_layout" }
+ic-subnet-tools = { path = "../subnet_tools" }
+ic-types = { path = "../../types/types" }
+reqwest = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+slog = { workspace = true }
+
+strum = { workspace = true }
+strum_macros = { workspace = true }
+tokio = { workspace = true }
+url = { workspace = true }
+
+[dev-dependencies]
+ic-test-utilities-tmpdir = { path = "../../test_utilities/tmpdir" }
+
+[[bin]]
+name = "subnet-merging-tool"
+path = "src/main.rs"