feat: Add a monotonic consumed_cycles_as_counter to CanisterMetrics

rs/execution_environment/src/canister_manager.rs

@@ -1375,6 +1375,13 @@ impl CanisterManager {
         );
 
         // Leftover cycles in the canister are considered `consumed`.
+        //
+        // Note that the canister's `consumed_cycles` gauge (rather than its monotonic
+        // `consumed_cycles_as_counter`) is what is moved into the subnet metrics
+        // below. The two are equal here: a canister can only be deleted while
+        // `Stopped` and with empty queues, so it has no callback and no execution
+        // left holding an outstanding prepayment (see
+        // `SystemState::outstanding_prepayments`).
         let leftover_cycles = self
             .cycles_account_manager
             .leftover_cycles_for_canister_to_deleted(

rs/execution_environment/src/scheduler.rs

@@ -1120,6 +1120,22 @@ impl SchedulerImpl {
 
                 // Abort all paused execution before the checkpoint.
                 abort_all_paused_executions(state, &self.exec_env, cost_schedule, &self.log);
+
+                // Backfill the monotonic `consumed_cycles_as_counter` of every
+                // canister from its `consumed_cycles` gauge, which predates it.
+                //
+                // Only done on checkpoint rounds, and only after the paused
+                // executions above have been aborted: a paused execution holds a
+                // prepayment that is not part of the replicated state, which would
+                // make the backfill overestimate the counter (see
+                // `SystemState::outstanding_prepayments`). Aborting materializes
+                // those prepayments into the canisters' task queues.
+                //
+                // Unconditional and idempotent, like
+                // `migrate_outcalls_cycles_to_use_cases` above: it is a no-op once a
+                // canister has been backfilled, and self-healing if a downgrade
+                // dropped the counter.
+                migrate_consumed_cycles_to_counters(state);
             }
             ExecutionRoundType::OrdinaryRound => {
                 self.abort_paused_executions_above_limit(state);
@@ -2154,3 +2170,46 @@ pub fn abort_all_paused_executions(
         abort_canister(canister, subnet_schedule, exec_env, cost_schedule, log);
     }
 }
+
+/// Backfills the monotonic `CanisterMetrics::consumed_cycles_as_counter` of every
+/// canister from its `consumed_cycles` gauge, which predates it and thus holds the
+/// full history. See `SystemState::migrate_consumed_cycles_to_counter`.
+///
+/// Must only be called with no paused executions left (i.e. on a checkpoint round,
+/// after `abort_all_paused_executions`); canisters that still have one are skipped,
+/// as their prepayment is not part of the replicated state.
+fn migrate_consumed_cycles_to_counters(state: &mut ReplicatedState) {
+    state.canisters_for_each_mut(|_id, canister| {
+        let metrics = canister.system_state.canister_metrics();
+        // A canister with a paused execution is skipped, its prepayment is not part
+        // of the replicated state. This should not happen when called as documented,
+        // but it is not worth a panic.
+        let Some(outstanding) = canister.system_state.outstanding_prepayments() else {
+            debug_assert!(false, "Called with a paused execution left over");
+            return;
+        };
+        let derived = metrics.consumed_cycles() - outstanding;
+        match derived.cmp(&metrics.consumed_cycles_as_counter()) {
+            // Not backfilled yet, or a downgrade dropped the counter. Only take a
+            // mutable reference here, so that this stays a read-only pass once every
+            // canister has been backfilled (`Arc::make_mut` clones the canister).
+            std::cmp::Ordering::Greater => {
+                Arc::make_mut(canister)
+                    .system_state
+                    .migrate_consumed_cycles_to_counter();
+            }
+            // The invariant that makes the backfill exact; see
+            // `SystemState::outstanding_prepayments`.
+            std::cmp::Ordering::Equal => {}
+            std::cmp::Ordering::Less => debug_assert!(
+                false,
+                "Canister {}: consumed cycles counter {} above the gauge {} net of the \
+                 {} outstanding prepayments",
+                canister.canister_id(),
+                metrics.consumed_cycles_as_counter(),
+                metrics.consumed_cycles(),
+                outstanding,
+            ),
+        }
+    });
+}

rs/execution_environment/src/scheduler/tests/metrics.rs

@@ -45,10 +45,11 @@ use ic_types_test_utils::ids::{canister_test_id, message_test_id, subnet_test_id
 use more_asserts::assert_ge;
 use std::time::Duration;
 
-/// Observes the state metrics at `height`, having first refreshed the derived
-/// consumed-cycles total that the `replicated_state_consumed_cycles_since_replica_started`
-/// gauge reads. Production refreshes it on every `commit_and_certify`, which this
-/// harness never does.
+/// Observes the state metrics at `height`, having first refreshed
+/// `SubnetMetrics::consumed_cycles_total_including_canisters`, the derived
+/// certified total that the assertions below compare the exported gauge against.
+/// Production refreshes it on every `commit_and_certify`, which this harness never
+/// does.
 fn observe_state_metrics(test: &mut SchedulerTest, height: u64) {
     test.state_mut().refresh_consumed_cycles();
     test.state_metrics().observe(
@@ -1104,6 +1105,141 @@ fn threshold_signature_agreements_metric_is_updated() {
     assert!(sign_with_threshold_contexts.is_empty());
 }
 
+/// Asserts the invariant that ties `CanisterMetrics::consumed_cycles` to its
+/// monotonic counterpart: the gauge exceeds the counter by exactly the prepayments
+/// whose refund is still outstanding. Returns the latter.
+fn assert_consumed_cycles_invariant(
+    test: &SchedulerTest,
+    canister_id: CanisterId,
+) -> NominalCycles {
+    let system_state = &test.canister_state(canister_id).system_state;
+    let outstanding = system_state
+        .outstanding_prepayments()
+        .expect("Canister has a paused execution");
+    let metrics = system_state.canister_metrics();
+    assert_eq!(
+        metrics.consumed_cycles() - outstanding,
+        metrics.consumed_cycles_as_counter(),
+        "consumed cycles gauge {} minus the {outstanding} outstanding prepayments \
+         must equal the counter {}",
+        metrics.consumed_cycles(),
+        metrics.consumed_cycles_as_counter(),
+    );
+    outstanding
+}
+
+/// Opens a call to an xnet canister, so that the caller is left with an outstanding
+/// prepayment for the response.
+fn call_xnet_canister(test: &mut SchedulerTest, canister: CanisterId) {
+    let xnet_canister = test.xnet_canister_id();
+    test.send_ingress(
+        canister,
+        ingress(1).call(other_side(xnet_canister, 1), on_response(1)),
+    );
+    test.execute_round(ExecutionRoundType::OrdinaryRound);
+}
+
+#[test]
+fn consumed_cycles_as_counter_matches_the_gauge_net_of_outstanding_prepayments() {
+    let mut test = SchedulerTestBuilder::new().build();
+    let canister = test.create_canister();
+
+    call_xnet_canister(&mut test, canister);
+
+    let outstanding = assert_consumed_cycles_invariant(&test, canister);
+    assert_ne!(outstanding, NominalCycles::zero());
+    assert_ne!(
+        test.canister_state(canister)
+            .system_state
+            .canister_metrics()
+            .consumed_cycles_as_counter(),
+        NominalCycles::zero()
+    );
+}
+
+#[test]
+fn checkpoint_round_backfills_consumed_cycles_as_counter() {
+    let mut test = SchedulerTestBuilder::new().build();
+    let canister = test.create_canister();
+
+    call_xnet_canister(&mut test, canister);
+    let outstanding = assert_consumed_cycles_invariant(&test, canister);
+    assert_ne!(outstanding, NominalCycles::zero());
+
+    // Pretend the canister was loaded from a checkpoint predating the counter.
+    test.canister_state_mut(canister)
+        .system_state
+        .reset_consumed_cycles_as_counter();
+    assert_eq!(
+        test.canister_state(canister)
+            .system_state
+            .canister_metrics()
+            .consumed_cycles_as_counter(),
+        NominalCycles::zero()
+    );
+
+    // An ordinary round does not backfill the counter, a checkpoint round does.
+    test.execute_round(ExecutionRoundType::OrdinaryRound);
+    assert_eq!(
+        test.canister_state(canister)
+            .system_state
+            .canister_metrics()
+            .consumed_cycles_as_counter(),
+        NominalCycles::zero()
+    );
+
+    test.execute_round(ExecutionRoundType::CheckpointRound);
+    assert_consumed_cycles_invariant(&test, canister);
+    assert_ne!(
+        test.canister_state(canister)
+            .system_state
+            .canister_metrics()
+            .consumed_cycles_as_counter(),
+        NominalCycles::zero()
+    );
+}
+
+/// A paused execution holds a prepayment that is not part of the replicated state,
+/// so it cannot be backfilled; but a checkpoint round aborts all paused executions
+/// before backfilling, materializing their prepayments into the task queues.
+#[test]
+fn checkpoint_round_backfills_consumed_cycles_as_counter_of_paused_canister() {
+    let mut test = SchedulerTestBuilder::new()
+        .with_scheduler_config(SchedulerConfig {
+            scheduler_cores: 2,
+            max_instructions_per_round: NumInstructions::from(100),
+            max_instructions_per_message: NumInstructions::from(1000),
+            max_instructions_per_slice: NumInstructions::from(100),
+            max_instructions_per_install_code_slice: NumInstructions::from(100),
+            ..SchedulerConfig::application_subnet()
+        })
+        .build();
+    let canister = test.create_canister();
+
+    test.send_ingress(canister, ingress(1000));
+    test.execute_round(ExecutionRoundType::OrdinaryRound);
+    assert!(test.canister_state(canister).has_paused_execution());
+    assert_eq!(
+        test.canister_state(canister)
+            .system_state
+            .outstanding_prepayments(),
+        None
+    );
+
+    // Pretend the canister was loaded from a checkpoint predating the counter.
+    test.canister_state_mut(canister)
+        .system_state
+        .reset_consumed_cycles_as_counter();
+
+    test.execute_round(ExecutionRoundType::CheckpointRound);
+
+    assert!(test.canister_state(canister).has_aborted_execution());
+    let outstanding = assert_consumed_cycles_invariant(&test, canister);
+    // The aborted execution's prepayment is outstanding, and is not part of the
+    // backfilled counter.
+    assert_ne!(outstanding, NominalCycles::zero());
+}
+
 #[test]
 fn consumed_cycles_ecdsa_outcalls_are_added_to_consumed_cycles_total() {
     for cost_schedule in [
@@ -1434,6 +1570,33 @@ fn consumed_cycles_for_instructions_are_updated_from_valid_canisters() {
     }
 }
 
+/// The exported total is the subnet-level aggregate (already monotonic) plus the
+/// canisters' `consumed_cycles_as_counter`; i.e. the certified
+/// `consumed_cycles_total_including_canisters`, net of the outstanding prepayments.
+#[test]
+fn consumed_cycles_total_is_exported_net_of_outstanding_prepayments() {
+    let mut test = SchedulerTestBuilder::new().build();
+    let canister = test.create_canister();
+
+    call_xnet_canister(&mut test, canister);
+    let outstanding = assert_consumed_cycles_invariant(&test, canister);
+    assert_ne!(outstanding, NominalCycles::zero());
+
+    observe_state_metrics(&mut test, 0);
+
+    let certified_total = test
+        .state()
+        .metadata
+        .subnet_metrics
+        .consumed_cycles_total_including_canisters();
+    let exported = fetch_gauge(
+        test.metrics_registry(),
+        "replicated_state_consumed_cycles_since_replica_started",
+    )
+    .unwrap();
+    assert_eq!((certified_total - outstanding).get() as f64, exported);
+}
+
 #[test]
 fn consumed_cycles_for_resource_allocations_are_updated_from_valid_canisters() {
     for cost_schedule in [

rs/protobuf/def/state/canister_state_bits/v1/canister_state_bits.proto

@@ -502,6 +502,15 @@ message CanisterStateBits {
   // These counters facilitate programming retrieval of metrics and performing
   // various aggregations on them more easily than their gauge counterparts.
   repeated ConsumedCyclesByUseCase consumed_cycles_by_use_cases_as_counters = 65;
+  // Total cycles consumed by the canister, presented as a counter: unlike
+  // `consumed_cycles`, which is lowered again whenever a refund is issued, this
+  // is only ever increased, by the actually consumed amount once the refund of a
+  // prepayment is known.
+  //
+  // Covers exactly the same use cases as `consumed_cycles`, i.e. everything
+  // except HTTPS outcalls, which are only tracked at the subnet level (and, for
+  // the canister, in `consumed_cycles_by_use_cases_as_counters`).
+  types.v1.NominalCycles consumed_cycles_as_counter = 71;
   CanisterHistory canister_history = 37;
   // Resource reservation cycles.
   state.queues.v1.Cycles reserved_balance = 38;

rs/protobuf/src/gen/state/state.canister_state_bits.v1.rs

@@ -750,6 +750,17 @@ pub struct CanisterStateBits {
     /// various aggregations on them more easily than their gauge counterparts.
     #[prost(message, repeated, tag = "65")]
     pub consumed_cycles_by_use_cases_as_counters: ::prost::alloc::vec::Vec<ConsumedCyclesByUseCase>,
+    /// Total cycles consumed by the canister, presented as a counter: unlike
+    /// `consumed_cycles`, which is lowered again whenever a refund is issued, this
+    /// is only ever increased, by the actually consumed amount once the refund of a
+    /// prepayment is known.
+    ///
+    /// Covers exactly the same use cases as `consumed_cycles`, i.e. everything
+    /// except HTTPS outcalls, which are only tracked at the subnet level (and, for
+    /// the canister, in `consumed_cycles_by_use_cases_as_counters`).
+    #[prost(message, optional, tag = "71")]
+    pub consumed_cycles_as_counter:
+        ::core::option::Option<super::super::super::types::v1::NominalCycles>,
     #[prost(message, optional, tag = "37")]
     pub canister_history: ::core::option::Option<CanisterHistory>,
     /// Resource reservation cycles.