chore: Cleanup

rs/embedders/src/wasm_utils/system_api_replacements.rs

@@ -7,9 +7,13 @@
 //!
 //! - properly report errors in a backwards-compatible way
 //! - convert between integer types and check for overflows
-//! - track accesses and dirty pages
+//! - mark written pages in the bytemap and enforce the accessed/dirty page limits
 //! - charge for instructions
 //!
+//! The accessed/dirty page counters only enforce the per-message limits; the
+//! instructions for touching a page are charged by the deterministic memory
+//! tracker from the SIGSEGV handler.
+//!
 
 use crate::{
     InternalErrorCode, WASM_PAGE_SIZE,

rs/embedders/src/wasmtime_embedder.rs

@@ -1181,8 +1181,6 @@ impl WasmtimeInstance {
         let access = self.page_accesses()?;
         self.set_instance_stats(&access);
 
-        // No need to charge for dirty wasm heap pages anymore: The DMT charges directly.
-
         match result {
             Ok(_) => Ok(InstanceRunResult {
                 exported_globals: self.get_exported_globals()?,

rs/embedders/tests/instrumentation.rs

@@ -280,14 +280,15 @@ fn instr_used(instance: &mut WasmtimeInstance) -> u64 {
         .get()
 }
 
-/// Returns the instruction overhead charged in-band by the deterministic memory
-/// tracker for the first access to Wasm pages.
+/// Returns the number of *OS pages* the deterministic memory tracker charges
+/// for in-band when the given Wasm pages are first written. Multiply by
+/// `page_overhead` to get instructions.
 ///
-/// - `n_heap_wasm_pages`: pages first *written* on the heap.
-///   Each triggers `mark_wasm_page_accessed` + `mark_wasm_page_dirty`:
-///   2 × (WASM_PAGE_SIZE / OS_PAGE_SIZE) = 2 × 16 = 32 instructions.
-/// - `n_stable_wasm_pages`: same as for heap: 32 instructions per page.
-fn deterministic_tracker_overhead(n_heap_wasm_pages: u64, n_stable_wasm_pages: u64) -> u64 {
+/// A first write to a Wasm page triggers both `mark_wasm_page_accessed` and
+/// `mark_wasm_page_dirty`, i.e. every OS page it covers is charged twice. The
+/// number of OS pages per Wasm page varies by platform (16 for 4 KiB pages on
+/// Linux, 4 for 16 KiB pages on arm64-darwin).
+fn dmt_write_charged_os_pages(n_heap_wasm_pages: u64, n_stable_wasm_pages: u64) -> u64 {
     let os_pages_per_wasm_page = (WASM_PAGE_SIZE_IN_BYTES / PAGE_SIZE) as u64;
     n_heap_wasm_pages * 2 * os_pages_per_wasm_page
         + n_stable_wasm_pages * 2 * os_pages_per_wasm_page
@@ -742,7 +743,7 @@ fn metering_loop() {
     );
 }
 
-fn run_charge_for_dirty_heap(wasm_memory_type: WasmMemoryType) {
+fn run_charge_for_heap_pages(wasm_memory_type: WasmMemoryType) {
     let memory = match wasm_memory_type {
         WasmMemoryType::Wasm32 => r#"(memory (export "memory") 10)"#,
         WasmMemoryType::Wasm64 => r#"(memory (export "memory") i64 10)"#,
@@ -804,32 +805,32 @@ fn run_charge_for_dirty_heap(wasm_memory_type: WasmMemoryType) {
 
     // Both stores target Wasm page 0 (bytes 0 and 4096 are within the 64KB page),
     // so only one heap page-first-write event occurs.
-    let overhead = deterministic_tracker_overhead(1, 0);
+    let dmt_pages = dmt_write_charged_os_pages(1, 0);
 
     let instructions_used = instr_used(&mut instance);
     // Function is 1 instruction.
     assert_eq!(
         instructions_used,
-        1 + 5 * cc + cg + 2 * cs + cl + overhead * cd
+        1 + 5 * cc + cg + 2 * cs + cl + dmt_pages * cd
     );
-    // Now run the same with insufficient instructions
-    // We should still succeed (to avoid potentially failing pre-upgrades
-    // of canisters that did not adjust their code to new metering)
+    // Now run the same with insufficient instructions. We should still succeed
+    // (to avoid potentially failing pre-upgrades of canisters that did not
+    // adjust their code to new metering)
     let mut instance = new_instance(&wat, 100);
     instance.run(func_ref("test")).unwrap();
 }
 
 #[test]
-fn charge_for_dirty_heap() {
-    run_charge_for_dirty_heap(WasmMemoryType::Wasm32);
+fn charge_for_heap_pages() {
+    run_charge_for_heap_pages(WasmMemoryType::Wasm32);
 }
 
 #[test]
-fn charge_for_dirty_heap_wasm64() {
-    run_charge_for_dirty_heap(WasmMemoryType::Wasm64);
+fn charge_for_heap_pages_wasm64() {
+    run_charge_for_heap_pages(WasmMemoryType::Wasm64);
 }
 
-fn run_charge_for_dirty_stable64_test() {
+fn run_charge_for_stable64_pages_test() {
     let wat = r#"
         (module
             (import "ic0" "stable64_grow"
@@ -911,23 +912,23 @@ fn run_charge_for_dirty_stable64_test() {
 
     // Both i64.stores hit heap Wasm page 0; the first stable64_write hits stable
     // Wasm page 0 (bytes 0 and 4096 are both within the 64KB page).
-    let overhead = deterministic_tracker_overhead(1, 1);
+    let dmt_pages = dmt_write_charged_os_pages(1, 1);
 
     let instructions_used = instr_used(&mut instance);
-    // 2 dirty stable pages and one heap
+    // One heap and one stable Wasm page, each accessed and dirtied.
     assert_eq!(
         instructions_used,
         // Function is 1 instruction.
-        1 + cdrop + ccall * 4 + csg + cc * 15 + cs * 2 + csw * 2 + csr + cl + cg + overhead * cd
+        1 + cdrop + ccall * 4 + csg + cc * 15 + cs * 2 + csw * 2 + csr + cl + cg + dmt_pages * cd
     );
 }
 
 #[test]
-fn charge_for_dirty_stable64_native() {
-    run_charge_for_dirty_stable64_test();
+fn charge_for_stable64_pages_native() {
+    run_charge_for_stable64_pages_test();
 }
 
-fn run_charge_for_dirty_stable_test() {
+fn run_charge_for_stable_pages_test() {
     let wat = r#"
         (module
             (import "ic0" "stable_grow"
@@ -1009,20 +1010,20 @@ fn run_charge_for_dirty_stable_test() {
 
     // Both i32.stores hit heap Wasm page 0; the first stable_write hits stable
     // Wasm page 0 (bytes 0 and 4096 are both within the 64KB page).
-    let overhead = deterministic_tracker_overhead(1, 1);
+    let dmt_pages = dmt_write_charged_os_pages(1, 1);
 
     let instructions_used = instr_used(&mut instance);
-    // 2 dirty stable pages and one heap
+    // One heap and one stable Wasm page, each accessed and dirtied.
     assert_eq!(
         instructions_used,
         // Function is 1 instruction.
-        1 + cdrop + ccall * 4 + csg + cc * 15 + cs * 2 + csw * 2 + csr + cl + cg + overhead * cd
+        1 + cdrop + ccall * 4 + csg + cc * 15 + cs * 2 + csw * 2 + csr + cl + cg + dmt_pages * cd
     );
 }
 
 #[test]
-fn charge_for_dirty_stable_native() {
-    run_charge_for_dirty_stable_test();
+fn charge_for_stable_pages_native() {
+    run_charge_for_stable_pages_test();
 }
 
 /// Helper method to generate a wasm module with tables in both
@@ -1164,7 +1165,7 @@ fn metering_wasm64_load_store_canister() {
     let drop = instruction_to_cost(&wasmparser::Operator::Drop, WasmMemoryType::Wasm64);
     // Both stores hit Wasm page 0 (bytes 0 and 4096 are within the 64KB page),
     // so only one heap page-first-write event occurs.
-    let overhead = deterministic_tracker_overhead(1, 0);
+    let dmt_pages = dmt_write_charged_os_pages(1, 0);
     let total_cost = 1
         + 2 * const_0
         + const_17
@@ -1173,7 +1174,7 @@ fn metering_wasm64_load_store_canister() {
         + 2 * store
         + load
         + drop
-        + overhead * page_overhead.get();
+        + dmt_pages * page_overhead.get();
     assert_eq!(instr_used_wasm64, total_cost);
 
     // Compute cost in Wasm32 mode and compare.
@@ -1243,7 +1244,7 @@ fn metering_wasm64_load_store_canister() {
         + 2 * store_wasm32
         + load_wasm32
         + drop_wasm32
-        + overhead * page_overhead.get();
+        + dmt_pages * page_overhead.get();
     assert_eq!(wasm_32_instructions, total_cost_wasm32);
 
     // Check that the cost in Wasm64 mode is higher than in Wasm32 mode.

rs/embedders/tests/wasmtime_random_memory_writes.rs

@@ -50,9 +50,10 @@ const TEST_DEFAULT_LOG_MEMORY_LIMIT: usize = 4 * 1024; // 4 KiB
 const OS_PAGES_PER_WASM_PAGE: usize =
     ic_replicated_state::canister_state::WASM_PAGE_SIZE_IN_BYTES / ic_sys::PAGE_SIZE;
 
-/// Returns the per-Wasm-page instruction charge applied by the deterministic
-/// memory tracker.
-fn dsm_charge_per_wasm_page() -> u64 {
+/// Returns the number of *instructions* the deterministic memory tracker
+/// charges for a single Wasm page event (accessed or dirty): `page_overhead`
+/// per OS page covered by the Wasm page.
+fn dmt_instructions_per_wasm_page() -> u64 {
     OS_PAGES_PER_WASM_PAGE as u64 * DEFAULT_PAGE_OVERHEAD.get()
 }
 
@@ -838,8 +839,8 @@ mod tests {
                     + ic_embedders::wasmtime_embedder::system_api_complexity::overhead::STABLE_READ
                         .get()
                     + STABLE_OP_BYTES
-                    + dsm_charge_per_wasm_page()
-                    + dsm_charge_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
             );
         }
 
@@ -862,8 +863,8 @@ mod tests {
                     + ic_embedders::wasmtime_embedder::system_api_complexity::overhead::STABLE64_READ
                         .get()
                     + STABLE_OP_BYTES
-                    + dsm_charge_per_wasm_page()
-                    + dsm_charge_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
             );
         }
 
@@ -885,8 +886,8 @@ mod tests {
                     + ic_embedders::wasmtime_embedder::system_api_complexity::overhead::STABLE_READ
                         .get()
                     + STABLE_OP_BYTES
-                    + dsm_charge_per_wasm_page()
-                    + dsm_charge_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
             );
         }
 
@@ -909,8 +910,9 @@ mod tests {
                     + ic_embedders::wasmtime_embedder::system_api_complexity::overhead::STABLE_WRITE
                         .get()
                     + STABLE_OP_BYTES
-                    + dsm_charge_per_wasm_page()
-                    + dsm_charge_per_wasm_page() + dsm_charge_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
             );
         }
 
@@ -932,8 +934,9 @@ mod tests {
                     + ic_embedders::wasmtime_embedder::system_api_complexity::overhead::STABLE_WRITE
                         .get()
                     + STABLE_OP_BYTES
-                    + dsm_charge_per_wasm_page()
-                    + dsm_charge_per_wasm_page() + dsm_charge_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
             );
         }
 
@@ -956,8 +959,9 @@ mod tests {
                     + ic_embedders::wasmtime_embedder::system_api_complexity::overhead::STABLE_WRITE
                         .get()
                     + STABLE_OP_BYTES
-                    + dsm_charge_per_wasm_page()
-                    + dsm_charge_per_wasm_page() + dsm_charge_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
             );
         }
 
@@ -979,8 +983,9 @@ mod tests {
                     + ic_embedders::wasmtime_embedder::system_api_complexity::overhead::STABLE_WRITE
                         .get()
                     + STABLE_OP_BYTES
-                    + dsm_charge_per_wasm_page()
-                    + dsm_charge_per_wasm_page() + dsm_charge_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
+                    + dmt_instructions_per_wasm_page()
             );
         }
     }
@@ -1005,10 +1010,12 @@ mod tests {
                 // The deterministic memory tracker charges at 64 KiB Wasm page
                 // granularity. Use InstanceStats to get the actual Wasm page
                 // counts and compute the exact tracker charge.
+                let per_page = dmt_instructions_per_wasm_page();
                 let tracker_charge =
                     |stats: &ic_interfaces::execution_environment::InstanceStats| -> u64 {
-                        stats.wasm_accessed_wasm_pages_count as u64 * dsm_charge_per_wasm_page()
-                            + stats.wasm_dirty_wasm_pages_count as u64 * dsm_charge_per_wasm_page()
+                        (stats.wasm_accessed_wasm_pages_count as u64
+                            + stats.wasm_dirty_wasm_pages_count as u64)
+                            * per_page
                     };
 
                 let (instructions_consumed_without_data, _dry_run_stats, dry_run_instance_stats) =

rs/execution_environment/EXECUTION_COST.md

@@ -26,21 +26,6 @@ by `DEFAULT_COST_TO_COMPILE_WASM_INSTRUCTION` and other compilation limits.
 
 TODO(EXC-2040): There is no script to derive the cost based on benchmark results.
 
-Heap Memory Overhead
---------------------
-
-Each Wasm heap memory page has an associated overhead defined
-by `DEFAULT_PAGE_OVERHEAD` and other costs.
-
-1. ✅ Runs daily on CI.
-2. ✅ Results are available in [Grafana](https://grafana.mainnet.dfinity.network/d/benchmarks-embedders-heap/benchmarks3a-embedders-heap).
-3. ✅ Raw benchmark:
-    * `bazel run //rs/embedders:heap_bench`
-4. ✅ Baseline comparison:
-    * `INCLUDE=heap ./rs/execution_environment/benches/run-all-benchmarks.sh`
-
-TODO(EXC-2040): There is no script to derive the cost based on benchmark results.
-
 Management Canister Calls
 -------------------------
 
@@ -60,18 +45,24 @@ Scripts: `rs/execution_environment/benches/management_canister/*`
 
 1. Run `run_snapshot_benchmarks_forever.sh` to generate the `MANAGEMENT_CANISTER.md` file.
 
-Stable Memory Overhead
-----------------------
+Memory Overhead
+---------------
 
-Each Wasm stable memory page has an associated overhead defined
-by `DEFAULT_PAGE_OVERHEAD` and other costs.
+Heap and stable memory pages share the same overhead, defined by
+`DEFAULT_PAGE_OVERHEAD` and other costs. It is charged per OS page by the
+deterministic memory tracker, once when a page is first accessed and once more
+when it is first dirtied. The two memories are benchmarked separately.
 
 1. ✅ Runs daily on CI.
-2. ✅ Results are available in [Grafana](https://grafana.mainnet.dfinity.network/d/benchmarks-embedders-stable-memory/benchmarks3a-embedders-stable-memory).
-3. ✅ Raw benchmark:
-    * `bazel run //rs/embedders:stable_memory_bench`
+2. ✅ Results are available in Grafana for:
+     * [heap](https://grafana.mainnet.dfinity.network/d/benchmarks-embedders-heap/benchmarks3a-embedders-heap)
+     * [stable memory](https://grafana.mainnet.dfinity.network/d/benchmarks-embedders-stable-memory/benchmarks3a-embedders-stable-memory)
+3. ✅ Raw benchmarks:
+    * Heap: `bazel run //rs/embedders:heap_bench`
+    * Stable memory: `bazel run //rs/embedders:stable_memory_bench`
 4. ✅ Baseline comparison:
-    * `INCLUDE=stable ./rs/execution_environment/benches/run-all-benchmarks.sh`
+    * Heap: `INCLUDE=heap ./rs/execution_environment/benches/run-all-benchmarks.sh`
+    * Stable memory: `INCLUDE=stable ./rs/execution_environment/benches/run-all-benchmarks.sh`
 
 TODO(EXC-2040): There is no script to derive the cost based on benchmark results.
 

rs/execution_environment/benches/lib/src/common.rs

@@ -70,28 +70,27 @@ lazy_static! {
         );
 }
 
-/// Returns the extra instruction overhead charged by the deterministic memory
-/// tracker for `n_wasm_pages` Wasm pages first accessed without dirty tracking
-/// (e.g. read-only accesses or non-replicated execution).
+/// Returns the number of *instructions* the deterministic memory tracker
+/// charges for first accessing `n_wasm_pages` Wasm pages without dirty tracking
+/// (read-only accesses or non-replicated execution).
 ///
 /// Each first-accessed Wasm page (64 KiB) triggers `mark_wasm_page_accessed`,
-/// which charges `page_overhead` instructions per OS page in-band via the
-/// SIGSEGV handler.  The number of OS pages per Wasm page varies by platform
+/// which charges `DEFAULT_PAGE_OVERHEAD` instructions per OS page in-band via
+/// the SIGSEGV handler. The number of OS pages per Wasm page varies by platform
 /// (4 KiB pages on Linux, 16 KiB on arm64-darwin).
-pub fn deterministic_tracker_overhead(n_wasm_pages: u64) -> u64 {
+pub fn dmt_access_instructions(n_wasm_pages: u64) -> u64 {
     const WASM_PAGE_SIZE: u64 = 65536;
     n_wasm_pages * (WASM_PAGE_SIZE / ic_sys::PAGE_SIZE as u64) * DEFAULT_PAGE_OVERHEAD.get()
 }
 
-/// Returns the extra instruction overhead charged by the deterministic memory
-/// tracker for `n_wasm_pages` Wasm heap pages first written in replicated
-/// execution (DirtyPageTracking::Track).
+/// Returns the number of *instructions* the deterministic memory tracker
+/// charges for first writing `n_wasm_pages` Wasm pages with dirty tracking
+/// (replicated execution, `DirtyPageTracking::Track`).
 ///
-/// Each first-written Wasm page triggers both `mark_wasm_page_accessed` and
-/// `mark_wasm_page_dirty` via the SIGSEGV handler, charging `page_overhead`
-/// instructions twice per OS page.
-pub fn deterministic_tracker_write_overhead(n_wasm_pages: u64) -> u64 {
-    2 * deterministic_tracker_overhead(n_wasm_pages)
+/// Each such page triggers both `mark_wasm_page_accessed` and
+/// `mark_wasm_page_dirty`, i.e. it is charged twice per OS page.
+pub fn dmt_write_instructions(n_wasm_pages: u64) -> u64 {
+    2 * dmt_access_instructions(n_wasm_pages)
 }
 
 /// Pieces needed to execute a benchmark.